From fad0eb091e58f345b922a49335c60bbeae6d5c6f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 13 Apr 2025 16:25:51 -0400 Subject: [PATCH 1/6] txgraph: reset quality when merging clusters (bugfix) --- src/txgraph.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/txgraph.cpp b/src/txgraph.cpp index 1b29a05e53..73676f66d0 100644 --- a/src/txgraph.cpp +++ b/src/txgraph.cpp @@ -1108,6 +1108,11 @@ void Cluster::ApplyDependencies(TxGraphImpl& graph, std::span Date: Sun, 13 Apr 2025 20:52:03 -0400 Subject: [PATCH 2/6] txgraph: 1-or-2-tx split-off clusters are optimal (optimization) --- src/txgraph.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/txgraph.cpp b/src/txgraph.cpp index 73676f66d0..247c264260 100644 --- a/src/txgraph.cpp +++ b/src/txgraph.cpp @@ -978,10 +978,11 @@ bool Cluster::Split(TxGraphImpl& graph) noexcept // Iterate over the connected components of this Cluster's m_depgraph. while (todo.Any()) { auto component = m_depgraph.FindConnectedComponent(todo); + auto split_quality = component.Count() <= 2 ? QualityLevel::OPTIMAL : new_quality; if (first && component == todo) { // The existing Cluster is an entire component. Leave it be, but update its quality. Assume(todo == m_depgraph.Positions()); - graph.SetClusterQuality(m_level, m_quality, m_setindex, new_quality); + graph.SetClusterQuality(m_level, m_quality, m_setindex, split_quality); // If this made the quality ACCEPTABLE or OPTIMAL, we need to compute and cache its // chunking. Updated(graph); @@ -996,7 +997,7 @@ bool Cluster::Split(TxGraphImpl& graph) noexcept for (auto i : component) { remap[i] = {new_cluster.get(), DepGraphIndex(-1)}; } - graph.InsertCluster(m_level, std::move(new_cluster), new_quality); + graph.InsertCluster(m_level, std::move(new_cluster), split_quality); todo -= component; } // Redistribute the transactions. From cfe9958852be0e0763c924bdbadc37e784f5aee5 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 13 Apr 2025 11:09:04 -0400 Subject: [PATCH 3/6] txgraph: track amount of work done in linearization (preparation) --- src/bench/cluster_linearize.cpp | 4 ++-- src/cluster_linearize.h | 9 +++++---- src/test/fuzz/cluster_linearize.cpp | 5 +++-- src/txgraph.cpp | 11 ++++++----- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/bench/cluster_linearize.cpp b/src/bench/cluster_linearize.cpp index cb06f3fc28..1cd36b5393 100644 --- a/src/bench/cluster_linearize.cpp +++ b/src/bench/cluster_linearize.cpp @@ -229,8 +229,8 @@ void BenchLinearizeOptimally(benchmark::Bench& bench, const std::array> Using(depgraph); uint64_t rng_seed = 0; bench.run([&] { - auto res = Linearize(depgraph, /*max_iterations=*/10000000, rng_seed++); - assert(res.second); + auto [_lin, optimal, _cost] = Linearize(depgraph, /*max_iterations=*/10000000, rng_seed++); + assert(optimal); }); }; diff --git a/src/cluster_linearize.h b/src/cluster_linearize.h index 17b7e6b27a..bec44d973e 100644 --- a/src/cluster_linearize.h +++ b/src/cluster_linearize.h @@ -1030,19 +1030,20 @@ public: * linearize. * @param[in] old_linearization An existing linearization for the cluster (which must be * topologically valid), or empty. - * @return A pair of: + * @return A tuple of: * - The resulting linearization. It is guaranteed to be at least as * good (in the feerate diagram sense) as old_linearization. * - A boolean indicating whether the result is guaranteed to be * optimal. + * - How many optimization steps were actually performed. * * Complexity: possibly O(N * min(max_iterations + N, sqrt(2^N))) where N=depgraph.TxCount(). */ template -std::pair, bool> Linearize(const DepGraph& depgraph, uint64_t max_iterations, uint64_t rng_seed, std::span old_linearization = {}) noexcept +std::tuple, bool, uint64_t> Linearize(const DepGraph& depgraph, uint64_t max_iterations, uint64_t rng_seed, std::span old_linearization = {}) noexcept { Assume(old_linearization.empty() || old_linearization.size() == depgraph.TxCount()); - if (depgraph.TxCount() == 0) return {{}, true}; + if (depgraph.TxCount() == 0) return {{}, true, 0}; uint64_t iterations_left = max_iterations; std::vector linearization; @@ -1113,7 +1114,7 @@ std::pair, bool> Linearize(const DepGraph& d } } - return {std::move(linearization), optimal}; + return {std::move(linearization), optimal, max_iterations - iterations_left}; } /** Improve a given linearization. diff --git a/src/test/fuzz/cluster_linearize.cpp b/src/test/fuzz/cluster_linearize.cpp index ef20eeaea0..4cd93e2079 100644 --- a/src/test/fuzz/cluster_linearize.cpp +++ b/src/test/fuzz/cluster_linearize.cpp @@ -1154,7 +1154,8 @@ FUZZ_TARGET(clusterlin_linearize) // Invoke Linearize(). iter_count &= 0x7ffff; - auto [linearization, optimal] = Linearize(depgraph, iter_count, rng_seed, old_linearization); + auto [linearization, optimal, cost] = Linearize(depgraph, iter_count, rng_seed, old_linearization); + assert(cost <= iter_count); SanityCheck(depgraph, linearization); auto chunking = ChunkLinearization(depgraph, linearization); @@ -1322,7 +1323,7 @@ FUZZ_TARGET(clusterlin_postlinearize_tree) // Try to find an even better linearization directly. This must not change the diagram for the // same reason. - auto [opt_linearization, _optimal] = Linearize(depgraph_tree, 100000, rng_seed, post_linearization); + auto [opt_linearization, _optimal, _cost] = Linearize(depgraph_tree, 100000, rng_seed, post_linearization); auto opt_chunking = ChunkLinearization(depgraph_tree, opt_linearization); auto cmp_opt = CompareChunks(opt_chunking, post_chunking); assert(cmp_opt == 0); diff --git a/src/txgraph.cpp b/src/txgraph.cpp index 247c264260..8decb5d009 100644 --- a/src/txgraph.cpp +++ b/src/txgraph.cpp @@ -189,8 +189,8 @@ public: void Merge(TxGraphImpl& graph, Cluster& cluster) noexcept; /** Given a span of (parent, child) pairs that all belong to this Cluster, apply them. */ void ApplyDependencies(TxGraphImpl& graph, std::span> to_apply) noexcept; - /** Improve the linearization of this Cluster. */ - void Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept; + /** Improve the linearization of this Cluster. Returns how much work was performed. */ + uint64_t Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept; /** For every chunk in the cluster, append its FeeFrac to ret. */ void AppendChunkFeerates(std::vector& ret) const noexcept; /** Add a TrimTxData entry (filling m_chunk_feerate, m_index, m_tx_size) for every @@ -1651,15 +1651,15 @@ void TxGraphImpl::ApplyDependencies(int level) noexcept clusterset.m_group_data = GroupData{}; } -void Cluster::Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept +uint64_t Cluster::Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept { // We can only relinearize Clusters that do not need splitting. Assume(!NeedsSplitting()); // No work is required for Clusters which are already optimally linearized. - if (IsOptimal()) return; + if (IsOptimal()) return 0; // Invoke the actual linearization algorithm (passing in the existing one). uint64_t rng_seed = graph.m_rng.rand64(); - auto [linearization, optimal] = Linearize(m_depgraph, max_iters, rng_seed, m_linearization); + auto [linearization, optimal, cost] = Linearize(m_depgraph, max_iters, rng_seed, m_linearization); // Postlinearize if the result isn't optimal already. This guarantees (among other things) // that the chunks of the resulting linearization are all connected. if (!optimal) PostLinearize(m_depgraph, linearization); @@ -1670,6 +1670,7 @@ void Cluster::Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept graph.SetClusterQuality(m_level, m_quality, m_setindex, new_quality); // Update the Entry objects. Updated(graph); + return cost; } void TxGraphImpl::MakeAcceptable(Cluster& cluster) noexcept From e96b00d99ebe27eeadba88841db32b2b8e741433 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 13 Apr 2025 11:16:26 -0400 Subject: [PATCH 4/6] txgraph: make number of acceptable iterations configurable (feature) --- src/bench/txgraph.cpp | 5 ++++- src/test/fuzz/txgraph.cpp | 4 +++- src/test/txgraph_tests.cpp | 12 ++++++++---- src/txgraph.cpp | 12 ++++++++---- src/txgraph.h | 5 +++-- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/bench/txgraph.cpp b/src/bench/txgraph.cpp index 2504d02bc3..5303386baa 100644 --- a/src/bench/txgraph.cpp +++ b/src/bench/txgraph.cpp @@ -46,6 +46,9 @@ void BenchTxGraphTrim(benchmark::Bench& bench) static constexpr int NUM_DEPS_PER_BOTTOM_TX = 100; /** Set a very large cluster size limit so that only the count limit is triggered. */ static constexpr int32_t MAX_CLUSTER_SIZE = 100'000 * 100; + /** Set a very high number for acceptable iterations, so that we certainly benchmark optimal + * linearization. */ + static constexpr uint64_t NUM_ACCEPTABLE_ITERS = 100'000'000; /** Refs to all top transactions. */ std::vector top_refs; @@ -57,7 +60,7 @@ void BenchTxGraphTrim(benchmark::Bench& bench) std::vector top_components; InsecureRandomContext rng(11); - auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE); + auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, NUM_ACCEPTABLE_ITERS); // Construct the top chains. for (int chain = 0; chain < NUM_TOP_CHAINS; ++chain) { diff --git a/src/test/fuzz/txgraph.cpp b/src/test/fuzz/txgraph.cpp index 561bf0b420..7e4409df6b 100644 --- a/src/test/fuzz/txgraph.cpp +++ b/src/test/fuzz/txgraph.cpp @@ -309,9 +309,11 @@ FUZZ_TARGET(txgraph) auto max_cluster_count = provider.ConsumeIntegralInRange(1, MAX_CLUSTER_COUNT_LIMIT); /** The maximum total size of transactions in a (non-oversized) cluster. */ auto max_cluster_size = provider.ConsumeIntegralInRange(1, 0x3fffff * MAX_CLUSTER_COUNT_LIMIT); + /** The number of iterations to consider a cluster acceptably linearized. */ + auto acceptable_iters = provider.ConsumeIntegralInRange(0, 10000); // Construct a real graph, and a vector of simulated graphs (main, and possibly staging). - auto real = MakeTxGraph(max_cluster_count, max_cluster_size); + auto real = MakeTxGraph(max_cluster_count, max_cluster_size, acceptable_iters); std::vector sims; sims.reserve(2); sims.emplace_back(max_cluster_count, max_cluster_size); diff --git a/src/test/txgraph_tests.cpp b/src/test/txgraph_tests.cpp index 75abfba7dc..63741e6cd7 100644 --- a/src/test/txgraph_tests.cpp +++ b/src/test/txgraph_tests.cpp @@ -13,6 +13,10 @@ BOOST_AUTO_TEST_SUITE(txgraph_tests) +/** The number used as acceptable_iters argument in these tests. High enough that everything + * should be optimal, always. */ +static constexpr uint64_t NUM_ACCEPTABLE_ITERS = 100'000'000; + BOOST_AUTO_TEST_CASE(txgraph_trim_zigzag) { // T T T T T T T T T T T T T T (50 T's) @@ -35,7 +39,7 @@ BOOST_AUTO_TEST_CASE(txgraph_trim_zigzag) static constexpr int32_t MAX_CLUSTER_SIZE = 100'000 * 100; // Create a new graph for the test. - auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE); + auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, NUM_ACCEPTABLE_ITERS); // Add all transactions and store their Refs. std::vector refs; @@ -98,7 +102,7 @@ BOOST_AUTO_TEST_CASE(txgraph_trim_flower) /** Set a very large cluster size limit so that only the count limit is triggered. */ static constexpr int32_t MAX_CLUSTER_SIZE = 100'000 * 100; - auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE); + auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, NUM_ACCEPTABLE_ITERS); // Add all transactions and store their Refs. std::vector refs; @@ -184,7 +188,7 @@ BOOST_AUTO_TEST_CASE(txgraph_trim_huge) std::vector top_components; FastRandomContext rng; - auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE); + auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, NUM_ACCEPTABLE_ITERS); // Construct the top chains. for (int chain = 0; chain < NUM_TOP_CHAINS; ++chain) { @@ -256,7 +260,7 @@ BOOST_AUTO_TEST_CASE(txgraph_trim_big_singletons) static constexpr int NUM_TOTAL_TX = 100; // Create a new graph for the test. - auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE); + auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, NUM_ACCEPTABLE_ITERS); // Add all transactions and store their Refs. std::vector refs; diff --git a/src/txgraph.cpp b/src/txgraph.cpp index 8decb5d009..6615bca50d 100644 --- a/src/txgraph.cpp +++ b/src/txgraph.cpp @@ -255,6 +255,9 @@ private: const DepGraphIndex m_max_cluster_count; /** This TxGraphImpl's maximum cluster size limit. */ const uint64_t m_max_cluster_size; + /** The number of linearization improvement steps needed per cluster to be considered + * acceptable. */ + const uint64_t m_acceptable_iters; /** Information about one group of Clusters to be merged. */ struct GroupEntry @@ -456,9 +459,10 @@ private: public: /** Construct a new TxGraphImpl with the specified limits. */ - explicit TxGraphImpl(DepGraphIndex max_cluster_count, uint64_t max_cluster_size) noexcept : + explicit TxGraphImpl(DepGraphIndex max_cluster_count, uint64_t max_cluster_size, uint64_t acceptable_iters) noexcept : m_max_cluster_count(max_cluster_count), m_max_cluster_size(max_cluster_size), + m_acceptable_iters(acceptable_iters), m_main_chunkindex(ChunkOrder(this)) { Assume(max_cluster_count >= 1); @@ -1677,7 +1681,7 @@ void TxGraphImpl::MakeAcceptable(Cluster& cluster) noexcept { // Relinearize the Cluster if needed. if (!cluster.NeedsSplitting() && !cluster.IsAcceptable() && !cluster.IsOversized()) { - cluster.Relinearize(*this, 10000); + cluster.Relinearize(*this, m_acceptable_iters); } } @@ -2892,7 +2896,7 @@ TxGraph::Ref::Ref(Ref&& other) noexcept std::swap(m_index, other.m_index); } -std::unique_ptr MakeTxGraph(unsigned max_cluster_count, uint64_t max_cluster_size) noexcept +std::unique_ptr MakeTxGraph(unsigned max_cluster_count, uint64_t max_cluster_size, uint64_t acceptable_iters) noexcept { - return std::make_unique(max_cluster_count, max_cluster_size); + return std::make_unique(max_cluster_count, max_cluster_size, acceptable_iters); } diff --git a/src/txgraph.h b/src/txgraph.h index 2c40cd28d2..4bddf95b86 100644 --- a/src/txgraph.h +++ b/src/txgraph.h @@ -247,7 +247,8 @@ public: /** Construct a new TxGraph with the specified limit on the number of transactions within a cluster, * and on the sum of transaction sizes within a cluster. max_cluster_count cannot exceed - * MAX_CLUSTER_COUNT_LIMIT. */ -std::unique_ptr MakeTxGraph(unsigned max_cluster_count, uint64_t max_cluster_size) noexcept; + * MAX_CLUSTER_COUNT_LIMIT. acceptable_iters controls how many linearization optimization + * steps will be performed per cluster before they are considered to be of acceptable quality. */ +std::unique_ptr MakeTxGraph(unsigned max_cluster_count, uint64_t max_cluster_size, uint64_t acceptable_iters) noexcept; #endif // BITCOIN_TXGRAPH_H From f3c2fc867fc4332dfed0a3766997433e1676dbe3 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 13 Apr 2025 17:16:27 -0400 Subject: [PATCH 5/6] txgraph: add work limit to DoWork(), try optimal (feature) This adds an `iters` parameter to DoWork(), which controls how much work it is allowed to do right now. Additionally, DoWork() won't stop at just getting everything ACCEPTABLE, but if there is work budget left, will also attempt to get every cluster linearized optimally. --- src/test/fuzz/txgraph.cpp | 31 +++++++++++++++++- src/txgraph.cpp | 68 ++++++++++++++++++++++++++++++++------- src/txgraph.h | 7 ++-- 3 files changed, 90 insertions(+), 16 deletions(-) diff --git a/src/test/fuzz/txgraph.cpp b/src/test/fuzz/txgraph.cpp index 7e4409df6b..d9629b11a4 100644 --- a/src/test/fuzz/txgraph.cpp +++ b/src/test/fuzz/txgraph.cpp @@ -58,6 +58,8 @@ struct SimTxGraph SetType modified; /** The configured maximum total size of transactions per cluster. */ uint64_t max_cluster_size; + /** Whether the corresponding real graph is known to be optimally linearized. */ + bool real_is_optimal{false}; /** Construct a new SimTxGraph with the specified maximum cluster count and size. */ explicit SimTxGraph(DepGraphIndex cluster_count, uint64_t cluster_size) : @@ -139,6 +141,7 @@ struct SimTxGraph { assert(graph.TxCount() < MAX_TRANSACTIONS); auto simpos = graph.AddTransaction(feerate); + real_is_optimal = false; MakeModified(simpos); assert(graph.Positions()[simpos]); simmap[simpos] = std::make_shared(); @@ -158,6 +161,7 @@ struct SimTxGraph if (chl_pos == MISSING) return; graph.AddDependencies(SetType::Singleton(par_pos), chl_pos); MakeModified(par_pos); + real_is_optimal = false; // This may invalidate our cached oversized value. if (oversized.has_value() && !*oversized) oversized = std::nullopt; } @@ -168,6 +172,7 @@ struct SimTxGraph auto pos = Find(ref); if (pos == MISSING) return; // No need to invoke MakeModified, because this equally affects main and staging. + real_is_optimal = false; graph.FeeRate(pos).fee = fee; } @@ -177,6 +182,7 @@ struct SimTxGraph auto pos = Find(ref); if (pos == MISSING) return; MakeModified(pos); + real_is_optimal = false; graph.RemoveTransactions(SetType::Singleton(pos)); simrevmap.erase(simmap[pos].get()); // Retain the TxGraph::Ref corresponding to this position, so the Ref destruction isn't @@ -203,6 +209,7 @@ struct SimTxGraph } else { MakeModified(pos); graph.RemoveTransactions(SetType::Singleton(pos)); + real_is_optimal = false; simrevmap.erase(simmap[pos].get()); simmap[pos].reset(); // This may invalidate our cached oversized value. @@ -467,6 +474,7 @@ FUZZ_TARGET(txgraph) if (top_sim.graph.Ancestors(pos_par)[pos_chl]) break; } top_sim.AddDependency(par, chl); + top_sim.real_is_optimal = false; real->AddDependency(*par, *chl); break; } else if ((block_builders.empty() || sims.size() > 1) && top_sim.removed.size() < 100 && command-- == 0) { @@ -721,7 +729,18 @@ FUZZ_TARGET(txgraph) break; } else if (command-- == 0) { // DoWork. - real->DoWork(); + uint64_t iters = provider.ConsumeIntegralInRange(0, alt ? 10000 : 255); + if (real->DoWork(iters)) { + for (unsigned level = 0; level < sims.size(); ++level) { + // DoWork() will not optimize oversized levels. + if (sims[level].IsOversized()) continue; + // DoWork() will not touch the main level if a builder is present. + if (level == 0 && !block_builders.empty()) continue; + // If neither of the two above conditions holds, and DoWork() returned + // then the level is optimal. + sims[level].real_is_optimal = true; + } + } break; } else if (sims.size() == 2 && !sims[0].IsOversized() && !sims[1].IsOversized() && command-- == 0) { // GetMainStagingDiagrams() @@ -1005,6 +1024,16 @@ FUZZ_TARGET(txgraph) } assert(todo.None()); + // If the real graph claims to be optimal (the last DoWork() call returned true), verify + // that calling Linearize on it does not improve it further. + if (sims[0].real_is_optimal) { + auto real_diagram = ChunkLinearization(sims[0].graph, vec1); + auto [sim_lin, _optimal, _cost] = Linearize(sims[0].graph, 300000, rng.rand64(), vec1); + auto sim_diagram = ChunkLinearization(sims[0].graph, sim_lin); + auto cmp = CompareChunks(real_diagram, sim_diagram); + assert(cmp == 0); + } + // For every transaction in the total ordering, find a random one before it and after it, // and compare their chunk feerates, which must be consistent with the ordering. for (size_t pos = 0; pos < vec1.size(); ++pos) { diff --git a/src/txgraph.cpp b/src/txgraph.cpp index 6615bca50d..345be02f0f 100644 --- a/src/txgraph.cpp +++ b/src/txgraph.cpp @@ -189,8 +189,9 @@ public: void Merge(TxGraphImpl& graph, Cluster& cluster) noexcept; /** Given a span of (parent, child) pairs that all belong to this Cluster, apply them. */ void ApplyDependencies(TxGraphImpl& graph, std::span> to_apply) noexcept; - /** Improve the linearization of this Cluster. Returns how much work was performed. */ - uint64_t Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept; + /** Improve the linearization of this Cluster. Returns how much work was performed and whether + * the Cluster's QualityLevel improved as a result. */ + std::pair Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept; /** For every chunk in the cluster, append its FeeFrac to ret. */ void AppendChunkFeerates(std::vector& ret) const noexcept; /** Add a TrimTxData entry (filling m_chunk_feerate, m_index, m_tx_size) for every @@ -592,7 +593,7 @@ public: void AddDependency(const Ref& parent, const Ref& child) noexcept final; void SetTransactionFee(const Ref&, int64_t fee) noexcept final; - void DoWork() noexcept final; + bool DoWork(uint64_t iters) noexcept final; void StartStaging() noexcept final; void CommitStaging() noexcept final; @@ -1655,12 +1656,12 @@ void TxGraphImpl::ApplyDependencies(int level) noexcept clusterset.m_group_data = GroupData{}; } -uint64_t Cluster::Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept +std::pair Cluster::Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept { // We can only relinearize Clusters that do not need splitting. Assume(!NeedsSplitting()); // No work is required for Clusters which are already optimally linearized. - if (IsOptimal()) return 0; + if (IsOptimal()) return {0, false}; // Invoke the actual linearization algorithm (passing in the existing one). uint64_t rng_seed = graph.m_rng.rand64(); auto [linearization, optimal, cost] = Linearize(m_depgraph, max_iters, rng_seed, m_linearization); @@ -1670,11 +1671,17 @@ uint64_t Cluster::Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept // Update the linearization. m_linearization = std::move(linearization); // Update the Cluster's quality. - auto new_quality = optimal ? QualityLevel::OPTIMAL : QualityLevel::ACCEPTABLE; - graph.SetClusterQuality(m_level, m_quality, m_setindex, new_quality); + bool improved = false; + if (optimal) { + graph.SetClusterQuality(m_level, m_quality, m_setindex, QualityLevel::OPTIMAL); + improved = true; + } else if (max_iters >= graph.m_acceptable_iters && !IsAcceptable()) { + graph.SetClusterQuality(m_level, m_quality, m_setindex, QualityLevel::ACCEPTABLE); + improved = true; + } // Update the Entry objects. Updated(graph); - return cost; + return {cost, improved}; } void TxGraphImpl::MakeAcceptable(Cluster& cluster) noexcept @@ -2478,13 +2485,50 @@ void TxGraphImpl::SanityCheck() const assert(actual_chunkindex == expected_chunkindex); } -void TxGraphImpl::DoWork() noexcept +bool TxGraphImpl::DoWork(uint64_t iters) noexcept { - for (int level = 0; level <= GetTopLevel(); ++level) { - if (level > 0 || m_main_chunkindex_observers == 0) { - MakeAllAcceptable(level); + uint64_t iters_done{0}; + // First linearize everything in NEEDS_RELINEARIZE to an acceptable level. If more budget + // remains after that, try to make everything optimal. + for (QualityLevel quality : {QualityLevel::NEEDS_RELINEARIZE, QualityLevel::ACCEPTABLE}) { + // First linearize staging, if it exists, then main. + for (int level = GetTopLevel(); level >= 0; --level) { + // Do not modify main if it has any observers. + if (level == 0 && m_main_chunkindex_observers != 0) continue; + ApplyDependencies(level); + auto& clusterset = GetClusterSet(level); + // Do not modify oversized levels. + if (clusterset.m_oversized == true) continue; + auto& queue = clusterset.m_clusters[int(quality)]; + while (!queue.empty()) { + if (iters_done >= iters) return false; + // Randomize the order in which we process, so that if the first cluster somehow + // needs more work than what iters allows, we don't keep spending it on the same + // one. + auto pos = m_rng.randrange(queue.size()); + auto iters_now = iters - iters_done; + if (quality == QualityLevel::NEEDS_RELINEARIZE) { + // If we're working with clusters that need relinearization still, only perform + // up to m_acceptable_iters iterations. If they become ACCEPTABLE, and we still + // have budget after all other clusters are ACCEPTABLE too, we'll spend the + // remaining budget on trying to make them OPTIMAL. + iters_now = std::min(iters_now, m_acceptable_iters); + } + auto [cost, improved] = queue[pos].get()->Relinearize(*this, iters_now); + iters_done += cost; + // If no improvement was made to the Cluster, it means we've essentially run out of + // budget. Even though it may be the case that iters_done < iters still, the + // linearizer decided there wasn't enough budget left to attempt anything with. + // To avoid an infinite loop that keeps trying clusters with minuscule budgets, + // stop here too. + if (!improved) return false; + } } } + // All possible work has been performed, so we can return true. Note that this does *not* mean + // that all clusters are optimally linearized now. It may be that there is nothing to do left + // because all non-optimal clusters are in oversized and/or observer-bearing levels. + return true; } void BlockBuilderImpl::Next() noexcept diff --git a/src/txgraph.h b/src/txgraph.h index 4bddf95b86..ef2a7bd3c0 100644 --- a/src/txgraph.h +++ b/src/txgraph.h @@ -94,9 +94,10 @@ public: virtual void SetTransactionFee(const Ref& arg, int64_t fee) noexcept = 0; /** TxGraph is internally lazy, and will not compute many things until they are needed. - * Calling DoWork will compute everything now, so that future operations are fast. This can be - * invoked while oversized. */ - virtual void DoWork() noexcept = 0; + * Calling DoWork will perform some work now (controlled by iters) so that future operations + * are fast, if there is any. Returns whether all currently-available work is done. This can + * be invoked while oversized, but oversized graphs will be skipped by this call. */ + virtual bool DoWork(uint64_t iters) noexcept = 0; /** Create a staging graph (which cannot exist already). This acts as if a full copy of * the transaction graph is made, upon which further modifications are made. This copy can From 62ed1f92efff42bc79c50935e6dbd9da4e072020 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 13 Jul 2025 13:32:35 -0400 Subject: [PATCH 6/6] txgraph: check that DoWork finds optimal if given high budget (tests) --- src/test/fuzz/cluster_linearize.cpp | 17 +------------ src/test/fuzz/txgraph.cpp | 39 +++++++++++++++++++++++------ src/test/util/cluster_linearize.h | 23 +++++++++++++++++ 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/src/test/fuzz/cluster_linearize.cpp b/src/test/fuzz/cluster_linearize.cpp index 4cd93e2079..9e8dc2f43d 100644 --- a/src/test/fuzz/cluster_linearize.cpp +++ b/src/test/fuzz/cluster_linearize.cpp @@ -1167,24 +1167,9 @@ FUZZ_TARGET(clusterlin_linearize) } // If the iteration count is sufficiently high, an optimal linearization must be found. - // Each linearization step can use up to 2^(k-1) iterations, with steps k=1..n. That sum is - // 2^n - 1. - const uint64_t n = depgraph.TxCount(); - if (n <= 19 && iter_count > (uint64_t{1} << n)) { + if (iter_count >= MaxOptimalLinearizationIters(depgraph.TxCount())) { assert(optimal); } - // Additionally, if the assumption of sqrt(2^k)+1 iterations per step holds, plus ceil(k/4) - // start-up cost per step, plus ceil(n^2/64) start-up cost overall, we can compute the upper - // bound for a whole linearization (summing for k=1..n) using the Python expression - // [sum((k+3)//4 + int(math.sqrt(2**k)) + 1 for k in range(1, n + 1)) + (n**2 + 63) // 64 for n in range(0, 35)]: - static constexpr uint64_t MAX_OPTIMAL_ITERS[] = { - 0, 4, 8, 12, 18, 26, 37, 51, 70, 97, 133, 182, 251, 346, 480, 666, 927, 1296, 1815, 2545, - 3576, 5031, 7087, 9991, 14094, 19895, 28096, 39690, 56083, 79263, 112041, 158391, 223936, - 316629, 447712 - }; - if (n < std::size(MAX_OPTIMAL_ITERS) && iter_count >= MAX_OPTIMAL_ITERS[n]) { - Assume(optimal); - } // If Linearize claims optimal result, run quality tests. if (optimal) { diff --git a/src/test/fuzz/txgraph.cpp b/src/test/fuzz/txgraph.cpp index d9629b11a4..e48d985294 100644 --- a/src/test/fuzz/txgraph.cpp +++ b/src/test/fuzz/txgraph.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -730,16 +731,38 @@ FUZZ_TARGET(txgraph) } else if (command-- == 0) { // DoWork. uint64_t iters = provider.ConsumeIntegralInRange(0, alt ? 10000 : 255); - if (real->DoWork(iters)) { - for (unsigned level = 0; level < sims.size(); ++level) { - // DoWork() will not optimize oversized levels. - if (sims[level].IsOversized()) continue; - // DoWork() will not touch the main level if a builder is present. - if (level == 0 && !block_builders.empty()) continue; - // If neither of the two above conditions holds, and DoWork() returned - // then the level is optimal. + bool ret = real->DoWork(iters); + uint64_t iters_for_optimal{0}; + for (unsigned level = 0; level < sims.size(); ++level) { + // DoWork() will not optimize oversized levels, or the main level if a builder + // is present. Note that this impacts the DoWork() return value, as true means + // that non-optimal clusters may remain within such oversized or builder-having + // levels. + if (sims[level].IsOversized()) continue; + if (level == 0 && !block_builders.empty()) continue; + // If neither of the two above conditions holds, and DoWork() returned true, + // then the level is optimal. + if (ret) { sims[level].real_is_optimal = true; } + // Compute how many iterations would be needed to make everything optimal. + for (auto component : sims[level].GetComponents()) { + auto iters_opt_this_cluster = MaxOptimalLinearizationIters(component.Count()); + if (iters_opt_this_cluster > acceptable_iters) { + // If the number of iterations required to linearize this cluster + // optimally exceeds acceptable_iters, DoWork() may process it in two + // stages: once to acceptable, and once to optimal. + iters_for_optimal += iters_opt_this_cluster + acceptable_iters; + } else { + iters_for_optimal += iters_opt_this_cluster; + } + } + } + if (!ret) { + // DoWork can only have more work left if the requested number of iterations + // was insufficient to linearize everything optimally within the levels it is + // allowed to touch. + assert(iters <= iters_for_optimal); } break; } else if (sims.size() == 2 && !sims[0].IsOversized() && !sims[1].IsOversized() && command-- == 0) { diff --git a/src/test/util/cluster_linearize.h b/src/test/util/cluster_linearize.h index a89a46552b..c79f414a0e 100644 --- a/src/test/util/cluster_linearize.h +++ b/src/test/util/cluster_linearize.h @@ -394,6 +394,29 @@ void SanityCheck(const DepGraph& depgraph, std::span