clusterlin: use 'cost' terminology instead of 'iters' (refactor)

This commit is contained in:
Pieter Wuille authored and Pieter Wuille committed 2026-02-24 10:08:47 -05:00
1 parent 9e7129df29
commit ecc9a84f85
12 files changed
+111 -91

No files matched your search

+3 -3
View File
@@ -55,7 +55,7 @@ void BenchLinearizeOptimallyTotal(benchmark::Bench& bench, const std::string& na
// Benchmark the total time to optimal. // Benchmark the total time to optimal.
uint64_t rng_seed = 0; uint64_t rng_seed = 0;
bench.name(bench_name).run([&] { bench.name(bench_name).run([&] {
auto [_lin, optimal, _cost] = Linearize(depgraph, /*max_iterations=*/10000000, rng_seed++, IndexTxOrder{}); auto [_lin, optimal, _cost] = Linearize(depgraph, /*max_cost=*/10000000, rng_seed++, IndexTxOrder{});
assert(optimal); assert(optimal);
}); });
} }
@@ -72,7 +72,7 @@ void BenchLinearizeOptimallyPerCost(benchmark::Bench& bench, const std::string&
// Determine the cost of 100 rng_seeds. // Determine the cost of 100 rng_seeds.
uint64_t total_cost = 0; uint64_t total_cost = 0;
for (uint64_t iter = 0; iter < 100; ++iter) { for (uint64_t iter = 0; iter < 100; ++iter) {
auto [_lin, optimal, cost] = Linearize(depgraph, /*max_iterations=*/10000000, /*rng_seed=*/iter, IndexTxOrder{}); auto [_lin, optimal, cost] = Linearize(depgraph, /*max_cost=*/10000000, /*rng_seed=*/iter, IndexTxOrder{});
total_cost += cost; total_cost += cost;
} }
@@ -80,7 +80,7 @@ void BenchLinearizeOptimallyPerCost(benchmark::Bench& bench, const std::string&
bench.name(bench_name).unit("cost").batch(total_cost).run([&] { bench.name(bench_name).unit("cost").batch(total_cost).run([&] {
uint64_t recompute_cost = 0; uint64_t recompute_cost = 0;
for (uint64_t iter = 0; iter < 100; ++iter) { for (uint64_t iter = 0; iter < 100; ++iter) {
auto [_lin, optimal, cost] = Linearize(depgraph, /*max_iterations=*/10000000, /*rng_seed=*/iter, IndexTxOrder{}); auto [_lin, optimal, cost] = Linearize(depgraph, /*max_cost=*/10000000, /*rng_seed=*/iter, IndexTxOrder{});
assert(optimal); assert(optimal);
recompute_cost += cost; recompute_cost += cost;
} }
+3 -3
View File
@@ -51,9 +51,9 @@ void BenchTxGraphTrim(benchmark::Bench& bench)
static constexpr int NUM_DEPS_PER_BOTTOM_TX = 100; static constexpr int NUM_DEPS_PER_BOTTOM_TX = 100;
/** Set a very large cluster size limit so that only the count limit is triggered. */ /** Set a very large cluster size limit so that only the count limit is triggered. */
static constexpr int32_t MAX_CLUSTER_SIZE = 100'000 * 100; static constexpr int32_t MAX_CLUSTER_SIZE = 100'000 * 100;
/** Set a very high number for acceptable iterations, so that we certainly benchmark optimal /** Set a very high number for acceptable cost, so that we certainly benchmark optimal
* linearization. */ * linearization. */
static constexpr uint64_t NUM_ACCEPTABLE_ITERS = 100'000'000; static constexpr uint64_t HIGH_ACCEPTABLE_COST = 100'000'000;
/** Refs to all top transactions. */ /** Refs to all top transactions. */
std::vector<TxGraph::Ref> top_refs; std::vector<TxGraph::Ref> top_refs;
@@ -65,7 +65,7 @@ void BenchTxGraphTrim(benchmark::Bench& bench)
std::vector<size_t> top_components; std::vector<size_t> top_components;
InsecureRandomContext rng(11); InsecureRandomContext rng(11);
auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, NUM_ACCEPTABLE_ITERS, PointerComparator); auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, HIGH_ACCEPTABLE_COST, PointerComparator);
// Construct the top chains. // Construct the top chains.
for (int chain = 0; chain < NUM_TOP_CHAINS; ++chain) { for (int chain = 0; chain < NUM_TOP_CHAINS; ++chain) {
+6 -6
View File
@@ -1742,7 +1742,7 @@ public:
/** Find or improve a linearization for a cluster. /** Find or improve a linearization for a cluster.
* *
* @param[in] depgraph Dependency graph of the cluster to be linearized. * @param[in] depgraph Dependency graph of the cluster to be linearized.
* @param[in] max_iterations Upper bound on the amount of work that will be done. * @param[in] max_cost Upper bound on the amount of work that will be done.
* @param[in] rng_seed A random number seed to control search order. This prevents peers * @param[in] rng_seed A random number seed to control search order. This prevents peers
* from predicting exactly which clusters would be hard for us to * from predicting exactly which clusters would be hard for us to
* linearize. * linearize.
@@ -1762,7 +1762,7 @@ public:
template<typename SetType> template<typename SetType>
std::tuple<std::vector<DepGraphIndex>, bool, uint64_t> Linearize( std::tuple<std::vector<DepGraphIndex>, bool, uint64_t> Linearize(
const DepGraph<SetType>& depgraph, const DepGraph<SetType>& depgraph,
uint64_t max_iterations, uint64_t max_cost,
uint64_t rng_seed, uint64_t rng_seed,
const StrongComparator<DepGraphIndex> auto& fallback_order, const StrongComparator<DepGraphIndex> auto& fallback_order,
std::span<const DepGraphIndex> old_linearization = {}, std::span<const DepGraphIndex> old_linearization = {},
@@ -1778,23 +1778,23 @@ std::tuple<std::vector<DepGraphIndex>, bool, uint64_t> Linearize(
} }
// Make improvement steps to it until we hit the max_iterations limit, or an optimal result // Make improvement steps to it until we hit the max_iterations limit, or an optimal result
// is found. // is found.
if (forest.GetCost() < max_iterations) { if (forest.GetCost() < max_cost) {
forest.StartOptimizing(); forest.StartOptimizing();
do { do {
if (!forest.OptimizeStep()) break; if (!forest.OptimizeStep()) break;
} while (forest.GetCost() < max_iterations); } while (forest.GetCost() < max_cost);
} }
// Make chunk minimization steps until we hit the max_iterations limit, or all chunks are // Make chunk minimization steps until we hit the max_iterations limit, or all chunks are
// minimal. // minimal.
bool optimal = false; bool optimal = false;
if (forest.GetCost() < max_iterations) { if (forest.GetCost() < max_cost) {
forest.StartMinimizing(); forest.StartMinimizing();
do { do {
if (!forest.MinimizeStep()) { if (!forest.MinimizeStep()) {
optimal = true; optimal = true;
break; break;
} }
} while (forest.GetCost() < max_iterations); } while (forest.GetCost() < max_cost);
} }
return {forest.GetLinearization(fallback_order), optimal, forest.GetCost()}; return {forest.GetLinearization(fallback_order), optimal, forest.GetCost()};
} }
+8 -2
View File
@@ -88,9 +88,15 @@ void TestOptimalLinearization(std::span<const uint8_t> enc, std::initializer_lis
is_topological = false; is_topological = false;
break; break;
} }
std::tie(lin, opt, cost) = Linearize(depgraph, 1000000000000, rng.rand64(), IndexTxOrder{}, lin, is_topological); std::tie(lin, opt, cost) = Linearize(
/*depgraph=*/depgraph,
/*max_cost=*/1000000000000,
/*rng_seed=*/rng.rand64(),
/*fallback_order=*/IndexTxOrder{},
/*old_linearization=*/lin,
/*is_topological=*/is_topological);
BOOST_CHECK(opt); BOOST_CHECK(opt);
BOOST_CHECK(cost <= MaxOptimalLinearizationIters(depgraph.TxCount())); BOOST_CHECK(cost <= MaxOptimalLinearizationCost(depgraph.TxCount()));
SanityCheck(depgraph, lin); SanityCheck(depgraph, lin);
BOOST_CHECK(std::ranges::equal(lin, optimal_linearization)); BOOST_CHECK(std::ranges::equal(lin, optimal_linearization));
} }
+16 -10
View File
@@ -984,7 +984,7 @@ FUZZ_TARGET(clusterlin_sfl)
// Verify that optimality is reached within an expected amount of work. This protects against // Verify that optimality is reached within an expected amount of work. This protects against
// hypothetical bugs that hugely increase the amount of work needed to reach optimality. // hypothetical bugs that hugely increase the amount of work needed to reach optimality.
assert(sfl.GetCost() <= MaxOptimalLinearizationIters(depgraph.TxCount())); assert(sfl.GetCost() <= MaxOptimalLinearizationCost(depgraph.TxCount()));
// The result must be as good as SimpleLinearize. // The result must be as good as SimpleLinearize.
auto [simple_linearization, simple_optimal] = SimpleLinearize(depgraph, MAX_SIMPLE_ITERATIONS / 10); auto [simple_linearization, simple_optimal] = SimpleLinearize(depgraph, MAX_SIMPLE_ITERATIONS / 10);
@@ -1011,15 +1011,15 @@ FUZZ_TARGET(clusterlin_linearize)
{ {
// Verify the behavior of Linearize(). // Verify the behavior of Linearize().
// Retrieve an RNG seed, an iteration count, a depgraph, and whether to make it connected from // Retrieve an RNG seed, a maximum amount of work, a depgraph, and whether to make it connected
// the fuzz input. // from the fuzz input.
SpanReader reader(buffer); SpanReader reader(buffer);
DepGraph<TestBitSet> depgraph; DepGraph<TestBitSet> depgraph;
uint64_t rng_seed{0}; uint64_t rng_seed{0};
uint64_t iter_count{0}; uint64_t max_cost{0};
uint8_t flags{7}; uint8_t flags{7};
try { try {
reader >> VARINT(iter_count) >> Using<DepGraphFormatter>(depgraph) >> rng_seed >> flags; reader >> VARINT(max_cost) >> Using<DepGraphFormatter>(depgraph) >> rng_seed >> flags;
} catch (const std::ios_base::failure&) {} } catch (const std::ios_base::failure&) {}
bool make_connected = flags & 1; bool make_connected = flags & 1;
// The following 3 booleans have 4 combinations: // The following 3 booleans have 4 combinations:
@@ -1043,8 +1043,14 @@ FUZZ_TARGET(clusterlin_linearize)
} }
// Invoke Linearize(). // Invoke Linearize().
iter_count &= 0x7ffff; max_cost &= 0x7ffff;
auto [linearization, optimal, cost] = Linearize(depgraph, iter_count, rng_seed, IndexTxOrder{}, old_linearization, /*is_topological=*/claim_topological_input); auto [linearization, optimal, cost] = Linearize(
/*depgraph=*/depgraph,
/*max_cost=*/max_cost,
/*rng_seed=*/rng_seed,
/*fallback_order=*/IndexTxOrder{},
/*old_linearization=*/old_linearization,
/*is_topological=*/claim_topological_input);
SanityCheck(depgraph, linearization); SanityCheck(depgraph, linearization);
auto chunking = ChunkLinearization(depgraph, linearization); auto chunking = ChunkLinearization(depgraph, linearization);
@@ -1056,8 +1062,8 @@ FUZZ_TARGET(clusterlin_linearize)
assert(cmp >= 0); assert(cmp >= 0);
} }
// If the iteration count is sufficiently high, an optimal linearization must be found. // If the maximum amount of work is sufficiently high, an optimal linearization must be found.
if (iter_count > MaxOptimalLinearizationIters(depgraph.TxCount())) { if (max_cost > MaxOptimalLinearizationCost(depgraph.TxCount())) {
assert(optimal); assert(optimal);
} }
@@ -1145,7 +1151,7 @@ FUZZ_TARGET(clusterlin_linearize)
// Redo from scratch with a different rng_seed. The resulting linearization should be // Redo from scratch with a different rng_seed. The resulting linearization should be
// deterministic, if both are optimal. // deterministic, if both are optimal.
auto [linearization2, optimal2, cost2] = Linearize(depgraph, MaxOptimalLinearizationIters(depgraph.TxCount()) + 1, rng_seed ^ 0x1337, IndexTxOrder{}); auto [linearization2, optimal2, cost2] = Linearize(depgraph, MaxOptimalLinearizationCost(depgraph.TxCount()) + 1, rng_seed ^ 0x1337, IndexTxOrder{});
assert(optimal2); assert(optimal2);
assert(linearization2 == linearization); assert(linearization2 == linearization);
} }
+16 -16
View File
@@ -325,8 +325,8 @@ FUZZ_TARGET(txgraph)
auto max_cluster_count = provider.ConsumeIntegralInRange<DepGraphIndex>(1, MAX_CLUSTER_COUNT_LIMIT); auto max_cluster_count = provider.ConsumeIntegralInRange<DepGraphIndex>(1, MAX_CLUSTER_COUNT_LIMIT);
/** The maximum total size of transactions in a (non-oversized) cluster. */ /** The maximum total size of transactions in a (non-oversized) cluster. */
auto max_cluster_size = provider.ConsumeIntegralInRange<uint64_t>(1, 0x3fffff * MAX_CLUSTER_COUNT_LIMIT); auto max_cluster_size = provider.ConsumeIntegralInRange<uint64_t>(1, 0x3fffff * MAX_CLUSTER_COUNT_LIMIT);
/** The number of iterations to consider a cluster acceptably linearized. */ /** The amount of work to consider a cluster acceptably linearized. */
auto acceptable_iters = provider.ConsumeIntegralInRange<uint64_t>(0, 10000); auto acceptable_cost = provider.ConsumeIntegralInRange<uint64_t>(0, 10000);
/** The set of uint64_t "txid"s that have been assigned before. */ /** The set of uint64_t "txid"s that have been assigned before. */
std::set<uint64_t> assigned_txids; std::set<uint64_t> assigned_txids;
@@ -342,7 +342,7 @@ FUZZ_TARGET(txgraph)
auto real = MakeTxGraph( auto real = MakeTxGraph(
/*max_cluster_count=*/max_cluster_count, /*max_cluster_count=*/max_cluster_count,
/*max_cluster_size=*/max_cluster_size, /*max_cluster_size=*/max_cluster_size,
/*acceptable_iters=*/acceptable_iters, /*acceptable_cost=*/acceptable_cost,
/*fallback_order=*/fallback_order); /*fallback_order=*/fallback_order);
std::vector<SimTxGraph> sims; std::vector<SimTxGraph> sims;
@@ -758,9 +758,9 @@ FUZZ_TARGET(txgraph)
break; break;
} else if (command-- == 0) { } else if (command-- == 0) {
// DoWork. // DoWork.
uint64_t iters = provider.ConsumeIntegralInRange<uint64_t>(0, alt ? 10000 : 255); uint64_t max_cost = provider.ConsumeIntegralInRange<uint64_t>(0, alt ? 10000 : 255);
bool ret = real->DoWork(iters); bool ret = real->DoWork(max_cost);
uint64_t iters_for_optimal{0}; uint64_t cost_for_optimal{0};
for (unsigned level = 0; level < sims.size(); ++level) { for (unsigned level = 0; level < sims.size(); ++level) {
// DoWork() will not optimize oversized levels, or the main level if a builder // DoWork() will not optimize oversized levels, or the main level if a builder
// is present. Note that this impacts the DoWork() return value, as true means // is present. Note that this impacts the DoWork() return value, as true means
@@ -773,24 +773,24 @@ FUZZ_TARGET(txgraph)
if (ret) { if (ret) {
sims[level].real_is_optimal = true; sims[level].real_is_optimal = true;
} }
// Compute how many iterations would be needed to make everything optimal. // Compute how much work would be needed to make everything optimal.
for (auto component : sims[level].GetComponents()) { for (auto component : sims[level].GetComponents()) {
auto iters_opt_this_cluster = MaxOptimalLinearizationIters(component.Count()); auto cost_opt_this_cluster = MaxOptimalLinearizationCost(component.Count());
if (iters_opt_this_cluster > acceptable_iters) { if (cost_opt_this_cluster > acceptable_cost) {
// If the number of iterations required to linearize this cluster // If the amount of work required to linearize this cluster
// optimally exceeds acceptable_iters, DoWork() may process it in two // optimally exceeds acceptable_cost, DoWork() may process it in two
// stages: once to acceptable, and once to optimal. // stages: once to acceptable, and once to optimal.
iters_for_optimal += iters_opt_this_cluster + acceptable_iters; cost_for_optimal += cost_opt_this_cluster + acceptable_cost;
} else { } else {
iters_for_optimal += iters_opt_this_cluster; cost_for_optimal += cost_opt_this_cluster;
} }
} }
} }
if (!ret) { if (!ret) {
// DoWork can only have more work left if the requested number of iterations // DoWork can only have more work left if the requested amount of work
// was insufficient to linearize everything optimally within the levels it is // was insufficient to linearize everything optimally within the levels it is
// allowed to touch. // allowed to touch.
assert(iters <= iters_for_optimal); assert(max_cost <= cost_for_optimal);
} }
break; break;
} else if (sims.size() == 2 && !sims[0].IsOversized() && !sims[1].IsOversized() && command-- == 0) { } else if (sims.size() == 2 && !sims[0].IsOversized() && !sims[1].IsOversized() && command-- == 0) {
@@ -1165,7 +1165,7 @@ FUZZ_TARGET(txgraph)
auto real_redo = MakeTxGraph( auto real_redo = MakeTxGraph(
/*max_cluster_count=*/max_cluster_count, /*max_cluster_count=*/max_cluster_count,
/*max_cluster_size=*/max_cluster_size, /*max_cluster_size=*/max_cluster_size,
/*acceptable_iters=*/acceptable_iters, /*acceptable_cost=*/acceptable_cost,
/*fallback_order=*/fallback_order); /*fallback_order=*/fallback_order);
/** Vector (indexed by SimTxGraph::Pos) of TxObjects in real_redo). */ /** Vector (indexed by SimTxGraph::Pos) of TxObjects in real_redo). */
std::vector<std::optional<SimTxObject>> txobjects_redo; std::vector<std::optional<SimTxObject>> txobjects_redo;
+8 -8
View File
@@ -15,9 +15,9 @@ BOOST_AUTO_TEST_SUITE(txgraph_tests)
namespace { namespace {
/** The number used as acceptable_iters argument in these tests. High enough that everything /** The number used as acceptable_cost argument in these tests. High enough that everything
* should be optimal, always. */ * should be optimal, always. */
constexpr uint64_t NUM_ACCEPTABLE_ITERS = 100'000'000; constexpr uint64_t HIGH_ACCEPTABLE_COST = 100'000'000;
std::strong_ordering PointerComparator(const TxGraph::Ref& a, const TxGraph::Ref& b) noexcept std::strong_ordering PointerComparator(const TxGraph::Ref& a, const TxGraph::Ref& b) noexcept
{ {
@@ -48,7 +48,7 @@ BOOST_AUTO_TEST_CASE(txgraph_trim_zigzag)
static constexpr int32_t MAX_CLUSTER_SIZE = 100'000 * 100; static constexpr int32_t MAX_CLUSTER_SIZE = 100'000 * 100;
// Create a new graph for the test. // Create a new graph for the test.
auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, NUM_ACCEPTABLE_ITERS, PointerComparator); auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, HIGH_ACCEPTABLE_COST, PointerComparator);
// Add all transactions and store their Refs. // Add all transactions and store their Refs.
std::vector<TxGraph::Ref> refs; std::vector<TxGraph::Ref> refs;
@@ -111,7 +111,7 @@ BOOST_AUTO_TEST_CASE(txgraph_trim_flower)
/** Set a very large cluster size limit so that only the count limit is triggered. */ /** Set a very large cluster size limit so that only the count limit is triggered. */
static constexpr int32_t MAX_CLUSTER_SIZE = 100'000 * 100; static constexpr int32_t MAX_CLUSTER_SIZE = 100'000 * 100;
auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, NUM_ACCEPTABLE_ITERS, PointerComparator); auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, HIGH_ACCEPTABLE_COST, PointerComparator);
// Add all transactions and store their Refs. // Add all transactions and store their Refs.
std::vector<TxGraph::Ref> refs; std::vector<TxGraph::Ref> refs;
@@ -197,7 +197,7 @@ BOOST_AUTO_TEST_CASE(txgraph_trim_huge)
std::vector<size_t> top_components; std::vector<size_t> top_components;
FastRandomContext rng; FastRandomContext rng;
auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, NUM_ACCEPTABLE_ITERS, PointerComparator); auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, HIGH_ACCEPTABLE_COST, PointerComparator);
// Construct the top chains. // Construct the top chains.
for (int chain = 0; chain < NUM_TOP_CHAINS; ++chain) { for (int chain = 0; chain < NUM_TOP_CHAINS; ++chain) {
@@ -270,7 +270,7 @@ BOOST_AUTO_TEST_CASE(txgraph_trim_big_singletons)
static constexpr int NUM_TOTAL_TX = 100; static constexpr int NUM_TOTAL_TX = 100;
// Create a new graph for the test. // Create a new graph for the test.
auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, NUM_ACCEPTABLE_ITERS, PointerComparator); auto graph = MakeTxGraph(MAX_CLUSTER_COUNT, MAX_CLUSTER_SIZE, HIGH_ACCEPTABLE_COST, PointerComparator);
// Add all transactions and store their Refs. // Add all transactions and store their Refs.
std::vector<TxGraph::Ref> refs; std::vector<TxGraph::Ref> refs;
@@ -304,7 +304,7 @@ BOOST_AUTO_TEST_CASE(txgraph_trim_big_singletons)
BOOST_AUTO_TEST_CASE(txgraph_chunk_chain) BOOST_AUTO_TEST_CASE(txgraph_chunk_chain)
{ {
// Create a new graph for the test. // Create a new graph for the test.
auto graph = MakeTxGraph(50, 1000, NUM_ACCEPTABLE_ITERS, PointerComparator); auto graph = MakeTxGraph(50, 1000, HIGH_ACCEPTABLE_COST, PointerComparator);
auto block_builder_checker = [&graph](std::vector<std::vector<TxGraph::Ref*>> expected_chunks) { auto block_builder_checker = [&graph](std::vector<std::vector<TxGraph::Ref*>> expected_chunks) {
std::vector<std::vector<TxGraph::Ref*>> chunks; std::vector<std::vector<TxGraph::Ref*>> chunks;
@@ -383,7 +383,7 @@ BOOST_AUTO_TEST_CASE(txgraph_staging)
/* Create a new graph for the test. /* Create a new graph for the test.
* The parameters are max_cluster_count, max_cluster_size, acceptable_iters * The parameters are max_cluster_count, max_cluster_size, acceptable_iters
*/ */
auto graph = MakeTxGraph(10, 1000, NUM_ACCEPTABLE_ITERS, PointerComparator); auto graph = MakeTxGraph(10, 1000, HIGH_ACCEPTABLE_COST, PointerComparator);
std::vector<TxGraph::Ref> refs; std::vector<TxGraph::Ref> refs;
refs.reserve(2); refs.reserve(2);
+4 -4
View File
@@ -394,13 +394,13 @@ void SanityCheck(const DepGraph<SetType>& depgraph, std::span<const DepGraphInde
} }
} }
inline uint64_t MaxOptimalLinearizationIters(DepGraphIndex cluster_count) inline uint64_t MaxOptimalLinearizationCost(DepGraphIndex cluster_count)
{ {
// These are the largest numbers seen returned as cost by Linearize(), in a large randomized // These are the largest numbers seen returned as cost by Linearize(), in a large randomized
// trial. There exist almost certainly far worse cases, but they are unlikely to be // trial. There exist almost certainly far worse cases, but they are unlikely to be
// encountered in randomized tests. The purpose of these numbers is guaranteeing that for // encountered in randomized tests. The purpose of these numbers is guaranteeing that for
// *some* reasonable cost bound, optimal linearizations are always found. // *some* reasonable cost bound, optimal linearizations are always found.
static constexpr uint64_t ITERS[65] = { static constexpr uint64_t COSTS[65] = {
0, 0,
0, 4, 10, 34, 76, 156, 229, 380, 0, 4, 10, 34, 76, 156, 229, 380,
441, 517, 678, 933, 1037, 1366, 1464, 1711, 441, 517, 678, 933, 1037, 1366, 1464, 1711,
@@ -411,9 +411,9 @@ inline uint64_t MaxOptimalLinearizationIters(DepGraphIndex cluster_count)
34046, 26227, 34662, 38019, 40814, 31113, 41448, 33968, 34046, 26227, 34662, 38019, 40814, 31113, 41448, 33968,
35024, 59207, 42872, 41277, 42365, 51833, 63410, 67035 35024, 59207, 42872, 41277, 42365, 51833, 63410, 67035
}; };
assert(cluster_count < std::size(ITERS)); assert(cluster_count < std::size(COSTS));
// Multiply the table number by two, to account for the fact that they are not absolutes. // Multiply the table number by two, to account for the fact that they are not absolutes.
return ITERS[cluster_count] * 2; return COSTS[cluster_count] * 2;
} }
} // namespace } // namespace
+35 -26
View File
@@ -217,7 +217,7 @@ public:
virtual void ApplyDependencies(TxGraphImpl& graph, int level, std::span<std::pair<GraphIndex, GraphIndex>> to_apply) noexcept = 0; virtual void ApplyDependencies(TxGraphImpl& graph, int level, std::span<std::pair<GraphIndex, GraphIndex>> to_apply) noexcept = 0;
/** Improve the linearization of this Cluster. Returns how much work was performed and whether /** Improve the linearization of this Cluster. Returns how much work was performed and whether
* the Cluster's QualityLevel improved as a result. */ * the Cluster's QualityLevel improved as a result. */
virtual std::pair<uint64_t, bool> Relinearize(TxGraphImpl& graph, int level, uint64_t max_iters) noexcept = 0; virtual std::pair<uint64_t, bool> Relinearize(TxGraphImpl& graph, int level, uint64_t max_cost) noexcept = 0;
/** For every chunk in the cluster, append its FeeFrac to ret. */ /** For every chunk in the cluster, append its FeeFrac to ret. */
virtual void AppendChunkFeerates(std::vector<FeeFrac>& ret) const noexcept = 0; virtual void AppendChunkFeerates(std::vector<FeeFrac>& ret) const noexcept = 0;
/** Add a TrimTxData entry (filling m_chunk_feerate, m_index, m_tx_size) for every /** Add a TrimTxData entry (filling m_chunk_feerate, m_index, m_tx_size) for every
@@ -296,7 +296,7 @@ public:
[[nodiscard]] bool Split(TxGraphImpl& graph, int level) noexcept final; [[nodiscard]] bool Split(TxGraphImpl& graph, int level) noexcept final;
void Merge(TxGraphImpl& graph, int level, Cluster& cluster) noexcept final; void Merge(TxGraphImpl& graph, int level, Cluster& cluster) noexcept final;
void ApplyDependencies(TxGraphImpl& graph, int level, std::span<std::pair<GraphIndex, GraphIndex>> to_apply) noexcept final; void ApplyDependencies(TxGraphImpl& graph, int level, std::span<std::pair<GraphIndex, GraphIndex>> to_apply) noexcept final;
std::pair<uint64_t, bool> Relinearize(TxGraphImpl& graph, int level, uint64_t max_iters) noexcept final; std::pair<uint64_t, bool> Relinearize(TxGraphImpl& graph, int level, uint64_t max_cost) noexcept final;
void AppendChunkFeerates(std::vector<FeeFrac>& ret) const noexcept final; void AppendChunkFeerates(std::vector<FeeFrac>& ret) const noexcept final;
uint64_t AppendTrimData(std::vector<TrimTxData>& ret, std::vector<std::pair<GraphIndex, GraphIndex>>& deps) const noexcept final; uint64_t AppendTrimData(std::vector<TrimTxData>& ret, std::vector<std::pair<GraphIndex, GraphIndex>>& deps) const noexcept final;
void GetAncestorRefs(const TxGraphImpl& graph, std::span<std::pair<Cluster*, DepGraphIndex>>& args, std::vector<TxGraph::Ref*>& output) noexcept final; void GetAncestorRefs(const TxGraphImpl& graph, std::span<std::pair<Cluster*, DepGraphIndex>>& args, std::vector<TxGraph::Ref*>& output) noexcept final;
@@ -353,7 +353,7 @@ public:
[[nodiscard]] bool Split(TxGraphImpl& graph, int level) noexcept final; [[nodiscard]] bool Split(TxGraphImpl& graph, int level) noexcept final;
void Merge(TxGraphImpl& graph, int level, Cluster& cluster) noexcept final; void Merge(TxGraphImpl& graph, int level, Cluster& cluster) noexcept final;
void ApplyDependencies(TxGraphImpl& graph, int level, std::span<std::pair<GraphIndex, GraphIndex>> to_apply) noexcept final; void ApplyDependencies(TxGraphImpl& graph, int level, std::span<std::pair<GraphIndex, GraphIndex>> to_apply) noexcept final;
std::pair<uint64_t, bool> Relinearize(TxGraphImpl& graph, int level, uint64_t max_iters) noexcept final; std::pair<uint64_t, bool> Relinearize(TxGraphImpl& graph, int level, uint64_t max_cost) noexcept final;
void AppendChunkFeerates(std::vector<FeeFrac>& ret) const noexcept final; void AppendChunkFeerates(std::vector<FeeFrac>& ret) const noexcept final;
uint64_t AppendTrimData(std::vector<TrimTxData>& ret, std::vector<std::pair<GraphIndex, GraphIndex>>& deps) const noexcept final; uint64_t AppendTrimData(std::vector<TrimTxData>& ret, std::vector<std::pair<GraphIndex, GraphIndex>>& deps) const noexcept final;
void GetAncestorRefs(const TxGraphImpl& graph, std::span<std::pair<Cluster*, DepGraphIndex>>& args, std::vector<TxGraph::Ref*>& output) noexcept final; void GetAncestorRefs(const TxGraphImpl& graph, std::span<std::pair<Cluster*, DepGraphIndex>>& args, std::vector<TxGraph::Ref*>& output) noexcept final;
@@ -400,9 +400,8 @@ private:
const DepGraphIndex m_max_cluster_count; const DepGraphIndex m_max_cluster_count;
/** This TxGraphImpl's maximum cluster size limit. */ /** This TxGraphImpl's maximum cluster size limit. */
const uint64_t m_max_cluster_size; const uint64_t m_max_cluster_size;
/** The number of linearization improvement steps needed per cluster to be considered /** The amount of linearization work needed per cluster to be considered acceptable. */
* acceptable. */ const uint64_t m_acceptable_cost;
const uint64_t m_acceptable_iters;
/** Fallback ordering for transactions. */ /** Fallback ordering for transactions. */
const std::function<std::strong_ordering(const TxGraph::Ref&, const TxGraph::Ref&)> m_fallback_order; const std::function<std::strong_ordering(const TxGraph::Ref&, const TxGraph::Ref&)> m_fallback_order;
@@ -636,12 +635,12 @@ public:
explicit TxGraphImpl( explicit TxGraphImpl(
DepGraphIndex max_cluster_count, DepGraphIndex max_cluster_count,
uint64_t max_cluster_size, uint64_t max_cluster_size,
uint64_t acceptable_iters, uint64_t acceptable_cost,
const std::function<std::strong_ordering(const TxGraph::Ref&, const TxGraph::Ref&)>& fallback_order const std::function<std::strong_ordering(const TxGraph::Ref&, const TxGraph::Ref&)>& fallback_order
) noexcept : ) noexcept :
m_max_cluster_count(max_cluster_count), m_max_cluster_count(max_cluster_count),
m_max_cluster_size(max_cluster_size), m_max_cluster_size(max_cluster_size),
m_acceptable_iters(acceptable_iters), m_acceptable_cost(acceptable_cost),
m_fallback_order(fallback_order), m_fallback_order(fallback_order),
m_main_chunkindex(ChunkOrder(this)) m_main_chunkindex(ChunkOrder(this))
{ {
@@ -803,7 +802,7 @@ public:
void AddDependency(const Ref& parent, const Ref& child) noexcept final; void AddDependency(const Ref& parent, const Ref& child) noexcept final;
void SetTransactionFee(const Ref&, int64_t fee) noexcept final; void SetTransactionFee(const Ref&, int64_t fee) noexcept final;
bool DoWork(uint64_t iters) noexcept final; bool DoWork(uint64_t max_cost) noexcept final;
void StartStaging() noexcept final; void StartStaging() noexcept final;
void CommitStaging() noexcept final; void CommitStaging() noexcept final;
@@ -2155,7 +2154,7 @@ void TxGraphImpl::ApplyDependencies(int level) noexcept
clusterset.m_group_data = GroupData{}; clusterset.m_group_data = GroupData{};
} }
std::pair<uint64_t, bool> GenericClusterImpl::Relinearize(TxGraphImpl& graph, int level, uint64_t max_iters) noexcept std::pair<uint64_t, bool> GenericClusterImpl::Relinearize(TxGraphImpl& graph, int level, uint64_t max_cost) noexcept
{ {
// We can only relinearize Clusters that do not need splitting. // We can only relinearize Clusters that do not need splitting.
Assume(!NeedsSplitting()); Assume(!NeedsSplitting());
@@ -2168,7 +2167,13 @@ std::pair<uint64_t, bool> GenericClusterImpl::Relinearize(TxGraphImpl& graph, in
const auto ref_b = graph.m_entries[m_mapping[b]].m_ref; const auto ref_b = graph.m_entries[m_mapping[b]].m_ref;
return graph.m_fallback_order(*ref_a, *ref_b); return graph.m_fallback_order(*ref_a, *ref_b);
}; };
auto [linearization, optimal, cost] = Linearize(m_depgraph, max_iters, rng_seed, fallback_order, m_linearization, /*is_topological=*/IsTopological()); auto [linearization, optimal, cost] = Linearize(
/*depgraph=*/m_depgraph,
/*max_cost=*/max_cost,
/*rng_seed=*/rng_seed,
/*fallback_order=*/fallback_order,
/*old_linearization=*/m_linearization,
/*is_topological=*/IsTopological());
// Postlinearize to improve the linearization (if optimal, only the sub-chunk order). // Postlinearize to improve the linearization (if optimal, only the sub-chunk order).
// This also guarantees that all chunks are connected (even when non-optimal). // This also guarantees that all chunks are connected (even when non-optimal).
PostLinearize(m_depgraph, linearization); PostLinearize(m_depgraph, linearization);
@@ -2179,7 +2184,7 @@ std::pair<uint64_t, bool> GenericClusterImpl::Relinearize(TxGraphImpl& graph, in
if (optimal) { if (optimal) {
graph.SetClusterQuality(level, m_quality, m_setindex, QualityLevel::OPTIMAL); graph.SetClusterQuality(level, m_quality, m_setindex, QualityLevel::OPTIMAL);
improved = true; improved = true;
} else if (max_iters >= graph.m_acceptable_iters && !IsAcceptable()) { } else if (max_cost >= graph.m_acceptable_cost && !IsAcceptable()) {
graph.SetClusterQuality(level, m_quality, m_setindex, QualityLevel::ACCEPTABLE); graph.SetClusterQuality(level, m_quality, m_setindex, QualityLevel::ACCEPTABLE);
improved = true; improved = true;
} else if (!IsTopological()) { } else if (!IsTopological()) {
@@ -2191,7 +2196,7 @@ std::pair<uint64_t, bool> GenericClusterImpl::Relinearize(TxGraphImpl& graph, in
return {cost, improved}; return {cost, improved};
} }
std::pair<uint64_t, bool> SingletonClusterImpl::Relinearize(TxGraphImpl& graph, int level, uint64_t max_iters) noexcept std::pair<uint64_t, bool> SingletonClusterImpl::Relinearize(TxGraphImpl& graph, int level, uint64_t max_cost) noexcept
{ {
// All singletons are optimal, oversized, or need splitting. Each of these precludes // All singletons are optimal, oversized, or need splitting. Each of these precludes
// Relinearize from being called. // Relinearize from being called.
@@ -2203,7 +2208,7 @@ void TxGraphImpl::MakeAcceptable(Cluster& cluster, int level) noexcept
{ {
// Relinearize the Cluster if needed. // Relinearize the Cluster if needed.
if (!cluster.NeedsSplitting() && !cluster.IsAcceptable() && !cluster.IsOversized()) { if (!cluster.NeedsSplitting() && !cluster.IsAcceptable() && !cluster.IsOversized()) {
cluster.Relinearize(*this, level, m_acceptable_iters); cluster.Relinearize(*this, level, m_acceptable_cost);
} }
} }
@@ -3105,9 +3110,9 @@ void TxGraphImpl::SanityCheck() const
assert(actual_chunkindex == expected_chunkindex); assert(actual_chunkindex == expected_chunkindex);
} }
bool TxGraphImpl::DoWork(uint64_t iters) noexcept bool TxGraphImpl::DoWork(uint64_t max_cost) noexcept
{ {
uint64_t iters_done{0}; uint64_t cost_done{0};
// First linearize everything in NEEDS_RELINEARIZE to an acceptable level. If more budget // First linearize everything in NEEDS_RELINEARIZE to an acceptable level. If more budget
// remains after that, try to make everything optimal. // remains after that, try to make everything optimal.
for (QualityLevel quality : {QualityLevel::NEEDS_FIX, QualityLevel::NEEDS_RELINEARIZE, QualityLevel::ACCEPTABLE}) { for (QualityLevel quality : {QualityLevel::NEEDS_FIX, QualityLevel::NEEDS_RELINEARIZE, QualityLevel::ACCEPTABLE}) {
@@ -3121,23 +3126,23 @@ bool TxGraphImpl::DoWork(uint64_t iters) noexcept
if (clusterset.m_oversized == true) continue; if (clusterset.m_oversized == true) continue;
auto& queue = clusterset.m_clusters[int(quality)]; auto& queue = clusterset.m_clusters[int(quality)];
while (!queue.empty()) { while (!queue.empty()) {
if (iters_done >= iters) return false; if (cost_done >= max_cost) return false;
// Randomize the order in which we process, so that if the first cluster somehow // Randomize the order in which we process, so that if the first cluster somehow
// needs more work than what iters allows, we don't keep spending it on the same // needs more work than what max_cost allows, we don't keep spending it on the same
// one. // one.
auto pos = m_rng.randrange<size_t>(queue.size()); auto pos = m_rng.randrange<size_t>(queue.size());
auto iters_now = iters - iters_done; auto cost_now = max_cost - cost_done;
if (quality == QualityLevel::NEEDS_FIX || quality == QualityLevel::NEEDS_RELINEARIZE) { if (quality == QualityLevel::NEEDS_FIX || quality == QualityLevel::NEEDS_RELINEARIZE) {
// If we're working with clusters that need relinearization still, only perform // If we're working with clusters that need relinearization still, only perform
// up to m_acceptable_iters iterations. If they become ACCEPTABLE, and we still // up to m_acceptable_cost work. If they become ACCEPTABLE, and we still
// have budget after all other clusters are ACCEPTABLE too, we'll spend the // have budget after all other clusters are ACCEPTABLE too, we'll spend the
// remaining budget on trying to make them OPTIMAL. // remaining budget on trying to make them OPTIMAL.
iters_now = std::min(iters_now, m_acceptable_iters); cost_now = std::min(cost_now, m_acceptable_cost);
} }
auto [cost, improved] = queue[pos].get()->Relinearize(*this, level, iters_now); auto [cost, improved] = queue[pos].get()->Relinearize(*this, level, cost_now);
iters_done += cost; cost_done += cost;
// If no improvement was made to the Cluster, it means we've essentially run out of // If no improvement was made to the Cluster, it means we've essentially run out of
// budget. Even though it may be the case that iters_done < iters still, the // budget. Even though it may be the case that cost_done < max_cost still, the
// linearizer decided there wasn't enough budget left to attempt anything with. // linearizer decided there wasn't enough budget left to attempt anything with.
// To avoid an infinite loop that keeps trying clusters with minuscule budgets, // To avoid an infinite loop that keeps trying clusters with minuscule budgets,
// stop here too. // stop here too.
@@ -3565,8 +3570,12 @@ TxGraph::Ref::Ref(Ref&& other) noexcept
std::unique_ptr<TxGraph> MakeTxGraph( std::unique_ptr<TxGraph> MakeTxGraph(
unsigned max_cluster_count, unsigned max_cluster_count,
uint64_t max_cluster_size, uint64_t max_cluster_size,
uint64_t acceptable_iters, uint64_t acceptable_cost,
const std::function<std::strong_ordering(const TxGraph::Ref&, const TxGraph::Ref&)>& fallback_order) noexcept const std::function<std::strong_ordering(const TxGraph::Ref&, const TxGraph::Ref&)>& fallback_order) noexcept
{ {
return std::make_unique<TxGraphImpl>(max_cluster_count, max_cluster_size, acceptable_iters, fallback_order); return std::make_unique<TxGraphImpl>(
/*max_cluster_count=*/max_cluster_count,
/*max_cluster_size=*/max_cluster_size,
/*acceptable_cost=*/acceptable_cost,
/*fallback_order=*/fallback_order);
} }
+4 -4
View File
@@ -102,10 +102,10 @@ public:
virtual void SetTransactionFee(const Ref& arg, int64_t fee) noexcept = 0; virtual void SetTransactionFee(const Ref& arg, int64_t fee) noexcept = 0;
/** TxGraph is internally lazy, and will not compute many things until they are needed. /** TxGraph is internally lazy, and will not compute many things until they are needed.
* Calling DoWork will perform some work now (controlled by iters) so that future operations * Calling DoWork will perform some work now (controlled by max_cost) so that future operations
* are fast, if there is any. Returns whether all currently-available work is done. This can * are fast, if there is any. Returns whether all currently-available work is done. This can
* be invoked while oversized, but oversized graphs will be skipped by this call. */ * be invoked while oversized, but oversized graphs will be skipped by this call. */
virtual bool DoWork(uint64_t iters) noexcept = 0; virtual bool DoWork(uint64_t max_cost) noexcept = 0;
/** Create a staging graph (which cannot exist already). This acts as if a full copy of /** Create a staging graph (which cannot exist already). This acts as if a full copy of
* the transaction graph is made, upon which further modifications are made. This copy can * the transaction graph is made, upon which further modifications are made. This copy can
@@ -257,7 +257,7 @@ public:
* and on the sum of transaction sizes within a cluster. * and on the sum of transaction sizes within a cluster.
* *
* - max_cluster_count cannot exceed MAX_CLUSTER_COUNT_LIMIT. * - max_cluster_count cannot exceed MAX_CLUSTER_COUNT_LIMIT.
* - acceptable_iters controls how many linearization optimization steps will be performed per * - acceptable_cost controls how much linearization optimization work will be performed per
* cluster before they are considered to be of acceptable quality. * cluster before they are considered to be of acceptable quality.
* - fallback_order determines how to break tie-breaks between transactions: * - fallback_order determines how to break tie-breaks between transactions:
* fallback_order(a, b) < 0 means a is "better" than b, and will (in case of ties) be placed * fallback_order(a, b) < 0 means a is "better" than b, and will (in case of ties) be placed
@@ -266,7 +266,7 @@ public:
std::unique_ptr<TxGraph> MakeTxGraph( std::unique_ptr<TxGraph> MakeTxGraph(
unsigned max_cluster_count, unsigned max_cluster_count,
uint64_t max_cluster_size, uint64_t max_cluster_size,
uint64_t acceptable_iters, uint64_t acceptable_cost,
const std::function<std::strong_ordering(const TxGraph::Ref&, const TxGraph::Ref&)>& fallback_order const std::function<std::strong_ordering(const TxGraph::Ref&, const TxGraph::Ref&)>& fallback_order
) noexcept; ) noexcept;
+4 -4
View File
@@ -179,7 +179,7 @@ CTxMemPool::CTxMemPool(Options opts, bilingual_str& error)
m_txgraph = MakeTxGraph( m_txgraph = MakeTxGraph(
/*max_cluster_count=*/m_opts.limits.cluster_count, /*max_cluster_count=*/m_opts.limits.cluster_count,
/*max_cluster_size=*/m_opts.limits.cluster_size_vbytes * WITNESS_SCALE_FACTOR, /*max_cluster_size=*/m_opts.limits.cluster_size_vbytes * WITNESS_SCALE_FACTOR,
/*acceptable_iters=*/ACCEPTABLE_ITERS, /*acceptable_cost=*/ACCEPTABLE_COST,
/*fallback_order=*/[&](const TxGraph::Ref& a, const TxGraph::Ref& b) noexcept { /*fallback_order=*/[&](const TxGraph::Ref& a, const TxGraph::Ref& b) noexcept {
const Txid& txid_a = static_cast<const CTxMemPoolEntry&>(a).GetTx().GetHash(); const Txid& txid_a = static_cast<const CTxMemPoolEntry&>(a).GetTx().GetHash();
const Txid& txid_b = static_cast<const CTxMemPoolEntry&>(b).GetTx().GetHash(); const Txid& txid_b = static_cast<const CTxMemPoolEntry&>(b).GetTx().GetHash();
@@ -221,7 +221,7 @@ void CTxMemPool::Apply(ChangeSet* changeset)
addNewTransaction(it); addNewTransaction(it);
} }
if (!m_txgraph->DoWork(POST_CHANGE_WORK)) { if (!m_txgraph->DoWork(/*max_cost=*/POST_CHANGE_COST)) {
LogDebug(BCLog::MEMPOOL, "Mempool in non-optimal ordering after addition(s)."); LogDebug(BCLog::MEMPOOL, "Mempool in non-optimal ordering after addition(s).");
} }
} }
@@ -380,7 +380,7 @@ void CTxMemPool::removeForReorg(CChain& chain, std::function<bool(txiter)> check
for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) {
assert(TestLockPointValidity(chain, it->GetLockPoints())); assert(TestLockPointValidity(chain, it->GetLockPoints()));
} }
if (!m_txgraph->DoWork(POST_CHANGE_WORK)) { if (!m_txgraph->DoWork(/*max_cost=*/POST_CHANGE_COST)) {
LogDebug(BCLog::MEMPOOL, "Mempool in non-optimal ordering after reorg."); LogDebug(BCLog::MEMPOOL, "Mempool in non-optimal ordering after reorg.");
} }
} }
@@ -425,7 +425,7 @@ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigne
} }
lastRollingFeeUpdate = GetTime(); lastRollingFeeUpdate = GetTime();
blockSinceLastRollingFeeBump = true; blockSinceLastRollingFeeBump = true;
if (!m_txgraph->DoWork(POST_CHANGE_WORK)) { if (!m_txgraph->DoWork(/*max_cost=*/POST_CHANGE_COST)) {
LogDebug(BCLog::MEMPOOL, "Mempool in non-optimal ordering after block."); LogDebug(BCLog::MEMPOOL, "Mempool in non-optimal ordering after block.");
} }
} }
+4 -5
View File
@@ -49,14 +49,13 @@ struct bilingual_str;
/** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */
static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF;
/** How many linearization iterations required for TxGraph clusters to have /** How much linearization cost required for TxGraph clusters to have
* "acceptable" quality, if they cannot be optimally linearized with fewer * "acceptable" quality, if they cannot be optimally linearized with less cost. */
* iterations. */ static constexpr uint64_t ACCEPTABLE_COST = 1'700;
static constexpr uint64_t ACCEPTABLE_ITERS = 1'700;
/** How much work we ask TxGraph to do after a mempool change occurs (either /** How much work we ask TxGraph to do after a mempool change occurs (either
* due to a changeset being applied, a new block being found, or a reorg). */ * due to a changeset being applied, a new block being found, or a reorg). */
static constexpr uint64_t POST_CHANGE_WORK = 5 * ACCEPTABLE_ITERS; static constexpr uint64_t POST_CHANGE_COST = 5 * ACCEPTABLE_COST;
/** /**
* Test whether the LockPoints height and time are still valid on the current chain * Test whether the LockPoints height and time are still valid on the current chain