fix: reorg behaviour in mempool tests to match real one

test/functional/mempool_ephemeral_dust.py

@@ -20,6 +20,9 @@ from test_framework.util import (
 from test_framework.wallet import (
     MiniWallet,
 )
+from test_framework.blocktools import (
+    create_empty_fork
+)
 
 class EphemeralDustTest(BitcoinTestFramework):
     def set_test_params(self):
@@ -64,6 +67,12 @@ class EphemeralDustTest(BitcoinTestFramework):
 
         return dusty_tx, sweep_tx
 
+    def trigger_reorg(self, fork_blocks):
+        """Trigger reorg of the fork blocks."""
+        for block in fork_blocks:
+            self.nodes[0].submitblock(block.serialize().hex())
+        assert_equal(self.nodes[0].getbestblockhash(), fork_blocks[-1].hash_hex)
+
     def run_test(self):
 
         node = self.nodes[0]
@@ -323,11 +332,15 @@ class EphemeralDustTest(BitcoinTestFramework):
 
         # Get dusty tx mined, then check that it makes it back into mempool on reorg
         # due to bypass_limits allowing 0-fee individually
+
+        # Prep for fork with empty blocks
+        fork_blocks = create_empty_fork(self.nodes[0])
+
         dusty_tx, _ = self.create_ephemeral_dust_package(tx_version=3)
         assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, dusty_tx["hex"])
 
-        block_res = self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_tx["hex"]], sync_fun=self.no_op)
+        self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_tx["hex"]], sync_fun=self.no_op)
-        self.nodes[0].invalidateblock(block_res["hash"])
+        self.trigger_reorg(fork_blocks)
         assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"]], sync=False)
 
         # Create a sweep that has dust of its own and leaves dusty_tx's dust unspent
@@ -335,9 +348,12 @@ class EphemeralDustTest(BitcoinTestFramework):
         self.add_output_to_create_multi_result(sweep_tx)
         assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, sweep_tx["hex"])
 
+        # Prep for fork with empty blocks
+        fork_blocks = create_empty_fork(self.nodes[0])
+
         # Mine the sweep then re-org, the sweep will not make it back in due to spend checks
-        block_res = self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_tx["hex"], sweep_tx["hex"]], sync_fun=self.no_op)
+        self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_tx["hex"], sweep_tx["hex"]], sync_fun=self.no_op)
-        self.nodes[0].invalidateblock(block_res["hash"])
+        self.trigger_reorg(fork_blocks)
         assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"]], sync=False)
 
         # Should re-enter if dust is swept
@@ -345,27 +361,39 @@ class EphemeralDustTest(BitcoinTestFramework):
         self.add_output_to_create_multi_result(sweep_tx_2)
         assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, sweep_tx_2["hex"])
 
-        reconsider_block_res = self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_tx["hex"], sweep_tx_2["hex"]], sync_fun=self.no_op)
+        # Prep for fork with empty blocks
-        self.nodes[0].invalidateblock(reconsider_block_res["hash"])
+        fork_blocks = create_empty_fork(self.nodes[0])
+
+        self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_tx["hex"], sweep_tx_2["hex"]], sync_fun=self.no_op)
+        self.trigger_reorg(fork_blocks)
         assert_mempool_contents(self, self.nodes[0], expected=[dusty_tx["tx"], sweep_tx_2["tx"]], sync=False)
 
         # TRUC transactions restriction for ephemeral dust disallows further spends of ancestor chains
         child_tx = self.wallet.create_self_transfer_multi(utxos_to_spend=sweep_tx_2["new_utxos"], version=3)
         assert_raises_rpc_error(-26, "TRUC-violation", self.nodes[0].sendrawtransaction, child_tx["hex"])
 
-        self.nodes[0].reconsiderblock(reconsider_block_res["hash"])
+        # Clean up the mempool
+        self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_tx["hex"], sweep_tx_2["hex"]], sync_fun=self.no_op)
         assert_equal(self.nodes[0].getrawmempool(), [])
 
         self.log.info("Test that ephemeral dust tx with fees or multi dust don't enter mempool via reorg")
         multi_dusty_tx, _ = self.create_ephemeral_dust_package(tx_version=3, num_dust_outputs=2)
-        block_res = self.generateblock(self.nodes[0], self.wallet.get_address(), [multi_dusty_tx["hex"]], sync_fun=self.no_op)
+
-        self.nodes[0].invalidateblock(block_res["hash"])
+        # Prep for fork with empty blocks
+        fork_blocks = create_empty_fork(self.nodes[0])
+
+        self.generateblock(self.nodes[0], self.wallet.get_address(), [multi_dusty_tx["hex"]], sync_fun=self.no_op)
+        self.trigger_reorg(fork_blocks)
         assert_equal(self.nodes[0].getrawmempool(), [])
 
         # With fee and one dust
         dusty_fee_tx, _ = self.create_ephemeral_dust_package(tx_version=3, dust_tx_fee=1)
-        block_res = self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_fee_tx["hex"]], sync_fun=self.no_op)
+
-        self.nodes[0].invalidateblock(block_res["hash"])
+        # Prep for fork with empty blocks
+        fork_blocks = create_empty_fork(self.nodes[0])
+
+        self.generateblock(self.nodes[0], self.wallet.get_address(), [dusty_fee_tx["hex"]], sync_fun=self.no_op)
+        self.trigger_reorg(fork_blocks)
         assert_equal(self.nodes[0].getrawmempool(), [])
 
         # Re-connect and make sure we have same state still

test/functional/mempool_packages.py

@@ -15,6 +15,7 @@ from test_framework.util import (
     assert_equal,
 )
 from test_framework.wallet import MiniWallet
+from test_framework.blocktools import create_empty_fork
 
 # custom limits for node1
 CUSTOM_CLUSTER_LIMIT = 10
@@ -33,6 +34,12 @@ class MempoolPackagesTest(BitcoinTestFramework):
             ],
         ]
 
+    def trigger_reorg(self, fork_blocks, node):
+        """Trigger reorg of the fork blocks."""
+        for block in fork_blocks:
+            node.submitblock(block.serialize().hex())
+        assert_equal(node.getbestblockhash(), fork_blocks[-1].hash_hex)
+
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
         self.wallet.rescan_utxos()
@@ -227,9 +234,16 @@ class MempoolPackagesTest(BitcoinTestFramework):
 
         # Test reorg handling
         # First, the basics:
+        fork_blocks = create_empty_fork(self.nodes[0])
+        mempool0 = self.nodes[0].getrawmempool(False)
+        self.generate(self.nodes[0], 1)
+        self.trigger_reorg(fork_blocks, self.nodes[0])
+
+        # Check if the txs are returned to the mempool
+        assert_equal(self.nodes[0].getrawmempool(), mempool0)
+
+        # Clean-up the mempool
         self.generate(self.nodes[0], 1)
-        self.nodes[1].invalidateblock(self.nodes[0].getbestblockhash())
-        self.nodes[1].reconsiderblock(self.nodes[0].getbestblockhash())
 
         # Now test the case where node1 has a transaction T in its mempool that
         # depends on transactions A and B which are in a mined block, and the
@@ -244,6 +258,9 @@ class MempoolPackagesTest(BitcoinTestFramework):
         # Tx1 and Tx7, and add to node1's mempool, then disconnect the
         # last block.
 
+        # Prep for fork
+        fork_blocks = create_empty_fork(self.nodes[0])
+
         # Create tx0 with 2 outputs
         tx0 = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], num_outputs=2)
 
@@ -261,8 +278,7 @@ class MempoolPackagesTest(BitcoinTestFramework):
         self.sync_mempools()
 
         # Now try to disconnect the tip on each node...
-        self.nodes[1].invalidateblock(self.nodes[1].getbestblockhash())
+        self.trigger_reorg(fork_blocks, self.nodes[0])
-        self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash())
         self.sync_blocks()
 
 if __name__ == '__main__':

test/functional/mempool_reorg.py

@@ -22,6 +22,13 @@ from test_framework.p2p import (
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, assert_raises_rpc_error
 from test_framework.wallet import MiniWallet
+from test_framework.blocktools import (
+    create_empty_fork,
+)
+
+# Number of blocks to create in temporary blockchain branch for reorg testing
+# needs to be long enough to allow MTP to move arbitrarily forward
+FORK_LENGTH = 20
 
 class MempoolCoinbaseTest(BitcoinTestFramework):
     def set_test_params(self):
@@ -33,6 +40,12 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
             []
         ]
 
+    def trigger_reorg(self, fork_blocks, node):
+        """Trigger reorg of the fork blocks."""
+        for block in fork_blocks:
+            node.submitblock(block.serialize().hex())
+        assert_equal(self.nodes[0].getbestblockhash(), fork_blocks[-1].hash_hex)
+
     def test_reorg_relay(self):
         self.log.info("Test that transactions from disconnected blocks are available for relay immediately")
         # Prevent time from moving forward
@@ -112,6 +125,10 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         self.wallet = MiniWallet(self.nodes[0])
         wallet = self.wallet
 
+        # Prevent clock from moving blocks further forward in time
+        now = int(time.time())
+        self.nodes[0].setmocktime(now)
+
         # Start with a 200 block chain
         assert_equal(self.nodes[0].getblockcount(), 200)
 
@@ -123,7 +140,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         # 1. Direct coinbase spend  :  spend_1
         # 2. Indirect (coinbase spend in chain, child in mempool) : spend_2 and spend_2_1
         # 3. Indirect (coinbase and child both in chain) : spend_3 and spend_3_1
-        # Use invalidateblock to make all of the above coinbase spends invalid (immature coinbase),
+        # Use re-org to make all of the above coinbase spends invalid (immature coinbase),
         # and make sure the mempool code behaves correctly.
         b = [self.nodes[0].getblockhash(n) for n in range(first_block, first_block+4)]
         coinbase_txids = [self.nodes[0].getblock(h)['tx'][0] for h in b]
@@ -135,11 +152,12 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         spend_2 = wallet.create_self_transfer(utxo_to_spend=utxo_2)
         spend_3 = wallet.create_self_transfer(utxo_to_spend=utxo_3)
 
-        self.log.info("Create another transaction which is time-locked to two blocks in the future")
+        self.log.info("Create another transaction which is time-locked to 300 seconds in the future")
+        future = now + 300
         utxo = wallet.get_utxo(txid=coinbase_txids[0])
         timelock_tx = wallet.create_self_transfer(
             utxo_to_spend=utxo,
-            locktime=self.nodes[0].getblockcount() + 2,
+            locktime=future,
         )['hex']
 
         self.log.info("Check that the time-locked transaction is too immature to spend")
@@ -160,10 +178,18 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         self.log.info("Broadcast and mine spend_3_1")
         spend_3_1_id = self.nodes[0].sendrawtransaction(spend_3_1['hex'])
         self.log.info("Generate a block")
-        last_block = self.generate(self.nodes[0], 1)
+
+        # Prep for fork, only go FORK_LENGTH  seconds into the MTP future max
+        fork_blocks = create_empty_fork(self.nodes[0], fork_length=FORK_LENGTH)
+
+        # Jump node and MTP 300 seconds and generate a slightly weaker chain than reorg one
+        self.nodes[0].setmocktime(future)
+        self.generate(self.nodes[0], FORK_LENGTH - 1)
+        block_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
+        assert(block_time >= now + 300)
+
         # generate() implicitly syncs blocks, so that peer 1 gets the block before timelock_tx
         # Otherwise, peer 1 would put the timelock_tx in m_lazy_recent_rejects
-
         self.log.info("The time-locked transaction can now be spent")
         timelock_tx_id = self.nodes[0].sendrawtransaction(timelock_tx)
 
@@ -174,15 +200,22 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         assert_equal(set(self.nodes[0].getrawmempool()), {spend_1_id, spend_2_1_id, timelock_tx_id})
         self.sync_all()
 
-        self.log.info("invalidate the last block")
+        self.trigger_reorg(fork_blocks, self.nodes[0])
-        for node in self.nodes:
+        self.sync_blocks()
-            node.invalidateblock(last_block[0])
+
+        # We went backwards in time to boot timelock_tx_id
+        fork_block_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
+        assert fork_block_time < block_time
+
         self.log.info("The time-locked transaction is now too immature and has been removed from the mempool")
         self.log.info("spend_3_1 has been re-orged out of the chain and is back in the mempool")
         assert_equal(set(self.nodes[0].getrawmempool()), {spend_1_id, spend_2_1_id, spend_3_1_id})
 
         self.log.info("Use invalidateblock to re-org back and make all those coinbase spends immature/invalid")
         b = self.nodes[0].getblockhash(first_block + 100)
+
+        # Use invalidateblock to go backwards in MTP time.
+        # invalidateblock actually moves MTP backwards, making timelock_tx_id valid again.
         for node in self.nodes:
             node.invalidateblock(b)
 

test/functional/mempool_resurrect.py

@@ -7,12 +7,18 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal
 from test_framework.wallet import MiniWallet
-
+from test_framework.blocktools import create_empty_fork
 
 class MempoolCoinbaseTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
 
+    def trigger_reorg(self, fork_blocks):
+        """Trigger reorg of the fork blocks."""
+        for block in fork_blocks:
+            self.nodes[0].submitblock(block.serialize().hex())
+        assert_equal(self.nodes[0].getbestblockhash(), fork_blocks[-1].hash_hex)
+
     def run_test(self):
         node = self.nodes[0]
         wallet = MiniWallet(node)
@@ -26,6 +32,9 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         # ... make sure all the transactions are put back in the mempool
         # Mine a new block
         # ... make sure all the transactions are confirmed again
+
+        # Prep for fork
+        fork_blocks = create_empty_fork(self.nodes[0])
         blocks = []
         spends1_ids = [wallet.send_self_transfer(from_node=node)['txid'] for _ in range(3)]
         blocks.extend(self.generate(node, 1))
@@ -40,8 +49,8 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         # Checks that all spend txns are contained in the mined blocks
         assert spends_ids < confirmed_txns
 
-        # Use invalidateblock to re-org back
+        # Trigger reorg
-        node.invalidateblock(blocks[0])
+        self.trigger_reorg(fork_blocks)
 
         # All txns should be back in mempool with 0 confirmations
         assert_equal(set(node.getrawmempool()), spends_ids)

test/functional/mempool_truc.py

@@ -18,6 +18,9 @@ from test_framework.wallet import (
     DEFAULT_FEE,
     MiniWallet,
 )
+from test_framework.blocktools import (
+    create_empty_fork,
+)
 
 MAX_REPLACEMENT_CANDIDATES = 100
 TRUC_MAX_VSIZE = 10000
@@ -50,6 +53,12 @@ class MempoolTRUC(BitcoinTestFramework):
         assert_equal(len(txids), len(mempool_contents))
         assert all([txid in txids for txid in mempool_contents])
 
+    def trigger_reorg(self, fork_blocks):
+        """Trigger reorg of the fork blocks."""
+        for block in fork_blocks:
+            self.nodes[0].submitblock(block.serialize().hex())
+        assert_equal(self.nodes[0].getbestblockhash(), fork_blocks[-1].hash_hex)
+
     @cleanup()
     def test_truc_max_vsize(self):
         node = self.nodes[0]
@@ -164,6 +173,9 @@ class MempoolTRUC(BitcoinTestFramework):
     @cleanup()
     def test_truc_reorg(self):
         node = self.nodes[0]
+
+        # Prep for fork
+        fork_blocks = create_empty_fork(node)
         self.log.info("Test that, during a reorg, TRUC rules are not enforced")
         self.check_mempool([])
 
@@ -182,9 +194,10 @@ class MempoolTRUC(BitcoinTestFramework):
         tx_chain_3 = self.wallet.create_self_transfer(utxo_to_spend=tx_chain_2["new_utxo"], version=3)
 
         tx_to_mine = [tx_v3_block["hex"], tx_v2_block["hex"], tx_v3_block2["hex"], tx_chain_1["hex"], tx_chain_2["hex"], tx_chain_3["hex"]]
-        block = self.generateblock(node, output="raw(42)", transactions=tx_to_mine)
+        self.generateblock(node, output="raw(42)", transactions=tx_to_mine)
 
         self.check_mempool([])
+
         tx_v2_from_v3 = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block["new_utxo"], version=2)
         tx_v3_from_v2 = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v2_block["new_utxo"], version=3)
         tx_v3_child_large = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=tx_v3_block2["new_utxo"], target_vsize=1250, version=3)
@@ -193,7 +206,7 @@ class MempoolTRUC(BitcoinTestFramework):
         self.check_mempool([tx_v2_from_v3["txid"], tx_v3_from_v2["txid"], tx_v3_child_large["txid"], tx_chain_4["txid"]])
 
         # Reorg should have all block transactions re-accepted, ignoring TRUC enforcement
-        node.invalidateblock(block["hash"])
+        self.trigger_reorg(fork_blocks)
         self.check_mempool([tx_v3_block["txid"], tx_v2_block["txid"], tx_v3_block2["txid"], tx_v2_from_v3["txid"], tx_v3_from_v2["txid"], tx_v3_child_large["txid"], tx_chain_1["txid"], tx_chain_2["txid"], tx_chain_3["txid"], tx_chain_4["txid"]])
 
     @cleanup(extra_args=["-limitclustercount=1"])
@@ -463,21 +476,20 @@ class MempoolTRUC(BitcoinTestFramework):
         node = self.nodes[0]
         self.log.info("Test that children of a TRUC transaction can be replaced individually, even if there are multiple due to reorg")
 
+        # Prep for fork
+        fork_blocks = create_empty_fork(node)
         ancestor_tx = self.wallet.send_self_transfer_multi(from_node=node, num_outputs=2, version=3)
         self.check_mempool([ancestor_tx["txid"]])
 
-        block = self.generate(node, 1)[0]
+        self.generate(node, 1)[0]
         self.check_mempool([])
 
         child_1 = self.wallet.send_self_transfer(from_node=node, version=3, utxo_to_spend=ancestor_tx["new_utxos"][0])
         child_2 = self.wallet.send_self_transfer(from_node=node, version=3, utxo_to_spend=ancestor_tx["new_utxos"][1])
         self.check_mempool([child_1["txid"], child_2["txid"]])
 
-        self.generate(node, 1)
-        self.check_mempool([])
-
         # Create a reorg, causing ancestor_tx to exceed the 1-child limit
-        node.invalidateblock(block)
+        self.trigger_reorg(fork_blocks)
         self.check_mempool([ancestor_tx["txid"], child_1["txid"], child_2["txid"]])
         assert_equal(node.getmempoolentry(ancestor_tx["txid"])["descendantcount"], 3)
 
@@ -562,10 +574,12 @@ class MempoolTRUC(BitcoinTestFramework):
         node = self.nodes[0]
         self.log.info("Test that sibling eviction is not allowed when multiple siblings exist")
 
+        # Prep for fork
+        fork_blocks = create_empty_fork(node)
         tx_with_multi_children = self.wallet.send_self_transfer_multi(from_node=node, num_outputs=3, version=3, confirmed_only=True)
         self.check_mempool([tx_with_multi_children["txid"]])
 
-        block_to_disconnect = self.generate(node, 1)[0]
+        self.generate(node, 1)
         self.check_mempool([])
 
         tx_with_sibling1 = self.wallet.send_self_transfer(from_node=node, version=3, utxo_to_spend=tx_with_multi_children["new_utxos"][0])
@@ -573,7 +587,7 @@ class MempoolTRUC(BitcoinTestFramework):
         self.check_mempool([tx_with_sibling1["txid"], tx_with_sibling2["txid"]])
 
         # Create a reorg, bringing tx_with_multi_children back into the mempool with a descendant count of 3.
-        node.invalidateblock(block_to_disconnect)
+        self.trigger_reorg(fork_blocks)
         self.check_mempool([tx_with_multi_children["txid"], tx_with_sibling1["txid"], tx_with_sibling2["txid"]])
         assert_equal(node.getmempoolentry(tx_with_multi_children["txid"])["descendantcount"], 3)
 

test/functional/mempool_updatefromblock.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2020-2022 The Bitcoin Core developers
+# Copyright (c) 2020-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test mempool descendants/ancestors information update.
@@ -27,6 +27,12 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework):
         self.num_nodes = 1
         self.extra_args = [['-limitclustersize=1000']]
 
+    def trigger_reorg(self, fork_blocks):
+        """Trigger reorg of the fork blocks."""
+        for block in fork_blocks:
+            self.nodes[0].submitblock(block.serialize().hex())
+        assert_equal(self.nodes[0].getbestblockhash(), fork_blocks[-1].hash_hex)
+
     def transaction_graph_test(self, size, *, n_tx_to_mine, fee=100_000):
         """Create an acyclic tournament (a type of directed graph) of transactions and use it for testing.
 
@@ -153,8 +159,7 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework):
 
         # Reorg back before the first block in the series, should drop something
         # but not all, and any time parent is dropped, child is also removed
-        for block in fork_blocks:
+        self.trigger_reorg(fork_blocks=fork_blocks)
-            self.nodes[0].submitblock(block.serialize().hex())
         mempool = self.nodes[0].getrawmempool()
         expected_parent_count = len(large_std_txs) - 2
         assert_equal(len(mempool), expected_parent_count * 2)