Merge bitcoin/bitcoin#34452: test: split interface_ipc.py

633d183119 test: misc interface_ipc_mining.py improvements (Sjors Provoost)
52ccd9215e test: split interface_ipc_mining.py into subtests (Sjors Provoost)
4e49fa2a68 test: add interface_ipc_mining.py (Sjors Provoost)
01a1ae889e test: move IPC helpers to ipc_util.py (Sjors Provoost)

Pull request description:

  This test has been growing too large, making it difficult to maintain. Especially when multiple pull requests change it.

  - move helper functions to `ipc_util.py`
  - move mining test to `interface_ipc_mining.py`, keeping only an interface sanity check in `interface_ipc.py`
  - split the tests in `interface_ipc_mining.py`
  - misc tweaks (to reduce churn in the above commits)

  Review hint:

  ```sh
  git show --color-moved=dimmed-zebra --color-moved-ws=ignore-space-change
  ```

ACKs for top commit:
  janb84:
    re ACK 633d183119
  fjahr:
    reACK 633d183119
  l0rinc:
    code review ACK 633d183119
  sedited:
    Re-ACK 633d183119

Tree-SHA512: 81bbbec1f03a6ff63068dbefc1bc4768fcd24313d84ba454bb2494fe4a65838dbaeb93ad7f02ed4c9932394f3d24638453bb51ce0e05f561dc613414beda37e4

test/functional/interface_ipc.py

@@ -4,44 +4,13 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test the IPC (multiprocess) interface."""
 import asyncio
-import inspect
-from contextlib import asynccontextmanager, AsyncExitStack
-from dataclasses import dataclass
-from io import BytesIO
-from pathlib import Path
-import shutil
-from test_framework.blocktools import NULL_OUTPOINT
-from test_framework.messages import (
-    CBlock,
-    CTransaction,
-    CTxIn,
-    CTxOut,
-    CTxInWitness,
-    ser_uint256,
-    COIN,
-)
-from test_framework.script import (
-    CScript,
-    CScriptNum,
-)
-from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import (
-    assert_equal,
-    assert_not_equal,
-)
-from test_framework.wallet import MiniWallet
-from typing import Optional
 
-# Stores the result of getCoinbaseTx()
+from test_framework.test_framework import BitcoinTestFramework
-@dataclass
+from test_framework.util import assert_equal
-class CoinbaseTxData:
+from test_framework.ipc_util import (
-    version: int
+    load_capnp_modules,
-    sequence: int
+    make_capnp_init_ctx,
-    scriptSigPrefix: bytes
+)
-    witness: Optional[bytes]
-    blockRewardRemaining: int
-    requiredOutputs: list[bytes]
-    lockTime: int
 
 # Test may be skipped and not have capnp installed
 try:
@@ -49,44 +18,6 @@ try:
 except ModuleNotFoundError:
     pass
 
-@asynccontextmanager
-async def destroying(obj, ctx):
-    """Call obj.destroy(ctx) at end of with: block. Similar to contextlib.closing."""
-    try:
-        yield obj
-    finally:
-        await obj.destroy(ctx)
-
-async def create_block_template(mining, stack, ctx, opts):
-    """Call mining.createNewBlock() and return template, then call template.destroy() when stack exits."""
-    return await stack.enter_async_context(destroying((await mining.createNewBlock(opts)).result, ctx))
-
-async def wait_next_template(template, stack, ctx, opts):
-    """Call template.waitNext() and return template, then call template.destroy() when stack exits."""
-    return await stack.enter_async_context(destroying((await template.waitNext(ctx, opts)).result, ctx))
-
-async def wait_and_do(wait_fn, do_fn):
-    """Call wait_fn, then sleep, then call do_fn in a parallel task. Wait for
-    both tasks to complete."""
-    wait_started = asyncio.Event()
-    result = None
-
-    async def wait():
-        nonlocal result
-        wait_started.set()
-        result = await wait_fn
-
-    async def do():
-        await wait_started.wait()
-        await asyncio.sleep(0.1)
-        # Let do_fn be either a callable or an awaitable object
-        if inspect.isawaitable(do_fn):
-            await do_fn
-        else:
-            do_fn()
-
-    await asyncio.gather(wait(), do())
-    return result
 
 class IPCInterfaceTest(BitcoinTestFramework):
 
@@ -94,91 +25,20 @@ class IPCInterfaceTest(BitcoinTestFramework):
         self.skip_if_no_ipc()
         self.skip_if_no_py_capnp()
 
-    def load_capnp_modules(self):
-        if capnp_bin := shutil.which("capnp"):
-            # Add the system cap'nproto path so include/capnp/c++.capnp can be found.
-            capnp_dir = Path(capnp_bin).resolve().parent.parent / "include"
-        else:
-            # If there is no system cap'nproto, the pycapnp module should have its own "bundled"
-            # includes at this location. If pycapnp was installed with bundled capnp,
-            # capnp/c++.capnp can be found here.
-            capnp_dir = Path(capnp.__path__[0]).parent
-        src_dir = Path(self.config['environment']['SRCDIR']) / "src"
-        mp_dir = src_dir / "ipc" / "libmultiprocess" / "include"
-        # List of import directories. Note: it is important for mp_dir to be
-        # listed first, in case there are other libmultiprocess installations on
-        # the system, to ensure that `import "/mp/proxy.capnp"` lines load the
-        # same file as capnp.load() loads directly below, and there are not
-        # "failed: Duplicate ID @0xcc316e3f71a040fb" errors.
-        imports = [str(mp_dir), str(capnp_dir), str(src_dir)]
-        return {
-            "proxy": capnp.load(str(mp_dir / "mp" / "proxy.capnp"), imports=imports),
-            "init": capnp.load(str(src_dir / "ipc" / "capnp" / "init.capnp"), imports=imports),
-            "echo": capnp.load(str(src_dir / "ipc" / "capnp" / "echo.capnp"), imports=imports),
-            "mining": capnp.load(str(src_dir / "ipc" / "capnp" / "mining.capnp"), imports=imports),
-        }
-
     def set_test_params(self):
-        self.num_nodes = 2
+        self.num_nodes = 1
 
     def setup_nodes(self):
-        self.extra_init = [{"ipcbind": True}, {}]
+        self.extra_init = [{"ipcbind": True}]
         super().setup_nodes()
         # Use this function to also load the capnp modules (we cannot use set_test_params for this,
         # as it is being called before knowing whether capnp is available).
-        self.capnp_modules = self.load_capnp_modules()
+        self.capnp_modules = load_capnp_modules(self.config)
-
-    async def make_capnp_init_ctx(self):
-        node = self.nodes[0]
-        # Establish a connection, and create Init proxy object.
-        connection = await capnp.AsyncIoStream.create_unix_connection(node.ipc_socket_path)
-        client = capnp.TwoPartyClient(connection)
-        init = client.bootstrap().cast_as(self.capnp_modules['init'].Init)
-        # Create a remote thread on the server for the IPC calls to be executed in.
-        threadmap = init.construct().threadMap
-        thread = threadmap.makeThread("pythread").result
-        ctx = self.capnp_modules['proxy'].Context()
-        ctx.thread = thread
-        # Return both.
-        return ctx, init
-
-    async def parse_and_deserialize_block(self, block_template, ctx):
-        block_data = BytesIO((await block_template.getBlock(ctx)).result)
-        block = CBlock()
-        block.deserialize(block_data)
-        return block
-
-    async def get_coinbase_raw_tx(self, block_template, ctx):
-        assert block_template is not None
-        coinbase_data = BytesIO((await block_template.getCoinbaseRawTx(ctx)).result)
-        tx = CTransaction()
-        tx.deserialize(coinbase_data)
-        return tx
-
-    async def parse_and_deserialize_coinbase(self, block_template, ctx) -> CoinbaseTxData:
-        assert block_template is not None
-        # Note: the template_capnp struct will be garbage-collected when this
-        # method returns, so it is important to copy any Data fields from it
-        # which need to be accessed later using the bytes() cast. Starting with
-        # pycapnp v2.2.0, Data fields have type `memoryview` and are ephemeral.
-        template_capnp = (await block_template.getCoinbaseTx(ctx)).result
-        witness: Optional[bytes] = None
-        if template_capnp._has("witness"):
-            witness = bytes(template_capnp.witness)
-        return CoinbaseTxData(
-            version=int(template_capnp.version),
-            sequence=int(template_capnp.sequence),
-            scriptSigPrefix=bytes(template_capnp.scriptSigPrefix),
-            witness=witness,
-            blockRewardRemaining=int(template_capnp.blockRewardRemaining),
-            requiredOutputs=[bytes(output) for output in template_capnp.requiredOutputs],
-            lockTime=int(template_capnp.lockTime),
-        )
 
     def run_echo_test(self):
         self.log.info("Running echo test")
         async def async_routine():
-            ctx, init = await self.make_capnp_init_ctx()
+            ctx, init = await make_capnp_init_ctx(self)
             self.log.debug("Create Echo proxy object")
             echo = init.makeEcho(ctx).result
             self.log.debug("Test a few invocations of echo")
@@ -189,70 +49,12 @@ class IPCInterfaceTest(BitcoinTestFramework):
             echo.destroy(ctx)
         asyncio.run(capnp.run(async_routine()))
 
-    async def build_coinbase_test(self, template, ctx, miniwallet):
-        self.log.debug("Build coinbase transaction using getCoinbaseTx()")
-        assert template is not None
-        coinbase_res = await self.parse_and_deserialize_coinbase(template, ctx)
-        coinbase_tx = CTransaction()
-        coinbase_tx.version = coinbase_res.version
-        coinbase_tx.vin = [CTxIn()]
-        coinbase_tx.vin[0].prevout = NULL_OUTPOINT
-        coinbase_tx.vin[0].nSequence = coinbase_res.sequence
-
-        # Verify there's no dummy extraNonce in the coinbase scriptSig
-        current_block_height = self.nodes[0].getchaintips()[0]["height"]
-        expected_scriptsig = CScript([CScriptNum(current_block_height + 1)])
-        assert_equal(coinbase_res.scriptSigPrefix.hex(), expected_scriptsig.hex())
-
-        # Typically a mining pool appends its name and an extraNonce
-        coinbase_tx.vin[0].scriptSig = coinbase_res.scriptSigPrefix
-
-        # We currently always provide a coinbase witness, even for empty
-        # blocks, but this may change, so always check:
-        has_witness = coinbase_res.witness is not None
-        if has_witness:
-            coinbase_tx.wit.vtxinwit = [CTxInWitness()]
-            coinbase_tx.wit.vtxinwit[0].scriptWitness.stack = [coinbase_res.witness]
-
-        # First output is our payout
-        coinbase_tx.vout = [CTxOut()]
-        coinbase_tx.vout[0].scriptPubKey = miniwallet.get_output_script()
-        coinbase_tx.vout[0].nValue = coinbase_res.blockRewardRemaining
-        # Add SegWit OP_RETURN. This is currently always present even for
-        # empty blocks, but this may change.
-        found_witness_op_return = False
-        # Compare SegWit OP_RETURN to getCoinbaseCommitment()
-        coinbase_commitment = (await template.getCoinbaseCommitment(ctx)).result
-        for output_data in coinbase_res.requiredOutputs:
-            output = CTxOut()
-            output.deserialize(BytesIO(output_data))
-            coinbase_tx.vout.append(output)
-            if output.scriptPubKey == coinbase_commitment:
-                found_witness_op_return = True
-
-        assert_equal(has_witness, found_witness_op_return)
-
-        coinbase_tx.nLockTime = coinbase_res.lockTime
-
-        # Compare to dummy coinbase transaction provided by the deprecated
-        # getCoinbaseRawTx()
-        coinbase_legacy = await self.get_coinbase_raw_tx(template, ctx)
-        assert_equal(coinbase_legacy.vout[0].nValue, coinbase_res.blockRewardRemaining)
-        # Swap dummy output for our own
-        coinbase_legacy.vout[0].scriptPubKey = coinbase_tx.vout[0].scriptPubKey
-        assert_equal(coinbase_tx.serialize().hex(), coinbase_legacy.serialize().hex())
-
-        return coinbase_tx
-
     def run_mining_test(self):
         self.log.info("Running mining test")
         block_hash_size = 32
-        block_header_size = 80
-        timeout = 1000.0 # 1000 milliseconds
-        miniwallet = MiniWallet(self.nodes[0])
 
         async def async_routine():
-            ctx, init = await self.make_capnp_init_ctx()
+            ctx, init = await make_capnp_init_ctx(self)
             self.log.debug("Create Mining proxy object")
             mining = init.makeMining(ctx).result
             self.log.debug("Test simple inspectors")
@@ -262,145 +64,7 @@ class IPCInterfaceTest(BitcoinTestFramework):
             assert blockref.hasResult
             assert_equal(len(blockref.result.hash), block_hash_size)
             current_block_height = self.nodes[0].getchaintips()[0]["height"]
-            assert blockref.result.height == current_block_height
+            assert_equal(blockref.result.height, current_block_height)
-            self.log.debug("Mine a block")
-            newblockref = (await wait_and_do(
-                mining.waitTipChanged(ctx, blockref.result.hash, timeout),
-                lambda: self.generate(self.nodes[0], 1))).result
-            assert_equal(len(newblockref.hash), block_hash_size)
-            assert_equal(newblockref.height, current_block_height + 1)
-            self.log.debug("Wait for timeout")
-            oldblockref = (await mining.waitTipChanged(ctx, newblockref.hash, timeout)).result
-            assert_equal(len(newblockref.hash), block_hash_size)
-            assert_equal(oldblockref.hash, newblockref.hash)
-            assert_equal(oldblockref.height, newblockref.height)
-
-            async with AsyncExitStack() as stack:
-                self.log.debug("Create a template")
-                opts = self.capnp_modules['mining'].BlockCreateOptions()
-                opts.useMempool = True
-                opts.blockReservedWeight = 4000
-                opts.coinbaseOutputMaxAdditionalSigops = 0
-                template = await create_block_template(mining, stack, ctx, opts)
-
-                self.log.debug("Test some inspectors of Template")
-                header = (await template.getBlockHeader(ctx)).result
-                assert_equal(len(header), block_header_size)
-                block = await self.parse_and_deserialize_block(template, ctx)
-                assert_equal(ser_uint256(block.hashPrevBlock), newblockref.hash)
-                assert len(block.vtx) >= 1
-                txfees = await template.getTxFees(ctx)
-                assert_equal(len(txfees.result), 0)
-                txsigops = await template.getTxSigops(ctx)
-                assert_equal(len(txsigops.result), 0)
-
-                self.log.debug("Wait for a new template")
-                waitoptions = self.capnp_modules['mining'].BlockWaitOptions()
-                waitoptions.timeout = timeout
-                waitoptions.feeThreshold = 1
-                template2 = await wait_and_do(
-                    wait_next_template(template, stack, ctx, waitoptions),
-                    lambda: self.generate(self.nodes[0], 1))
-                block2 = await self.parse_and_deserialize_block(template2, ctx)
-                assert_equal(len(block2.vtx), 1)
-
-                self.log.debug("Wait for another, but time out")
-                template3 = await template2.waitNext(ctx, waitoptions)
-                assert_equal(template3._has("result"), False)
-
-                self.log.debug("Wait for another, get one after increase in fees in the mempool")
-                template4 = await wait_and_do(
-                    wait_next_template(template2, stack, ctx, waitoptions),
-                    lambda: miniwallet.send_self_transfer(fee_rate=10, from_node=self.nodes[0]))
-                block3 = await self.parse_and_deserialize_block(template4, ctx)
-                assert_equal(len(block3.vtx), 2)
-
-                self.log.debug("Wait again, this should return the same template, since the fee threshold is zero")
-                waitoptions.feeThreshold = 0
-                template5 = await wait_next_template(template4, stack, ctx, waitoptions)
-                block4 = await self.parse_and_deserialize_block(template5, ctx)
-                assert_equal(len(block4.vtx), 2)
-                waitoptions.feeThreshold = 1
-
-                self.log.debug("Wait for another, get one after increase in fees in the mempool")
-                template6 = await wait_and_do(
-                    wait_next_template(template5, stack, ctx, waitoptions),
-                    lambda: miniwallet.send_self_transfer(fee_rate=10, from_node=self.nodes[0]))
-                block4 = await self.parse_and_deserialize_block(template6, ctx)
-                assert_equal(len(block4.vtx), 3)
-
-                self.log.debug("Wait for another, but time out, since the fee threshold is set now")
-                template7 = await template6.waitNext(ctx, waitoptions)
-                assert_equal(template7._has("result"), False)
-
-                self.log.debug("interruptWait should abort the current wait")
-                async def wait_for_block():
-                    new_waitoptions = self.capnp_modules['mining'].BlockWaitOptions()
-                    new_waitoptions.timeout = waitoptions.timeout * 60 # 1 minute wait
-                    new_waitoptions.feeThreshold = 1
-                    template7 = await template6.waitNext(ctx, new_waitoptions)
-                    assert_equal(template7._has("result"), False)
-                await wait_and_do(wait_for_block(), template6.interruptWait())
-
-            current_block_height = self.nodes[0].getchaintips()[0]["height"]
-            check_opts = self.capnp_modules['mining'].BlockCheckOptions()
-            async with destroying((await mining.createNewBlock(opts)).result, ctx) as template:
-                block = await self.parse_and_deserialize_block(template, ctx)
-                balance = miniwallet.get_balance()
-                coinbase = await self.build_coinbase_test(template, ctx, miniwallet)
-                # Reduce payout for balance comparison simplicity
-                coinbase.vout[0].nValue = COIN
-                block.vtx[0] = coinbase
-                block.hashMerkleRoot = block.calc_merkle_root()
-                original_version = block.nVersion
-                self.log.debug("Submit a block with a bad version")
-                block.nVersion = 0
-                block.solve()
-                check = await mining.checkBlock(block.serialize(), check_opts)
-                assert_equal(check.result, False)
-                assert_equal(check.reason, "bad-version(0x00000000)")
-                submitted = (await template.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize())).result
-                assert_equal(submitted, False)
-                self.log.debug("Submit a valid block")
-                block.nVersion = original_version
-                block.solve()
-
-                self.log.debug("First call checkBlock()")
-                block_valid = (await mining.checkBlock(block.serialize(), check_opts)).result
-                assert_equal(block_valid, True)
-
-                # The remote template block will be mutated, capture the original:
-                remote_block_before = await self.parse_and_deserialize_block(template, ctx)
-
-                self.log.debug("Submitted coinbase must include witness")
-                assert_not_equal(coinbase.serialize_without_witness().hex(), coinbase.serialize().hex())
-                submitted = (await template.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize_without_witness())).result
-                assert_equal(submitted, False)
-
-                self.log.debug("Even a rejected submitBlock() mutates the template's block")
-                # Can be used by clients to download and inspect the (rejected)
-                # reconstructed block.
-                remote_block_after = await self.parse_and_deserialize_block(template, ctx)
-                assert_not_equal(remote_block_before.serialize().hex(), remote_block_after.serialize().hex())
-
-                self.log.debug("Submit again, with the witness")
-                submitted = (await template.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize())).result
-                assert_equal(submitted, True)
-
-            self.log.debug("Block should propagate")
-            # Check that the IPC node actually updates its own chain
-            assert_equal(self.nodes[0].getchaintips()[0]["height"], current_block_height + 1)
-            # Stalls if a regression causes submitBlock() to accept an invalid block:
-            self.sync_all()
-            # Check that the other node accepts the block
-            assert_equal(self.nodes[0].getchaintips()[0], self.nodes[1].getchaintips()[0])
-
-            miniwallet.rescan_utxos()
-            assert_equal(miniwallet.get_balance(), balance + 1)
-            self.log.debug("Check block should fail now, since it is a duplicate")
-            check = await mining.checkBlock(block.serialize(), check_opts)
-            assert_equal(check.result, False)
-            assert_equal(check.reason, "inconclusive-not-best-prevblk")
 
         asyncio.run(capnp.run(async_routine()))
 

test/functional/interface_ipc_mining.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python3
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test the IPC (multiprocess) Mining interface."""
+import asyncio
+from contextlib import AsyncExitStack
+from io import BytesIO
+from test_framework.blocktools import NULL_OUTPOINT
+from test_framework.messages import (
+    CTransaction,
+    CTxIn,
+    CTxOut,
+    CTxInWitness,
+    ser_uint256,
+    COIN,
+)
+from test_framework.script import (
+    CScript,
+    CScriptNum,
+)
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    assert_not_equal
+)
+from test_framework.wallet import MiniWallet
+from test_framework.ipc_util import (
+    destroying,
+    mining_create_block_template,
+    load_capnp_modules,
+    make_capnp_init_ctx,
+    mining_get_block,
+    mining_get_coinbase_tx,
+    mining_get_coinbase_raw_tx,
+    mining_wait_next_template,
+    wait_and_do,
+)
+
+# Test may be skipped and not have capnp installed
+try:
+    import capnp  # type: ignore[import] # noqa: F401
+except ModuleNotFoundError:
+    pass
+
+
+class IPCMiningTest(BitcoinTestFramework):
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_ipc()
+        self.skip_if_no_py_capnp()
+
+    def set_test_params(self):
+        self.num_nodes = 2
+
+    def setup_nodes(self):
+        self.extra_init = [{"ipcbind": True}, {}]
+        super().setup_nodes()
+        # Use this function to also load the capnp modules (we cannot use set_test_params for this,
+        # as it is being called before knowing whether capnp is available).
+        self.capnp_modules = load_capnp_modules(self.config)
+
+    async def build_coinbase_test(self, template, ctx, miniwallet):
+        self.log.debug("Build coinbase transaction using getCoinbaseTx()")
+        assert template is not None
+        coinbase_res = await mining_get_coinbase_tx(template, ctx)
+        coinbase_tx = CTransaction()
+        coinbase_tx.version = coinbase_res.version
+        coinbase_tx.vin = [CTxIn()]
+        coinbase_tx.vin[0].prevout = NULL_OUTPOINT
+        coinbase_tx.vin[0].nSequence = coinbase_res.sequence
+
+        # Verify there's no dummy extraNonce in the coinbase scriptSig
+        current_block_height = self.nodes[0].getchaintips()[0]["height"]
+        expected_scriptsig = CScript([CScriptNum(current_block_height + 1)])
+        assert_equal(coinbase_res.scriptSigPrefix.hex(), expected_scriptsig.hex())
+
+        # Typically a mining pool appends its name and an extraNonce
+        coinbase_tx.vin[0].scriptSig = coinbase_res.scriptSigPrefix
+
+        # We currently always provide a coinbase witness, even for empty
+        # blocks, but this may change, so always check:
+        has_witness = coinbase_res.witness is not None
+        if has_witness:
+            coinbase_tx.wit.vtxinwit = [CTxInWitness()]
+            coinbase_tx.wit.vtxinwit[0].scriptWitness.stack = [coinbase_res.witness]
+
+        # First output is our payout
+        coinbase_tx.vout = [CTxOut()]
+        coinbase_tx.vout[0].scriptPubKey = miniwallet.get_output_script()
+        coinbase_tx.vout[0].nValue = coinbase_res.blockRewardRemaining
+        # Add SegWit OP_RETURN. This is currently always present even for
+        # empty blocks, but this may change.
+        found_witness_op_return = False
+        # Compare SegWit OP_RETURN to getCoinbaseCommitment()
+        coinbase_commitment = (await template.getCoinbaseCommitment(ctx)).result
+        for output_data in coinbase_res.requiredOutputs:
+            output = CTxOut()
+            output.deserialize(BytesIO(output_data))
+            coinbase_tx.vout.append(output)
+            if output.scriptPubKey == coinbase_commitment:
+                found_witness_op_return = True
+
+        assert_equal(has_witness, found_witness_op_return)
+
+        coinbase_tx.nLockTime = coinbase_res.lockTime
+        # Compare to dummy coinbase transaction provided by the deprecated
+        # getCoinbaseRawTx()
+        coinbase_legacy = await mining_get_coinbase_raw_tx(template, ctx)
+        assert_equal(coinbase_legacy.vout[0].nValue, coinbase_res.blockRewardRemaining)
+        # Swap dummy output for our own
+        coinbase_legacy.vout[0].scriptPubKey = coinbase_tx.vout[0].scriptPubKey
+        assert_equal(coinbase_tx.serialize().hex(), coinbase_legacy.serialize().hex())
+
+        return coinbase_tx
+
+    async def make_mining_ctx(self):
+        """Create IPC context and Mining proxy object."""
+        ctx, init = await make_capnp_init_ctx(self)
+        self.log.debug("Create Mining proxy object")
+        mining = init.makeMining(ctx).result
+        return ctx, mining
+
+    def run_mining_interface_test(self):
+        """Test Mining interface methods."""
+        self.log.info("Running Mining interface test")
+        block_hash_size = 32
+        timeout = 1000.0 # 1000 milliseconds
+
+        async def async_routine():
+            ctx, mining = await self.make_mining_ctx()
+            blockref = await mining.getTip(ctx)
+            current_block_height = self.nodes[0].getchaintips()[0]["height"]
+            assert_equal(blockref.result.height, current_block_height)
+
+            self.log.debug("Mine a block")
+            newblockref = (await wait_and_do(
+                mining.waitTipChanged(ctx, blockref.result.hash, timeout),
+                lambda: self.generate(self.nodes[0], 1))).result
+            assert_equal(len(newblockref.hash), block_hash_size)
+            assert_equal(newblockref.height, current_block_height + 1)
+            self.log.debug("Wait for timeout")
+            oldblockref = (await mining.waitTipChanged(ctx, newblockref.hash, timeout)).result
+            assert_equal(len(newblockref.hash), block_hash_size)
+            assert_equal(oldblockref.hash, newblockref.hash)
+            assert_equal(oldblockref.height, newblockref.height)
+
+        asyncio.run(capnp.run(async_routine()))
+
+    def run_block_template_test(self):
+        """Test BlockTemplate interface methods."""
+        self.log.info("Running BlockTemplate interface test")
+        block_header_size = 80
+        timeout = 1000.0 # 1000 milliseconds
+
+        async def async_routine():
+            ctx, mining = await self.make_mining_ctx()
+            blockref = await mining.getTip(ctx)
+
+            async with AsyncExitStack() as stack:
+                self.log.debug("Create a template")
+                template = await mining_create_block_template(mining, stack, ctx, self.default_block_create_options)
+
+                self.log.debug("Test some inspectors of Template")
+                header = (await template.getBlockHeader(ctx)).result
+                assert_equal(len(header), block_header_size)
+                block = await mining_get_block(template, ctx)
+                assert_equal(ser_uint256(block.hashPrevBlock), blockref.result.hash)
+                assert len(block.vtx) >= 1
+                txfees = await template.getTxFees(ctx)
+                assert_equal(len(txfees.result), 0)
+                txsigops = await template.getTxSigops(ctx)
+                assert_equal(len(txsigops.result), 0)
+
+                self.log.debug("Wait for a new template")
+                waitoptions = self.capnp_modules['mining'].BlockWaitOptions()
+                waitoptions.timeout = timeout
+                waitoptions.feeThreshold = 1
+                template2 = await wait_and_do(
+                    mining_wait_next_template(template, stack, ctx, waitoptions),
+                    lambda: self.generate(self.nodes[0], 1))
+                block2 = await mining_get_block(template2, ctx)
+                assert_equal(len(block2.vtx), 1)
+
+                self.log.debug("Wait for another, but time out")
+                template3 = await template2.waitNext(ctx, waitoptions)
+                assert_equal(template3._has("result"), False)
+
+                self.log.debug("Wait for another, get one after increase in fees in the mempool")
+                template4 = await wait_and_do(
+                    mining_wait_next_template(template2, stack, ctx, waitoptions),
+                    lambda: self.miniwallet.send_self_transfer(fee_rate=10, from_node=self.nodes[0]))
+                block3 = await mining_get_block(template4, ctx)
+                assert_equal(len(block3.vtx), 2)
+
+                self.log.debug("Wait again, this should return the same template, since the fee threshold is zero")
+                waitoptions.feeThreshold = 0
+                template5 = await mining_wait_next_template(template4, stack, ctx, waitoptions)
+                block4 = await mining_get_block(template5, ctx)
+                assert_equal(len(block4.vtx), 2)
+                waitoptions.feeThreshold = 1
+
+                self.log.debug("Wait for another, get one after increase in fees in the mempool")
+                template6 = await wait_and_do(
+                    mining_wait_next_template(template5, stack, ctx, waitoptions),
+                    lambda: self.miniwallet.send_self_transfer(fee_rate=10, from_node=self.nodes[0]))
+                block4 = await mining_get_block(template6, ctx)
+                assert_equal(len(block4.vtx), 3)
+
+                self.log.debug("Wait for another, but time out, since the fee threshold is set now")
+                template7 = await template6.waitNext(ctx, waitoptions)
+                assert_equal(template7._has("result"), False)
+
+                self.log.debug("interruptWait should abort the current wait")
+                async def wait_for_block():
+                    new_waitoptions = self.capnp_modules['mining'].BlockWaitOptions()
+                    new_waitoptions.timeout = timeout * 60 # 1 minute wait
+                    new_waitoptions.feeThreshold = 1
+                    template7 = await template6.waitNext(ctx, new_waitoptions)
+                    assert_equal(template7._has("result"), False)
+                await wait_and_do(wait_for_block(), template6.interruptWait())
+
+        asyncio.run(capnp.run(async_routine()))
+
+    def run_coinbase_and_submission_test(self):
+        """Test coinbase construction (getCoinbaseTx, getCoinbaseCommitment) and block submission (submitSolution)."""
+        self.log.info("Running coinbase construction and submission test")
+
+        async def async_routine():
+            ctx, mining = await self.make_mining_ctx()
+
+            current_block_height = self.nodes[0].getchaintips()[0]["height"]
+            check_opts = self.capnp_modules['mining'].BlockCheckOptions()
+
+            async with destroying((await mining.createNewBlock(self.default_block_create_options)).result, ctx) as template:
+                block = await mining_get_block(template, ctx)
+                balance = self.miniwallet.get_balance()
+                coinbase = await self.build_coinbase_test(template, ctx, self.miniwallet)
+                # Reduce payout for balance comparison simplicity
+                coinbase.vout[0].nValue = COIN
+                block.vtx[0] = coinbase
+                block.hashMerkleRoot = block.calc_merkle_root()
+                original_version = block.nVersion
+
+                self.log.debug("Submit a block with a bad version")
+                block.nVersion = 0
+                block.solve()
+                check = await mining.checkBlock(block.serialize(), check_opts)
+                assert_equal(check.result, False)
+                assert_equal(check.reason, "bad-version(0x00000000)")
+                submitted = (await template.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize())).result
+                assert_equal(submitted, False)
+                self.log.debug("Submit a valid block")
+                block.nVersion = original_version
+                block.solve()
+
+                self.log.debug("First call checkBlock()")
+                block_valid = (await mining.checkBlock(block.serialize(), check_opts)).result
+                assert_equal(block_valid, True)
+
+                # The remote template block will be mutated, capture the original:
+                remote_block_before = await mining_get_block(template, ctx)
+
+                self.log.debug("Submitted coinbase must include witness")
+                assert_not_equal(coinbase.serialize_without_witness().hex(), coinbase.serialize().hex())
+                submitted = (await template.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize_without_witness())).result
+                assert_equal(submitted, False)
+
+                self.log.debug("Even a rejected submitSolution() mutates the template's block")
+                # Can be used by clients to download and inspect the (rejected)
+                # reconstructed block.
+                remote_block_after = await mining_get_block(template, ctx)
+                assert_not_equal(remote_block_before.serialize().hex(), remote_block_after.serialize().hex())
+
+                self.log.debug("Submit again, with the witness")
+                submitted = (await template.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize())).result
+                assert_equal(submitted, True)
+
+            self.log.debug("Block should propagate")
+            # Check that the IPC node actually updates its own chain
+            assert_equal(self.nodes[0].getchaintips()[0]["height"], current_block_height + 1)
+            # Stalls if a regression causes submitSolution() to accept an invalid block:
+            self.sync_all()
+            # Check that the other node accepts the block
+            assert_equal(self.nodes[0].getchaintips()[0], self.nodes[1].getchaintips()[0])
+
+            self.miniwallet.rescan_utxos()
+            assert_equal(self.miniwallet.get_balance(), balance + 1)
+            self.log.debug("Check block should fail now, since it is a duplicate")
+            check = await mining.checkBlock(block.serialize(), check_opts)
+            assert_equal(check.result, False)
+            assert_equal(check.reason, "inconclusive-not-best-prevblk")
+
+        asyncio.run(capnp.run(async_routine()))
+
+    def run_test(self):
+        self.miniwallet = MiniWallet(self.nodes[0])
+        self.default_block_create_options = self.capnp_modules['mining'].BlockCreateOptions(
+            useMempool=True,
+            blockReservedWeight=4000,
+            coinbaseOutputMaxAdditionalSigops=0
+        )
+
+        self.run_mining_interface_test()
+        self.run_block_template_test()
+        self.run_coinbase_and_submission_test()
+
+
+if __name__ == '__main__':
+    IPCMiningTest(__file__).main()

test/functional/test_framework/ipc_util.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Shared utilities for IPC (multiprocess) interface tests."""
+import asyncio
+import inspect
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from io import BytesIO
+from pathlib import Path
+import shutil
+from typing import Optional
+
+from test_framework.messages import CBlock, CTransaction
+
+# Test may be skipped and not have capnp installed
+try:
+    import capnp  # type: ignore[import] # noqa: F401
+except ModuleNotFoundError:
+    pass
+
+
+# Stores the result of getCoinbaseTx()
+@dataclass
+class CoinbaseTxData:
+    version: int
+    sequence: int
+    scriptSigPrefix: bytes
+    witness: Optional[bytes]
+    blockRewardRemaining: int
+    requiredOutputs: list[bytes]
+    lockTime: int
+
+
+@asynccontextmanager
+async def destroying(obj, ctx):
+    """Call obj.destroy(ctx) at end of with: block. Similar to contextlib.closing."""
+    try:
+        yield obj
+    finally:
+        await obj.destroy(ctx)
+
+
+async def wait_and_do(wait_fn, do_fn):
+    """Call wait_fn, then sleep, then call do_fn in a parallel task. Wait for
+    both tasks to complete."""
+    wait_started = asyncio.Event()
+    result = None
+
+    async def wait():
+        nonlocal result
+        wait_started.set()
+        result = await wait_fn
+
+    async def do():
+        await wait_started.wait()
+        await asyncio.sleep(0.1)
+        # Let do_fn be either a callable or an awaitable object
+        if inspect.isawaitable(do_fn):
+            await do_fn
+        else:
+            do_fn()
+
+    await asyncio.gather(wait(), do())
+    return result
+
+
+def load_capnp_modules(config):
+    if capnp_bin := shutil.which("capnp"):
+        # Add the system cap'nproto path so include/capnp/c++.capnp can be found.
+        capnp_dir = Path(capnp_bin).resolve().parent.parent / "include"
+    else:
+        # If there is no system cap'nproto, the pycapnp module should have its own "bundled"
+        # includes at this location. If pycapnp was installed with bundled capnp,
+        # capnp/c++.capnp can be found here.
+        capnp_dir = Path(capnp.__path__[0]).parent
+    src_dir = Path(config['environment']['SRCDIR']) / "src"
+    mp_dir = src_dir / "ipc" / "libmultiprocess" / "include"
+    # List of import directories. Note: it is important for mp_dir to be
+    # listed first, in case there are other libmultiprocess installations on
+    # the system, to ensure that `import "/mp/proxy.capnp"` lines load the
+    # same file as capnp.load() loads directly below, and there are not
+    # "failed: Duplicate ID @0xcc316e3f71a040fb" errors.
+    imports = [str(mp_dir), str(capnp_dir), str(src_dir)]
+    return {
+        "proxy": capnp.load(str(mp_dir / "mp" / "proxy.capnp"), imports=imports),
+        "init": capnp.load(str(src_dir / "ipc" / "capnp" / "init.capnp"), imports=imports),
+        "echo": capnp.load(str(src_dir / "ipc" / "capnp" / "echo.capnp"), imports=imports),
+        "mining": capnp.load(str(src_dir / "ipc" / "capnp" / "mining.capnp"), imports=imports),
+    }
+
+
+async def make_capnp_init_ctx(self):
+    node = self.nodes[0]
+    # Establish a connection, and create Init proxy object.
+    connection = await capnp.AsyncIoStream.create_unix_connection(node.ipc_socket_path)
+    client = capnp.TwoPartyClient(connection)
+    init = client.bootstrap().cast_as(self.capnp_modules['init'].Init)
+    # Create a remote thread on the server for the IPC calls to be executed in.
+    threadmap = init.construct().threadMap
+    thread = threadmap.makeThread("pythread").result
+    ctx = self.capnp_modules['proxy'].Context()
+    ctx.thread = thread
+    # Return both.
+    return ctx, init
+
+
+async def mining_create_block_template(mining, stack, ctx, opts):
+    """Call mining.createNewBlock() and return template, then call template.destroy() when stack exits."""
+    return await stack.enter_async_context(destroying((await mining.createNewBlock(opts)).result, ctx))
+
+
+async def mining_wait_next_template(template, stack, ctx, opts):
+    """Call template.waitNext() and return template, then call template.destroy() when stack exits."""
+    return await stack.enter_async_context(destroying((await template.waitNext(ctx, opts)).result, ctx))
+
+
+async def mining_get_block(block_template, ctx):
+    block_data = BytesIO((await block_template.getBlock(ctx)).result)
+    block = CBlock()
+    block.deserialize(block_data)
+    return block
+
+
+async def mining_get_coinbase_raw_tx(block_template, ctx):
+    assert block_template is not None
+    coinbase_data = BytesIO((await block_template.getCoinbaseRawTx(ctx)).result)
+    tx = CTransaction()
+    tx.deserialize(coinbase_data)
+    return tx
+
+
+async def mining_get_coinbase_tx(block_template, ctx) -> CoinbaseTxData:
+    assert block_template is not None
+    # Note: the template_capnp struct will be garbage-collected when this
+    # method returns, so it is important to copy any Data fields from it
+    # which need to be accessed later using the bytes() cast. Starting with
+    # pycapnp v2.2.0, Data fields have type `memoryview` and are ephemeral.
+    template_capnp = (await block_template.getCoinbaseTx(ctx)).result
+    witness: Optional[bytes] = None
+    if template_capnp._has("witness"):
+        witness = bytes(template_capnp.witness)
+    return CoinbaseTxData(
+        version=int(template_capnp.version),
+        sequence=int(template_capnp.sequence),
+        scriptSigPrefix=bytes(template_capnp.scriptSigPrefix),
+        witness=witness,
+        blockRewardRemaining=int(template_capnp.blockRewardRemaining),
+        requiredOutputs=[bytes(output) for output in template_capnp.requiredOutputs],
+        lockTime=int(template_capnp.lockTime),
+    )

test/functional/test_runner.py

@@ -356,6 +356,7 @@ BASE_SCRIPTS = [
     'mempool_cluster.py',
     'feature_logging.py',
     'interface_ipc.py',
+    'interface_ipc_mining.py',
     'feature_anchors.py',
     'mempool_datacarrier.py',
     'feature_coinstatsindex.py',