Merge bitcoin/bitcoin#29032: signet: omit commitment for some trivial challenges

6ee32aaaca test: signet tool genpsbt and solvepsbt commands (Sjors Provoost)
0a99d99fe4 signet: miner skips PSBT step for OP_TRUE (Sjors Provoost)
cdfb70e5a6 signet: split decode_psbt miner helper (Sjors Provoost)

Pull request description:

  [BIP325](https://github.com/bitcoin/bips/blob/master/bip-0325.mediawiki) mentions the following rule:

  > In the special case where an empty solution is valid (ie scriptSig and scriptWitness are both empty) this additional commitment can optionally be left out. This special case is to allow non-signet-aware block generation code to be used to test a custom signet chain where the challenge is trivially true.

  Such a signet can be created using e.g. `-signetchallenge=51` (`OP_TRUE`). However `contrib/signet/miner` won't omit the commitment.

  This PR improves the miner by skipping the PSBT for known trivial scripts (just `OP_TRUE` and trivial pushes for now). This prevents it from appending the 4 byte signet header to the witness commitment, as allowed by the above rule.

  ---

  Previously the script would fail with `PSBT signing failed`, making it difficult to mine. This is no longer the case.

ACKs for top commit:
  achow101:
    ACK 6ee32aaaca
  theStack:
    re-ACK 6ee32aaaca
  danielabrozzoni:
    ACK 6ee32aaaca

Tree-SHA512: e47fbf471f2909286a6c1c073799ea388b9c19551afcce96cf9af45cc48d25c02f1e48e08861a88b604361e2c107a759d5baf393da8a37360de419f31651758a

contrib/signet/README.md

@@ -80,3 +80,4 @@ These steps can instead be done explicitly:
 
 This is intended to allow you to replace part of the pipeline for further experimentation (eg, to sign the block with a hardware wallet).
 
+For custom signets with a trivial challenge such as `OP_TRUE` and `OP_2` the walletprocesspsbt step can be skipped.

contrib/signet/miner

@@ -18,7 +18,7 @@ PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__fi
 PATH_BASE_TEST_FUNCTIONAL = os.path.abspath(os.path.join(PATH_BASE_CONTRIB_SIGNET, "..", "..", "test", "functional"))
 sys.path.insert(0, PATH_BASE_TEST_FUNCTIONAL)
 
-from test_framework.blocktools import get_witness_script, script_BIP34_coinbase_height # noqa: E402
+from test_framework.blocktools import get_witness_script, script_BIP34_coinbase_height, SIGNET_HEADER # noqa: E402
 from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_binary, from_hex, ser_string, ser_uint256, tx_from_hex, MAX_SEQUENCE_NONFINAL # noqa: E402
 from test_framework.psbt import PSBT, PSBTMap, PSBT_GLOBAL_UNSIGNED_TX, PSBT_IN_FINAL_SCRIPTSIG, PSBT_IN_FINAL_SCRIPTWITNESS, PSBT_IN_NON_WITNESS_UTXO, PSBT_IN_SIGHASH_TYPE # noqa: E402
 from test_framework.script import CScript, CScriptOp # noqa: E402
@@ -28,7 +28,6 @@ logging.basicConfig(
     level=logging.INFO,
     datefmt='%Y-%m-%d %H:%M:%S')
 
-SIGNET_HEADER = b"\xec\xc7\xda\xa2"
 PSBT_SIGNET_BLOCK = b"\xfc\x06signetb"    # proprietary PSBT global field holding the block being signed
 RE_MULTIMINER = re.compile(r"^(\d+)(-(\d+))?/(\d+)$")
 
@@ -66,22 +65,31 @@ def signet_txs(block, challenge):
 
     return spend, to_spend
 
-def decode_psbt(b64psbt):
+def decode_challenge_psbt(b64psbt):
     psbt = PSBT.from_base64(b64psbt)
 
     assert len(psbt.tx.vin) == 1
     assert len(psbt.tx.vout) == 1
     assert PSBT_SIGNET_BLOCK in psbt.g.map
+    return psbt
 
+def get_block_from_psbt(psbt):
+    return from_binary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK])
+
+def get_solution_from_psbt(psbt, emptyok=False):
     scriptSig = psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTSIG, b"")
     scriptWitness = psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTWITNESS, b"\x00")
-
+    if emptyok and len(scriptSig) == 0 and scriptWitness == b"\x00":
-    return from_binary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness
+        return None
+    return ser_string(scriptSig) + scriptWitness
 
 def finish_block(block, signet_solution, grind_cmd):
-    block.vtx[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER + signet_solution)
+    if signet_solution is None:
-    block.vtx[0].rehash()
+        pass # Don't need to add a signet commitment if there's no signet signature needed
-    block.hashMerkleRoot = block.calc_merkle_root()
+    else:
+        block.vtx[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER + signet_solution)
+        block.vtx[0].rehash()
+        block.hashMerkleRoot = block.calc_merkle_root()
     if grind_cmd is None:
         block.solve()
     else:
@@ -93,10 +101,7 @@ def finish_block(block, signet_solution, grind_cmd):
         block.rehash()
     return block
 
-def generate_psbt(tmpl, reward_spk, *, blocktime=None, poolid=None):
+def new_block(tmpl, reward_spk, *, blocktime=None, poolid=None):
-    signet_spk = tmpl["signet_challenge"]
-    signet_spk_bin = bytes.fromhex(signet_spk)
-
     scriptSig = script_BIP34_coinbase_height(tmpl["height"])
     if poolid is not None:
         scriptSig = CScript(b"" + scriptSig + CScriptOp.encode_op_pushdata(poolid))
@@ -125,8 +130,14 @@ def generate_psbt(tmpl, reward_spk, *, blocktime=None, poolid=None):
     block.vtx[0].wit.vtxinwit = [cbwit]
     block.vtx[0].vout.append(CTxOut(0, bytes(get_witness_script(witroot, witnonce))))
 
-    signme, spendme = signet_txs(block, signet_spk_bin)
+    block.vtx[0].rehash()
+    block.hashMerkleRoot = block.calc_merkle_root()
 
+    return block
+
+def generate_psbt(block, signet_spk):
+    signet_spk_bin = bytes.fromhex(signet_spk)
+    signme, spendme = signet_txs(block, signet_spk_bin)
     psbt = PSBT()
     psbt.g = PSBTMap( {PSBT_GLOBAL_UNSIGNED_TX: signme.serialize(),
                        PSBT_SIGNET_BLOCK: block.serialize()
@@ -175,12 +186,16 @@ def get_reward_addr_spk(args, height):
 def do_genpsbt(args):
     poolid = get_poolid(args)
     tmpl = json.load(sys.stdin)
+    signet_spk = tmpl["signet_challenge"]
     _, reward_spk = get_reward_addr_spk(args, tmpl["height"])
-    psbt = generate_psbt(tmpl, reward_spk, poolid=poolid)
+    block = new_block(tmpl, reward_spk, poolid=poolid)
+    psbt = generate_psbt(block, signet_spk)
     print(psbt)
 
 def do_solvepsbt(args):
-    block, signet_solution = decode_psbt(sys.stdin.read())
+    psbt = decode_challenge_psbt(sys.stdin.read())
+    block = get_block_from_psbt(psbt)
+    signet_solution = get_solution_from_psbt(psbt, emptyok=True)
     block = finish_block(block, signet_solution, args.grind_cmd)
     print(block.serialize().hex())
 
@@ -223,6 +238,21 @@ def seconds_to_hms(s):
         out = "-" + out
     return out
 
+def trivial_challenge(spkhex):
+    """
+    BIP325 allows omitting the signet commitment when scriptSig and
+    scriptWitness are both empty. This is the case for trivial
+    challenges such as OP_TRUE or a single data push.
+    """
+    spk = bytes.fromhex(spkhex)
+    if len(spk) == 1 and 0x51 <= spk[0] <= 0x60:
+        # OP_TRUE/OP_1...OP_16
+        return True
+    elif 2 <= len(spk) <= 76 and spk[0] + 1 == len(spk):
+        # Single fixed push of 1-75 bytes
+        return True
+    return False
+
 class Generate:
     INTERVAL = 600.0*2016/2015 # 10 minutes, adjusted for the off-by-one bug
 
@@ -323,14 +353,22 @@ class Generate:
         return tmpl
 
     def mine(self, bcli, grind_cmd, tmpl, reward_spk):
-        psbt = generate_psbt(tmpl, reward_spk, blocktime=self.mine_time, poolid=self.poolid)
+        block = new_block(tmpl, reward_spk, blocktime=self.mine_time, poolid=self.poolid)
-        input_stream = os.linesep.join([psbt, "true", "ALL"]).encode('utf8')
+
-        psbt_signed = json.loads(bcli("-stdin", "walletprocesspsbt", input=input_stream))
+        signet_spk = tmpl["signet_challenge"]
-        if not psbt_signed.get("complete",False):
+        if trivial_challenge(signet_spk):
-            logging.debug("Generated PSBT: %s" % (psbt,))
+            signet_solution = None
-            sys.stderr.write("PSBT signing failed\n")
+        else:
-            return None
+            psbt = generate_psbt(block, signet_spk)
-        block, signet_solution = decode_psbt(psbt_signed["psbt"])
+            input_stream = os.linesep.join([psbt, "true", "ALL"]).encode('utf8')
+            psbt_signed = json.loads(bcli("-stdin", "walletprocesspsbt", input=input_stream))
+            if not psbt_signed.get("complete",False):
+               logging.debug("Generated PSBT: %s" % (psbt,))
+               sys.stderr.write("PSBT signing failed\n")
+               return None
+            psbt = decode_challenge_psbt(psbt_signed["psbt"])
+            signet_solution = get_solution_from_psbt(psbt)
+
         return finish_block(block, signet_solution, grind_cmd)
 
 def do_generate(args):

test/functional/test_framework/blocktools.py

@@ -81,6 +81,9 @@ DIFF_4_N_BITS = 0x1c3fffc0
 DIFF_4_TARGET = int(DIFF_1_TARGET / 4)
 assert_equal(uint256_from_compact(DIFF_4_N_BITS), DIFF_4_TARGET)
 
+# From BIP325
+SIGNET_HEADER = b"\xec\xc7\xda\xa2"
+
 def nbits_str(nbits):
     return f"{nbits:08x}"
 

test/functional/tool_signet_miner.py

@@ -4,15 +4,16 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test signet miner tool"""
 
+import json
 import os.path
 import shlex
 import subprocess
 import sys
 import time
 
-from test_framework.blocktools import DIFF_1_N_BITS
+from test_framework.blocktools import DIFF_1_N_BITS, SIGNET_HEADER
 from test_framework.key import ECKey
-from test_framework.script_util import key_to_p2wpkh_script
+from test_framework.script_util import CScript, key_to_p2wpkh_script
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
@@ -23,31 +24,49 @@ from test_framework.wallet_util import bytes_to_wif
 
 CHALLENGE_PRIVATE_KEY = (42).to_bytes(32, 'big')
 
+def get_segwit_commitment(node):
+    coinbase = node.getblock(node.getbestblockhash(), 2)['tx'][0]
+    commitment = coinbase['vout'][1]['scriptPubKey']['hex']
+    assert_equal(commitment[0:12], '6a24aa21a9ed')
+    return commitment
+
+def get_signet_commitment(segwit_commitment):
+    for el in CScript.fromhex(segwit_commitment):
+        if isinstance(el, bytes) and el[0:4] == SIGNET_HEADER:
+            return el[4:].hex()
+    return None
 
 class SignetMinerTest(BitcoinTestFramework):
     def set_test_params(self):
         self.chain = "signet"
         self.setup_clean_chain = True
-        self.num_nodes = 1
+        self.num_nodes = 4
 
         # generate and specify signet challenge (simple p2wpkh script)
         privkey = ECKey()
         privkey.set(CHALLENGE_PRIVATE_KEY, True)
         pubkey = privkey.get_pubkey().get_bytes()
         challenge = key_to_p2wpkh_script(pubkey)
-        self.extra_args = [[f'-signetchallenge={challenge.hex()}']]
+
+        self.extra_args = [
+            [f'-signetchallenge={challenge.hex()}'],
+            ["-signetchallenge=51"], # OP_TRUE
+            ["-signetchallenge=60"], # OP_16
+            ["-signetchallenge=202cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"], # sha256("hello")
+        ]
 
     def skip_test_if_missing_module(self):
         self.skip_if_no_cli()
         self.skip_if_no_wallet()
         self.skip_if_no_bitcoin_util()
 
-    def run_test(self):
+    def setup_network(self):
-        node = self.nodes[0]
+        self.setup_nodes()
-        # import private key needed for signing block
+        # Nodes with different signet networks are not connected
-        wallet_importprivkey(node, bytes_to_wif(CHALLENGE_PRIVATE_KEY), "now")
 
-        # generate block with signet miner tool
+    # generate block with signet miner tool
+    def mine_block(self, node):
+        n_blocks = node.getblockcount()
         base_dir = self.config["environment"]["SRCDIR"]
         signet_miner_path = os.path.join(base_dir, "contrib", "signet", "miner")
         rpc_argv = node.binaries.rpc_argv() + [f"-datadir={node.cli.datadir}"]
@@ -63,7 +82,73 @@ class SignetMinerTest(BitcoinTestFramework):
                 f'--set-block-time={int(time.time())}',
                 '--poolnum=99',
             ], check=True, stderr=subprocess.STDOUT)
-        assert_equal(node.getblockcount(), 1)
+        assert_equal(node.getblockcount(), n_blocks + 1)
+
+    # generate block using the signet miner tool genpsbt and solvepsbt commands
+    def mine_block_manual(self, node, *, sign):
+        n_blocks = node.getblockcount()
+        base_dir = self.config["environment"]["SRCDIR"]
+        signet_miner_path = os.path.join(base_dir, "contrib", "signet", "miner")
+        rpc_argv = node.binaries.rpc_argv() + [f"-datadir={node.cli.datadir}"]
+        util_argv = node.binaries.util_argv() + ["grind"]
+        base_cmd = [
+            sys.executable,
+            signet_miner_path,
+            f'--cli={shlex.join(rpc_argv)}',
+        ]
+
+        template = node.getblocktemplate(dict(rules=["signet","segwit"]))
+        genpsbt = subprocess.run(base_cmd + [
+                'genpsbt',
+                f'--address={node.getnewaddress()}',
+                '--poolnum=98',
+            ], check=True, input=json.dumps(template).encode('utf8'), capture_output=True)
+        psbt = genpsbt.stdout.decode('utf8').strip()
+        if sign:
+            self.log.debug("Sign the PSBT")
+            res = node.walletprocesspsbt(psbt=psbt, sign=True, sighashtype='ALL')
+            assert res['complete']
+            psbt = res['psbt']
+        solvepsbt = subprocess.run(base_cmd + [
+                'solvepsbt',
+                f'--grind-cmd={shlex.join(util_argv)}',
+            ], check=True, input=psbt.encode('utf8'), capture_output=True)
+        node.submitblock(solvepsbt.stdout.decode('utf8').strip())
+        assert_equal(node.getblockcount(), n_blocks + 1)
+
+    def run_test(self):
+        self.log.info("Signet node with single signature challenge")
+        node = self.nodes[0]
+        # import private key needed for signing block
+        wallet_importprivkey(node, bytes_to_wif(CHALLENGE_PRIVATE_KEY), 0)
+        self.mine_block(node)
+        # MUST include signet commitment
+        assert get_signet_commitment(get_segwit_commitment(node))
+
+        self.log.info("Mine manually using genpsbt and solvepsbt")
+        self.mine_block_manual(node, sign=True)
+        assert get_signet_commitment(get_segwit_commitment(node))
+
+        node = self.nodes[1]
+        self.log.info("Signet node with trivial challenge (OP_TRUE)")
+        self.mine_block(node)
+        # MAY omit signet commitment (BIP 325). Do so for better compatibility
+        # with signet unaware mining software and hardware.
+        assert get_signet_commitment(get_segwit_commitment(node)) is None
+
+        node = self.nodes[2]
+        self.log.info("Signet node with trivial challenge (OP_16)")
+        self.mine_block(node)
+        assert get_signet_commitment(get_segwit_commitment(node)) is None
+
+        node = self.nodes[3]
+        self.log.info("Signet node with trivial challenge (push sha256 hash)")
+        self.mine_block(node)
+        assert get_signet_commitment(get_segwit_commitment(node)) is None
+
+        self.log.info("Manual mining with a trivial challenge doesn't require a PSBT")
+        self.mine_block_manual(node, sign=False)
+        assert get_signet_commitment(get_segwit_commitment(node)) is None
 
 
 if __name__ == "__main__":