Merge bitcoin/bitcoin#28374: test: python cryptography required for BIP 324 functional tests

c534c08710 [test/crypto] Add FSChaCha20Poly1305 AEAD python implementation (stratospher)
c2a458f1c2 [test/crypto] Add FSChaCha20 python implementation (stratospher)
c4ea5f6288 [test/crypto] Add RFC 8439's ChaCha20Poly1305 AEAD (stratospher)
9fc6e0355e [test/crypto] Add Poly1305 python implementation (stratospher)
fec2ca6c9a [test/crypto] Use chacha20_block function in `data_to_num3072` (stratospher)
0cde60da3a [test/crypto] Add ChaCha20 python implementation (stratospher)
69d3f50ab6 [test/crypto] Add HMAC-based Key Derivation Function (HKDF) (stratospher)
08a4a56cbc [test] Move test framework crypto functions to crypto/ (stratospher)

Pull request description:

  split off from #24748 to keep commits related to cryptography and functional test framework changes separate.

  This PR adds python implementation and unit tests for HKDF, ChaCha20, Poly1305, ChaCha20Poly1305 AEAD, FSChaCha20 and FSChaCha20Poly1305 AEAD.

  They're based on https://github.com/bitcoin/bips/blob/cc177ab7bc5abcdcdf9c956ee88afd1052053328/bip-0324/reference.py for easy review.

ACKs for top commit:
  sipa:
    utACK c534c08710
  achow101:
    ACK c534c08710
  theStack:
    re-ACK c534c08710

Tree-SHA512: 08a0a422d2937eadcf0edfede37e535e6bc4c2e4b192441bbf9bc26dd3f03fa3388effd22f0527c55af173933d0b50e5b2b3d36f2b62d0aca3098728ef06970e

test/functional/feature_taproot.py

@@ -104,7 +104,7 @@ from test_framework.key import (
     tweak_add_privkey,
     ECKey,
 )
-from test_framework import secp256k1
+from test_framework.crypto import secp256k1
 from test_framework.address import (
     hash160,
     program_to_witness,

test/functional/feature_utxo_set_hash.py

@@ -11,7 +11,7 @@ from test_framework.messages import (
     COutPoint,
     from_hex,
 )
-from test_framework.muhash import MuHash3072
+from test_framework.crypto.muhash import MuHash3072
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal
 from test_framework.wallet import MiniWallet

test/functional/test_framework/blockfilter.py

@@ -4,7 +4,7 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Helper routines relevant for compact block filters (BIP158).
 """
-from .siphash import siphash
+from .crypto.siphash import siphash
 
 
 def bip158_basic_element_hash(script_pub_key, N, block_hash):

test/functional/test_framework/crypto/bip324_cipher.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+"""Test-only implementation of ChaCha20 Poly1305 AEAD Construction in RFC 8439 and FSChaCha20Poly1305 for BIP 324
+
+It is designed for ease of understanding, not performance.
+
+WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for
+anything but tests.
+"""
+
+import unittest
+
+from .chacha20 import chacha20_block, REKEY_INTERVAL
+from .poly1305 import Poly1305
+
+
+def pad16(x):
+    if len(x) % 16 == 0:
+        return b''
+    return b'\x00' * (16 - (len(x) % 16))
+
+
+def aead_chacha20_poly1305_encrypt(key, nonce, aad, plaintext):
+    """Encrypt a plaintext using ChaCha20Poly1305."""
+    ret = bytearray()
+    msg_len = len(plaintext)
+    for i in range((msg_len + 63) // 64):
+        now = min(64, msg_len - 64 * i)
+        keystream = chacha20_block(key, nonce, i + 1)
+        for j in range(now):
+            ret.append(plaintext[j + 64 * i] ^ keystream[j])
+    poly1305 = Poly1305(chacha20_block(key, nonce, 0)[:32])
+    mac_data = aad + pad16(aad)
+    mac_data += ret + pad16(ret)
+    mac_data += len(aad).to_bytes(8, 'little') + msg_len.to_bytes(8, 'little')
+    ret += poly1305.tag(mac_data)
+    return bytes(ret)
+
+
+def aead_chacha20_poly1305_decrypt(key, nonce, aad, ciphertext):
+    """Decrypt a ChaCha20Poly1305 ciphertext."""
+    if len(ciphertext) < 16:
+        return None
+    msg_len = len(ciphertext) - 16
+    poly1305 = Poly1305(chacha20_block(key, nonce, 0)[:32])
+    mac_data = aad + pad16(aad)
+    mac_data += ciphertext[:-16] + pad16(ciphertext[:-16])
+    mac_data += len(aad).to_bytes(8, 'little') + msg_len.to_bytes(8, 'little')
+    if ciphertext[-16:] != poly1305.tag(mac_data):
+        return None
+    ret = bytearray()
+    for i in range((msg_len + 63) // 64):
+        now = min(64, msg_len - 64 * i)
+        keystream = chacha20_block(key, nonce, i + 1)
+        for j in range(now):
+            ret.append(ciphertext[j + 64 * i] ^ keystream[j])
+    return bytes(ret)
+
+
+class FSChaCha20Poly1305:
+    """Rekeying wrapper AEAD around ChaCha20Poly1305."""
+    def __init__(self, initial_key):
+        self._key = initial_key
+        self._packet_counter = 0
+
+    def _crypt(self, aad, text, is_decrypt):
+        nonce = ((self._packet_counter % REKEY_INTERVAL).to_bytes(4, 'little') +
+                 (self._packet_counter // REKEY_INTERVAL).to_bytes(8, 'little'))
+        if is_decrypt:
+            ret = aead_chacha20_poly1305_decrypt(self._key, nonce, aad, text)
+        else:
+            ret = aead_chacha20_poly1305_encrypt(self._key, nonce, aad, text)
+        if (self._packet_counter + 1) % REKEY_INTERVAL == 0:
+            rekey_nonce = b"\xFF\xFF\xFF\xFF" + nonce[4:]
+            self._key = aead_chacha20_poly1305_encrypt(self._key, rekey_nonce, b"", b"\x00" * 32)[:32]
+        self._packet_counter += 1
+        return ret
+
+    def decrypt(self, aad, ciphertext):
+        return self._crypt(aad, ciphertext, True)
+
+    def encrypt(self, aad, plaintext):
+        return self._crypt(aad, plaintext, False)
+
+
+# Test vectors from RFC8439 consisting of plaintext, aad, 32 byte key, 12 byte nonce and ciphertext
+AEAD_TESTS = [
+    # RFC 8439 Example from section 2.8.2
+    ["4c616469657320616e642047656e746c656d656e206f662074686520636c6173"
+     "73206f66202739393a204966204920636f756c64206f6666657220796f75206f"
+     "6e6c79206f6e652074697020666f7220746865206675747572652c2073756e73"
+     "637265656e20776f756c642062652069742e",
+     "50515253c0c1c2c3c4c5c6c7",
+     "808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9f",
+     [7, 0x4746454443424140],
+     "d31a8d34648e60db7b86afbc53ef7ec2a4aded51296e08fea9e2b5a736ee62d6"
+     "3dbea45e8ca9671282fafb69da92728b1a71de0a9e060b2905d6a5b67ecd3b36"
+     "92ddbd7f2d778b8c9803aee328091b58fab324e4fad675945585808b4831d7bc"
+     "3ff4def08e4b7a9de576d26586cec64b61161ae10b594f09e26a7e902ecbd060"
+     "0691"],
+    # RFC 8439 Test vector A.5
+    ["496e7465726e65742d4472616674732061726520647261667420646f63756d65"
+     "6e74732076616c696420666f722061206d6178696d756d206f6620736978206d"
+     "6f6e74687320616e64206d617920626520757064617465642c207265706c6163"
+     "65642c206f72206f62736f6c65746564206279206f7468657220646f63756d65"
+     "6e747320617420616e792074696d652e20497420697320696e617070726f7072"
+     "6961746520746f2075736520496e7465726e65742d4472616674732061732072"
+     "65666572656e6365206d6174657269616c206f7220746f206369746520746865"
+     "6d206f74686572207468616e206173202fe2809c776f726b20696e2070726f67"
+     "726573732e2fe2809d",
+     "f33388860000000000004e91",
+     "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0",
+     [0, 0x0807060504030201],
+     "64a0861575861af460f062c79be643bd5e805cfd345cf389f108670ac76c8cb2"
+     "4c6cfc18755d43eea09ee94e382d26b0bdb7b73c321b0100d4f03b7f355894cf"
+     "332f830e710b97ce98c8a84abd0b948114ad176e008d33bd60f982b1ff37c855"
+     "9797a06ef4f0ef61c186324e2b3506383606907b6a7c02b0f9f6157b53c867e4"
+     "b9166c767b804d46a59b5216cde7a4e99040c5a40433225ee282a1b0a06c523e"
+     "af4534d7f83fa1155b0047718cbc546a0d072b04b3564eea1b422273f548271a"
+     "0bb2316053fa76991955ebd63159434ecebb4e466dae5a1073a6727627097a10"
+     "49e617d91d361094fa68f0ff77987130305beaba2eda04df997b714d6c6f2c29"
+     "a6ad5cb4022b02709beead9d67890cbb22392336fea1851f38"],
+    # Test vectors exercising aad and plaintext which are multiples of 16 bytes.
+    ["8d2d6a8befd9716fab35819eaac83b33269afb9f1a00fddf66095a6c0cd91951"
+     "a6b7ad3db580be0674c3f0b55f618e34",
+     "",
+     "72ddc73f07101282bbbcf853b9012a9f9695fc5d36b303a97fd0845d0314e0c3",
+     [0x3432b75f, 0xb3585537eb7f4024],
+     "f760b8224fb2a317b1b07875092606131232a5b86ae142df5df1c846a7f6341a"
+     "f2564483dd77f836be45e6230808ffe402a6f0a3e8be074b3d1f4ea8a7b09451"],
+    ["",
+     "36970d8a704c065de16250c18033de5a400520ac1b5842b24551e5823a3314f3"
+     "946285171e04a81ebfbe3566e312e74ab80e94c7dd2ff4e10de0098a58d0f503",
+     "77adda51d6730b9ad6c995658cbd49f581b2547e7c0c08fcc24ceec797461021",
+     [0x1f90da88, 0x75dafa3ef84471a4],
+     "aaae5bb81e8407c94b2ae86ae0c7efbe"],
+]
+
+FSAEAD_TESTS = [
+    ["d6a4cb04ef0f7c09c1866ed29dc24d820e75b0491032a51b4c3366f9ca35c19e"
+     "a3047ec6be9d45f9637b63e1cf9eb4c2523a5aab7b851ebeba87199db0e839cf"
+     "0d5c25e50168306377aedbe9089fd2463ded88b83211cf51b73b150608cc7a60"
+     "0d0f11b9a742948482e1b109d8faf15b450aa7322e892fa2208c6691e3fecf4c"
+     "711191b14d75a72147",
+     "786cb9b6ebf44288974cf0",
+     "5c9e1c3951a74fba66708bf9d2c217571684556b6a6a3573bff2847d38612654",
+     500,
+     "9dcebbd3281ea3dd8e9a1ef7d55a97abd6743e56ebc0c190cb2c4e14160b385e"
+     "0bf508dddf754bd02c7c208447c131ce23e47a4a14dfaf5dd8bc601323950f75"
+     "4e05d46e9232f83fc5120fbbef6f5347a826ec79a93820718d4ec7a2b7cfaaa4"
+     "4b21e16d726448b62f803811aff4f6d827ed78e738ce8a507b81a8ae13131192"
+     "8039213de18a5120dc9b7370baca878f50ff254418de3da50c"],
+    ["8349b7a2690b63d01204800c288ff1138a1d473c832c90ea8b3fc102d0bb3adc"
+     "44261b247c7c3d6760bfbe979d061c305f46d94c0582ac3099f0bf249f8cb234",
+     "",
+     "3bd2093fcbcb0d034d8c569583c5425c1a53171ea299f8cc3bbf9ae3530adfce",
+     60000,
+     "30a6757ff8439b975363f166a0fa0e36722ab35936abd704297948f45083f4d4"
+     "99433137ce931f7fca28a0acd3bc30f57b550acbc21cbd45bbef0739d9caf30c"
+     "14b94829deb27f0b1923a2af704ae5d6"],
+]
+
+
+class TestFrameworkAEAD(unittest.TestCase):
+    def test_aead(self):
+        """ChaCha20Poly1305 AEAD test vectors."""
+        for test_vector in AEAD_TESTS:
+            hex_plain, hex_aad, hex_key, hex_nonce, hex_cipher = test_vector
+            plain = bytes.fromhex(hex_plain)
+            aad = bytes.fromhex(hex_aad)
+            key = bytes.fromhex(hex_key)
+            nonce = hex_nonce[0].to_bytes(4, 'little') + hex_nonce[1].to_bytes(8, 'little')
+
+            ciphertext = aead_chacha20_poly1305_encrypt(key, nonce, aad, plain)
+            self.assertEqual(hex_cipher, ciphertext.hex())
+            plaintext = aead_chacha20_poly1305_decrypt(key, nonce, aad, ciphertext)
+            self.assertEqual(plain, plaintext)
+
+    def test_fschacha20poly1305aead(self):
+        "FSChaCha20Poly1305 AEAD test vectors."
+        for test_vector in FSAEAD_TESTS:
+            hex_plain, hex_aad, hex_key, msg_idx, hex_cipher = test_vector
+            plain = bytes.fromhex(hex_plain)
+            aad = bytes.fromhex(hex_aad)
+            key = bytes.fromhex(hex_key)
+
+            enc_aead = FSChaCha20Poly1305(key)
+            dec_aead = FSChaCha20Poly1305(key)
+
+            for _ in range(msg_idx):
+                enc_aead.encrypt(b"", b"")
+            ciphertext = enc_aead.encrypt(aad, plain)
+            self.assertEqual(hex_cipher, ciphertext.hex())
+
+            for _ in range(msg_idx):
+                dec_aead.decrypt(b"", bytes(16))
+            plaintext = dec_aead.decrypt(aad, ciphertext)
+            self.assertEqual(plain, plaintext)

test/functional/test_framework/crypto/chacha20.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+"""Test-only implementation of ChaCha20 cipher and FSChaCha20 for BIP 324
+
+It is designed for ease of understanding, not performance.
+
+WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for
+anything but tests.
+"""
+
+import unittest
+
+CHACHA20_INDICES = (
+    (0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15),
+    (0, 5, 10, 15), (1, 6, 11, 12), (2, 7, 8, 13), (3, 4, 9, 14)
+)
+
+CHACHA20_CONSTANTS = (0x61707865, 0x3320646e, 0x79622d32, 0x6b206574)
+REKEY_INTERVAL = 224 # packets
+
+
+def rotl32(v, bits):
+    """Rotate the 32-bit value v left by bits bits."""
+    bits %= 32  # Make sure the term below does not throw an exception
+    return ((v << bits) & 0xffffffff) | (v >> (32 - bits))
+
+
+def chacha20_doubleround(s):
+    """Apply a ChaCha20 double round to 16-element state array s.
+    See https://cr.yp.to/chacha/chacha-20080128.pdf and https://tools.ietf.org/html/rfc8439
+    """
+    for a, b, c, d in CHACHA20_INDICES:
+        s[a] = (s[a] + s[b]) & 0xffffffff
+        s[d] = rotl32(s[d] ^ s[a], 16)
+        s[c] = (s[c] + s[d]) & 0xffffffff
+        s[b] = rotl32(s[b] ^ s[c], 12)
+        s[a] = (s[a] + s[b]) & 0xffffffff
+        s[d] = rotl32(s[d] ^ s[a], 8)
+        s[c] = (s[c] + s[d]) & 0xffffffff
+        s[b] = rotl32(s[b] ^ s[c], 7)
+
+
+def chacha20_block(key, nonce, cnt):
+    """Compute the 64-byte output of the ChaCha20 block function.
+    Takes as input a 32-byte key, 12-byte nonce, and 32-bit integer counter.
+    """
+    # Initial state.
+    init = [0] * 16
+    init[:4] = CHACHA20_CONSTANTS[:4]
+    init[4:12] = [int.from_bytes(key[i:i+4], 'little') for i in range(0, 32, 4)]
+    init[12] = cnt
+    init[13:16] = [int.from_bytes(nonce[i:i+4], 'little') for i in range(0, 12, 4)]
+    # Perform 20 rounds.
+    state = list(init)
+    for _ in range(10):
+        chacha20_doubleround(state)
+    # Add initial values back into state.
+    for i in range(16):
+        state[i] = (state[i] + init[i]) & 0xffffffff
+    # Produce byte output
+    return b''.join(state[i].to_bytes(4, 'little') for i in range(16))
+
+class FSChaCha20:
+    """Rekeying wrapper stream cipher around ChaCha20."""
+    def __init__(self, initial_key, rekey_interval=REKEY_INTERVAL):
+        self._key = initial_key
+        self._rekey_interval = rekey_interval
+        self._block_counter = 0
+        self._chunk_counter = 0
+        self._keystream = b''
+
+    def _get_keystream_bytes(self, nbytes):
+        while len(self._keystream) < nbytes:
+            nonce = ((0).to_bytes(4, 'little') + (self._chunk_counter // self._rekey_interval).to_bytes(8, 'little'))
+            self._keystream += chacha20_block(self._key, nonce, self._block_counter)
+            self._block_counter += 1
+        ret = self._keystream[:nbytes]
+        self._keystream = self._keystream[nbytes:]
+        return ret
+
+    def crypt(self, chunk):
+        ks = self._get_keystream_bytes(len(chunk))
+        ret = bytes([ks[i] ^ chunk[i] for i in range(len(chunk))])
+        if ((self._chunk_counter + 1) % self._rekey_interval) == 0:
+            self._key = self._get_keystream_bytes(32)
+            self._block_counter = 0
+            self._keystream = b''
+        self._chunk_counter += 1
+        return ret
+
+
+# Test vectors from RFC7539/8439 consisting of 32 byte key, 12 byte nonce, block counter
+# and 64 byte output after applying `chacha20_block` function
+CHACHA20_TESTS = [
+    ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0x09000000, 0x4a000000], 1,
+     "10f1e7e4d13b5915500fdd1fa32071c4c7d1f4c733c068030422aa9ac3d46c4e"
+     "d2826446079faa0914c2d705d98b02a2b5129cd1de164eb9cbd083e8a2503c4e"],
+    ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0], 0,
+     "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7"
+     "da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586"],
+    ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0], 1,
+     "9f07e7be5551387a98ba977c732d080dcb0f29a048e3656912c6533e32ee7aed"
+     "29b721769ce64e43d57133b074d839d531ed1f28510afb45ace10a1f4b794d6f"],
+    ["0000000000000000000000000000000000000000000000000000000000000001", [0, 0], 1,
+     "3aeb5224ecf849929b9d828db1ced4dd832025e8018b8160b82284f3c949aa5a"
+     "8eca00bbb4a73bdad192b5c42f73f2fd4e273644c8b36125a64addeb006c13a0"],
+    ["00ff000000000000000000000000000000000000000000000000000000000000", [0, 0], 2,
+     "72d54dfbf12ec44b362692df94137f328fea8da73990265ec1bbbea1ae9af0ca"
+     "13b25aa26cb4a648cb9b9d1be65b2c0924a66c54d545ec1b7374f4872e99f096"],
+    ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0x200000000000000], 0,
+     "c2c64d378cd536374ae204b9ef933fcd1a8b2288b3dfa49672ab765b54ee27c7"
+     "8a970e0e955c14f3a88e741b97c286f75f8fc299e8148362fa198a39531bed6d"],
+    ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0, 0x4a000000], 1,
+     "224f51f3401bd9e12fde276fb8631ded8c131f823d2c06e27e4fcaec9ef3cf78"
+     "8a3b0aa372600a92b57974cded2b9334794cba40c63e34cdea212c4cf07d41b7"],
+    ["0000000000000000000000000000000000000000000000000000000000000001", [0, 0], 0,
+     "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41"
+     "bbe2a0b6ea7566d2a5d1e7e20d42af2c53d792b1c43fea817e9ad275ae546963"],
+    ["0000000000000000000000000000000000000000000000000000000000000000", [0, 1], 0,
+     "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32"
+     "111e4caf237ee53ca8ad6426194a88545ddc497a0b466e7d6bbdb0041b2f586b"],
+    ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0, 0x0706050403020100], 0,
+     "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c1"
+     "34a4547b733b46413042c9440049176905d3be59ea1c53f15916155c2be8241a"],
+]
+
+FSCHACHA20_TESTS = [
+    ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f",
+     "0000000000000000000000000000000000000000000000000000000000000000", 256,
+     "a93df4ef03011f3db95f60d996e1785df5de38fc39bfcb663a47bb5561928349"],
+    ["01", "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 5, "ea"],
+    ["e93fdb5c762804b9a706816aca31e35b11d2aa3080108ef46a5b1f1508819c0a",
+     "8ec4c3ccdaea336bdeb245636970be01266509b33f3d2642504eaf412206207a", 4096,
+     "8bfaa4eacff308fdb4a94a5ff25bd9d0c1f84b77f81239f67ff39d6e1ac280c9"],
+]
+
+
+class TestFrameworkChacha(unittest.TestCase):
+    def test_chacha20(self):
+        """ChaCha20 test vectors."""
+        for test_vector in CHACHA20_TESTS:
+            hex_key, nonce, counter, hex_output = test_vector
+            key = bytes.fromhex(hex_key)
+            nonce_bytes = nonce[0].to_bytes(4, 'little') + nonce[1].to_bytes(8, 'little')
+            keystream = chacha20_block(key, nonce_bytes, counter)
+            self.assertEqual(hex_output, keystream.hex())
+
+    def test_fschacha20(self):
+        """FSChaCha20 test vectors."""
+        for test_vector in FSCHACHA20_TESTS:
+            hex_plaintext, hex_key, rekey_interval, hex_ciphertext_after_rotation = test_vector
+            plaintext = bytes.fromhex(hex_plaintext)
+            key = bytes.fromhex(hex_key)
+            fsc20 = FSChaCha20(key, rekey_interval)
+            for _ in range(rekey_interval):
+                fsc20.crypt(plaintext)
+
+            ciphertext = fsc20.crypt(plaintext)
+            self.assertEqual(hex_ciphertext_after_rotation, ciphertext.hex())

test/functional/test_framework/crypto/ellswift.py

@@ -12,7 +12,7 @@ import os
 import random
 import unittest
 
-from test_framework.secp256k1 import FE, G, GE
+from test_framework.crypto.secp256k1 import FE, G, GE
 
 # Precomputed constant square root of -3 (mod p).
 MINUS_3_SQRT = FE(-3).sqrt()