diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index 9d40fc6fd4..32d10264d6 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -35,6 +35,7 @@ add_executable(bench_bitcoin mempool_eviction.cpp mempool_stress.cpp merkle_root.cpp + obfuscation.cpp parse_hex.cpp peer_eviction.cpp poly1305.cpp @@ -52,7 +53,6 @@ add_executable(bench_bitcoin txorphanage.cpp util_time.cpp verify_script.cpp - xor.cpp ) include(TargetDataSources) diff --git a/src/bench/xor.cpp b/src/bench/obfuscation.cpp similarity index 53% rename from src/bench/xor.cpp rename to src/bench/obfuscation.cpp index fc9dc5d172..178be56a5d 100644 --- a/src/bench/xor.cpp +++ b/src/bench/obfuscation.cpp @@ -4,21 +4,22 @@ #include #include -#include -#include +#include #include #include -static void Xor(benchmark::Bench& bench) +static void ObfuscationBench(benchmark::Bench& bench) { FastRandomContext frc{/*fDeterministic=*/true}; auto data{frc.randbytes(1024)}; - auto key{frc.randbytes(31)}; + const Obfuscation obfuscation{frc.randbytes()}; + size_t offset{0}; bench.batch(data.size()).unit("byte").run([&] { - util::Xor(data, key); + obfuscation(data, offset++); // mutated differently each time + ankerl::nanobench::doNotOptimizeAway(data); }); } -BENCHMARK(Xor, benchmark::PriorityLevel::HIGH); +BENCHMARK(ObfuscationBench, benchmark::PriorityLevel::HIGH); diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 1c35e11863..b13699572c 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -173,7 +174,7 @@ void CDBBatch::Clear() void CDBBatch::WriteImpl(std::span key, DataStream& ssValue) { leveldb::Slice slKey(CharCast(key.data()), key.size()); - ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); + dbwrapper_private::GetObfuscation(parent)(ssValue); leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size()); m_impl_batch->batch.Put(slKey, slValue); } @@ -248,24 +249,14 @@ CDBWrapper::CDBWrapper(const DBParams& params) LogPrintf("Finished database compaction of %s\n", fs::PathToString(params.path)); } - // The base-case obfuscation key, which is a noop. - obfuscate_key = std::vector(OBFUSCATE_KEY_NUM_BYTES, '\000'); - - bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key); - - if (!key_exists && params.obfuscate && IsEmpty()) { - // Initialize non-degenerate obfuscation if it won't upset - // existing, non-obfuscated data. - std::vector new_key = CreateObfuscateKey(); - - // Write `new_key` so we don't obfuscate the key with itself - Write(OBFUSCATE_KEY_KEY, new_key); - obfuscate_key = new_key; - - LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key)); + assert(!m_obfuscation); // Needed for unobfuscated Read()/Write() below + if (!Read(OBFUSCATION_KEY_KEY, m_obfuscation) && params.obfuscate && IsEmpty()) { + // Generate, write and read back the new obfuscation key, making sure we don't obfuscate the key itself + Write(OBFUSCATION_KEY_KEY, FastRandomContext{}.randbytes(Obfuscation::KEY_SIZE)); + Read(OBFUSCATION_KEY_KEY, m_obfuscation); + LogInfo("Wrote new obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey()); } - - LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key)); + LogInfo("Using obfuscation key for %s: %s", fs::PathToString(params.path), m_obfuscation.HexKey()); } CDBWrapper::~CDBWrapper() @@ -310,25 +301,6 @@ size_t CDBWrapper::DynamicMemoryUsage() const return parsed.value(); } -// Prefixed with null character to avoid collisions with other keys -// -// We must use a string constructor which specifies length so that we copy -// past the null-terminator. -const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14); - -const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8; - -/** - * Returns a string (consisting of 8 random bytes) suitable for use as an - * obfuscating XOR key. - */ -std::vector CDBWrapper::CreateObfuscateKey() const -{ - std::vector ret(OBFUSCATE_KEY_NUM_BYTES); - GetRandBytes(ret); - return ret; -} - std::optional CDBWrapper::ReadImpl(std::span key) const { leveldb::Slice slKey(CharCast(key.data()), key.size()); @@ -412,9 +384,9 @@ void CDBIterator::Next() { m_impl_iter->iter->Next(); } namespace dbwrapper_private { -const std::vector& GetObfuscateKey(const CDBWrapper &w) +const Obfuscation& GetObfuscation(const CDBWrapper& w) { - return w.obfuscate_key; + return w.m_obfuscation; } } // namespace dbwrapper_private diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 789b5be8fc..c5d4940486 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -18,7 +18,6 @@ #include #include #include -#include static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024; @@ -63,8 +62,7 @@ namespace dbwrapper_private { * Database obfuscation should be considered an implementation detail of the * specific database. */ -const std::vector& GetObfuscateKey(const CDBWrapper &w); - +const Obfuscation& GetObfuscation(const CDBWrapper&); }; // namespace dbwrapper_private bool DestroyDB(const std::string& path_str); @@ -166,7 +164,7 @@ public: template bool GetValue(V& value) { try { DataStream ssValue{GetValueImpl()}; - ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); + dbwrapper_private::GetObfuscation(parent)(ssValue); ssValue >> value; } catch (const std::exception&) { return false; @@ -179,7 +177,7 @@ struct LevelDBContext; class CDBWrapper { - friend const std::vector& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w); + friend const Obfuscation& dbwrapper_private::GetObfuscation(const CDBWrapper&); private: //! holds all leveldb-specific fields of this class std::unique_ptr m_db_context; @@ -187,16 +185,11 @@ private: //! the name of this database std::string m_name; - //! a key used for optional XOR-obfuscation of the database - std::vector obfuscate_key; + //! optional XOR-obfuscation of the database + Obfuscation m_obfuscation; - //! the key under which the obfuscation key is stored - static const std::string OBFUSCATE_KEY_KEY; - - //! the length of the obfuscate key in number of bytes - static const unsigned int OBFUSCATE_KEY_NUM_BYTES; - - std::vector CreateObfuscateKey() const; + //! obfuscation key storage key, null-prefixed to avoid collisions + inline static const std::string OBFUSCATION_KEY_KEY{"\000obfuscate_key", 14}; // explicit size to avoid truncation at leading \0 //! path to filesystem storage const fs::path m_path; @@ -228,7 +221,7 @@ public: } try { DataStream ssValue{MakeByteSpan(*strValue)}; - ssValue.Xor(obfuscate_key); + m_obfuscation(ssValue); ssValue >> value; } catch (const std::exception&) { return false; diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index ba205a9c69..4f6a4977bd 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -779,13 +780,13 @@ void BlockManager::UnlinkPrunedFiles(const std::set& setFilesToPrune) const AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const { - return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_xor_key}; + return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_obfuscation}; } /** Open an undo file (rev?????.dat) */ AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const { - return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_xor_key}; + return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_obfuscation}; } fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const @@ -1123,7 +1124,7 @@ static auto InitBlocksdirXorKey(const BlockManager::Options& opts) { // Bytes are serialized without length indicator, so this is also the exact // size of the XOR-key file. - std::array xor_key{}; + std::array obfuscation{}; // Consider this to be the first run if the blocksdir contains only hidden // files (those which start with a .). Checking for a fully-empty dir would @@ -1140,14 +1141,14 @@ static auto InitBlocksdirXorKey(const BlockManager::Options& opts) if (opts.use_xor && first_run) { // Only use random fresh key when the boolean option is set and on the // very first start of the program. - FastRandomContext{}.fillrand(xor_key); + FastRandomContext{}.fillrand(obfuscation); } const fs::path xor_key_path{opts.blocks_dir / "xor.dat"}; if (fs::exists(xor_key_path)) { // A pre-existing xor key file has priority. AutoFile xor_key_file{fsbridge::fopen(xor_key_path, "rb")}; - xor_key_file >> xor_key; + xor_key_file >> obfuscation; } else { // Create initial or missing xor key file AutoFile xor_key_file{fsbridge::fopen(xor_key_path, @@ -1157,7 +1158,7 @@ static auto InitBlocksdirXorKey(const BlockManager::Options& opts) "wbx" #endif )}; - xor_key_file << xor_key; + xor_key_file << obfuscation; if (xor_key_file.fclose() != 0) { throw std::runtime_error{strprintf("Error closing XOR key file %s: %s", fs::PathToString(xor_key_path), @@ -1165,20 +1166,20 @@ static auto InitBlocksdirXorKey(const BlockManager::Options& opts) } } // If the user disabled the key, it must be zero. - if (!opts.use_xor && xor_key != decltype(xor_key){}) { + if (!opts.use_xor && obfuscation != decltype(obfuscation){}) { throw std::runtime_error{ strprintf("The blocksdir XOR-key can not be disabled when a random key was already stored! " "Stored key: '%s', stored path: '%s'.", - HexStr(xor_key), fs::PathToString(xor_key_path)), + HexStr(obfuscation), fs::PathToString(xor_key_path)), }; } - LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(xor_key)); - return std::vector{xor_key.begin(), xor_key.end()}; + LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(obfuscation)); + return Obfuscation{obfuscation}; } BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts) : m_prune_mode{opts.prune_target > 0}, - m_xor_key{InitBlocksdirXorKey(opts)}, + m_obfuscation{InitBlocksdirXorKey(opts)}, m_opts{std::move(opts)}, m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}}, m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}}, diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index f9a03eaf83..cee0eb61ed 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -235,7 +235,7 @@ private: const bool m_prune_mode; - const std::vector m_xor_key; + const Obfuscation m_obfuscation; /** Dirty block index entries. */ std::set m_dirty_blockindex; diff --git a/src/node/mempool_persist.cpp b/src/node/mempool_persist.cpp index ff47172c27..5b0f80be6c 100644 --- a/src/node/mempool_persist.cpp +++ b/src/node/mempool_persist.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -59,15 +60,17 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active try { uint64_t version; file >> version; - std::vector xor_key; + if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) { - // Leave XOR-key empty + file.SetObfuscation({}); } else if (version == MEMPOOL_DUMP_VERSION) { - file >> xor_key; + Obfuscation obfuscation; + file >> obfuscation; + file.SetObfuscation(obfuscation); } else { return false; } - file.SetXor(xor_key); + uint64_t total_txns_to_load; file >> total_txns_to_load; uint64_t txns_tried = 0; @@ -179,12 +182,13 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock const uint64_t version{pool.m_opts.persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION}; file << version; - std::vector xor_key(8); if (!pool.m_opts.persist_v1_dat) { - FastRandomContext{}.fillrand(xor_key); - file << xor_key; + const Obfuscation obfuscation{FastRandomContext{}.randbytes()}; + file << obfuscation; + file.SetObfuscation(obfuscation); + } else { + file.SetObfuscation({}); } - file.SetXor(xor_key); uint64_t mempool_transactions_to_write(vinfo.size()); file << mempool_transactions_to_write; diff --git a/src/random.h b/src/random.h index c702309d0c..330c10a36b 100644 --- a/src/random.h +++ b/src/random.h @@ -301,6 +301,15 @@ public: return ret; } + /** Generate fixed-size random bytes. */ + template + std::array randbytes() noexcept + { + std::array ret; + Impl().fillrand(MakeWritableByteSpan(ret)); + return ret; + } + /** Generate a random 32-bit integer. */ uint32_t rand32() noexcept { return Impl().template randbits<32>(); } diff --git a/src/streams.cpp b/src/streams.cpp index 2c873e50d3..0364c2134f 100644 --- a/src/streams.cpp +++ b/src/streams.cpp @@ -6,11 +6,11 @@ #include #include #include +#include #include -AutoFile::AutoFile(std::FILE* file, std::vector data_xor) - : m_file{file}, m_xor{std::move(data_xor)} +AutoFile::AutoFile(std::FILE* file, const Obfuscation& obfuscation) : m_file{file}, m_obfuscation{obfuscation} { if (!IsNull()) { auto pos{std::ftell(m_file)}; @@ -21,12 +21,12 @@ AutoFile::AutoFile(std::FILE* file, std::vector data_xor) std::size_t AutoFile::detail_fread(std::span dst) { if (!m_file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr"); - size_t ret = std::fread(dst.data(), 1, dst.size(), m_file); - if (!m_xor.empty()) { - if (!m_position.has_value()) throw std::ios_base::failure("AutoFile::read: position unknown"); - util::Xor(dst.subspan(0, ret), m_xor, *m_position); + const size_t ret = std::fread(dst.data(), 1, dst.size(), m_file); + if (m_obfuscation) { + if (!m_position) throw std::ios_base::failure("AutoFile::read: position unknown"); + m_obfuscation(dst.subspan(0, ret), *m_position); } - if (m_position.has_value()) *m_position += ret; + if (m_position) *m_position += ret; return ret; } @@ -81,7 +81,7 @@ void AutoFile::ignore(size_t nSize) void AutoFile::write(std::span src) { if (!m_file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr"); - if (m_xor.empty()) { + if (!m_obfuscation) { if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) { throw std::ios_base::failure("AutoFile::write: write failed"); } @@ -101,9 +101,9 @@ void AutoFile::write(std::span src) void AutoFile::write_buffer(std::span src) { if (!m_file) throw std::ios_base::failure("AutoFile::write_buffer: file handle is nullptr"); - if (m_xor.size()) { + if (m_obfuscation) { if (!m_position) throw std::ios_base::failure("AutoFile::write_buffer: obfuscation position unknown"); - util::Xor(src, m_xor, *m_position); // obfuscate in-place + m_obfuscation(src, *m_position); // obfuscate in-place } if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) { throw std::ios_base::failure("AutoFile::write_buffer: write failed"); diff --git a/src/streams.h b/src/streams.h index ae9cb94bae..36af8dd615 100644 --- a/src/streams.h +++ b/src/streams.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -24,30 +25,8 @@ #include #include #include -#include #include -namespace util { -inline void Xor(std::span write, std::span key, size_t key_offset = 0) -{ - if (key.size() == 0) { - return; - } - key_offset %= key.size(); - - for (size_t i = 0, j = key_offset; i != write.size(); i++) { - write[i] ^= key[j++]; - - // This potentially acts on very many bytes of data, so it's - // important that we calculate `j`, i.e. the `key` index in this - // way instead of doing a %, which would effectively be a division - // for each byte Xor'd -- much slower than need be. - if (j == key.size()) - j = 0; - } -} -} // namespace util - /* Minimal stream for overwriting and/or appending to an existing byte vector * * The referenced vector will grow as necessary @@ -265,23 +244,13 @@ public: return (*this); } - template + template DataStream& operator>>(T&& obj) { ::Unserialize(*this, obj); return (*this); } - /** - * XOR the contents of this stream with a certain key. - * - * @param[in] key The key used to XOR the data in this stream. - */ - void Xor(const std::vector& key) - { - util::Xor(MakeWritableByteSpan(*this), MakeByteSpan(key)); - } - /** Compute total memory usage of this object (own memory + any dynamic memory). */ size_t GetMemoryUsage() const noexcept; }; @@ -402,12 +371,12 @@ class AutoFile { protected: std::FILE* m_file; - std::vector m_xor; + Obfuscation m_obfuscation; std::optional m_position; bool m_was_written{false}; public: - explicit AutoFile(std::FILE* file, std::vector data_xor={}); + explicit AutoFile(std::FILE* file, const Obfuscation& obfuscation = {}); ~AutoFile() { @@ -455,7 +424,7 @@ public: bool IsNull() const { return m_file == nullptr; } /** Continue with a different XOR key */ - void SetXor(std::vector data_xor) { m_xor = data_xor; } + void SetObfuscation(const Obfuscation& obfuscation) { m_obfuscation = obfuscation; } /** Implementation detail, only used internally. */ std::size_t detail_fread(std::span dst); diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 3a86036327..cd0f347b66 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -9,39 +9,62 @@ #include #include +#include #include using util::ToString; -// Test if a string consists entirely of null characters -static bool is_null_key(const std::vector& key) { - bool isnull = true; - - for (unsigned int i = 0; i < key.size(); i++) - isnull &= (key[i] == '\x00'); - - return isnull; -} - BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(dbwrapper) { // Perform tests both obfuscated and non-obfuscated. for (const bool obfuscate : {false, true}) { - fs::path ph = m_args.GetDataDirBase() / (obfuscate ? "dbwrapper_obfuscate_true" : "dbwrapper_obfuscate_false"); - CDBWrapper dbw({.path = ph, .cache_bytes = 1 << 20, .memory_only = true, .wipe_data = false, .obfuscate = obfuscate}); - uint8_t key{'k'}; - uint256 in = m_rng.rand256(); - uint256 res; + constexpr size_t CACHE_SIZE{1_MiB}; + const fs::path path{m_args.GetDataDirBase() / "dbwrapper"}; - // Ensure that we're doing real obfuscation when obfuscate=true - BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw))); + Obfuscation obfuscation; + std::vector> key_values{}; - BOOST_CHECK(dbw.Write(key, in)); - BOOST_CHECK(dbw.Read(key, res)); - BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + // Write values + { + CDBWrapper dbw{{.path = path, .cache_bytes = CACHE_SIZE, .wipe_data = true, .obfuscate = obfuscate}}; + BOOST_CHECK_EQUAL(obfuscate, !dbw.IsEmpty()); + + // Ensure that we're doing real obfuscation when obfuscate=true + obfuscation = dbwrapper_private::GetObfuscation(dbw); + BOOST_CHECK_EQUAL(obfuscate, dbwrapper_private::GetObfuscation(dbw)); + + for (uint8_t k{0}; k < 10; ++k) { + uint8_t key{k}; + uint256 value{m_rng.rand256()}; + BOOST_CHECK(dbw.Write(key, value)); + key_values.emplace_back(key, value); + } + } + + // Verify that the obfuscation key is never obfuscated + { + CDBWrapper dbw{{.path = path, .cache_bytes = CACHE_SIZE, .obfuscate = false}}; + BOOST_CHECK_EQUAL(obfuscation, dbwrapper_private::GetObfuscation(dbw)); + } + + // Read back the values + { + CDBWrapper dbw{{.path = path, .cache_bytes = CACHE_SIZE, .obfuscate = obfuscate}}; + + // Ensure obfuscation is read back correctly + BOOST_CHECK_EQUAL(obfuscation, dbwrapper_private::GetObfuscation(dbw)); + BOOST_CHECK_EQUAL(obfuscate, dbwrapper_private::GetObfuscation(dbw)); + + // Verify all written values + for (const auto& [key, expected_value] : key_values) { + uint256 read_value{}; + BOOST_CHECK(dbw.Read(key, read_value)); + BOOST_CHECK_EQUAL(read_value, expected_value); + } + } } } @@ -57,7 +80,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_basic_data) bool res_bool; // Ensure that we're doing real obfuscation when obfuscate=true - BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw))); + BOOST_CHECK_EQUAL(obfuscate, dbwrapper_private::GetObfuscation(dbw)); //Simulate block raw data - "b + block hash" std::string key_block = "b" + m_rng.rand256().ToString(); @@ -116,13 +139,13 @@ BOOST_AUTO_TEST_CASE(dbwrapper_basic_data) std::string file_option_tag = "F"; uint8_t filename_length = m_rng.randbits(8); std::string filename = "randomfilename"; - std::string key_file_option = strprintf("%s%01x%s", file_option_tag,filename_length,filename); + std::string key_file_option = strprintf("%s%01x%s", file_option_tag, filename_length, filename); bool in_file_bool = m_rng.randbool(); BOOST_CHECK(dbw.Write(key_file_option, in_file_bool)); BOOST_CHECK(dbw.Read(key_file_option, res_bool)); BOOST_CHECK_EQUAL(res_bool, in_file_bool); - } + } } // Test batch operations @@ -231,8 +254,8 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) BOOST_CHECK(odbw.Read(key, res2)); BOOST_CHECK_EQUAL(res2.ToString(), in.ToString()); - BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data - BOOST_CHECK(is_null_key(dbwrapper_private::GetObfuscateKey(odbw))); // The key should be an empty string + BOOST_CHECK(!odbw.IsEmpty()); + BOOST_CHECK(!dbwrapper_private::GetObfuscation(odbw)); // The key should be an empty string uint256 in2 = m_rng.rand256(); uint256 res3; @@ -269,7 +292,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) // Check that the key/val we wrote with unobfuscated wrapper doesn't exist uint256 res2; BOOST_CHECK(!odbw.Read(key, res2)); - BOOST_CHECK(!is_null_key(dbwrapper_private::GetObfuscateKey(odbw))); + BOOST_CHECK(dbwrapper_private::GetObfuscation(odbw)); uint256 in2 = m_rng.rand256(); uint256 res3; diff --git a/src/test/fuzz/autofile.cpp b/src/test/fuzz/autofile.cpp index 8d17624da6..0de855668d 100644 --- a/src/test/fuzz/autofile.cpp +++ b/src/test/fuzz/autofile.cpp @@ -4,9 +4,10 @@ #include #include -#include #include +#include #include +#include #include #include @@ -18,9 +19,10 @@ FUZZ_TARGET(autofile) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; + const auto key_bytes{ConsumeFixedLengthByteVector(fuzzed_data_provider, Obfuscation::KEY_SIZE)}; AutoFile auto_file{ fuzzed_file_provider.open(), - ConsumeRandomLengthByteVector(fuzzed_data_provider), + Obfuscation{std::span{key_bytes}.first()}, }; LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { diff --git a/src/test/fuzz/buffered_file.cpp b/src/test/fuzz/buffered_file.cpp index a6a042a25c..75ef8f313a 100644 --- a/src/test/fuzz/buffered_file.cpp +++ b/src/test/fuzz/buffered_file.cpp @@ -4,9 +4,10 @@ #include #include -#include #include +#include #include +#include #include #include @@ -20,9 +21,10 @@ FUZZ_TARGET(buffered_file) FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; std::optional opt_buffered_file; + const auto key_bytes{ConsumeFixedLengthByteVector(fuzzed_data_provider, Obfuscation::KEY_SIZE)}; AutoFile fuzzed_file{ fuzzed_file_provider.open(), - ConsumeRandomLengthByteVector(fuzzed_data_provider), + Obfuscation{std::span{key_bytes}.first()}, }; try { auto n_buf_size = fuzzed_data_provider.ConsumeIntegralInRange(0, 4096); diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index 3d8b543e64..538d41125a 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -58,7 +58,7 @@ BOOST_AUTO_TEST_CASE(fastrandom_tests_deterministic) BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); BOOST_CHECK_EQUAL(ctx1.rand64(), ctx2.rand64()); BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); - BOOST_CHECK(ctx1.randbytes(17) == ctx2.randbytes(17)); + BOOST_CHECK(std::ranges::equal(ctx1.randbytes(17), ctx2.randbytes<17>())); // check vector/array behavior symmetry BOOST_CHECK(ctx1.rand256() == ctx2.rand256()); BOOST_CHECK_EQUAL(ctx1.randbits(7), ctx2.randbits(7)); BOOST_CHECK(ctx1.randbytes(128) == ctx2.randbytes(128)); diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 9e8281d26a..ce496df5a9 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -8,27 +8,120 @@ #include #include #include +#include #include #include using namespace std::string_literals; +using namespace util::hex_literals; BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup) +// Test that obfuscation can be properly reverted even with random chunk sizes. +BOOST_AUTO_TEST_CASE(xor_roundtrip_random_chunks) +{ + auto apply_random_xor_chunks{[&](std::span target, const Obfuscation& obfuscation) { + for (size_t offset{0}; offset < target.size();) { + const size_t chunk_size{1 + m_rng.randrange(target.size() - offset)}; + obfuscation(target.subspan(offset, chunk_size), offset); + offset += chunk_size; + } + }}; + + for (size_t test{0}; test < 100; ++test) { + const size_t write_size{1 + m_rng.randrange(100U)}; + const std::vector original{m_rng.randbytes(write_size)}; + std::vector roundtrip{original}; + + const auto key_bytes{m_rng.randbool() ? m_rng.randbytes() : std::array{}}; + const Obfuscation obfuscation{key_bytes}; + apply_random_xor_chunks(roundtrip, obfuscation); + + const bool key_all_zeros{std::ranges::all_of( + std::span{key_bytes}.first(std::min(write_size, Obfuscation::KEY_SIZE)), [](auto b) { return b == std::byte{0}; })}; + BOOST_CHECK(key_all_zeros ? original == roundtrip : original != roundtrip); + + apply_random_xor_chunks(roundtrip, obfuscation); + BOOST_CHECK(original == roundtrip); + } +} + +// Compares optimized obfuscation against a trivial, byte-by-byte reference implementation +// with random offsets to ensure proper handling of key wrapping. +BOOST_AUTO_TEST_CASE(xor_bytes_reference) +{ + auto expected_xor{[](std::span target, std::span obfuscation, size_t key_offset) { + for (auto& b : target) { + b ^= obfuscation[key_offset++ % obfuscation.size()]; + } + }}; + + for (size_t test{0}; test < 100; ++test) { + const size_t write_size{1 + m_rng.randrange(100U)}; + const size_t key_offset{m_rng.randrange(3 * Obfuscation::KEY_SIZE)}; // Make sure the key can wrap around + const size_t write_offset{std::min(write_size, m_rng.randrange(Obfuscation::KEY_SIZE * 2))}; // Write unaligned data + + const auto key_bytes{m_rng.randbool() ? m_rng.randbytes() : std::array{}}; + const Obfuscation obfuscation{key_bytes}; + std::vector expected{m_rng.randbytes(write_size)}; + std::vector actual{expected}; + + expected_xor(std::span{expected}.subspan(write_offset), key_bytes, key_offset); + obfuscation(std::span{actual}.subspan(write_offset), key_offset); + + BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), actual.begin(), actual.end()); + } +} + +BOOST_AUTO_TEST_CASE(obfuscation_hexkey) +{ + const auto key_bytes{m_rng.randbytes()}; + + const Obfuscation obfuscation{key_bytes}; + BOOST_CHECK_EQUAL(obfuscation.HexKey(), HexStr(key_bytes)); +} + +BOOST_AUTO_TEST_CASE(obfuscation_serialize) +{ + const Obfuscation original{m_rng.randbytes()}; + + // Serialization + DataStream ds; + ds << original; + + BOOST_CHECK_EQUAL(ds.size(), 1 + Obfuscation::KEY_SIZE); // serialized as a vector + + // Deserialization + Obfuscation recovered{}; + ds >> recovered; + + BOOST_CHECK_EQUAL(recovered.HexKey(), original.HexKey()); +} + +BOOST_AUTO_TEST_CASE(obfuscation_empty) +{ + const Obfuscation null_obf{}; + BOOST_CHECK(!null_obf); + + const Obfuscation non_null_obf{"ff00ff00ff00ff00"_hex}; + BOOST_CHECK(non_null_obf); +} + BOOST_AUTO_TEST_CASE(xor_file) { fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"}; auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }}; const std::vector test1{1, 2, 3}; const std::vector test2{4, 5}; - const std::vector xor_pat{std::byte{0xff}, std::byte{0x00}}; + const Obfuscation obfuscation{"ff00ff00ff00ff00"_hex}; + { // Check errors for missing file - AutoFile xor_file{raw_file("rb"), xor_pat}; - BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullpt"}); - BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullpt"}); - BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullpt"}); + AutoFile xor_file{raw_file("rb"), obfuscation}; + BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullptr"}); + BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullptr"}); + BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullptr"}); } { #ifdef __MINGW64__ @@ -37,7 +130,7 @@ BOOST_AUTO_TEST_CASE(xor_file) #else const char* mode = "wbx"; #endif - AutoFile xor_file{raw_file(mode), xor_pat}; + AutoFile xor_file{raw_file(mode), obfuscation}; xor_file << test1 << test2; BOOST_REQUIRE_EQUAL(xor_file.fclose(), 0); } @@ -51,7 +144,7 @@ BOOST_AUTO_TEST_CASE(xor_file) BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"}); } { - AutoFile xor_file{raw_file("rb"), xor_pat}; + AutoFile xor_file{raw_file("rb"), obfuscation}; std::vector read1, read2; xor_file >> read1 >> read2; BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1)); @@ -60,7 +153,7 @@ BOOST_AUTO_TEST_CASE(xor_file) BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); } { - AutoFile xor_file{raw_file("rb"), xor_pat}; + AutoFile xor_file{raw_file("rb"), obfuscation}; std::vector read2; // Check that ignore works xor_file.ignore(4); @@ -76,7 +169,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_writer) { unsigned char a(1); unsigned char b(2); - unsigned char bytes[] = { 3, 4, 5, 6 }; + unsigned char bytes[] = {3, 4, 5, 6}; std::vector vch; // Each test runs twice. Serializing a second time at the same starting @@ -223,34 +316,26 @@ BOOST_AUTO_TEST_CASE(bitstream_reader_writer) BOOST_AUTO_TEST_CASE(streams_serializedata_xor) { - std::vector in; - // Degenerate case { - DataStream ds{in}; - ds.Xor({0x00, 0x00}); + DataStream ds{}; + Obfuscation{}(ds); BOOST_CHECK_EQUAL(""s, ds.str()); } - in.push_back(std::byte{0x0f}); - in.push_back(std::byte{0xf0}); - - // Single character key { - DataStream ds{in}; - ds.Xor({0xff}); + const Obfuscation obfuscation{"ffffffffffffffff"_hex}; + + DataStream ds{"0ff0"_hex}; + obfuscation(ds); BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str()); } - // Multi character key - - in.clear(); - in.push_back(std::byte{0xf0}); - in.push_back(std::byte{0x0f}); - { - DataStream ds{in}; - ds.Xor({0xff, 0x0f}); + const Obfuscation obfuscation{"ff0fff0fff0fff0f"_hex}; + + DataStream ds{"f00f"_hex}; + obfuscation(ds); BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str()); } } @@ -563,7 +648,7 @@ BOOST_AUTO_TEST_CASE(buffered_reader_matches_autofile_random_content) const FlatFilePos pos{0, 0}; const FlatFileSeq test_file{m_args.GetDataDirBase(), "buffered_file_test_random", node::BLOCKFILE_CHUNK_SIZE}; - const std::vector obfuscation{m_rng.randbytes(8)}; + const Obfuscation obfuscation{m_rng.randbytes()}; // Write out the file with random content { @@ -618,7 +703,7 @@ BOOST_AUTO_TEST_CASE(buffered_writer_matches_autofile_random_content) const FlatFileSeq test_buffered{m_args.GetDataDirBase(), "buffered_write_test", node::BLOCKFILE_CHUNK_SIZE}; const FlatFileSeq test_direct{m_args.GetDataDirBase(), "direct_write_test", node::BLOCKFILE_CHUNK_SIZE}; - const std::vector obfuscation{m_rng.randbytes(8)}; + const Obfuscation obfuscation{m_rng.randbytes()}; { DataBuffer test_data{m_rng.randbytes(file_size)}; diff --git a/src/util/obfuscation.h b/src/util/obfuscation.h new file mode 100644 index 0000000000..db7527064b --- /dev/null +++ b/src/util/obfuscation.h @@ -0,0 +1,115 @@ +// Copyright (c) 2025-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_OBFUSCATION_H +#define BITCOIN_UTIL_OBFUSCATION_H + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +class Obfuscation +{ +public: + using KeyType = uint64_t; + static constexpr size_t KEY_SIZE{sizeof(KeyType)}; + + Obfuscation() { SetRotations(0); } + explicit Obfuscation(std::span key_bytes) + { + SetRotations(ToKey(key_bytes)); + } + + operator bool() const { return m_rotations[0] != 0; } + + void operator()(std::span target, size_t key_offset = 0) const + { + if (!*this) return; + + KeyType rot_key{m_rotations[key_offset % KEY_SIZE]}; // Continue obfuscation from where we left off + if (target.size() > KEY_SIZE) { + // Obfuscate until 64-bit alignment boundary + if (const auto misalign{std::bit_cast(target.data()) % KEY_SIZE}) { + const size_t alignment{std::min(KEY_SIZE - misalign, target.size())}; + XorWord(target.first(alignment), rot_key); + + target = {std::assume_aligned(target.data() + alignment), target.size() - alignment}; + rot_key = m_rotations[(key_offset + alignment) % KEY_SIZE]; + } + // Aligned obfuscation in 64-byte chunks + for (constexpr auto unroll{8}; target.size() >= KEY_SIZE * unroll; target = target.subspan(KEY_SIZE * unroll)) { + for (size_t i{0}; i < unroll; ++i) { + XorWord(target.subspan(i * KEY_SIZE, KEY_SIZE), rot_key); + } + } + // Aligned obfuscation in 64-bit chunks + for (; target.size() >= KEY_SIZE; target = target.subspan(KEY_SIZE)) { + XorWord(target.first(), rot_key); + } + } + XorWord(target, rot_key); + } + + template + void Serialize(Stream& s) const + { + // Use vector serialization for convenient compact size prefix. + std::vector bytes{KEY_SIZE}; + std::memcpy(bytes.data(), &m_rotations[0], KEY_SIZE); + s << bytes; + } + + template + void Unserialize(Stream& s) + { + std::vector bytes{KEY_SIZE}; + s >> bytes; + if (bytes.size() != KEY_SIZE) throw std::ios_base::failure(strprintf("Obfuscation key size should be exactly %s bytes long", KEY_SIZE)); + SetRotations(ToKey(std::span(bytes))); + } + + std::string HexKey() const + { + return HexStr(std::bit_cast>(m_rotations[0])); + } + +private: + // Cached key rotations for different offsets. + std::array m_rotations; + + void SetRotations(KeyType key) + { + for (size_t i{0}; i < KEY_SIZE; ++i) { + int key_rotation_bits{int(CHAR_BIT * i)}; + if constexpr (std::endian::native == std::endian::big) key_rotation_bits *= -1; + m_rotations[i] = std::rotr(key, key_rotation_bits); + } + } + + static KeyType ToKey(std::span key_span) + { + KeyType key{}; + std::memcpy(&key, key_span.data(), KEY_SIZE); + return key; + } + + static void XorWord(std::span target, KeyType key) + { + assert(target.size() <= KEY_SIZE); + if (target.empty()) return; + KeyType raw{}; + std::memcpy(&raw, target.data(), target.size()); + raw ^= key; + std::memcpy(target.data(), &raw, target.size()); + } +}; + +#endif // BITCOIN_UTIL_OBFUSCATION_H