index: Check availability of undo data for indices

src/index/base.h

@@ -122,9 +122,6 @@ protected:
 
     void ChainStateFlushed(const kernel::ChainstateRole& role, const CBlockLocator& locator) override;
 
-    /// Return custom notification options for index.
-    [[nodiscard]] virtual interfaces::Chain::NotifyOptions CustomOptions() { return {}; }
-
     /// Initialize internal state from the database and block index.
     [[nodiscard]] virtual bool CustomInit(const std::optional<interfaces::BlockRef>& block) { return true; }
 
@@ -151,6 +148,9 @@ public:
     /// Get the name of the index for display in logs.
     const std::string& GetName() const LIFETIMEBOUND { return m_name; }
 
+    /// Return custom notification options for index.
+    [[nodiscard]] virtual interfaces::Chain::NotifyOptions CustomOptions() { return {}; }
+
     /// Blocks the current thread until the index is caught up to the current
     /// state of the block chain. This only blocks if the index has gotten in
     /// sync once and only needs to process blocks in the ValidationInterface

src/index/blockfilterindex.h

@@ -63,8 +63,6 @@ private:
     std::optional<uint256> ReadFilterHeader(int height, const uint256& expected_block_hash);
 
 protected:
-    interfaces::Chain::NotifyOptions CustomOptions() override;
-
     bool CustomInit(const std::optional<interfaces::BlockRef>& block) override;
 
     bool CustomCommit(CDBBatch& batch) override;
@@ -80,6 +78,8 @@ public:
     explicit BlockFilterIndex(std::unique_ptr<interfaces::Chain> chain, BlockFilterType filter_type,
                               size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
 
+    interfaces::Chain::NotifyOptions CustomOptions() override;
+
     BlockFilterType GetFilterType() const { return m_filter_type; }
 
     /** Get a single filter by block. */

src/index/coinstatsindex.h

@@ -52,8 +52,6 @@ private:
     bool AllowPrune() const override { return true; }
 
 protected:
-    interfaces::Chain::NotifyOptions CustomOptions() override;
-
     bool CustomInit(const std::optional<interfaces::BlockRef>& block) override;
 
     bool CustomCommit(CDBBatch& batch) override;
@@ -68,6 +66,8 @@ public:
     // Constructs the index, which becomes available to be queried.
     explicit CoinStatsIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
 
+    interfaces::Chain::NotifyOptions CustomOptions() override;
+
     // Look up stats for a specific block using CBlockIndex
     std::optional<kernel::CCoinsStats> LookUpStats(const CBlockIndex& block_index) const;
 };

src/init.cpp

@@ -2265,41 +2265,70 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
 
 bool StartIndexBackgroundSync(NodeContext& node)
 {
-    // Find the oldest block among all indexes.
-    // This block is used to verify that we have the required blocks' data stored on disk,
-    // starting from that point up to the current tip.
-    // indexes_start_block='nullptr' means "start from height 0".
-    std::optional<const CBlockIndex*> indexes_start_block;
-    std::string older_index_name;
     ChainstateManager& chainman = *Assert(node.chainman);
     const Chainstate& chainstate = WITH_LOCK(::cs_main, return chainman.ValidatedChainstate());
     const CChain& index_chain = chainstate.m_chain;
+    const int current_height = WITH_LOCK(::cs_main, return index_chain.Height());
 
-    for (auto index : node.indexes) {
+    // Skip checking data availability if we have not synced any blocks yet
-        const IndexSummary& summary = index->GetSummary();
+    if (current_height > 0) {
-        if (summary.synced) continue;
+        // Before starting index sync, verify that all required block data is available
+        // on disk from each index's current sync position up to the chain tip.
+        //
+        // This is done separately for undo and block data: First we verify block + undo
+        // data existence from tip down to the lowest height required by any index that
+        // needs undo data (e.g., coinstatsindex, blockfilterindex). Then, if any
+        // block-only index needs to sync from a lower height than previously covered,
+        // verify block data existence down to that lower height.
+        //
+        // This avoids checking undo data for blocks where no index requires it,
+        // though currently block and undo data availability are synchronized on disk
+        // under normal circumstances.
+        std::optional<const CBlockIndex*> block_start;
+        std::string block_start_name;
+        std::optional<const CBlockIndex*> undo_start;
+        std::string undo_start_name;
 
-        // Get the last common block between the index best block and the active chain
+        for (const auto& index : node.indexes) {
-        LOCK(::cs_main);
+            const IndexSummary& summary = index->GetSummary();
-        const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(summary.best_block_hash);
+            if (summary.synced) continue;
-        if (!index_chain.Contains(pindex)) {
+
-            pindex = index_chain.FindFork(pindex);
+            // Get the last common block between the index best block and the active chain
+            const CBlockIndex* pindex = nullptr;
+            {
+                LOCK(::cs_main);
+                pindex = chainman.m_blockman.LookupBlockIndex(summary.best_block_hash);
+                if (!index_chain.Contains(pindex)) {
+                    pindex = index_chain.FindFork(pindex);
+                }
+            }
+            if (!pindex) {
+                pindex = index_chain.Genesis();
+            }
+
+            bool need_undo = index->CustomOptions().connect_undo_data;
+            auto& op_start_index = need_undo ? undo_start : block_start;
+            auto& name_index = need_undo ? undo_start_name : block_start_name;
+
+            if (op_start_index && pindex->nHeight >= op_start_index.value()->nHeight) continue;
+            op_start_index = pindex;
+            name_index = summary.name;
         }
 
-        if (!indexes_start_block || !pindex || pindex->nHeight < indexes_start_block.value()->nHeight) {
+        // Verify all blocks needed to sync to current tip are present including undo data.
-            indexes_start_block = pindex;
+        if (undo_start) {
-            older_index_name = summary.name;
+            LOCK(::cs_main);
-            if (!pindex) break; // Starting from genesis so no need to look for earlier block.
+            if (!chainman.m_blockman.CheckBlockDataAvailability(*index_chain.Tip(), *Assert(undo_start.value()), BlockStatus{BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO})) {
+                return InitError(Untranslated(strprintf("%s best block of the index goes beyond pruned data (including undo data). Please disable the index or reindex (which will download the whole blockchain again)", undo_start_name)));
+            }
         }
-    };
 
-    // Verify all blocks needed to sync to current tip are present.
+        // Verify all blocks needed to sync to current tip are present unless we already checked all of them above.
-    if (indexes_start_block) {
+        if (block_start && !(undo_start && undo_start.value()->nHeight <= block_start.value()->nHeight)) {
-        LOCK(::cs_main);
+            LOCK(::cs_main);
-        const CBlockIndex* start_block = *indexes_start_block;
+            if (!chainman.m_blockman.CheckBlockDataAvailability(*index_chain.Tip(), *Assert(block_start.value()), BlockStatus{BLOCK_HAVE_DATA})) {
-        if (!start_block) start_block = chainman.ActiveChain().Genesis();
+                return InitError(Untranslated(strprintf("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)", block_start_name)));
-        if (!chainman.m_blockman.CheckBlockDataAvailability(*index_chain.Tip(), *Assert(start_block))) {
+            }
-            return InitError(Untranslated(strprintf("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)", older_index_name)));
         }
     }
 

src/node/blockstorage.h

@@ -411,7 +411,8 @@ public:
     /** Calculate the amount of disk space the block & undo files currently use */
     uint64_t CalculateCurrentUsage();
 
-    //! Check if all blocks in the [upper_block, lower_block] range have data available.
+    //! Check if all blocks in the [upper_block, lower_block] range have data available as
+    //! defined by the status mask.
     //! The caller is responsible for ensuring that lower_block is an ancestor of upper_block
     //! (part of the same chain).
     bool CheckBlockDataAvailability(const CBlockIndex& upper_block, const CBlockIndex& lower_block, BlockStatus block_status = BLOCK_HAVE_DATA) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);

src/test/blockmanager_tests.cpp

@@ -149,6 +149,7 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_block_data_availability, TestChain100Setup)
     // detect this by using a status mask.
     first_available_block->nStatus &= ~BLOCK_HAVE_UNDO;
     BOOST_CHECK(!blockman.CheckBlockDataAvailability(tip, *first_available_block, BlockStatus{BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO}));
+    BOOST_CHECK(blockman.CheckBlockDataAvailability(tip, *first_available_block, BlockStatus{BLOCK_HAVE_DATA}));
 }
 
 BOOST_FIXTURE_TEST_CASE(blockmanager_block_data_part, TestChain100Setup)

test/functional/feature_index_prune.py

@@ -130,8 +130,8 @@ class FeatureIndexPruneTest(BitcoinTestFramework):
             self.stop_node(i)
 
         self.log.info("make sure we get an init error when starting the nodes again with the indices")
-        filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"
+        filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data (including undo data). Please disable the index or reindex (which will download the whole blockchain again)"
-        stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"
+        stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data (including undo data). Please disable the index or reindex (which will download the whole blockchain again)"
         end_msg = f"{os.linesep}Error: A fatal internal error occurred, see debug.log for details: Failed to start indexes, shutting down…"
         for i, msg in enumerate([filter_msg, stats_msg, filter_msg]):
             self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg+end_msg)