From 3822aa11d41e65e9033dd82785534bb68b1fdcf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 31 Mar 2025 19:31:18 +0200 Subject: [PATCH 1/4] Introduce fork cache --- nano/core_test/CMakeLists.txt | 1 + nano/core_test/fork_cache.cpp | 310 +++++++++++++++++++++++++++++++++ nano/lib/blocks.hpp | 9 +- nano/lib/stats_enums.hpp | 9 +- nano/node/CMakeLists.txt | 2 + nano/node/active_elections.cpp | 18 +- nano/node/election.cpp | 11 ++ nano/node/fork_cache.cpp | 84 +++++++++ nano/node/fork_cache.hpp | 67 +++++++ nano/node/fwd.hpp | 1 + nano/node/node.cpp | 15 ++ nano/node/node.hpp | 2 + nano/node/nodeconfig.hpp | 2 + 13 files changed, 524 insertions(+), 7 deletions(-) create mode 100644 nano/core_test/fork_cache.cpp create mode 100644 nano/node/fork_cache.cpp create mode 100644 nano/node/fork_cache.hpp diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index b3275cf63..e4be77ff5 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -25,6 +25,7 @@ add_executable( enums.cpp epochs.cpp fair_queue.cpp + fork_cache.cpp ipc.cpp ledger.cpp ledger_confirm.cpp diff --git a/nano/core_test/fork_cache.cpp b/nano/core_test/fork_cache.cpp new file mode 100644 index 000000000..b11591f2f --- /dev/null +++ b/nano/core_test/fork_cache.cpp @@ -0,0 +1,310 @@ +#include +#include +#include + +#include + +#include + +TEST (fork_cache, construction) +{ + nano::test::system system; + nano::fork_cache_config cfg; + nano::fork_cache fork_cache{ cfg, system.stats }; + ASSERT_EQ (0, fork_cache.size ()); + ASSERT_FALSE (fork_cache.contains (nano::qualified_root{})); +} + +/* + * Inserts a single block to cache, ensures it can be retrieved + */ +TEST (fork_cache, one) +{ + nano::test::system system; + nano::fork_cache_config cfg; + nano::fork_cache fork_cache{ cfg, system.stats }; + + auto block = std::make_shared (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + nano::qualified_root root = block->qualified_root (); + + fork_cache.put (block); + ASSERT_EQ (1, fork_cache.size ()); + ASSERT_TRUE (fork_cache.contains (root)); + + auto blocks = fork_cache.get (root); + ASSERT_EQ (1, blocks.size ()); + ASSERT_EQ (block, blocks.front ()); +} + +/* + * Inserts multiple blocks with same root, ensures all are retrievable + */ +TEST (fork_cache, multiple_forks) +{ + nano::test::system system; + nano::fork_cache_config cfg; + nano::fork_cache fork_cache{ cfg, system.stats }; + + // Create several blocks with the same qualified root (same previous and account) + auto block1 = std::make_shared (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + auto block2 = std::make_shared (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio * 2, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + auto block3 = std::make_shared (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio * 3, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + + nano::qualified_root root = block1->qualified_root (); + ASSERT_EQ (root, block2->qualified_root ()); + ASSERT_EQ (root, block3->qualified_root ()); + + fork_cache.put (block1); + fork_cache.put (block2); + fork_cache.put (block3); + + ASSERT_EQ (1, fork_cache.size ()); // Only one root in the cache + ASSERT_TRUE (fork_cache.contains (root)); + + auto blocks = fork_cache.get (root); + ASSERT_EQ (3, blocks.size ()); + + // Check if all blocks are present + ASSERT_TRUE (std::find (blocks.begin (), blocks.end (), block1) != blocks.end ()); + ASSERT_TRUE (std::find (blocks.begin (), blocks.end (), block2) != blocks.end ()); + ASSERT_TRUE (std::find (blocks.begin (), blocks.end (), block3) != blocks.end ()); +} + +/* + * Inserts multiple blocks with different roots, ensures all can be retrieved + */ +TEST (fork_cache, multiple_roots) +{ + nano::test::system system; + nano::fork_cache_config cfg; + nano::fork_cache fork_cache{ cfg, system.stats }; + + // Create blocks with different roots + auto block1 = std::make_shared (nano::dev::genesis_key.pub, nano::test::random_hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + + nano::keypair key2; + auto block2 = std::make_shared (key2.pub, nano::test::random_hash (), key2.pub, nano::Knano_ratio, nano::test::random_hash (), key2.prv, key2.pub, 0); + + nano::keypair key3; + auto block3 = std::make_shared (key3.pub, nano::test::random_hash (), key3.pub, nano::Knano_ratio, nano::test::random_hash (), key3.prv, key3.pub, 0); + + nano::qualified_root root1 = block1->qualified_root (); + nano::qualified_root root2 = block2->qualified_root (); + nano::qualified_root root3 = block3->qualified_root (); + + // Make sure roots are different + ASSERT_NE (root1, root2); + ASSERT_NE (root1, root3); + ASSERT_NE (root2, root3); + + fork_cache.put (block1); + fork_cache.put (block2); + fork_cache.put (block3); + + ASSERT_EQ (3, fork_cache.size ()); + ASSERT_TRUE (fork_cache.contains (root1)); + ASSERT_TRUE (fork_cache.contains (root2)); + ASSERT_TRUE (fork_cache.contains (root3)); + + auto blocks1 = fork_cache.get (root1); + ASSERT_EQ (1, blocks1.size ()); + ASSERT_EQ (block1, blocks1.front ()); + + auto blocks2 = fork_cache.get (root2); + ASSERT_EQ (1, blocks2.size ()); + ASSERT_EQ (block2, blocks2.front ()); + + auto blocks3 = fork_cache.get (root3); + ASSERT_EQ (1, blocks3.size ()); + ASSERT_EQ (block3, blocks3.front ()); +} + +/* + * Tests duplicate block handling (same block twice) + */ +TEST (fork_cache, duplicate_block) +{ + nano::test::system system; + nano::fork_cache_config cfg; + nano::fork_cache fork_cache{ cfg, system.stats }; + + auto block = std::make_shared (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + nano::qualified_root root = block->qualified_root (); + + // Insert the same block twice + fork_cache.put (block); + ASSERT_EQ (1, fork_cache.size ()); + ASSERT_EQ (1, fork_cache.get (root).size ()); + + // Check the stats for insert count + ASSERT_EQ (1, system.stats.count (nano::stat::type::fork_cache, nano::stat::detail::insert)); + + fork_cache.put (block); + ASSERT_EQ (1, fork_cache.size ()); + + // Block should only be added once to the deque + auto blocks = fork_cache.get (root); + ASSERT_EQ (1, blocks.size ()); + ASSERT_EQ (block, blocks.front ()); +} + +/* + * Tests that when max_forks_per_root is exceeded, oldest entries are dropped + */ +TEST (fork_cache, overfill_per_root) +{ + nano::test::system system; + nano::fork_cache_config cfg; + cfg.max_forks_per_root = 2; + nano::fork_cache fork_cache{ cfg, system.stats }; + + // Create several blocks with the same qualified root + auto block1 = std::make_shared (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + auto block2 = std::make_shared (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio * 2, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + auto block3 = std::make_shared (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio * 3, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + + nano::qualified_root root = block1->qualified_root (); + + // Insert all three blocks + fork_cache.put (block1); + fork_cache.put (block2); + fork_cache.put (block3); + + ASSERT_EQ (1, fork_cache.size ()); + auto blocks = fork_cache.get (root); + ASSERT_EQ (2, blocks.size ()); // Only 2 blocks should be kept + + // The oldest block (block1) should have been removed + ASSERT_FALSE (std::find (blocks.begin (), blocks.end (), block1) != blocks.end ()); + ASSERT_TRUE (std::find (blocks.begin (), blocks.end (), block2) != blocks.end ()); + ASSERT_TRUE (std::find (blocks.begin (), blocks.end (), block3) != blocks.end ()); +} + +/* + * Tests that when max_size is exceeded, oldest root entries are dropped + */ +TEST (fork_cache, overfill_total) +{ + nano::test::system system; + nano::fork_cache_config cfg; + cfg.max_size = 2; + nano::fork_cache fork_cache{ cfg, system.stats }; + + // Create blocks with different roots + auto block1 = std::make_shared (nano::dev::genesis_key.pub, nano::test::random_hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + + nano::keypair key2; + auto block2 = std::make_shared (key2.pub, nano::test::random_hash (), key2.pub, nano::Knano_ratio, nano::test::random_hash (), key2.prv, key2.pub, 0); + + nano::keypair key3; + auto block3 = std::make_shared (key3.pub, nano::test::random_hash (), key3.pub, nano::Knano_ratio, nano::test::random_hash (), key3.prv, key3.pub, 0); + + nano::qualified_root root1 = block1->qualified_root (); + nano::qualified_root root2 = block2->qualified_root (); + nano::qualified_root root3 = block3->qualified_root (); + + // Make sure roots are different + ASSERT_NE (root1, root2); + ASSERT_NE (root1, root3); + ASSERT_NE (root2, root3); + + // Insert all three blocks + fork_cache.put (block1); + fork_cache.put (block2); + fork_cache.put (block3); + + ASSERT_EQ (2, fork_cache.size ()); // Only 2 roots should be kept + + // The oldest root (root1) should have been removed + ASSERT_FALSE (fork_cache.contains (root1)); + ASSERT_TRUE (fork_cache.contains (root2)); + ASSERT_TRUE (fork_cache.contains (root3)); +} + +/* + * Tests a more complex scenario with multiple roots and multiple forks per root + */ +TEST (fork_cache, complex_scenario) +{ + nano::test::system system; + nano::fork_cache_config cfg; + cfg.max_size = 2; + cfg.max_forks_per_root = 2; + nano::fork_cache fork_cache{ cfg, system.stats }; + + // Create multiple blocks for first root + auto const previous1 = nano::test::random_hash (); + auto block1a = std::make_shared (nano::dev::genesis_key.pub, previous1, nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + auto block1b = std::make_shared (nano::dev::genesis_key.pub, previous1, nano::dev::genesis_key.pub, nano::Knano_ratio * 2, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + auto block1c = std::make_shared (nano::dev::genesis_key.pub, previous1, nano::dev::genesis_key.pub, nano::Knano_ratio * 3, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0); + + nano::qualified_root root1 = block1a->qualified_root (); + + // Create blocks for second root + auto const previous2 = nano::test::random_hash (); + nano::keypair key2; + auto block2a = std::make_shared (key2.pub, previous2, key2.pub, nano::Knano_ratio, nano::test::random_hash (), key2.prv, key2.pub, 0); + auto block2b = std::make_shared (key2.pub, previous2, key2.pub, nano::Knano_ratio * 2, nano::test::random_hash (), key2.prv, key2.pub, 0); + + nano::qualified_root root2 = block2a->qualified_root (); + + // Create block for third root + nano::keypair key3; + auto const previous3 = nano::test::random_hash (); + auto block3 = std::make_shared (key3.pub, previous3, key3.pub, nano::Knano_ratio, nano::test::random_hash (), key3.prv, key3.pub, 0); + + nano::qualified_root root3 = block3->qualified_root (); + + // Make sure roots are different + ASSERT_NE (root1, root2); + ASSERT_NE (root1, root3); + ASSERT_NE (root2, root3); + + // Insert blocks for first root + fork_cache.put (block1a); + fork_cache.put (block1b); + fork_cache.put (block1c); + + // First root should have max_forks_per_root=2 blocks, with the oldest dropped + ASSERT_EQ (1, fork_cache.size ()); + auto blocks1 = fork_cache.get (root1); + ASSERT_EQ (2, blocks1.size ()); + ASSERT_FALSE (std::find (blocks1.begin (), blocks1.end (), block1a) != blocks1.end ()); // Oldest should be dropped + ASSERT_TRUE (std::find (blocks1.begin (), blocks1.end (), block1b) != blocks1.end ()); + ASSERT_TRUE (std::find (blocks1.begin (), blocks1.end (), block1c) != blocks1.end ()); + + // Insert blocks for second root + fork_cache.put (block2a); + fork_cache.put (block2b); + + // Still within max_size=2, so both roots should be present + ASSERT_EQ (2, fork_cache.size ()); + ASSERT_TRUE (fork_cache.contains (root1)); + ASSERT_TRUE (fork_cache.contains (root2)); + + auto blocks2 = fork_cache.get (root2); + ASSERT_EQ (2, blocks2.size ()); + + // Insert block for third root + fork_cache.put (block3); + + // Should exceed max_size, oldest root (root1) should be dropped + ASSERT_EQ (2, fork_cache.size ()); + ASSERT_FALSE (fork_cache.contains (root1)); // Oldest root dropped + ASSERT_TRUE (fork_cache.contains (root2)); + ASSERT_TRUE (fork_cache.contains (root3)); +} + +/* + * Tests getting a non-existent root + */ +TEST (fork_cache, nonexistent) +{ + nano::test::system system; + nano::fork_cache_config cfg; + nano::fork_cache fork_cache{ cfg, system.stats }; + + nano::qualified_root nonexistent_root; + auto blocks = fork_cache.get (nonexistent_root); + ASSERT_TRUE (blocks.empty ()); +} \ No newline at end of file diff --git a/nano/lib/blocks.hpp b/nano/lib/blocks.hpp index 5bdb0a4af..69f8ed327 100644 --- a/nano/lib/blocks.hpp +++ b/nano/lib/blocks.hpp @@ -353,7 +353,14 @@ class state_block : public nano::block { public: state_block () = default; - state_block (nano::account const &, nano::block_hash const &, nano::account const &, nano::amount const &, nano::link const &, nano::raw_key const &, nano::public_key const &, uint64_t); + state_block (nano::account const & account, + nano::block_hash const & previous, + nano::account const & representative, + nano::amount const & balance, + nano::link const & link, + nano::raw_key const & prv, + nano::public_key const & pub, + uint64_t work); state_block (bool &, nano::stream &); state_block (bool &, boost::property_tree::ptree const &); virtual ~state_block () = default; diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 228e8a545..8d6aba5a8 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -121,6 +121,7 @@ enum class type process_confirmed, online_reps, pruning, + fork_cache, _last // Must be the last enum }; @@ -478,18 +479,17 @@ enum class detail activate_full, scanned, - // active + // active_elections insert, insert_failed, transition_priority, transition_priority_failed, election_cleanup, activate_immediately, - - // active_elections started, stopped, confirm_dependent, + forks_cached, // unchecked put, @@ -677,6 +677,9 @@ enum class detail pruned_count, collect_targets, + // fork_cache + overfill_entry, + _last // Must be the last enum }; diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index c6d42d5e2..d26e5b8b9 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -71,6 +71,8 @@ add_library( epoch_upgrader.hpp epoch_upgrader.cpp fair_queue.hpp + fork_cache.hpp + fork_cache.cpp fwd.hpp ipc/action_handler.hpp ipc/action_handler.cpp diff --git a/nano/node/active_elections.cpp b/nano/node/active_elections.cpp index 1d4810a54..9845e0d28 100644 --- a/nano/node/active_elections.cpp +++ b/nano/node/active_elections.cpp @@ -400,8 +400,8 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr< auto const root = block_a->qualified_root (); auto const hash = block_a->hash (); - auto const existing = roots.get ().find (root); - if (existing == roots.get ().end ()) + + if (auto existing = roots.get ().find (root); existing == roots.get ().end ()) { if (!recently_confirmed.exists (root)) { @@ -479,9 +479,21 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr< { debug_assert (result.election); - node.vote_cache_processor.trigger (hash); + // Notifications node.observers.active_started.notify (hash); vacancy_updated.notify (); + + // Let the election know about already observed votes + node.vote_cache_processor.trigger (hash); + + // Let the election know about already observed forks + auto forks = node.fork_cache.get (root); + node.stats.add (nano::stat::type::active_elections, nano::stat::detail::forks_cached, forks.size ()); + + for (auto const & fork : forks) + { + publish (fork); + } } // Votes are generated for inserted or ongoing elections diff --git a/nano/node/election.cpp b/nano/node/election.cpp index fc0025a8d..6ccf519b9 100644 --- a/nano/node/election.cpp +++ b/nano/node/election.cpp @@ -456,6 +456,17 @@ void nano::election::confirm_if_quorum (nano::unique_lock & lock_a) { status.winner = block_l; remove_votes (status_winner_hash_l); + + node.logger.debug (nano::log::type::election, "Winning fork changed from {} to {} for root: {} (behavior: {}, state: {}, voters: {}, blocks: {}, duration: {}ms)", + status_winner_hash_l, + winner_hash_l, + qualified_root, + to_string (behavior_m), + to_string (state_m), + status.voter_count, + status.block_count, + duration ().count ()); + node.block_processor.force (block_l); } if (have_quorum (tally_l)) diff --git a/nano/node/fork_cache.cpp b/nano/node/fork_cache.cpp new file mode 100644 index 000000000..b223cf905 --- /dev/null +++ b/nano/node/fork_cache.cpp @@ -0,0 +1,84 @@ +#include +#include + +#include + +nano::fork_cache::fork_cache (nano::fork_cache_config const & config_a, nano::stats & stats_a) : + config{ config_a }, + stats{ stats_a } +{ +} + +void nano::fork_cache::put (std::shared_ptr block) +{ + release_assert (block != nullptr); + + std::lock_guard guard{ mutex }; + + // Add the new block to the cache, duplicates are prevented by the multi_index container + auto [it, added] = roots.push_back ({ block->qualified_root () }); + release_assert (it != roots.end ()); + stats.inc (nano::stat::type::fork_cache, added ? nano::stat::detail::insert : nano::stat::detail::duplicate); + + // Check if we already have this hash + bool exists = std::find_if (it->forks.begin (), it->forks.end (), [&block] (auto const & fork) { + return fork->hash () == block->hash (); + }) + != it->forks.end (); + + if (exists) + { + return; + } + + it->forks.push_back (block); + + // Check if we have too many forks for this root + if (it->forks.size () > config.max_forks_per_root) + { + stats.inc (nano::stat::type::fork_cache, nano::stat::detail::overfill_entry); + it->forks.pop_front (); // Remove the oldest entry + } + + // Check if we're at capacity + if (roots.size () > config.max_size) + { + // Remove oldest entry (first in sequence) + stats.inc (nano::stat::type::fork_cache, nano::stat::detail::overfill); + roots.pop_front (); // Remove the oldest entry + } +} + +std::deque> nano::fork_cache::get (nano::qualified_root const & root) const +{ + std::lock_guard guard{ mutex }; + + if (auto it = roots.get ().find (root); it != roots.get ().end ()) + { + return it->forks; + } + return {}; +} + +size_t nano::fork_cache::size () const +{ + std::lock_guard guard{ mutex }; + + return roots.size (); +} + +bool nano::fork_cache::contains (nano::qualified_root const & root) const +{ + std::lock_guard guard{ mutex }; + + return roots.get ().count (root) > 0; +} + +nano::container_info nano::fork_cache::container_info () const +{ + std::lock_guard guard{ mutex }; + + nano::container_info result; + result.put ("roots", roots); + return result; +} \ No newline at end of file diff --git a/nano/node/fork_cache.hpp b/nano/node/fork_cache.hpp new file mode 100644 index 000000000..2c149050d --- /dev/null +++ b/nano/node/fork_cache.hpp @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace mi = boost::multi_index; + +namespace nano +{ +class fork_cache_config final +{ +public: + size_t max_size{ 1024 * 16 }; + size_t max_forks_per_root{ 10 }; +}; + +class fork_cache final +{ +public: + fork_cache (fork_cache_config const &, nano::stats &); + + void put (std::shared_ptr fork); + std::deque> get (nano::qualified_root const & root) const; + + size_t size () const; + bool contains (nano::qualified_root const & root) const; + + nano::container_info container_info () const; + +private: + fork_cache_config const & config; + nano::stats & stats; + + struct entry + { + nano::qualified_root root; + mutable std::deque> forks; + }; + + // clang-format off + class tag_sequenced {}; + class tag_root {}; + + using ordered_forks = boost::multi_index_container>, + mi::hashed_unique, + mi::member> + >>; + // clang-format on + + ordered_forks roots; + + mutable std::mutex mutex; +}; +} \ No newline at end of file diff --git a/nano/node/fwd.hpp b/nano/node/fwd.hpp index 95d1e1153..bc798fbb8 100644 --- a/nano/node/fwd.hpp +++ b/nano/node/fwd.hpp @@ -19,6 +19,7 @@ class bootstrap_service; class confirming_set; class election; class election_status; +class fork_cache; class ledger_notifications; class local_block_broadcaster; class local_vote_history; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index f96449fc8..4f7c27d54 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -147,6 +148,8 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy port_mapping{ *port_mapping_impl }, block_processor_impl{ std::make_unique (config, ledger, ledger_notifications, unchecked, stats, logger) }, block_processor{ *block_processor_impl }, + fork_cache_impl{ std::make_unique (config.fork_cache, stats) }, + fork_cache{ *fork_cache_impl }, confirming_set_impl{ std::make_unique (config.confirming_set, ledger, ledger_notifications, stats, logger) }, confirming_set{ *confirming_set_impl }, bucketing_impl{ std::make_unique () }, @@ -229,6 +232,17 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy active.recently_confirmed.erase (hash); }); + // Cache forks + ledger_notifications.blocks_processed.add ([this] (auto const & batch) { + for (auto const & [result, context] : batch) + { + if (result == nano::block_status::fork) + { + fork_cache.put (context.block); + } + } + }); + // Announce new blocks via websocket ledger_notifications.blocks_processed.add ([this] (auto const & batch) { auto const transaction = ledger.tx_begin_read (); @@ -998,6 +1012,7 @@ nano::container_info nano::node::container_info () const info.add ("http_callbacks", http_callbacks.container_info ()); info.add ("pruning", pruning.container_info ()); info.add ("vote_rebroadcaster", vote_rebroadcaster.container_info ()); + info.add ("fork_cache", fork_cache.container_info ()); return info; } diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 1a06c18a6..a4666c6e4 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -147,6 +147,8 @@ public: nano::port_mapping & port_mapping; std::unique_ptr block_processor_impl; nano::block_processor & block_processor; + std::unique_ptr fork_cache_impl; + nano::fork_cache & fork_cache; std::unique_ptr confirming_set_impl; nano::confirming_set & confirming_set; std::unique_ptr bucketing_impl; diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index 22dcb9b4d..a58f88d70 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -164,6 +165,7 @@ public: nano::backlog_scan_config backlog_scan; nano::bounded_backlog_config bounded_backlog; nano::vote_rebroadcaster_config vote_rebroadcaster; + nano::fork_cache_config fork_cache; public: /** Entry is ignored if it cannot be parsed as a valid address:port */ From 53b3e21529683595e531dbf20b0e0f19d904568c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:48:31 +0200 Subject: [PATCH 2/4] Dev run timing adjustments --- nano/node/bootstrap/bootstrap_service.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nano/node/bootstrap/bootstrap_service.cpp b/nano/node/bootstrap/bootstrap_service.cpp index ec0f7d1a2..6f4578821 100644 --- a/nano/node/bootstrap/bootstrap_service.cpp +++ b/nano/node/bootstrap/bootstrap_service.cpp @@ -789,7 +789,7 @@ void nano::bootstrap_service::cleanup_and_sync () } // Reinsert known dependencies into the priority set - if (sync_dependencies_interval.elapse (nano::is_dev_run () ? 1s : 60s)) + if (sync_dependencies_interval.elapse (nano::is_dev_run () ? 500ms : 60s)) { stats.inc (nano::stat::type::bootstrap, nano::stat::detail::sync_dependencies); accounts.sync_dependencies (); @@ -803,7 +803,7 @@ void nano::bootstrap_service::run_timeouts () { stats.inc (nano::stat::type::bootstrap, nano::stat::detail::loop_cleanup); cleanup_and_sync (); - condition.wait_for (lock, 5s, [this] () { return stopped; }); + condition.wait_for (lock, nano::is_dev_run () ? 500ms : 5s, [this] () { return stopped; }); } } From d682454665268eaddce7a76ed5de102f57f3f769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:14:00 +0200 Subject: [PATCH 3/4] Fork cache config implementation --- nano/node/bootstrap/bootstrap_server.cpp | 2 ++ nano/node/fork_cache.cpp | 21 +++++++++++++++++++++ nano/node/fork_cache.hpp | 4 ++++ nano/node/nodeconfig.cpp | 10 ++++++++++ 4 files changed, 37 insertions(+) diff --git a/nano/node/bootstrap/bootstrap_server.cpp b/nano/node/bootstrap/bootstrap_server.cpp index aac4e8a2e..f6e6ae037 100644 --- a/nano/node/bootstrap/bootstrap_server.cpp +++ b/nano/node/bootstrap/bootstrap_server.cpp @@ -439,6 +439,7 @@ nano::stat::detail nano::to_stat_detail (nano::asc_pull_type type) nano::error nano::bootstrap_server_config::serialize (nano::tomlconfig & toml) const { + toml.put ("enable", enable, "Enable bootstrap server. \ntype:bool"); toml.put ("max_queue", max_queue, "Maximum number of queued requests per peer. \ntype:uint64"); toml.put ("threads", threads, "Number of threads to process requests. \ntype:uint64"); toml.put ("batch_size", batch_size, "Maximum number of requests to process in a single batch. \ntype:uint64"); @@ -449,6 +450,7 @@ nano::error nano::bootstrap_server_config::serialize (nano::tomlconfig & toml) c nano::error nano::bootstrap_server_config::deserialize (nano::tomlconfig & toml) { + toml.get ("enable", enable); toml.get ("max_queue", max_queue); toml.get ("threads", threads); toml.get ("batch_size", batch_size); diff --git a/nano/node/fork_cache.cpp b/nano/node/fork_cache.cpp index b223cf905..65e1d5167 100644 --- a/nano/node/fork_cache.cpp +++ b/nano/node/fork_cache.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -81,4 +82,24 @@ nano::container_info nano::fork_cache::container_info () const nano::container_info result; result.put ("roots", roots); return result; +} + +/* + * fork_cache_config + */ + +nano::error nano::fork_cache_config::deserialize (nano::tomlconfig & toml) +{ + toml.get ("max_size", max_size); + toml.get ("max_forks_per_root", max_forks_per_root); + + return toml.get_error (); +} + +nano::error nano::fork_cache_config::serialize (nano::tomlconfig & toml) const +{ + toml.put ("max_size", max_size, "Maximum number of roots in the cache. Each root can have multiple forks. \ntype:uint64"); + toml.put ("max_forks_per_root", max_forks_per_root, "Maximum number of forks per root. \ntype:uint64"); + + return toml.get_error (); } \ No newline at end of file diff --git a/nano/node/fork_cache.hpp b/nano/node/fork_cache.hpp index 2c149050d..cfa0585d3 100644 --- a/nano/node/fork_cache.hpp +++ b/nano/node/fork_cache.hpp @@ -20,6 +20,10 @@ namespace nano { class fork_cache_config final { +public: + nano::error deserialize (nano::tomlconfig &); + nano::error serialize (nano::tomlconfig &) const; + public: size_t max_size{ 1024 * 16 }; size_t max_forks_per_root{ 10 }; diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index fb28b1b6c..677b5d555 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -274,6 +274,10 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const bounded_backlog.serialize (bounded_backlog_l); toml.put_child ("bounded_backlog", bounded_backlog_l); + nano::tomlconfig fork_cache_l; + fork_cache.serialize (fork_cache_l); + toml.put_child ("fork_cache", fork_cache_l); + return toml.get_error (); } @@ -431,6 +435,12 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml) bounded_backlog.deserialize (config_l); } + if (toml.has_key ("fork_cache")) + { + auto config_l = toml.get_required_child ("fork_cache"); + fork_cache.deserialize (config_l); + } + /* * Values */ From 684c7f904190461eef96dd5be9aac7d419d4d612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:12:50 +0200 Subject: [PATCH 4/4] Bootstrap poison test --- nano/core_test/node.cpp | 143 ++++++++++++++++++++++- nano/node/bootstrap/bootstrap_server.cpp | 10 ++ nano/node/bootstrap/bootstrap_server.hpp | 1 + 3 files changed, 151 insertions(+), 3 deletions(-) diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index 6643d6c99..df32b9909 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -3232,8 +3232,6 @@ TEST (node, dependency_graph_frontier) ASSERT_TIMELY_EQ (15s, node2.ledger.cemented_count (), node2.ledger.block_count ()); } -namespace nano -{ TEST (node, deferred_dependent_elections) { nano::test::system system; @@ -3355,7 +3353,6 @@ TEST (node, deferred_dependent_elections) ASSERT_TIMELY (5s, node.block_confirmed (send2->hash ())); ASSERT_TIMELY (5s, node.active.active (receive->qualified_root ())); } -} // Test that a node configured with `enable_pruning` and `max_pruning_age = 1s` will automatically // prune old confirmed blocks without explicitly saying `node.ledger_pruning` in the unit test @@ -3668,3 +3665,143 @@ TEST (node, bounded_backlog) ASSERT_TIMELY_EQ (20s, node.ledger.block_count (), 11); // 10 + genesis } + +// This test checks that a bootstrapping node can resolve a fork when a "poisoned" node +// attempts to feed it the incorrect side of a fork. +// The scenario involves: +// 1. A bootstrapping node (node_boot) - the node being tested +// 2. A poisoned node (node_poison) that has bootstrap serving enabled with an incorrect block +// 3. A representative node (node_rep) with enough voting weight but bootstrap serving disabled +// 4. A non-representative node (node_correct) that serves the correct side of the fork +TEST (node, bootstrap_poison) +{ + nano::test::system system; + + // Create the representative node with bootstrap serving disabled + nano::node_config rep_config = system.default_config (); + rep_config.bootstrap_server.enable = false; // Disable bootstrap serving + rep_config.bootstrap.enable = false; // Disable bootstrap from the network + // Disable schedulers + rep_config.priority_scheduler.enable = false; + rep_config.hinted_scheduler.enable = false; + rep_config.optimistic_scheduler.enable = false; + rep_config.backlog_scan.enable = false; + auto & node_rep = *system.add_node (rep_config); + + // Create the poisoned node with bootstrap serving enabled + nano::node_config poison_config = system.default_config (); + poison_config.bootstrap_server.enable = true; // Enable bootstrap serving + poison_config.bootstrap.enable = false; // Disable bootstrap from the network + // Disable schedulers + poison_config.priority_scheduler.enable = false; + poison_config.hinted_scheduler.enable = false; + poison_config.optimistic_scheduler.enable = false; + poison_config.backlog_scan.enable = false; + auto & node_poison = *system.add_node (poison_config); + + // Representative node needs to hold the genesis key to have voting weight + system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); + + // Create keys for our test accounts + nano::keypair key1; + + // Create and process blocks on representative node (the correct chain) + nano::block_builder builder; + + // First send from genesis to key1 + auto send1 = builder.send () + .previous (nano::dev::genesis->hash ()) + .destination (key1.pub) + .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*system.work.generate (nano::dev::genesis->hash ())) + .build (); + + ASSERT_EQ (nano::block_status::progress, node_rep.process (send1)); + ASSERT_EQ (nano::block_status::progress, node_poison.process (send1)); + + // Valid open block for key1 (correct version) + auto open_correct = builder.open () + .source (send1->hash ()) + .representative (key1.pub) + .account (key1.pub) + .sign (key1.prv, key1.pub) + .work (*system.work.generate (key1.pub)) + .build (); + + // Fork of the open block (incorrect version) - using a different representative + nano::keypair bad_rep; + auto open_fork = builder.open () + .source (send1->hash ()) + .representative (bad_rep.pub) // Different representative + .account (key1.pub) + .sign (key1.prv, key1.pub) + .work (*system.work.generate (key1.pub)) + .build (); + + // Process the correct open block on the representative node + ASSERT_EQ (nano::block_status::progress, node_rep.process (open_correct)); + + // Process the forked open block on the poisoned node + ASSERT_EQ (nano::block_status::progress, node_poison.process (open_fork)); + + // Confirm the correct block on the representative node + nano::test::confirm (node_rep, { open_correct }); + + // Now create a bootstrapping node that will try to sync from both nodes + nano::node_config node_config = system.default_config (); + node_config.bootstrap.account_sets.cooldown = 100ms; // Short cooldown between requests to speed up the test + node_config.bootstrap.request_timeout = 250ms; + node_config.bootstrap.frontier_rate_limit = 100; + // Disable schedulers + node_config.priority_scheduler.enable = false; + node_config.hinted_scheduler.enable = false; + node_config.optimistic_scheduler.enable = false; + node_config.backlog_scan.enable = false; + nano::node_flags node_flags; + auto & node = *system.add_node (node_config, node_flags); + ASSERT_EQ (node.network.size (), 2); + + std::cout << "Main node: " << node.identifier () << std::endl; + std::cout << "Waiting for: " << open_fork->hash ().to_string () << std::endl; + + // The node should initially get the incorrect block from the poisoned node + ASSERT_TIMELY (15s, node.block (open_fork->hash ()) != nullptr); + ASSERT_NEVER (1s, node.stats.count (nano::stat::type::ledger, nano::stat::detail::fork) > 0); + + // Create another non-rep node that will serve the correct side of the fork + nano::node_config correct_config = system.default_config (); + correct_config.bootstrap_server.enable = true; // Enable bootstrap serving + correct_config.bootstrap.enable = false; // Disable bootstrap from the network + correct_config.priority_scheduler.enable = false; + correct_config.hinted_scheduler.enable = false; + correct_config.optimistic_scheduler.enable = false; + auto & node_correct = *system.add_node (correct_config); + ASSERT_EQ (node.network.size (), 3); + + // Process the correct open block on the non-representative node + ASSERT_EQ (nano::block_status::progress, node_correct.process (send1)); + ASSERT_EQ (nano::block_status::progress, node_correct.process (open_correct)); + + // The node should at some point notice that there is a forked block + ASSERT_TIMELY (15s, node.stats.count (nano::stat::type::ledger, nano::stat::detail::fork) > 0); + + // Should no longer be needed, fork should be cached + node_correct.stop (); + + // We need an election active to force the correct side of the fork + ASSERT_TRUE (nano::test::start_election (system, node, open_fork->hash ())); + + // Wait for the node to resolve the fork conflict + ASSERT_TIMELY (15s, node.block (open_correct->hash ()) != nullptr); + + // Verify that the node got the correct block and not the fork + ASSERT_NE (nullptr, node.block (open_correct->hash ())); + ASSERT_EQ (nullptr, node.block (open_fork->hash ())); + + // Verify the account information on the bootstrap node is correct + nano::account_info account_info; + ASSERT_FALSE (node.store.account.get (node.store.tx_begin_read (), key1.pub, account_info)); + ASSERT_EQ (account_info.head, open_correct->hash ()); + ASSERT_EQ (account_info.representative, key1.pub); // Correct representative +} \ No newline at end of file diff --git a/nano/node/bootstrap/bootstrap_server.cpp b/nano/node/bootstrap/bootstrap_server.cpp index f6e6ae037..36c3daacb 100644 --- a/nano/node/bootstrap/bootstrap_server.cpp +++ b/nano/node/bootstrap/bootstrap_server.cpp @@ -37,6 +37,11 @@ void nano::bootstrap_server::start () { debug_assert (threads.empty ()); + if (!config.enable) + { + return; + } + for (auto i = 0u; i < config.threads; ++i) { threads.push_back (std::thread ([this] () { @@ -107,6 +112,11 @@ bool nano::bootstrap_server::verify (const nano::asc_pull_req & message) const bool nano::bootstrap_server::request (nano::asc_pull_req const & message, std::shared_ptr const & channel) { + if (!config.enable) + { + return false; + } + if (!verify (message)) { stats.inc (nano::stat::type::bootstrap_server, nano::stat::detail::invalid); diff --git a/nano/node/bootstrap/bootstrap_server.hpp b/nano/node/bootstrap/bootstrap_server.hpp index 597ee2854..1a348dd98 100644 --- a/nano/node/bootstrap/bootstrap_server.hpp +++ b/nano/node/bootstrap/bootstrap_server.hpp @@ -22,6 +22,7 @@ public: nano::error serialize (nano::tomlconfig &) const; public: + bool enable{ true }; size_t max_queue{ 16 }; size_t threads{ 1 }; size_t batch_size{ 64 };