Merge pull request #4871 from pwojcikdev/fork-cache

Fork cache

nano/core_test/CMakeLists.txt

@@ -25,6 +25,7 @@ add_executable(
   enums.cpp
   epochs.cpp
   fair_queue.cpp
+  fork_cache.cpp
   ipc.cpp
   ledger.cpp
   ledger_confirm.cpp

nano/core_test/fork_cache.cpp

@@ -0,0 +1,310 @@
+#include <nano/node/fork_cache.hpp>
+#include <nano/test_common/system.hpp>
+#include <nano/test_common/testutil.hpp>
+
+#include <gtest/gtest.h>
+
+#include <map>
+
+TEST (fork_cache, construction)
+{
+	nano::test::system system;
+	nano::fork_cache_config cfg;
+	nano::fork_cache fork_cache{ cfg, system.stats };
+	ASSERT_EQ (0, fork_cache.size ());
+	ASSERT_FALSE (fork_cache.contains (nano::qualified_root{}));
+}
+
+/*
+ * Inserts a single block to cache, ensures it can be retrieved
+ */
+TEST (fork_cache, one)
+{
+	nano::test::system system;
+	nano::fork_cache_config cfg;
+	nano::fork_cache fork_cache{ cfg, system.stats };
+
+	auto block = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+	nano::qualified_root root = block->qualified_root ();
+
+	fork_cache.put (block);
+	ASSERT_EQ (1, fork_cache.size ());
+	ASSERT_TRUE (fork_cache.contains (root));
+
+	auto blocks = fork_cache.get (root);
+	ASSERT_EQ (1, blocks.size ());
+	ASSERT_EQ (block, blocks.front ());
+}
+
+/*
+ * Inserts multiple blocks with same root, ensures all are retrievable
+ */
+TEST (fork_cache, multiple_forks)
+{
+	nano::test::system system;
+	nano::fork_cache_config cfg;
+	nano::fork_cache fork_cache{ cfg, system.stats };
+
+	// Create several blocks with the same qualified root (same previous and account)
+	auto block1 = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+	auto block2 = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio * 2, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+	auto block3 = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio * 3, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+
+	nano::qualified_root root = block1->qualified_root ();
+	ASSERT_EQ (root, block2->qualified_root ());
+	ASSERT_EQ (root, block3->qualified_root ());
+
+	fork_cache.put (block1);
+	fork_cache.put (block2);
+	fork_cache.put (block3);
+
+	ASSERT_EQ (1, fork_cache.size ()); // Only one root in the cache
+	ASSERT_TRUE (fork_cache.contains (root));
+
+	auto blocks = fork_cache.get (root);
+	ASSERT_EQ (3, blocks.size ());
+
+	// Check if all blocks are present
+	ASSERT_TRUE (std::find (blocks.begin (), blocks.end (), block1) != blocks.end ());
+	ASSERT_TRUE (std::find (blocks.begin (), blocks.end (), block2) != blocks.end ());
+	ASSERT_TRUE (std::find (blocks.begin (), blocks.end (), block3) != blocks.end ());
+}
+
+/*
+ * Inserts multiple blocks with different roots, ensures all can be retrieved
+ */
+TEST (fork_cache, multiple_roots)
+{
+	nano::test::system system;
+	nano::fork_cache_config cfg;
+	nano::fork_cache fork_cache{ cfg, system.stats };
+
+	// Create blocks with different roots
+	auto block1 = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::test::random_hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+
+	nano::keypair key2;
+	auto block2 = std::make_shared<nano::state_block> (key2.pub, nano::test::random_hash (), key2.pub, nano::Knano_ratio, nano::test::random_hash (), key2.prv, key2.pub, 0);
+
+	nano::keypair key3;
+	auto block3 = std::make_shared<nano::state_block> (key3.pub, nano::test::random_hash (), key3.pub, nano::Knano_ratio, nano::test::random_hash (), key3.prv, key3.pub, 0);
+
+	nano::qualified_root root1 = block1->qualified_root ();
+	nano::qualified_root root2 = block2->qualified_root ();
+	nano::qualified_root root3 = block3->qualified_root ();
+
+	// Make sure roots are different
+	ASSERT_NE (root1, root2);
+	ASSERT_NE (root1, root3);
+	ASSERT_NE (root2, root3);
+
+	fork_cache.put (block1);
+	fork_cache.put (block2);
+	fork_cache.put (block3);
+
+	ASSERT_EQ (3, fork_cache.size ());
+	ASSERT_TRUE (fork_cache.contains (root1));
+	ASSERT_TRUE (fork_cache.contains (root2));
+	ASSERT_TRUE (fork_cache.contains (root3));
+
+	auto blocks1 = fork_cache.get (root1);
+	ASSERT_EQ (1, blocks1.size ());
+	ASSERT_EQ (block1, blocks1.front ());
+
+	auto blocks2 = fork_cache.get (root2);
+	ASSERT_EQ (1, blocks2.size ());
+	ASSERT_EQ (block2, blocks2.front ());
+
+	auto blocks3 = fork_cache.get (root3);
+	ASSERT_EQ (1, blocks3.size ());
+	ASSERT_EQ (block3, blocks3.front ());
+}
+
+/*
+ * Tests duplicate block handling (same block twice)
+ */
+TEST (fork_cache, duplicate_block)
+{
+	nano::test::system system;
+	nano::fork_cache_config cfg;
+	nano::fork_cache fork_cache{ cfg, system.stats };
+
+	auto block = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+	nano::qualified_root root = block->qualified_root ();
+
+	// Insert the same block twice
+	fork_cache.put (block);
+	ASSERT_EQ (1, fork_cache.size ());
+	ASSERT_EQ (1, fork_cache.get (root).size ());
+
+	// Check the stats for insert count
+	ASSERT_EQ (1, system.stats.count (nano::stat::type::fork_cache, nano::stat::detail::insert));
+
+	fork_cache.put (block);
+	ASSERT_EQ (1, fork_cache.size ());
+
+	// Block should only be added once to the deque
+	auto blocks = fork_cache.get (root);
+	ASSERT_EQ (1, blocks.size ());
+	ASSERT_EQ (block, blocks.front ());
+}
+
+/*
+ * Tests that when max_forks_per_root is exceeded, oldest entries are dropped
+ */
+TEST (fork_cache, overfill_per_root)
+{
+	nano::test::system system;
+	nano::fork_cache_config cfg;
+	cfg.max_forks_per_root = 2;
+	nano::fork_cache fork_cache{ cfg, system.stats };
+
+	// Create several blocks with the same qualified root
+	auto block1 = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+	auto block2 = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio * 2, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+	auto block3 = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::Knano_ratio * 3, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+
+	nano::qualified_root root = block1->qualified_root ();
+
+	// Insert all three blocks
+	fork_cache.put (block1);
+	fork_cache.put (block2);
+	fork_cache.put (block3);
+
+	ASSERT_EQ (1, fork_cache.size ());
+	auto blocks = fork_cache.get (root);
+	ASSERT_EQ (2, blocks.size ()); // Only 2 blocks should be kept
+
+	// The oldest block (block1) should have been removed
+	ASSERT_FALSE (std::find (blocks.begin (), blocks.end (), block1) != blocks.end ());
+	ASSERT_TRUE (std::find (blocks.begin (), blocks.end (), block2) != blocks.end ());
+	ASSERT_TRUE (std::find (blocks.begin (), blocks.end (), block3) != blocks.end ());
+}
+
+/*
+ * Tests that when max_size is exceeded, oldest root entries are dropped
+ */
+TEST (fork_cache, overfill_total)
+{
+	nano::test::system system;
+	nano::fork_cache_config cfg;
+	cfg.max_size = 2;
+	nano::fork_cache fork_cache{ cfg, system.stats };
+
+	// Create blocks with different roots
+	auto block1 = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::test::random_hash (), nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+
+	nano::keypair key2;
+	auto block2 = std::make_shared<nano::state_block> (key2.pub, nano::test::random_hash (), key2.pub, nano::Knano_ratio, nano::test::random_hash (), key2.prv, key2.pub, 0);
+
+	nano::keypair key3;
+	auto block3 = std::make_shared<nano::state_block> (key3.pub, nano::test::random_hash (), key3.pub, nano::Knano_ratio, nano::test::random_hash (), key3.prv, key3.pub, 0);
+
+	nano::qualified_root root1 = block1->qualified_root ();
+	nano::qualified_root root2 = block2->qualified_root ();
+	nano::qualified_root root3 = block3->qualified_root ();
+
+	// Make sure roots are different
+	ASSERT_NE (root1, root2);
+	ASSERT_NE (root1, root3);
+	ASSERT_NE (root2, root3);
+
+	// Insert all three blocks
+	fork_cache.put (block1);
+	fork_cache.put (block2);
+	fork_cache.put (block3);
+
+	ASSERT_EQ (2, fork_cache.size ()); // Only 2 roots should be kept
+
+	// The oldest root (root1) should have been removed
+	ASSERT_FALSE (fork_cache.contains (root1));
+	ASSERT_TRUE (fork_cache.contains (root2));
+	ASSERT_TRUE (fork_cache.contains (root3));
+}
+
+/*
+ * Tests a more complex scenario with multiple roots and multiple forks per root
+ */
+TEST (fork_cache, complex_scenario)
+{
+	nano::test::system system;
+	nano::fork_cache_config cfg;
+	cfg.max_size = 2;
+	cfg.max_forks_per_root = 2;
+	nano::fork_cache fork_cache{ cfg, system.stats };
+
+	// Create multiple blocks for first root
+	auto const previous1 = nano::test::random_hash ();
+	auto block1a = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, previous1, nano::dev::genesis_key.pub, nano::Knano_ratio, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+	auto block1b = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, previous1, nano::dev::genesis_key.pub, nano::Knano_ratio * 2, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+	auto block1c = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, previous1, nano::dev::genesis_key.pub, nano::Knano_ratio * 3, nano::test::random_hash (), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
+
+	nano::qualified_root root1 = block1a->qualified_root ();
+
+	// Create blocks for second root
+	auto const previous2 = nano::test::random_hash ();
+	nano::keypair key2;
+	auto block2a = std::make_shared<nano::state_block> (key2.pub, previous2, key2.pub, nano::Knano_ratio, nano::test::random_hash (), key2.prv, key2.pub, 0);
+	auto block2b = std::make_shared<nano::state_block> (key2.pub, previous2, key2.pub, nano::Knano_ratio * 2, nano::test::random_hash (), key2.prv, key2.pub, 0);
+
+	nano::qualified_root root2 = block2a->qualified_root ();
+
+	// Create block for third root
+	nano::keypair key3;
+	auto const previous3 = nano::test::random_hash ();
+	auto block3 = std::make_shared<nano::state_block> (key3.pub, previous3, key3.pub, nano::Knano_ratio, nano::test::random_hash (), key3.prv, key3.pub, 0);
+
+	nano::qualified_root root3 = block3->qualified_root ();
+
+	// Make sure roots are different
+	ASSERT_NE (root1, root2);
+	ASSERT_NE (root1, root3);
+	ASSERT_NE (root2, root3);
+
+	// Insert blocks for first root
+	fork_cache.put (block1a);
+	fork_cache.put (block1b);
+	fork_cache.put (block1c);
+
+	// First root should have max_forks_per_root=2 blocks, with the oldest dropped
+	ASSERT_EQ (1, fork_cache.size ());
+	auto blocks1 = fork_cache.get (root1);
+	ASSERT_EQ (2, blocks1.size ());
+	ASSERT_FALSE (std::find (blocks1.begin (), blocks1.end (), block1a) != blocks1.end ()); // Oldest should be dropped
+	ASSERT_TRUE (std::find (blocks1.begin (), blocks1.end (), block1b) != blocks1.end ());
+	ASSERT_TRUE (std::find (blocks1.begin (), blocks1.end (), block1c) != blocks1.end ());
+
+	// Insert blocks for second root
+	fork_cache.put (block2a);
+	fork_cache.put (block2b);
+
+	// Still within max_size=2, so both roots should be present
+	ASSERT_EQ (2, fork_cache.size ());
+	ASSERT_TRUE (fork_cache.contains (root1));
+	ASSERT_TRUE (fork_cache.contains (root2));
+
+	auto blocks2 = fork_cache.get (root2);
+	ASSERT_EQ (2, blocks2.size ());
+
+	// Insert block for third root
+	fork_cache.put (block3);
+
+	// Should exceed max_size, oldest root (root1) should be dropped
+	ASSERT_EQ (2, fork_cache.size ());
+	ASSERT_FALSE (fork_cache.contains (root1)); // Oldest root dropped
+	ASSERT_TRUE (fork_cache.contains (root2));
+	ASSERT_TRUE (fork_cache.contains (root3));
+}
+
+/*
+ * Tests getting a non-existent root
+ */
+TEST (fork_cache, nonexistent)
+{
+	nano::test::system system;
+	nano::fork_cache_config cfg;
+	nano::fork_cache fork_cache{ cfg, system.stats };
+
+	nano::qualified_root nonexistent_root;
+	auto blocks = fork_cache.get (nonexistent_root);
+	ASSERT_TRUE (blocks.empty ());
+}

nano/core_test/node.cpp

@@ -3232,8 +3232,6 @@ TEST (node, dependency_graph_frontier)
 	ASSERT_TIMELY_EQ (15s, node2.ledger.cemented_count (), node2.ledger.block_count ());
 }
 
-namespace nano
-{
 TEST (node, deferred_dependent_elections)
 {
 	nano::test::system system;
@@ -3355,7 +3353,6 @@ TEST (node, deferred_dependent_elections)
 	ASSERT_TIMELY (5s, node.block_confirmed (send2->hash ()));
 	ASSERT_TIMELY (5s, node.active.active (receive->qualified_root ()));
 }
-}
 
 // Test that a node configured with `enable_pruning` and `max_pruning_age = 1s` will automatically
 // prune old confirmed blocks without explicitly saying `node.ledger_pruning` in the unit test
@@ -3668,3 +3665,143 @@ TEST (node, bounded_backlog)
 
 	ASSERT_TIMELY_EQ (20s, node.ledger.block_count (), 11); // 10 + genesis
 }
+
+// This test checks that a bootstrapping node can resolve a fork when a "poisoned" node
+// attempts to feed it the incorrect side of a fork.
+// The scenario involves:
+// 1. A bootstrapping node (node_boot) - the node being tested
+// 2. A poisoned node (node_poison) that has bootstrap serving enabled with an incorrect block
+// 3. A representative node (node_rep) with enough voting weight but bootstrap serving disabled
+// 4. A non-representative node (node_correct) that serves the correct side of the fork
+TEST (node, bootstrap_poison)
+{
+	nano::test::system system;
+
+	// Create the representative node with bootstrap serving disabled
+	nano::node_config rep_config = system.default_config ();
+	rep_config.bootstrap_server.enable = false; // Disable bootstrap serving
+	rep_config.bootstrap.enable = false; // Disable bootstrap from the network
+	// Disable schedulers
+	rep_config.priority_scheduler.enable = false;
+	rep_config.hinted_scheduler.enable = false;
+	rep_config.optimistic_scheduler.enable = false;
+	rep_config.backlog_scan.enable = false;
+	auto & node_rep = *system.add_node (rep_config);
+
+	// Create the poisoned node with bootstrap serving enabled
+	nano::node_config poison_config = system.default_config ();
+	poison_config.bootstrap_server.enable = true; // Enable bootstrap serving
+	poison_config.bootstrap.enable = false; // Disable bootstrap from the network
+	// Disable schedulers
+	poison_config.priority_scheduler.enable = false;
+	poison_config.hinted_scheduler.enable = false;
+	poison_config.optimistic_scheduler.enable = false;
+	poison_config.backlog_scan.enable = false;
+	auto & node_poison = *system.add_node (poison_config);
+
+	// Representative node needs to hold the genesis key to have voting weight
+	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
+
+	// Create keys for our test accounts
+	nano::keypair key1;
+
+	// Create and process blocks on representative node (the correct chain)
+	nano::block_builder builder;
+
+	// First send from genesis to key1
+	auto send1 = builder.send ()
+				 .previous (nano::dev::genesis->hash ())
+				 .destination (key1.pub)
+				 .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio)
+				 .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
+				 .work (*system.work.generate (nano::dev::genesis->hash ()))
+				 .build ();
+
+	ASSERT_EQ (nano::block_status::progress, node_rep.process (send1));
+	ASSERT_EQ (nano::block_status::progress, node_poison.process (send1));
+
+	// Valid open block for key1 (correct version)
+	auto open_correct = builder.open ()
+						.source (send1->hash ())
+						.representative (key1.pub)
+						.account (key1.pub)
+						.sign (key1.prv, key1.pub)
+						.work (*system.work.generate (key1.pub))
+						.build ();
+
+	// Fork of the open block (incorrect version) - using a different representative
+	nano::keypair bad_rep;
+	auto open_fork = builder.open ()
+					 .source (send1->hash ())
+					 .representative (bad_rep.pub) // Different representative
+					 .account (key1.pub)
+					 .sign (key1.prv, key1.pub)
+					 .work (*system.work.generate (key1.pub))
+					 .build ();
+
+	// Process the correct open block on the representative node
+	ASSERT_EQ (nano::block_status::progress, node_rep.process (open_correct));
+
+	// Process the forked open block on the poisoned node
+	ASSERT_EQ (nano::block_status::progress, node_poison.process (open_fork));
+
+	// Confirm the correct block on the representative node
+	nano::test::confirm (node_rep, { open_correct });
+
+	// Now create a bootstrapping node that will try to sync from both nodes
+	nano::node_config node_config = system.default_config ();
+	node_config.bootstrap.account_sets.cooldown = 100ms; // Short cooldown between requests to speed up the test
+	node_config.bootstrap.request_timeout = 250ms;
+	node_config.bootstrap.frontier_rate_limit = 100;
+	// Disable schedulers
+	node_config.priority_scheduler.enable = false;
+	node_config.hinted_scheduler.enable = false;
+	node_config.optimistic_scheduler.enable = false;
+	node_config.backlog_scan.enable = false;
+	nano::node_flags node_flags;
+	auto & node = *system.add_node (node_config, node_flags);
+	ASSERT_EQ (node.network.size (), 2);
+
+	std::cout << "Main node: " << node.identifier () << std::endl;
+	std::cout << "Waiting for: " << open_fork->hash ().to_string () << std::endl;
+
+	// The node should initially get the incorrect block from the poisoned node
+	ASSERT_TIMELY (15s, node.block (open_fork->hash ()) != nullptr);
+	ASSERT_NEVER (1s, node.stats.count (nano::stat::type::ledger, nano::stat::detail::fork) > 0);
+
+	// Create another non-rep node that will serve the correct side of the fork
+	nano::node_config correct_config = system.default_config ();
+	correct_config.bootstrap_server.enable = true; // Enable bootstrap serving
+	correct_config.bootstrap.enable = false; // Disable bootstrap from the network
+	correct_config.priority_scheduler.enable = false;
+	correct_config.hinted_scheduler.enable = false;
+	correct_config.optimistic_scheduler.enable = false;
+	auto & node_correct = *system.add_node (correct_config);
+	ASSERT_EQ (node.network.size (), 3);
+
+	// Process the correct open block on the non-representative node
+	ASSERT_EQ (nano::block_status::progress, node_correct.process (send1));
+	ASSERT_EQ (nano::block_status::progress, node_correct.process (open_correct));
+
+	// The node should at some point notice that there is a forked block
+	ASSERT_TIMELY (15s, node.stats.count (nano::stat::type::ledger, nano::stat::detail::fork) > 0);
+
+	// Should no longer be needed, fork should be cached
+	node_correct.stop ();
+
+	// We need an election active to force the correct side of the fork
+	ASSERT_TRUE (nano::test::start_election (system, node, open_fork->hash ()));
+
+	// Wait for the node to resolve the fork conflict
+	ASSERT_TIMELY (15s, node.block (open_correct->hash ()) != nullptr);
+
+	// Verify that the node got the correct block and not the fork
+	ASSERT_NE (nullptr, node.block (open_correct->hash ()));
+	ASSERT_EQ (nullptr, node.block (open_fork->hash ()));
+
+	// Verify the account information on the bootstrap node is correct
+	nano::account_info account_info;
+	ASSERT_FALSE (node.store.account.get (node.store.tx_begin_read (), key1.pub, account_info));
+	ASSERT_EQ (account_info.head, open_correct->hash ());
+	ASSERT_EQ (account_info.representative, key1.pub); // Correct representative
+}

nano/lib/blocks.hpp

@@ -353,7 +353,14 @@ class state_block : public nano::block
 {
 public:
 	state_block () = default;
-	state_block (nano::account const &, nano::block_hash const &, nano::account const &, nano::amount const &, nano::link const &, nano::raw_key const &, nano::public_key const &, uint64_t);
+	state_block (nano::account const & account,
+	nano::block_hash const & previous,
+	nano::account const & representative,
+	nano::amount const & balance,
+	nano::link const & link,
+	nano::raw_key const & prv,
+	nano::public_key const & pub,
+	uint64_t work);
 	state_block (bool &, nano::stream &);
 	state_block (bool &, boost::property_tree::ptree const &);
 	virtual ~state_block () = default;

nano/lib/stats_enums.hpp

@@ -121,6 +121,7 @@ enum class type
 	process_confirmed,
 	online_reps,
 	pruning,
+	fork_cache,
 
 	_last // Must be the last enum
 };
@@ -478,18 +479,17 @@ enum class detail
 	activate_full,
 	scanned,
 
-	// active
+	// active_elections
 	insert,
 	insert_failed,
 	transition_priority,
 	transition_priority_failed,
 	election_cleanup,
 	activate_immediately,
-
-	// active_elections
 	started,
 	stopped,
 	confirm_dependent,
+	forks_cached,
 
 	// unchecked
 	put,
@@ -677,6 +677,9 @@ enum class detail
 	pruned_count,
 	collect_targets,
 
+	// fork_cache
+	overfill_entry,
+
 	_last // Must be the last enum
 };
 

nano/node/CMakeLists.txt

@@ -71,6 +71,8 @@ add_library(
   epoch_upgrader.hpp
   epoch_upgrader.cpp
   fair_queue.hpp
+  fork_cache.hpp
+  fork_cache.cpp
   fwd.hpp
   ipc/action_handler.hpp
   ipc/action_handler.cpp

nano/node/active_elections.cpp

@@ -400,8 +400,8 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr<
 
 	auto const root = block_a->qualified_root ();
 	auto const hash = block_a->hash ();
-	auto const existing = roots.get<tag_root> ().find (root);
+
-	if (existing == roots.get<tag_root> ().end ())
+	if (auto existing = roots.get<tag_root> ().find (root); existing == roots.get<tag_root> ().end ())
 	{
 		if (!recently_confirmed.exists (root))
 		{
@@ -479,9 +479,21 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr<
 	{
 		debug_assert (result.election);
 
-		node.vote_cache_processor.trigger (hash);
+		// Notifications
 		node.observers.active_started.notify (hash);
 		vacancy_updated.notify ();
+
+		// Let the election know about already observed votes
+		node.vote_cache_processor.trigger (hash);
+
+		// Let the election know about already observed forks
+		auto forks = node.fork_cache.get (root);
+		node.stats.add (nano::stat::type::active_elections, nano::stat::detail::forks_cached, forks.size ());
+
+		for (auto const & fork : forks)
+		{
+			publish (fork);
+		}
 	}
 
 	// Votes are generated for inserted or ongoing elections

nano/node/bootstrap/bootstrap_server.cpp

@@ -37,6 +37,11 @@ void nano::bootstrap_server::start ()
 {
 	debug_assert (threads.empty ());
 
+	if (!config.enable)
+	{
+		return;
+	}
+
 	for (auto i = 0u; i < config.threads; ++i)
 	{
 		threads.push_back (std::thread ([this] () {
@@ -107,6 +112,11 @@ bool nano::bootstrap_server::verify (const nano::asc_pull_req & message) const
 
 bool nano::bootstrap_server::request (nano::asc_pull_req const & message, std::shared_ptr<nano::transport::channel> const & channel)
 {
+	if (!config.enable)
+	{
+		return false;
+	}
+
 	if (!verify (message))
 	{
 		stats.inc (nano::stat::type::bootstrap_server, nano::stat::detail::invalid);
@@ -439,6 +449,7 @@ nano::stat::detail nano::to_stat_detail (nano::asc_pull_type type)
 
 nano::error nano::bootstrap_server_config::serialize (nano::tomlconfig & toml) const
 {
+	toml.put ("enable", enable, "Enable bootstrap server. \ntype:bool");
 	toml.put ("max_queue", max_queue, "Maximum number of queued requests per peer. \ntype:uint64");
 	toml.put ("threads", threads, "Number of threads to process requests. \ntype:uint64");
 	toml.put ("batch_size", batch_size, "Maximum number of requests to process in a single batch. \ntype:uint64");
@@ -449,6 +460,7 @@ nano::error nano::bootstrap_server_config::serialize (nano::tomlconfig & toml) c
 
 nano::error nano::bootstrap_server_config::deserialize (nano::tomlconfig & toml)
 {
+	toml.get ("enable", enable);
 	toml.get ("max_queue", max_queue);
 	toml.get ("threads", threads);
 	toml.get ("batch_size", batch_size);

nano/node/bootstrap/bootstrap_server.hpp

@@ -22,6 +22,7 @@ public:
 	nano::error serialize (nano::tomlconfig &) const;
 
 public:
+	bool enable{ true };
 	size_t max_queue{ 16 };
 	size_t threads{ 1 };
 	size_t batch_size{ 64 };

nano/node/bootstrap/bootstrap_service.cpp

@@ -789,7 +789,7 @@ void nano::bootstrap_service::cleanup_and_sync ()
 	}
 
 	// Reinsert known dependencies into the priority set
-	if (sync_dependencies_interval.elapse (nano::is_dev_run () ? 1s : 60s))
+	if (sync_dependencies_interval.elapse (nano::is_dev_run () ? 500ms : 60s))
 	{
 		stats.inc (nano::stat::type::bootstrap, nano::stat::detail::sync_dependencies);
 		accounts.sync_dependencies ();
@@ -803,7 +803,7 @@ void nano::bootstrap_service::run_timeouts ()
 	{
 		stats.inc (nano::stat::type::bootstrap, nano::stat::detail::loop_cleanup);
 		cleanup_and_sync ();
-		condition.wait_for (lock, 5s, [this] () { return stopped; });
+		condition.wait_for (lock, nano::is_dev_run () ? 500ms : 5s, [this] () { return stopped; });
 	}
 }
 

nano/node/election.cpp

@@ -456,6 +456,17 @@ void nano::election::confirm_if_quorum (nano::unique_lock<nano::mutex> & lock_a)
 	{
 		status.winner = block_l;
 		remove_votes (status_winner_hash_l);
+
+		node.logger.debug (nano::log::type::election, "Winning fork changed from {} to {} for root: {} (behavior: {}, state: {}, voters: {}, blocks: {}, duration: {}ms)",
+		status_winner_hash_l,
+		winner_hash_l,
+		qualified_root,
+		to_string (behavior_m),
+		to_string (state_m),
+		status.voter_count,
+		status.block_count,
+		duration ().count ());
+
 		node.block_processor.force (block_l);
 	}
 	if (have_quorum (tally_l))

nano/node/fork_cache.cpp

@@ -0,0 +1,105 @@
+#include <nano/lib/stats.hpp>
+#include <nano/lib/tomlconfig.hpp>
+#include <nano/node/fork_cache.hpp>
+
+#include <boost/range/iterator_range.hpp>
+
+nano::fork_cache::fork_cache (nano::fork_cache_config const & config_a, nano::stats & stats_a) :
+	config{ config_a },
+	stats{ stats_a }
+{
+}
+
+void nano::fork_cache::put (std::shared_ptr<nano::block> block)
+{
+	release_assert (block != nullptr);
+
+	std::lock_guard guard{ mutex };
+
+	// Add the new block to the cache, duplicates are prevented by the multi_index container
+	auto [it, added] = roots.push_back ({ block->qualified_root () });
+	release_assert (it != roots.end ());
+	stats.inc (nano::stat::type::fork_cache, added ? nano::stat::detail::insert : nano::stat::detail::duplicate);
+
+	// Check if we already have this hash
+	bool exists = std::find_if (it->forks.begin (), it->forks.end (), [&block] (auto const & fork) {
+		return fork->hash () == block->hash ();
+	})
+	!= it->forks.end ();
+
+	if (exists)
+	{
+		return;
+	}
+
+	it->forks.push_back (block);
+
+	// Check if we have too many forks for this root
+	if (it->forks.size () > config.max_forks_per_root)
+	{
+		stats.inc (nano::stat::type::fork_cache, nano::stat::detail::overfill_entry);
+		it->forks.pop_front (); // Remove the oldest entry
+	}
+
+	// Check if we're at capacity
+	if (roots.size () > config.max_size)
+	{
+		// Remove oldest entry (first in sequence)
+		stats.inc (nano::stat::type::fork_cache, nano::stat::detail::overfill);
+		roots.pop_front (); // Remove the oldest entry
+	}
+}
+
+std::deque<std::shared_ptr<nano::block>> nano::fork_cache::get (nano::qualified_root const & root) const
+{
+	std::lock_guard guard{ mutex };
+
+	if (auto it = roots.get<tag_root> ().find (root); it != roots.get<tag_root> ().end ())
+	{
+		return it->forks;
+	}
+	return {};
+}
+
+size_t nano::fork_cache::size () const
+{
+	std::lock_guard guard{ mutex };
+
+	return roots.size ();
+}
+
+bool nano::fork_cache::contains (nano::qualified_root const & root) const
+{
+	std::lock_guard guard{ mutex };
+
+	return roots.get<tag_root> ().count (root) > 0;
+}
+
+nano::container_info nano::fork_cache::container_info () const
+{
+	std::lock_guard guard{ mutex };
+
+	nano::container_info result;
+	result.put ("roots", roots);
+	return result;
+}
+
+/*
+ * fork_cache_config
+ */
+
+nano::error nano::fork_cache_config::deserialize (nano::tomlconfig & toml)
+{
+	toml.get ("max_size", max_size);
+	toml.get ("max_forks_per_root", max_forks_per_root);
+
+	return toml.get_error ();
+}
+
+nano::error nano::fork_cache_config::serialize (nano::tomlconfig & toml) const
+{
+	toml.put ("max_size", max_size, "Maximum number of roots in the cache. Each root can have multiple forks. \ntype:uint64");
+	toml.put ("max_forks_per_root", max_forks_per_root, "Maximum number of forks per root. \ntype:uint64");
+
+	return toml.get_error ();
+}

nano/node/fork_cache.hpp

@@ -0,0 +1,71 @@
+#pragma once
+
+#include <nano/lib/blocks.hpp>
+#include <nano/lib/numbers.hpp>
+#include <nano/lib/numbers_templ.hpp>
+#include <nano/node/fwd.hpp>
+
+#include <boost/multi_index/hashed_index.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/sequenced_index.hpp>
+#include <boost/multi_index_container.hpp>
+
+#include <deque>
+#include <memory>
+#include <mutex>
+
+namespace mi = boost::multi_index;
+
+namespace nano
+{
+class fork_cache_config final
+{
+public:
+	nano::error deserialize (nano::tomlconfig &);
+	nano::error serialize (nano::tomlconfig &) const;
+
+public:
+	size_t max_size{ 1024 * 16 };
+	size_t max_forks_per_root{ 10 };
+};
+
+class fork_cache final
+{
+public:
+	fork_cache (fork_cache_config const &, nano::stats &);
+
+	void put (std::shared_ptr<nano::block> fork);
+	std::deque<std::shared_ptr<nano::block>> get (nano::qualified_root const & root) const;
+
+	size_t size () const;
+	bool contains (nano::qualified_root const & root) const;
+
+	nano::container_info container_info () const;
+
+private:
+	fork_cache_config const & config;
+	nano::stats & stats;
+
+	struct entry
+	{
+		nano::qualified_root root;
+		mutable std::deque<std::shared_ptr<nano::block>> forks;
+	};
+
+	// clang-format off
+	class tag_sequenced {};
+	class tag_root {};
+
+	using ordered_forks = boost::multi_index_container<entry,
+	mi::indexed_by<
+		mi::sequenced<mi::tag<tag_sequenced>>,
+		mi::hashed_unique<mi::tag<tag_root>,
+			mi::member<entry, nano::qualified_root, &entry::root>>
+	>>;
+	// clang-format on
+
+	ordered_forks roots;
+
+	mutable std::mutex mutex;
+};
+}

nano/node/fwd.hpp

@@ -19,6 +19,7 @@ class bootstrap_service;
 class confirming_set;
 class election;
 class election_status;
+class fork_cache;
 class ledger_notifications;
 class local_block_broadcaster;
 class local_vote_history;

nano/node/node.cpp

@@ -20,6 +20,7 @@
 #include <nano/node/daemonconfig.hpp>
 #include <nano/node/election_status.hpp>
 #include <nano/node/endpoint.hpp>
+#include <nano/node/fork_cache.hpp>
 #include <nano/node/ledger_notifications.hpp>
 #include <nano/node/local_block_broadcaster.hpp>
 #include <nano/node/local_vote_history.hpp>
@@ -147,6 +148,8 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
 	port_mapping{ *port_mapping_impl },
 	block_processor_impl{ std::make_unique<nano::block_processor> (config, ledger, ledger_notifications, unchecked, stats, logger) },
 	block_processor{ *block_processor_impl },
+	fork_cache_impl{ std::make_unique<nano::fork_cache> (config.fork_cache, stats) },
+	fork_cache{ *fork_cache_impl },
 	confirming_set_impl{ std::make_unique<nano::confirming_set> (config.confirming_set, ledger, ledger_notifications, stats, logger) },
 	confirming_set{ *confirming_set_impl },
 	bucketing_impl{ std::make_unique<nano::bucketing> () },
@@ -229,6 +232,17 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
 		active.recently_confirmed.erase (hash);
 	});
 
+	// Cache forks
+	ledger_notifications.blocks_processed.add ([this] (auto const & batch) {
+		for (auto const & [result, context] : batch)
+		{
+			if (result == nano::block_status::fork)
+			{
+				fork_cache.put (context.block);
+			}
+		}
+	});
+
 	// Announce new blocks via websocket
 	ledger_notifications.blocks_processed.add ([this] (auto const & batch) {
 		auto const transaction = ledger.tx_begin_read ();
@@ -998,6 +1012,7 @@ nano::container_info nano::node::container_info () const
 	info.add ("http_callbacks", http_callbacks.container_info ());
 	info.add ("pruning", pruning.container_info ());
 	info.add ("vote_rebroadcaster", vote_rebroadcaster.container_info ());
+	info.add ("fork_cache", fork_cache.container_info ());
 	return info;
 }
 

nano/node/node.hpp

@@ -147,6 +147,8 @@ public:
 	nano::port_mapping & port_mapping;
 	std::unique_ptr<nano::block_processor> block_processor_impl;
 	nano::block_processor & block_processor;
+	std::unique_ptr<nano::fork_cache> fork_cache_impl;
+	nano::fork_cache & fork_cache;
 	std::unique_ptr<nano::confirming_set> confirming_set_impl;
 	nano::confirming_set & confirming_set;
 	std::unique_ptr<nano::bucketing> bucketing_impl;

nano/node/nodeconfig.cpp

@@ -274,6 +274,10 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
 	bounded_backlog.serialize (bounded_backlog_l);
 	toml.put_child ("bounded_backlog", bounded_backlog_l);
 
+	nano::tomlconfig fork_cache_l;
+	fork_cache.serialize (fork_cache_l);
+	toml.put_child ("fork_cache", fork_cache_l);
+
 	return toml.get_error ();
 }
 
@@ -431,6 +435,12 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 			bounded_backlog.deserialize (config_l);
 		}
 
+		if (toml.has_key ("fork_cache"))
+		{
+			auto config_l = toml.get_required_child ("fork_cache");
+			fork_cache.deserialize (config_l);
+		}
+
 		/*
 		 * Values
 		 */

nano/node/nodeconfig.hpp

@@ -15,6 +15,7 @@
 #include <nano/node/bootstrap/bootstrap_server.hpp>
 #include <nano/node/bounded_backlog.hpp>
 #include <nano/node/confirming_set.hpp>
+#include <nano/node/fork_cache.hpp>
 #include <nano/node/ipc/ipc_config.hpp>
 #include <nano/node/local_block_broadcaster.hpp>
 #include <nano/node/message_processor.hpp>
@@ -164,6 +165,7 @@ public:
 	nano::backlog_scan_config backlog_scan;
 	nano::bounded_backlog_config bounded_backlog;
 	nano::vote_rebroadcaster_config vote_rebroadcaster;
+	nano::fork_cache_config fork_cache;
 
 public:
 	/** Entry is ignored if it cannot be parsed as a valid address:port */