Bootstrap stale elections

nano/core_test/active_elections.cpp

@@ -1482,3 +1482,40 @@ TEST (active_elections, broadcast_block_on_activation)
 	ASSERT_TIMELY (5s, node1->active.active (send1->qualified_root ()));
 	ASSERT_TIMELY (5s, node2->block_or_pruned_exists (send1->hash ()));
 }
+
+TEST (active_elections, bootstrap_stale)
+{
+	nano::test::system system;
+
+	// Configure node with short stale threshold for testing
+	nano::node_config node_config = system.default_config ();
+	node_config.active_elections.bootstrap_stale_threshold = 2s; // Short threshold for faster testing
+
+	auto & node = *system.add_node (node_config);
+
+	// Create a test block
+	nano::keypair key;
+	nano::state_block_builder builder;
+	auto send = builder.make_block ()
+				.account (nano::dev::genesis_key.pub)
+				.previous (nano::dev::genesis->hash ())
+				.representative (nano::dev::genesis_key.pub)
+				.balance (nano::dev::constants.genesis_amount - nano::Knano_ratio)
+				.link (key.pub)
+				.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
+				.work (*system.work.generate (nano::dev::genesis->hash ()))
+				.build ();
+
+	// Process the block and start an election
+	node.process_active (send);
+
+	// Ensure election starts
+	std::shared_ptr<nano::election> election;
+	ASSERT_TIMELY (5s, (election = node.active.election (send->qualified_root ())) != nullptr);
+
+	// Check initial state
+	ASSERT_EQ (0, node.stats.count (nano::stat::type::active_elections, nano::stat::detail::bootstrap_stale));
+
+	// Wait for bootstrap_stale_threshold to pass and the statistic to be incremented
+	ASSERT_TIMELY (5s, node.stats.count (nano::stat::type::active_elections, nano::stat::detail::bootstrap_stale) > 0);
+}

nano/lib/stats_enums.hpp

@@ -490,6 +490,7 @@ enum class detail
 	stopped,
 	confirm_dependent,
 	forks_cached,
+	bootstrap_stale,
 
 	// unchecked
 	put,

nano/node/active_elections.cpp

@@ -4,6 +4,7 @@
 #include <nano/lib/numbers.hpp>
 #include <nano/lib/threading.hpp>
 #include <nano/node/active_elections.hpp>
+#include <nano/node/bootstrap/bootstrap_service.hpp>
 #include <nano/node/confirmation_solicitor.hpp>
 #include <nano/node/confirming_set.hpp>
 #include <nano/node/election.hpp>
@@ -120,7 +121,8 @@ void nano::active_elections::run ()
 		node.stats.inc (nano::stat::type::active, nano::stat::detail::loop);
 
 		tick_elections (lock);
-		debug_assert (lock.owns_lock ());
+		debug_assert (!lock.owns_lock ());
+		lock.lock ();
 
 		auto const min_sleep = node.network_params.network.aec_loop_interval / 2;
 		auto const wakeup = std::max (stamp + node.network_params.network.aec_loop_interval, std::chrono::steady_clock::now () + min_sleep);
@@ -270,36 +272,39 @@ void nano::active_elections::tick_elections (nano::unique_lock<nano::mutex> & lo
 {
 	debug_assert (lock.owns_lock ());
 
-	auto const elections_l = list_active_impl ();
+	auto const election_list = list_active_impl ();
 
 	lock.unlock ();
 
 	nano::confirmation_solicitor solicitor (node.network, node.config);
 	solicitor.prepare (node.rep_crawler.principal_representatives (std::numeric_limits<std::size_t>::max ()));
 
-	std::size_t unconfirmed_count_l (0);
 	nano::timer<std::chrono::milliseconds> elapsed (nano::timer_state::started);
 
-	/*
-	 * Loop through active elections in descending order of proof-of-work difficulty, requesting confirmation
-	 *
-	 * Only up to a certain amount of elections are queued for confirmation request and block rebroadcasting. The remaining elections can still be confirmed if votes arrive
-	 * Elections extending the soft config.size limit are flushed after a certain time-to-live cutoff
-	 * Flushed elections are later re-activated via frontier confirmation
-	 */
-	for (auto const & election_l : elections_l)
+	std::deque<std::shared_ptr<nano::election>> stale_elections;
+	for (auto const & election : election_list)
 	{
-		bool const confirmed_l (election_l->confirmed ());
-		unconfirmed_count_l += !confirmed_l;
-
-		if (election_l->transition_time (solicitor))
+		if (election->transition_time (solicitor))
 		{
-			erase (election_l->qualified_root);
+			erase (election->qualified_root);
+		}
+		else if (election->duration () > config.bootstrap_stale_threshold)
+		{
+			stale_elections.push_back (election);
 		}
 	}
 
 	solicitor.flush ();
-	lock.lock ();
+
+	if (bootstrap_stale_interval.elapse (config.bootstrap_stale_threshold / 2))
+	{
+		node.stats.add (nano::stat::type::active_elections, nano::stat::detail::bootstrap_stale, stale_elections.size ());
+
+		for (auto const & election : stale_elections)
+		{
+			node.bootstrap.prioritize (election->account ());
+		}
+	}
 }
 
 void nano::active_elections::cleanup_election (nano::unique_lock<nano::mutex> & lock_a, std::shared_ptr<nano::election> election)
@@ -637,7 +642,8 @@ nano::error nano::active_elections_config::serialize (nano::tomlconfig & toml) c
 	toml.put ("optimistic_limit_percentage", optimistic_limit_percentage, "Limit of optimistic elections as percentage of `active_elections_size`. \ntype:uint64");
 	toml.put ("confirmation_history_size", confirmation_history_size, "Maximum confirmation history size. If tracking the rate of block confirmations, the websocket feature is recommended instead. \ntype:uint64");
 	toml.put ("confirmation_cache", confirmation_cache, "Maximum number of confirmed elections kept in cache to prevent restarting an election. \ntype:uint64");
-
+	toml.put ("max_election_winners", max_election_winners, "Maximum size of election winner details set. \ntype:uint64");
+	toml.put ("bootstrap_stale_threshold", bootstrap_stale_threshold.count (), "Time after which additional bootstrap attempts are made to find missing blocks for an election. \ntype:seconds");
 	return toml.get_error ();
 }
 
@@ -648,6 +654,8 @@ nano::error nano::active_elections_config::deserialize (nano::tomlconfig & toml)
 	toml.get ("optimistic_limit_percentage", optimistic_limit_percentage);
 	toml.get ("confirmation_history_size", confirmation_history_size);
 	toml.get ("confirmation_cache", confirmation_cache);
+	toml.get ("max_election_winners", max_election_winners);
+	toml.get_duration ("bootstrap_stale_threshold", bootstrap_stale_threshold);
 
 	return toml.get_error ();
 }

nano/node/active_elections.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <nano/lib/enum_util.hpp>
+#include <nano/lib/interval.hpp>
 #include <nano/lib/numbers.hpp>
 #include <nano/lib/observer_set.hpp>
 #include <nano/node/election_behavior.hpp>
@@ -50,6 +51,8 @@ public:
 	std::size_t confirmation_cache{ 65536 };
 	// Maximum size of election winner details set
 	std::size_t max_election_winners{ 1024 * 16 };
+
+	std::chrono::seconds bootstrap_stale_threshold{ 60s };
 };
 
 /**
@@ -164,6 +167,8 @@ private:
 	bool stopped{ false };
 	std::thread thread;
 
+	nano::interval bootstrap_stale_interval;
+
 	friend class election;
 
 public: // Tests

nano/node/bootstrap/bootstrap_service.cpp

@@ -163,6 +163,12 @@ void nano::bootstrap_service::reset ()
 	throttle.reset ();
 }
 
+void nano::bootstrap_service::prioritize (nano::account const & account)
+{
+	nano::lock_guard<nano::mutex> lock{ mutex };
+	accounts.priority_set (account);
+}
+
 bool nano::bootstrap_service::send (std::shared_ptr<nano::transport::channel> const & channel, async_tag tag)
 {
 	debug_assert (tag.type != query_type::invalid);

nano/node/bootstrap/bootstrap_service.hpp

@@ -46,6 +46,11 @@ public:
 	 */
 	void reset ();
 
+	/**
+	 * Adds an account to the priority set
+	 */
+	void prioritize (nano::account const & account);
+
 	std::size_t blocked_size () const;
 	std::size_t priority_size () const;
 	std::size_t score_size () const;

nano/node/election.cpp

@@ -805,6 +805,12 @@ void nano::election::force_confirm ()
 	confirm_once (lock);
 }
 
+nano::account nano::election::account () const
+{
+	nano::lock_guard<nano::mutex> guard{ mutex };
+	return status.winner->account ();
+}
+
 std::unordered_map<nano::block_hash, std::shared_ptr<nano::block>> nano::election::blocks () const
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };

nano/node/election.hpp

@@ -142,6 +142,7 @@ public: // Information
 	nano::root const root;
 	nano::qualified_root const qualified_root;
 
+	nano::account account () const;
 	std::vector<nano::vote_with_weight_info> votes_with_weight () const;
 	nano::election_behavior behavior () const;
 	nano::election_state state () const;