Optimistic elections (#4111)

nano/core_test/CMakeLists.txt

@@ -33,6 +33,7 @@ add_executable(
   network.cpp
   network_filter.cpp
   node.cpp
+  optimistic_scheduler.cpp
   processing_queue.cpp
   processor_service.cpp
   peer_container.cpp

nano/core_test/active_transactions.cpp

@@ -428,6 +428,7 @@ TEST (active_transactions, inactive_votes_cache_election_start)
 	nano::test::system system;
 	nano::node_config node_config (nano::test::get_available_port (), system.logging);
 	node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
+	node_config.optimistic_scheduler.enabled = false;
 	auto & node = *system.add_node (node_config);
 	nano::block_hash latest (node.latest (nano::dev::genesis_key.pub));
 	nano::keypair key1, key2;
@@ -1418,6 +1419,7 @@ TEST (active_transactions, limit_vote_hinted_elections)
 	nano::node_config config = system.default_config ();
 	const int aec_limit = 10;
 	config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
+	config.optimistic_scheduler.enabled = false;
 	config.active_elections_size = aec_limit;
 	config.active_elections_hinted_limit_percentage = 10; // Should give us a limit of 1 hinted election
 	auto & node = *system.add_node (config);

nano/core_test/optimistic_scheduler.cpp

@@ -0,0 +1,105 @@
+#include <nano/node/election.hpp>
+#include <nano/test_common/chains.hpp>
+#include <nano/test_common/system.hpp>
+#include <nano/test_common/testutil.hpp>
+
+#include <gtest/gtest.h>
+
+#include <chrono>
+
+using namespace std::chrono_literals;
+
+/*
+ * Ensure account gets activated for a single unconfirmed account chain
+ */
+TEST (optimistic_scheduler, activate_one)
+{
+	nano::test::system system{};
+	auto & node = *system.add_node ();
+
+	// Needs to be greater than optimistic scheduler `gap_threshold`
+	const int howmany_blocks = 64;
+
+	auto chains = nano::test::setup_chains (system, node, /* single chain */ 1, howmany_blocks, nano::dev::genesis_key, /* do not confirm */ false);
+	auto & [account, blocks] = chains.front ();
+
+	// Confirm block towards at the beginning the chain, so gap between confirmation and account frontier is larger than `gap_threshold`
+	ASSERT_TRUE (nano::test::confirm (node, { blocks.at (11) }));
+	ASSERT_TIMELY (5s, nano::test::confirmed (node, { blocks.at (11) }));
+
+	// Ensure unconfirmed account head block gets activated
+	auto const & block = blocks.back ();
+	ASSERT_TIMELY (5s, node.active.active (block->hash ()));
+	ASSERT_TRUE (node.active.election (block->qualified_root ())->behavior () == nano::election_behavior::optimistic);
+}
+
+/*
+ * Ensure account gets activated for a single unconfirmed account chain with nothing yet confirmed
+ */
+TEST (optimistic_scheduler, activate_one_zero_conf)
+{
+	nano::test::system system{};
+	auto & node = *system.add_node ();
+
+	// Can be smaller than optimistic scheduler `gap_threshold`
+	// This is meant to activate short account chains (eg. binary tree spam leaf accounts)
+	const int howmany_blocks = 6;
+
+	auto chains = nano::test::setup_chains (system, node, /* single chain */ 1, howmany_blocks, nano::dev::genesis_key, /* do not confirm */ false);
+	auto & [account, blocks] = chains.front ();
+
+	// Ensure unconfirmed account head block gets activated
+	auto const & block = blocks.back ();
+	ASSERT_TIMELY (5s, node.active.active (block->hash ()));
+	ASSERT_TRUE (node.active.election (block->qualified_root ())->behavior () == nano::election_behavior::optimistic);
+}
+
+/*
+ * Ensure account gets activated for a multiple unconfirmed account chains
+ */
+TEST (optimistic_scheduler, activate_many)
+{
+	nano::test::system system{};
+	auto & node = *system.add_node ();
+
+	// Needs to be greater than optimistic scheduler `gap_threshold`
+	const int howmany_blocks = 64;
+	const int howmany_chains = 16;
+
+	auto chains = nano::test::setup_chains (system, node, howmany_chains, howmany_blocks, nano::dev::genesis_key, /* do not confirm */ false);
+
+	// Ensure all unconfirmed accounts head block gets activated
+	ASSERT_TIMELY (5s, std::all_of (chains.begin (), chains.end (), [&] (auto const & entry) {
+		auto const & [account, blocks] = entry;
+		auto const & block = blocks.back ();
+		return node.active.active (block->hash ()) && node.active.election (block->qualified_root ())->behavior () == nano::election_behavior::optimistic;
+	}));
+}
+
+/*
+ * Ensure accounts with some blocks already confirmed and with less than `gap_threshold` blocks do not get activated
+ */
+TEST (optimistic_scheduler, under_gap_threshold)
+{
+	nano::test::system system{};
+	nano::node_config config = system.default_config ();
+	config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
+	auto & node = *system.add_node (config);
+
+	// Must be smaller than optimistic scheduler `gap_threshold`
+	const int howmany_blocks = 64;
+
+	auto chains = nano::test::setup_chains (system, node, /* single chain */ 1, howmany_blocks, nano::dev::genesis_key, /* do not confirm */ false);
+	auto & [account, blocks] = chains.front ();
+
+	// Confirm block towards the end of the chain, so gap between confirmation and account frontier is less than `gap_threshold`
+	ASSERT_TRUE (nano::test::confirm (node, { blocks.at (55) }));
+	ASSERT_TIMELY (5s, nano::test::confirmed (node, { blocks.at (55) }));
+
+	// Manually trigger backlog scan
+	node.backlog.trigger ();
+
+	// Ensure unconfirmed account head block gets activated
+	auto const & block = blocks.back ();
+	ASSERT_NEVER (3s, node.active.active (block->hash ()));
+}

nano/core_test/toml.cpp

@@ -264,6 +264,10 @@ TEST (toml, daemon_config_deserialize_defaults)
 	ASSERT_EQ (conf.node.rocksdb_config.enable, defaults.node.rocksdb_config.enable);
 	ASSERT_EQ (conf.node.rocksdb_config.memory_multiplier, defaults.node.rocksdb_config.memory_multiplier);
 	ASSERT_EQ (conf.node.rocksdb_config.io_threads, defaults.node.rocksdb_config.io_threads);
+
+	ASSERT_EQ (conf.node.optimistic_scheduler.enabled, defaults.node.optimistic_scheduler.enabled);
+	ASSERT_EQ (conf.node.optimistic_scheduler.gap_threshold, defaults.node.optimistic_scheduler.gap_threshold);
+	ASSERT_EQ (conf.node.optimistic_scheduler.max_size, defaults.node.optimistic_scheduler.max_size);
 }
 
 TEST (toml, optional_child)
@@ -519,6 +523,11 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	max_databases = 999
 	map_size = 999
 
+	[node.optimistic_scheduler]
+	enabled = false
+	gap_threshold = 999
+	max_size = 999
+
 	[node.rocksdb]
 	enable = true
 	memory_multiplier = 3
@@ -682,6 +691,10 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	ASSERT_EQ (nano::rocksdb_config::using_rocksdb_in_tests (), defaults.node.rocksdb_config.enable);
 	ASSERT_NE (conf.node.rocksdb_config.memory_multiplier, defaults.node.rocksdb_config.memory_multiplier);
 	ASSERT_NE (conf.node.rocksdb_config.io_threads, defaults.node.rocksdb_config.io_threads);
+
+	ASSERT_NE (conf.node.optimistic_scheduler.enabled, defaults.node.optimistic_scheduler.enabled);
+	ASSERT_NE (conf.node.optimistic_scheduler.gap_threshold, defaults.node.optimistic_scheduler.gap_threshold);
+	ASSERT_NE (conf.node.optimistic_scheduler.max_size, defaults.node.optimistic_scheduler.max_size);
 }
 
 /** There should be no required values **/

nano/lib/config.hpp

@@ -251,6 +251,7 @@ public:
 			telemetry_cache_cutoff = 2000ms;
 			telemetry_request_interval = 500ms;
 			telemetry_broadcast_interval = 500ms;
+			optimistic_activation_delay = 2s;
 		}
 	}
 
@@ -302,6 +303,9 @@ public:
 	/** Telemetry data older than this value is considered stale */
 	std::chrono::milliseconds telemetry_cache_cutoff{ 1000 * 130 }; // 2 * `telemetry_broadcast_interval` + some margin
 
+	/** How much to delay activation of optimistic elections to avoid interfering with election scheduler */
+	std::chrono::seconds optimistic_activation_delay{ 30 };
+
 	/** Returns the network this object contains values for */
 	nano::networks network () const
 	{

nano/lib/stats_enums.hpp

@@ -42,6 +42,7 @@ enum class type : uint8_t
 	backlog,
 	unchecked,
 	election_scheduler,
+	optimistic_scheduler,
 
 	_last // Must be the last enum
 };
@@ -159,6 +160,7 @@ enum class detail : uint8_t
 	// election types
 	normal,
 	hinted,
+	optimistic,
 
 	// received messages
 	invalid_header,

nano/lib/threading.cpp

@@ -107,6 +107,8 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
 			break;
 		case nano::thread_role::name::telemetry:
 			thread_role_name_string = "Telemetry";
+		case nano::thread_role::name::optimistic_scheduler:
+			thread_role_name_string = "Optimistic";
 			break;
 		default:
 			debug_assert (false && "nano::thread_role::get_string unhandled thread role");

nano/lib/threading.hpp

@@ -48,6 +48,7 @@ namespace thread_role
 		vote_generator_queue,
 		bootstrap_server,
 		telemetry,
+		optimistic_scheduler,
 	};
 
 	/*

nano/lib/utility.hpp

@@ -203,4 +203,24 @@ constexpr TARGET_TYPE narrow_cast (SOURCE_TYPE const & val)
 
 // Issue #3748
 void sort_options_description (const boost::program_options::options_description & source, boost::program_options::options_description & target);
+
+using clock = std::chrono::steady_clock;
+
+/**
+ * Check whether time elapsed between `last` and `now` is greater than `duration`
+ */
+template <typename Duration>
+bool elapsed (nano::clock::time_point const & last, Duration duration, nano::clock::time_point const & now)
+{
+	return last + duration < now;
+}
+
+/**
+ * Check whether time elapsed since `last` is greater than `duration`
+ */
+template <typename Duration>
+bool elapsed (nano::clock::time_point const & last, Duration duration)
+{
+	return elapsed (last, duration, nano::clock::now ());
+}
 }

nano/node/CMakeLists.txt

@@ -143,6 +143,8 @@ add_library(
   openclconfig.cpp
   openclwork.hpp
   openclwork.cpp
+  optimistic_scheduler.hpp
+  optimistic_scheduler.cpp
   peer_exclusion.hpp
   peer_exclusion.cpp
   portmapping.hpp

nano/node/active_transactions.cpp

@@ -196,6 +196,11 @@ int64_t nano::active_transactions::limit (nano::election_behavior behavior) cons
 			const uint64_t limit = node.config.active_elections_hinted_limit_percentage * node.config.active_elections_size / 100;
 			return static_cast<int64_t> (limit);
 		}
+		case nano::election_behavior::optimistic:
+		{
+			const uint64_t limit = node.config.active_elections_optimistic_limit_percentage * node.config.active_elections_size / 100;
+			return static_cast<int64_t> (limit);
+		}
 	}
 
 	debug_assert (false, "unknown election behavior");
@@ -210,8 +215,8 @@ int64_t nano::active_transactions::vacancy (nano::election_behavior behavior) co
 		case nano::election_behavior::normal:
 			return limit () - static_cast<int64_t> (roots.size ());
 		case nano::election_behavior::hinted:
-			return limit (nano::election_behavior::hinted) - count_by_behavior[nano::election_behavior::hinted];
+		case nano::election_behavior::optimistic:
-			;
+			return limit (behavior) - count_by_behavior[behavior];
 	}
 	debug_assert (false); // Unknown enum
 	return 0;
@@ -261,6 +266,7 @@ void nano::active_transactions::request_confirm (nano::unique_lock<nano::mutex>
 
 void nano::active_transactions::cleanup_election (nano::unique_lock<nano::mutex> & lock_a, std::shared_ptr<nano::election> election)
 {
+	debug_assert (!mutex.try_lock ());
 	debug_assert (lock_a.owns_lock ());
 
 	node.stats.inc (completion_type (*election), nano::to_stat_detail (election->behavior ()));
@@ -368,6 +374,7 @@ nano::election_insertion_result nano::active_transactions::insert (const std::sh
 
 nano::election_insertion_result nano::active_transactions::insert_impl (nano::unique_lock<nano::mutex> & lock_a, std::shared_ptr<nano::block> const & block_a, nano::election_behavior election_behavior_a, std::function<void (std::shared_ptr<nano::block> const &)> const & confirmation_action_a)
 {
+	debug_assert (!mutex.try_lock ());
 	debug_assert (lock_a.owns_lock ());
 	debug_assert (block_a->has_sideband ());
 	nano::election_insertion_result result;
@@ -594,7 +601,7 @@ bool nano::active_transactions::publish (std::shared_ptr<nano::block> const & bl
 			{
 				cache->fill (election);
 			}
-			node.stats.inc (nano::stat::type::election, nano::stat::detail::election_block_conflict);
+			node.stats.inc (nano::stat::type::active, nano::stat::detail::election_block_conflict);
 		}
 	}
 	return result;
@@ -645,8 +652,6 @@ void nano::active_transactions::add_inactive_vote_cache (nano::block_hash const
 	if (node.ledger.weight (vote->account) > node.minimum_principal_weight ())
 	{
 		node.inactive_vote_cache.vote (hash, vote);
-
-		node.stats.inc (nano::stat::type::vote_cache, nano::stat::detail::vote_processed);
 	}
 }
 
@@ -676,6 +681,7 @@ std::unique_ptr<nano::container_info_component> nano::collect_container_info (ac
 	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "election_winner_details", active_transactions.election_winner_details_size (), sizeof (decltype (active_transactions.election_winner_details)::value_type) }));
 	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "normal", static_cast<std::size_t> (active_transactions.count_by_behavior[nano::election_behavior::normal]), 0 }));
 	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "hinted", static_cast<std::size_t> (active_transactions.count_by_behavior[nano::election_behavior::hinted]), 0 }));
+	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "optimistic", static_cast<std::size_t> (active_transactions.count_by_behavior[nano::election_behavior::optimistic]), 0 }));
 
 	composite->add_component (active_transactions.recently_confirmed.collect_container_info ("recently_confirmed"));
 	composite->add_component (active_transactions.recently_cemented.collect_container_info ("recently_cemented"));

nano/node/election.cpp

@@ -124,9 +124,23 @@ bool nano::election::state_change (nano::election::state_t expected_a, nano::ele
 	return result;
 }
 
+std::chrono::milliseconds nano::election::confirm_req_time () const
+{
+	switch (behavior ())
+	{
+		case election_behavior::normal:
+		case election_behavior::hinted:
+			return base_latency () * 5;
+		case election_behavior::optimistic:
+			return base_latency () * 2;
+	}
+	debug_assert (false);
+	return {};
+}
+
 void nano::election::send_confirm_req (nano::confirmation_solicitor & solicitor_a)
 {
-	if ((base_latency () * 5) < (std::chrono::steady_clock::now () - last_req))
+	if (confirm_req_time () < (std::chrono::steady_clock::now () - last_req))
 	{
 		nano::lock_guard<nano::mutex> guard{ mutex };
 		if (!solicitor_a.add (*this))
@@ -225,8 +239,10 @@ std::chrono::milliseconds nano::election::time_to_live () const
 		case election_behavior::normal:
 			return std::chrono::milliseconds (5 * 60 * 1000);
 		case election_behavior::hinted:
+		case election_behavior::optimistic:
 			return std::chrono::milliseconds (30 * 1000);
 	}
+	debug_assert (false);
 	return {};
 }
 
@@ -646,6 +662,10 @@ nano::stat::detail nano::to_stat_detail (nano::election_behavior behavior)
 		{
 			return nano::stat::detail::hinted;
 		}
+		case nano::election_behavior::optimistic:
+		{
+			return nano::stat::detail::optimistic;
+		}
 	}
 
 	debug_assert (false, "unknown election behavior");

nano/node/election.hpp

@@ -46,7 +46,19 @@ public:
 enum class election_behavior
 {
 	normal,
-	hinted
+	/**
+	 * Hinted elections:
+	 * - shorter timespan
+	 * - limited space inside AEC
+	 */
+	hinted,
+	/**
+	 * Optimistic elections:
+	 * - shorter timespan
+	 * - limited space inside AEC
+	 * - more frequent confirmation requests
+	 */
+	optimistic,
 };
 
 nano::stat::detail to_stat_detail (nano::election_behavior);
@@ -161,10 +173,14 @@ private:
 	void remove_block (nano::block_hash const &);
 	bool replace_by_weight (nano::unique_lock<nano::mutex> & lock_a, nano::block_hash const &);
 	std::chrono::milliseconds time_to_live () const;
-	/*
+	/**
 	 * Calculates minimum time delay between subsequent votes when processing non-final votes
 	 */
 	std::chrono::seconds cooldown_time (nano::uint128_t weight) const;
+	/**
+	 * Calculates time delay between broadcasting confirmation requests
+	 */
+	std::chrono::milliseconds confirm_req_time () const;
 
 private:
 	std::unordered_map<nano::block_hash, std::shared_ptr<nano::block>> last_blocks;

nano/node/election_scheduler.cpp

@@ -156,7 +156,6 @@ void nano::election_scheduler::run ()
 				auto block = priority.top ();
 				priority.pop ();
 				lock.unlock ();
-				std::shared_ptr<nano::election> election;
 				stats.inc (nano::stat::type::election_scheduler, nano::stat::detail::insert_priority);
 				auto result = node.active.insert (block);
 				if (result.inserted)

nano/node/hinted_scheduler.cpp

@@ -74,7 +74,7 @@ bool nano::hinted_scheduler::run_one (nano::uint128_t const & minimum_tally)
 				// We check for AEC vacancy inside our predicate
 				auto result = node.active.insert (block, nano::election_behavior::hinted);
 
-				stats.inc (nano::stat::type::hinting, result.inserted ? nano::stat::detail::hinted : nano::stat::detail::insert_failed);
+				stats.inc (nano::stat::type::hinting, result.inserted ? nano::stat::detail::insert : nano::stat::detail::insert_failed);
 
 				return result.inserted; // Return whether block was inserted
 			}

nano/node/node.cpp

@@ -195,6 +195,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co
 	generator{ config, ledger, wallets, vote_processor, history, network, stats, /* non-final */ false },
 	final_generator{ config, ledger, wallets, vote_processor, history, network, stats, /* final */ true },
 	active (*this, confirmation_height_processor),
+	optimistic{ config.optimistic_scheduler, *this, ledger, active, network_params.network, stats },
 	scheduler{ *this, stats },
 	hinting{ nano::nodeconfig_to_hinted_scheduler_config (config), *this, inactive_vote_cache, active, online_reps, stats },
 	aggregator (config, stats, generator, final_generator, history, ledger, wallets, active),
@@ -216,6 +217,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co
 
 	backlog.activate_callback.add ([this] (nano::transaction const & transaction, nano::account const & account, nano::account_info const & account_info, nano::confirmation_height_info const & conf_info) {
 		scheduler.activate (account, transaction);
+		optimistic.activate (account, account_info, conf_info);
 	});
 
 	if (!init_error ())
@@ -224,6 +226,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co
 		active.vacancy_update = [this] () {
 			scheduler.notify ();
 			hinting.notify ();
+			optimistic.notify ();
 		};
 
 		wallets.observer = [this] (bool active) {
@@ -684,6 +687,7 @@ void nano::node::start ()
 	active.start ();
 	generator.start ();
 	final_generator.start ();
+	optimistic.start ();
 	scheduler.start ();
 	backlog.start ();
 	hinting.start ();
@@ -711,6 +715,7 @@ void nano::node::stop ()
 	aggregator.stop ();
 	vote_processor.stop ();
 	scheduler.stop ();
+	optimistic.stop ();
 	hinting.stop ();
 	active.stop ();
 	generator.stop ();

nano/node/node.hpp

@@ -22,6 +22,7 @@
 #include <nano/node/node_observers.hpp>
 #include <nano/node/nodeconfig.hpp>
 #include <nano/node/online_reps.hpp>
+#include <nano/node/optimistic_scheduler.hpp>
 #include <nano/node/portmapping.hpp>
 #include <nano/node/repcrawler.hpp>
 #include <nano/node/request_aggregator.hpp>
@@ -181,6 +182,7 @@ public:
 	nano::vote_generator generator;
 	nano::vote_generator final_generator;
 	nano::active_transactions active;
+	nano::optimistic_scheduler optimistic;
 	nano::election_scheduler scheduler;
 	nano::hinted_scheduler hinting;
 	nano::request_aggregator aggregator;

nano/node/nodeconfig.cpp

@@ -194,6 +194,10 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
 	lmdb_config.serialize_toml (lmdb_l);
 	toml.put_child ("lmdb", lmdb_l);
 
+	nano::tomlconfig optimistic_l;
+	optimistic_scheduler.serialize (optimistic_l);
+	toml.put_child ("optimistic_scheduler", optimistic_l);
+
 	return toml.get_error ();
 }
 
@@ -245,6 +249,12 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 			rocksdb_config.deserialize_toml (rocksdb_config_l);
 		}
 
+		if (toml.has_key ("optimistic_scheduler"))
+		{
+			auto config_l = toml.get_required_child ("optimistic_scheduler");
+			optimistic_scheduler.deserialize (config_l);
+		}
+
 		if (toml.has_key ("work_peers"))
 		{
 			work_peers.clear ();

nano/node/nodeconfig.hpp

@@ -9,6 +9,7 @@
 #include <nano/lib/stats.hpp>
 #include <nano/node/ipc/ipc_config.hpp>
 #include <nano/node/logging.hpp>
+#include <nano/node/optimistic_scheduler.hpp>
 #include <nano/node/websocketconfig.hpp>
 #include <nano/secure/common.hpp>
 
@@ -44,6 +45,7 @@ public:
 	nano::account random_representative () const;
 	nano::network_params network_params;
 	std::optional<uint16_t> peering_port{};
+	nano::optimistic_scheduler_config optimistic_scheduler;
 	nano::logging logging;
 	std::vector<std::pair<std::string, uint16_t>> work_peers;
 	std::vector<std::pair<std::string, uint16_t>> secondary_work_peers{ { "127.0.0.1", 8076 } }; /* Default of nano-pow-server */
@@ -85,8 +87,12 @@ public:
 	/** Timeout for initiated async operations */
 	std::chrono::seconds tcp_io_timeout{ (network_params.network.is_dev_network () && !is_sanitizer_build ()) ? std::chrono::seconds (5) : std::chrono::seconds (15) };
 	std::chrono::nanoseconds pow_sleep_interval{ 0 };
+	// TODO: Move related settings to `active_transactions_config` class
 	std::size_t active_elections_size{ 5000 };
-	std::size_t active_elections_hinted_limit_percentage{ 20 }; // Limit of hinted elections as percentage of active_elections_size
+	/** Limit of hinted elections as percentage of `active_elections_size` */
+	std::size_t active_elections_hinted_limit_percentage{ 20 };
+	/** Limit of optimistic elections as percentage of `active_elections_size` */
+	std::size_t active_elections_optimistic_limit_percentage{ 10 };
 	/** Default maximum incoming TCP connections, including realtime network & bootstrap */
 	unsigned tcp_incoming_connections_max{ 2048 };
 	bool use_memory_pools{ true };

nano/node/optimistic_scheduler.cpp

@@ -0,0 +1,184 @@
+#include <nano/lib/stats.hpp>
+#include <nano/lib/tomlconfig.hpp>
+#include <nano/node/node.hpp>
+#include <nano/node/optimistic_scheduler.hpp>
+
+nano::optimistic_scheduler::optimistic_scheduler (optimistic_scheduler_config const & config_a, nano::node & node_a, nano::ledger & ledger_a, nano::active_transactions & active_a, nano::network_constants const & network_constants_a, nano::stats & stats_a) :
+	config{ config_a },
+	node{ node_a },
+	ledger{ ledger_a },
+	active{ active_a },
+	network_constants{ network_constants_a },
+	stats{ stats_a }
+{
+}
+
+nano::optimistic_scheduler::~optimistic_scheduler ()
+{
+	// Thread must be stopped before destruction
+	debug_assert (!thread.joinable ());
+}
+
+void nano::optimistic_scheduler::start ()
+{
+	if (!config.enabled)
+	{
+		return;
+	}
+
+	debug_assert (!thread.joinable ());
+
+	thread = std::thread{ [this] () {
+		nano::thread_role::set (nano::thread_role::name::optimistic_scheduler);
+		run ();
+	} };
+}
+
+void nano::optimistic_scheduler::stop ()
+{
+	{
+		nano::lock_guard<nano::mutex> guard{ mutex };
+		stopped = true;
+	}
+	notify ();
+	nano::join_or_pass (thread);
+}
+
+void nano::optimistic_scheduler::notify ()
+{
+	condition.notify_all ();
+}
+
+bool nano::optimistic_scheduler::activate_predicate (const nano::account_info & account_info, const nano::confirmation_height_info & conf_info) const
+{
+	// Chain with a big enough gap between account frontier and confirmation frontier
+	if (account_info.block_count - conf_info.height > config.gap_threshold)
+	{
+		return true;
+	}
+	// Account with nothing confirmed yet
+	if (conf_info.height == 0)
+	{
+		return true;
+	}
+	return false;
+}
+
+bool nano::optimistic_scheduler::activate (const nano::account & account, const nano::account_info & account_info, const nano::confirmation_height_info & conf_info)
+{
+	if (!config.enabled)
+	{
+		return false;
+	}
+
+	debug_assert (account_info.block_count >= conf_info.height);
+	if (activate_predicate (account_info, conf_info))
+	{
+		{
+			nano::lock_guard<nano::mutex> lock{ mutex };
+
+			// Prevent duplicate candidate accounts
+			if (candidates.get<tag_account> ().contains (account))
+			{
+				return false; // Not activated
+			}
+			// Limit candidates container size
+			if (candidates.size () >= config.max_size)
+			{
+				return false; // Not activated
+			}
+
+			stats.inc (nano::stat::type::optimistic_scheduler, nano::stat::detail::activated);
+			candidates.push_back ({ account, nano::clock::now () });
+		}
+		return true; // Activated
+	}
+	return false; // Not activated
+}
+
+bool nano::optimistic_scheduler::predicate () const
+{
+	debug_assert (!mutex.try_lock ());
+
+	if (active.vacancy (nano::election_behavior::optimistic) <= 0)
+	{
+		return false;
+	}
+	if (candidates.empty ())
+	{
+		return false;
+	}
+
+	auto candidate = candidates.front ();
+	bool result = nano::elapsed (candidate.timestamp, network_constants.optimistic_activation_delay);
+	return result;
+}
+
+void nano::optimistic_scheduler::run ()
+{
+	nano::unique_lock<nano::mutex> lock{ mutex };
+	while (!stopped)
+	{
+		stats.inc (nano::stat::type::optimistic_scheduler, nano::stat::detail::loop);
+
+		if (predicate ())
+		{
+			auto transaction = ledger.store.tx_begin_read ();
+
+			while (predicate ())
+			{
+				debug_assert (!candidates.empty ());
+				auto candidate = candidates.front ();
+				candidates.pop_front ();
+				lock.unlock ();
+
+				run_one (transaction, candidate);
+
+				lock.lock ();
+			}
+		}
+
+		condition.wait_for (lock, network_constants.optimistic_activation_delay / 2, [this] () {
+			return stopped || predicate ();
+		});
+	}
+}
+
+void nano::optimistic_scheduler::run_one (nano::transaction const & transaction, entry const & candidate)
+{
+	auto block = ledger.head_block (transaction, candidate.account);
+	if (block)
+	{
+		// Ensure block is not already confirmed
+		if (!node.block_confirmed_or_being_confirmed (block->hash ()))
+		{
+			// Try to insert it into AEC
+			// We check for AEC vacancy inside our predicate
+			auto result = node.active.insert (block, nano::election_behavior::optimistic);
+
+			stats.inc (nano::stat::type::optimistic_scheduler, result.inserted ? nano::stat::detail::insert : nano::stat::detail::insert_failed);
+		}
+	}
+}
+
+/*
+ * optimistic_scheduler_config
+ */
+
+nano::error nano::optimistic_scheduler_config::deserialize (nano::tomlconfig & toml)
+{
+	toml.get ("enabled", enabled);
+	toml.get ("gap_threshold", gap_threshold);
+	toml.get ("max_size", max_size);
+
+	return toml.get_error ();
+}
+
+nano::error nano::optimistic_scheduler_config::serialize (nano::tomlconfig & toml) const
+{
+	toml.put ("enable", enabled, "Enable or disable optimistic elections\ntype:bool");
+	toml.put ("gap_threshold", gap_threshold, "Minimum difference between confirmation frontier and account frontier to become a candidate for optimistic confirmation\ntype:uint64");
+	toml.put ("max_size", max_size, "Maximum number of candidates stored in memory\ntype:uint64");
+
+	return toml.get_error ();
+}

nano/node/optimistic_scheduler.hpp

@@ -0,0 +1,108 @@
+#pragma once
+
+#include <nano/lib/locks.hpp>
+#include <nano/lib/numbers.hpp>
+#include <nano/lib/timer.hpp>
+#include <nano/lib/utility.hpp>
+#include <nano/secure/common.hpp>
+
+#include <boost/multi_index/hashed_index.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/sequenced_index.hpp>
+#include <boost/multi_index_container.hpp>
+
+#include <condition_variable>
+#include <memory>
+#include <queue>
+#include <thread>
+#include <vector>
+
+namespace mi = boost::multi_index;
+
+namespace nano
+{
+class node;
+class ledger;
+class active_transactions;
+
+class optimistic_scheduler_config final
+{
+public:
+	nano::error deserialize (nano::tomlconfig & toml);
+	nano::error serialize (nano::tomlconfig & toml) const;
+
+public:
+	bool enabled{ true };
+
+	/** Minimum difference between confirmation frontier and account frontier to become a candidate for optimistic confirmation */
+	std::size_t gap_threshold{ 32 };
+
+	/** Maximum number of candidates stored in memory */
+	std::size_t max_size{ 1024 * 64 };
+};
+
+class optimistic_scheduler final
+{
+	struct entry;
+
+public:
+	optimistic_scheduler (optimistic_scheduler_config const &, nano::node &, nano::ledger &, nano::active_transactions &, nano::network_constants const & network_constants, nano::stats &);
+	~optimistic_scheduler ();
+
+	void start ();
+	void stop ();
+
+	/**
+	 * Called from backlog population to process accounts with unconfirmed blocks
+	 */
+	bool activate (nano::account const &, nano::account_info const &, nano::confirmation_height_info const &);
+
+	/**
+	 * Notify about changes in AEC vacancy
+	 */
+	void notify ();
+
+private:
+	bool activate_predicate (nano::account_info const &, nano::confirmation_height_info const &) const;
+
+	bool predicate () const;
+	void run ();
+	void run_one (nano::transaction const &, entry const & candidate);
+
+private: // Dependencies
+	optimistic_scheduler_config const & config;
+	nano::node & node;
+	nano::ledger & ledger;
+	nano::active_transactions & active;
+	nano::network_constants const & network_constants;
+	nano::stats & stats;
+
+private:
+	struct entry
+	{
+		nano::account account;
+		nano::clock::time_point timestamp;
+	};
+
+	// clang-format off
+	class tag_sequenced {};
+	class tag_account {};
+
+	using ordered_candidates = boost::multi_index_container<entry,
+	mi::indexed_by<
+		mi::sequenced<mi::tag<tag_sequenced>>,
+		mi::hashed_unique<mi::tag<tag_account>,
+			mi::member<entry, nano::account, &entry::account>>
+	>>;
+	// clang-format on
+
+	/** Accounts eligible for optimistic scheduling */
+	ordered_candidates candidates;
+
+	bool stopped{ false };
+	nano::condition_variable condition;
+	mutable nano::mutex mutex;
+	std::thread thread;
+};
+}

nano/test_common/chains.cpp

@@ -2,7 +2,7 @@
 
 using namespace std::chrono_literals;
 
-nano::block_list_t nano::test::setup_chain (nano::test::system & system, nano::node & node, int count, nano::keypair target)
+nano::block_list_t nano::test::setup_chain (nano::test::system & system, nano::node & node, int count, nano::keypair target, bool confirm)
 {
 	auto latest = node.latest (target.pub);
 	auto balance = node.balance (target.pub);
@@ -30,14 +30,17 @@ nano::block_list_t nano::test::setup_chain (nano::test::system & system, nano::n
 
 	EXPECT_TRUE (nano::test::process (node, blocks));
 
+	if (confirm)
+	{
 		// Confirm whole chain at once
 		EXPECT_TIMELY (5s, nano::test::confirm (node, { blocks.back () }));
 		EXPECT_TIMELY (5s, nano::test::confirmed (node, blocks));
+	}
 
 	return blocks;
 }
 
-std::vector<std::pair<nano::account, nano::block_list_t>> nano::test::setup_chains (nano::test::system & system, nano::node & node, int chain_count, int block_count, nano::keypair source)
+std::vector<std::pair<nano::account, nano::block_list_t>> nano::test::setup_chains (nano::test::system & system, nano::node & node, int chain_count, int block_count, nano::keypair source, bool confirm)
 {
 	auto latest = node.latest (source.pub);
 	auto balance = node.balance (source.pub);
@@ -69,12 +72,16 @@ std::vector<std::pair<nano::account, nano::block_list_t>> nano::test::setup_chai
 
 		latest = send->hash ();
 
-		// Ensure blocks are in the ledger and confirmed
 		EXPECT_TRUE (nano::test::process (node, { send, open }));
+
+		if (confirm)
+		{
+			// Ensure blocks are in the ledger and confirmed
 			EXPECT_TIMELY (5s, nano::test::confirm (node, { send, open }));
 			EXPECT_TIMELY (5s, nano::test::confirmed (node, { send, open }));
+		}
 
-		auto added_blocks = nano::test::setup_chain (system, node, block_count, key);
+		auto added_blocks = nano::test::setup_chain (system, node, block_count, key, confirm);
 
 		auto blocks = block_list_t{ open };
 		blocks.insert (blocks.end (), added_blocks.begin (), added_blocks.end ());

nano/test_common/chains.hpp

@@ -17,13 +17,13 @@ namespace nano::test
  * Creates `count` random 1 raw send blocks in a `target` account chain
  * @returns created blocks
  */
-nano::block_list_t setup_chain (nano::test::system & system, nano::node & node, int count, nano::keypair target = nano::dev::genesis_key);
+nano::block_list_t setup_chain (nano::test::system & system, nano::node & node, int count, nano::keypair target = nano::dev::genesis_key, bool confirm = true);
 
 /**
  * Creates `chain_count` account chains, each with `block_count` 1 raw random send blocks, all accounts are seeded from `source` account
  * @returns list of created accounts and their blocks
  */
-std::vector<std::pair<nano::account, nano::block_list_t>> setup_chains (nano::test::system & system, nano::node & node, int chain_count, int block_count, nano::keypair source = nano::dev::genesis_key);
+std::vector<std::pair<nano::account, nano::block_list_t>> setup_chains (nano::test::system & system, nano::node & node, int chain_count, int block_count, nano::keypair source = nano::dev::genesis_key, bool confirm = true);
 
 /**
  * Creates `count` 1 raw send blocks from `source` account, each to randomly created account