Merge pull request #4790 from pwojcikdev/backlog-scan-reworks

Backlog scan refactoring

nano/core_test/active_elections.cpp

@@ -170,7 +170,7 @@ TEST (active_elections, DISABLED_keep_local)
 	// Bound to 2, won't drop wallet created transactions, but good to test dropping remote
 	node_config.active_elections.size = 2;
 	// Disable frontier confirmation to allow the test to finish before
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 
 	auto & node = *system.add_node (node_config);
 	auto & wallet (*system.wallet (0));
@@ -330,7 +330,7 @@ TEST (inactive_votes_cache, existing_vote)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node = *system.add_node (node_config);
 	nano::block_hash latest (node.latest (nano::dev::genesis_key.pub));
 	nano::keypair key;
@@ -384,7 +384,7 @@ TEST (inactive_votes_cache, multiple_votes)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node = *system.add_node (node_config);
 	nano::keypair key1;
 	nano::block_builder builder;
@@ -437,7 +437,7 @@ TEST (inactive_votes_cache, election_start)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	node_config.priority_scheduler.enable = false;
 	node_config.optimistic_scheduler.enable = false;
 	auto & node = *system.add_node (node_config);
@@ -543,7 +543,7 @@ TEST (active_elections, vote_replays)
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
 	node_config.enable_voting = false;
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node = *system.add_node (node_config);
 	nano::keypair key;
 	nano::state_block_builder builder;
@@ -699,7 +699,7 @@ TEST (active_elections, republish_winner)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node1 = *system.add_node (node_config);
 	node_config.peering_port = system.get_available_port ();
 	auto & node2 = *system.add_node (node_config);
@@ -765,7 +765,7 @@ TEST (active_elections, fork_filter_cleanup)
 	nano::test::system system{};
 
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 
 	auto & node1 = *system.add_node (node_config);
 	nano::keypair key{};
@@ -846,7 +846,7 @@ TEST (active_elections, fork_replacement_tally)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node1 (*system.add_node (node_config));
 
 	size_t const reps_count = 20;
@@ -1003,7 +1003,7 @@ TEST (active_elections, confirmation_consistency)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node = *system.add_node (node_config);
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 	for (unsigned i = 0; i < 10; ++i)
@@ -1095,7 +1095,7 @@ TEST (active_elections, activate_account_chain)
 	nano::test::system system;
 	nano::node_flags flags;
 	nano::node_config config = system.default_config ();
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto & node = *system.add_node (config, flags);
 
 	nano::keypair key;
@@ -1187,7 +1187,7 @@ TEST (active_elections, activate_inactive)
 	nano::test::system system;
 	nano::node_flags flags;
 	nano::node_config config = system.default_config ();
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto & node = *system.add_node (config, flags);
 
 	nano::keypair key;
@@ -1337,7 +1337,7 @@ TEST (active_elections, limit_vote_hinted_elections)
 	nano::test::system system;
 	nano::node_config config = system.default_config ();
 	const int aec_limit = 10;
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	config.optimistic_scheduler.enable = false;
 	config.active_elections.size = aec_limit;
 	config.active_elections.hinted_limit_percentage = 10; // Should give us a limit of 1 hinted election

nano/core_test/backlog.cpp

@@ -22,10 +22,12 @@ TEST (backlog, population)
 	nano::test::system system{};
 	auto & node = *system.add_node ();
 
-	node.backlog.activate_callback.add ([&] (nano::secure::transaction const & transaction, nano::account const & account) {
+	node.backlog_scan.batch_activated.add ([&] (auto const & batch) {
 		nano::lock_guard<nano::mutex> lock{ mutex };
-
+		for (auto const & info : batch)
-		activated.insert (account);
+		{
+			activated.insert (info.account);
+		}
 	});
 
 	auto blocks = nano::test::setup_independent_blocks (system, node, 256);

nano/core_test/bootstrap.cpp

@@ -299,7 +299,7 @@ TEST (bootstrap, frontier_scan)
 	config.bootstrap.enable_scan = false;
 	config.bootstrap.enable_dependency_walker = false;
 	// Disable election activation
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	config.priority_scheduler.enable = false;
 	config.optimistic_scheduler.enable = false;
 	config.hinted_scheduler.enable = false;
@@ -395,7 +395,7 @@ TEST (bootstrap, frontier_scan_pending)
 	config.bootstrap.enable_scan = false;
 	config.bootstrap.enable_dependency_walker = false;
 	// Disable election activation
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	config.priority_scheduler.enable = false;
 	config.optimistic_scheduler.enable = false;
 	config.hinted_scheduler.enable = false;
@@ -477,7 +477,7 @@ TEST (bootstrap, frontier_scan_cannot_prioritize)
 	config.bootstrap.enable_scan = false;
 	config.bootstrap.enable_dependency_walker = false;
 	// Disable election activation
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	config.priority_scheduler.enable = false;
 	config.optimistic_scheduler.enable = false;
 	config.hinted_scheduler.enable = false;

nano/core_test/confirming_set.cpp

@@ -100,7 +100,7 @@ TEST (confirmation_callback, observer_callbacks)
 	nano::test::system system;
 	nano::node_flags node_flags;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = system.add_node (node_config, node_flags);
 
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
@@ -145,7 +145,7 @@ TEST (confirmation_callback, confirmed_history)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	node_config.bootstrap.enable = false;
 	auto node = system.add_node (node_config);
 
@@ -217,7 +217,7 @@ TEST (confirmation_callback, dependent_election)
 	nano::test::system system;
 	nano::node_flags node_flags;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = system.add_node (node_config, node_flags);
 
 	nano::block_hash latest (node->latest (nano::dev::genesis_key.pub));

nano/core_test/election.cpp

@@ -36,7 +36,7 @@ TEST (election, quorum_minimum_flip_success)
 
 	nano::node_config node_config = system.default_config ();
 	node_config.online_weight_minimum = nano::dev::constants.genesis_amount;
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 
 	auto & node1 = *system.add_node (node_config);
 	auto const latest_hash = nano::dev::genesis->hash ();
@@ -86,7 +86,7 @@ TEST (election, quorum_minimum_flip_fail)
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
 	node_config.online_weight_minimum = nano::dev::constants.genesis_amount;
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node = *system.add_node (node_config);
 	nano::state_block_builder builder;
 
@@ -137,7 +137,7 @@ TEST (election, quorum_minimum_confirm_success)
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
 	node_config.online_weight_minimum = nano::dev::constants.genesis_amount;
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node1 = *system.add_node (node_config);
 	nano::keypair key1;
 	nano::block_builder builder;
@@ -167,7 +167,7 @@ TEST (election, quorum_minimum_confirm_fail)
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
 	node_config.online_weight_minimum = nano::dev::constants.genesis_amount;
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node1 = *system.add_node (node_config);
 
 	nano::block_builder builder;
@@ -205,7 +205,7 @@ TEST (election, quorum_minimum_update_weight_before_quorum_checks)
 	nano::test::system system;
 
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 
 	auto & node1 = *system.add_node (node_config);
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);

nano/core_test/election_scheduler.cpp

@@ -178,7 +178,7 @@ TEST (election_scheduler, no_vacancy)
 
 	nano::node_config config = system.default_config ();
 	config.active_elections.size = 1;
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto & node = *system.add_node (config);
 
 	nano::state_block_builder builder{};

nano/core_test/ledger.cpp

@@ -1018,7 +1018,7 @@ TEST (votes, add_existing)
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
 	node_config.online_weight_minimum = nano::dev::constants.genesis_amount;
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node1 = *system.add_node (node_config);
 	nano::keypair key1;
 	nano::block_builder builder;
@@ -4267,7 +4267,7 @@ TEST (ledger, unchecked_epoch_invalid)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node1 (*system.add_node (node_config));
 	nano::keypair destination;
 	nano::block_builder builder;

nano/core_test/ledger_confirm.cpp

@@ -58,7 +58,7 @@ TEST (ledger_confirm, multiple_accounts)
 	nano::test::system system;
 	nano::node_flags node_flags;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = system.add_node (node_config, node_flags);
 	nano::keypair key1;
 	nano::keypair key2;
@@ -232,7 +232,7 @@ TEST (ledger_confirm, send_receive_between_2_accounts)
 	nano::test::system system;
 	nano::node_flags node_flags;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = system.add_node (node_config, node_flags);
 	nano::keypair key1;
 	nano::block_hash latest (node->latest (nano::dev::genesis_key.pub));
@@ -361,7 +361,7 @@ TEST (ledger_confirm, send_receive_self)
 	nano::test::system system;
 	nano::node_flags node_flags;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = system.add_node (node_config, node_flags);
 	nano::block_hash latest (node->latest (nano::dev::genesis_key.pub));
 
@@ -449,7 +449,7 @@ TEST (ledger_confirm, all_block_types)
 	nano::test::system system;
 	nano::node_flags node_flags;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = system.add_node (node_config, node_flags);
 	nano::block_hash latest (node->latest (nano::dev::genesis_key.pub));
 	nano::keypair key1;

nano/core_test/node.cpp

@@ -263,7 +263,7 @@ TEST (node, auto_bootstrap)
 {
 	nano::test::system system;
 	nano::node_config config (system.get_available_port ());
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	nano::node_flags node_flags;
 	node_flags.disable_bootstrap_bulk_push_client = true;
 	node_flags.disable_lazy_bootstrap = true;
@@ -291,7 +291,7 @@ TEST (node, auto_bootstrap_reverse)
 {
 	nano::test::system system;
 	nano::node_config config (system.get_available_port ());
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	nano::node_flags node_flags;
 	node_flags.disable_bootstrap_bulk_push_client = true;
 	node_flags.disable_lazy_bootstrap = true;
@@ -364,7 +364,7 @@ TEST (node, search_receivable_confirmed)
 {
 	nano::test::system system;
 	nano::node_config node_config (system.get_available_port ());
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = system.add_node (node_config);
 	nano::keypair key2;
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
@@ -393,7 +393,7 @@ TEST (node, search_receivable_pruned)
 {
 	nano::test::system system;
 	nano::node_config node_config (system.get_available_port ());
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node1 = system.add_node (node_config);
 	nano::node_flags node_flags;
 	node_flags.enable_pruning = true;
@@ -699,7 +699,7 @@ TEST (node, fork_multi_flip)
 	nano::test::system system;
 	nano::node_flags node_flags;
 	nano::node_config node_config (system.get_available_port ());
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node1 (*system.add_node (node_config, node_flags, type));
 	node_config.peering_port = system.get_available_port ();
 	node_config.bootstrap.account_sets.cooldown = 100ms; // Reduce cooldown to speed up fork resolution
@@ -752,7 +752,7 @@ TEST (node, fork_bootstrap_flip)
 {
 	nano::test::system system;
 	nano::node_config config1{ system.get_available_port () };
-	config1.backlog_population.enable = false;
+	config1.backlog_scan.enable = false;
 	nano::node_flags node_flags;
 	node_flags.disable_bootstrap_bulk_push_client = true;
 	node_flags.disable_lazy_bootstrap = true;
@@ -1159,7 +1159,7 @@ TEST (node, DISABLED_broadcast_elected)
 	nano::node_flags node_flags;
 	nano::test::system system;
 	nano::node_config node_config (system.get_available_port ());
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node0 = system.add_node (node_config, node_flags, type);
 	node_config.peering_port = system.get_available_port ();
 	auto node1 = system.add_node (node_config, node_flags, type);
@@ -1286,7 +1286,7 @@ TEST (node, rep_self_vote)
 	nano::test::system system;
 	nano::node_config node_config (system.get_available_port ());
 	node_config.online_weight_minimum = std::numeric_limits<nano::uint128_t>::max ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node0 = system.add_node (node_config);
 	nano::keypair rep_big;
 	nano::block_builder builder;
@@ -1689,7 +1689,7 @@ TEST (node, DISABLED_local_votes_cache)
 {
 	nano::test::system system;
 	nano::node_config node_config (system.get_available_port ());
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	node_config.receive_minimum = nano::dev::constants.genesis_amount;
 	auto & node (*system.add_node (node_config));
 	nano::state_block_builder builder;
@@ -1773,7 +1773,7 @@ TEST (node, DISABLED_local_votes_cache_batch)
 {
 	nano::test::system system;
 	nano::node_config node_config (system.get_available_port ());
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node (*system.add_node (node_config));
 	ASSERT_GE (node.network_params.voting.max_cache, 2);
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
@@ -1847,7 +1847,7 @@ TEST (node, DISABLED_local_votes_cache_generate_new_vote)
 {
 	nano::test::system system;
 	nano::node_config node_config (system.get_available_port ());
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node (*system.add_node (node_config));
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 
@@ -1901,7 +1901,7 @@ TEST (node, DISABLED_local_votes_cache_fork)
 	node_flags.disable_legacy_bootstrap = true;
 	node_flags.disable_wallet_bootstrap = true;
 	nano::node_config node_config (system.get_available_port ());
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node1 (*system.add_node (node_config, node_flags));
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 	auto send1 = nano::state_block_builder ()
@@ -2150,7 +2150,7 @@ TEST (node, epoch_conflict_confirm)
 {
 	nano::test::system system;
 	nano::node_config node_config (system.get_available_port ());
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node0 = *system.add_node (node_config);
 	node_config.peering_port = system.get_available_port ();
 	auto & node1 = *system.add_node (node_config);
@@ -2661,7 +2661,7 @@ TEST (node, bidirectional_tcp)
 	node_flags.disable_lazy_bootstrap = true;
 	node_flags.disable_wallet_bootstrap = true;
 	nano::node_config node_config (system.get_available_port ());
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node1 = system.add_node (node_config, node_flags);
 	node_config.peering_port = system.get_available_port ();
 	node_config.tcp_incoming_connections_max = 0; // Disable incoming TCP connections for node 2
@@ -2854,7 +2854,7 @@ TEST (node, rollback_gap_source)
 {
 	nano::test::system system;
 	nano::node_config node_config (system.get_available_port ());
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node = *system.add_node (node_config);
 	nano::state_block_builder builder;
 	nano::keypair key;
@@ -2922,7 +2922,7 @@ TEST (node, dependency_graph)
 {
 	nano::test::system system;
 	nano::node_config config (system.get_available_port ());
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto & node = *system.add_node (config);
 
 	nano::state_block_builder builder;
@@ -3120,10 +3120,10 @@ TEST (node, dependency_graph_frontier)
 {
 	nano::test::system system;
 	nano::node_config config (system.get_available_port ());
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto & node1 = *system.add_node (config);
 	config.peering_port = system.get_available_port ();
-	config.backlog_population.enable = true;
+	config.backlog_scan.enable = true;
 	auto & node2 = *system.add_node (config);
 
 	nano::state_block_builder builder;
@@ -3287,9 +3287,9 @@ TEST (node, deferred_dependent_elections)
 {
 	nano::test::system system;
 	nano::node_config node_config_1{ system.get_available_port () };
-	node_config_1.backlog_population.enable = false;
+	node_config_1.backlog_scan.enable = false;
 	nano::node_config node_config_2{ system.get_available_port () };
-	node_config_2.backlog_population.enable = false;
+	node_config_2.backlog_scan.enable = false;
 	nano::node_flags flags;
 	flags.disable_request_loop = true;
 	auto & node = *system.add_node (node_config_1, flags);

nano/core_test/numbers.cpp

@@ -649,3 +649,98 @@ TEST (uint512_union, hash)
 		}
 	}
 }
+
+TEST (sat_math, add_sat)
+{
+	// Test uint128_t
+	{
+		nano::uint128_t max = std::numeric_limits<nano::uint128_t>::max ();
+		nano::uint128_t one = 1;
+		nano::uint128_t large_val = max - 100;
+
+		// Normal addition
+		ASSERT_EQ (nano::add_sat (one, one), nano::uint128_t (2));
+
+		// Saturation at max
+		ASSERT_EQ (nano::add_sat (max, one), max);
+		ASSERT_EQ (nano::add_sat (large_val, nano::uint128_t (200)), max);
+		ASSERT_EQ (nano::add_sat (max, max), max);
+	}
+	// Test uint256_t
+	{
+		nano::uint256_t max = std::numeric_limits<nano::uint256_t>::max ();
+		nano::uint256_t one = 1;
+		nano::uint256_t large_val = max - 100;
+
+		// Normal addition
+		ASSERT_EQ (nano::add_sat (one, one), nano::uint256_t (2));
+
+		// Saturation at max
+		ASSERT_EQ (nano::add_sat (max, one), max);
+		ASSERT_EQ (nano::add_sat (large_val, nano::uint256_t (200)), max);
+		ASSERT_EQ (nano::add_sat (max, max), max);
+	}
+	// Test uint512_t
+	{
+		nano::uint512_t max = std::numeric_limits<nano::uint512_t>::max ();
+		nano::uint512_t one = 1;
+		nano::uint512_t large_val = max - 100;
+
+		// Normal addition
+		ASSERT_EQ (nano::add_sat (one, one), nano::uint512_t (2));
+
+		// Saturation at max
+		ASSERT_EQ (nano::add_sat (max, one), max);
+		ASSERT_EQ (nano::add_sat (large_val, nano::uint512_t (200)), max);
+		ASSERT_EQ (nano::add_sat (max, max), max);
+	}
+}
+
+TEST (sat_math, sub_sat)
+{
+	// Test uint128_t
+	{
+		nano::uint128_t max = std::numeric_limits<nano::uint128_t>::max ();
+		nano::uint128_t min = std::numeric_limits<nano::uint128_t>::min ();
+		nano::uint128_t one = 1;
+		nano::uint128_t hundred (100);
+
+		// Normal subtraction
+		ASSERT_EQ (nano::sub_sat (hundred, one), nano::uint128_t (99));
+
+		// Saturation at min
+		ASSERT_EQ (nano::sub_sat (min, one), min);
+		ASSERT_EQ (nano::sub_sat (hundred, nano::uint128_t (200)), min);
+		ASSERT_EQ (nano::sub_sat (min, max), min);
+	}
+	// Test uint256_t
+	{
+		nano::uint256_t max = std::numeric_limits<nano::uint256_t>::max ();
+		nano::uint256_t min = std::numeric_limits<nano::uint256_t>::min ();
+		nano::uint256_t one = 1;
+		nano::uint256_t hundred (100);
+
+		// Normal subtraction
+		ASSERT_EQ (nano::sub_sat (hundred, one), nano::uint256_t (99));
+
+		// Saturation at min
+		ASSERT_EQ (nano::sub_sat (min, one), min);
+		ASSERT_EQ (nano::sub_sat (hundred, nano::uint256_t (200)), min);
+		ASSERT_EQ (nano::sub_sat (min, max), min);
+	}
+	// Test uint512_t
+	{
+		nano::uint512_t max = std::numeric_limits<nano::uint512_t>::max ();
+		nano::uint512_t min = std::numeric_limits<nano::uint512_t>::min ();
+		nano::uint512_t one = 1;
+		nano::uint512_t hundred (100);
+
+		// Normal subtraction
+		ASSERT_EQ (nano::sub_sat (hundred, one), nano::uint512_t (99));
+
+		// Saturation at min
+		ASSERT_EQ (nano::sub_sat (min, one), min);
+		ASSERT_EQ (nano::sub_sat (hundred, nano::uint512_t (200)), min);
+		ASSERT_EQ (nano::sub_sat (min, max), min);
+	}
+}

nano/core_test/optimistic_scheduler.cpp

@@ -85,7 +85,7 @@ TEST (optimistic_scheduler, under_gap_threshold)
 {
 	nano::test::system system{};
 	nano::node_config config = system.default_config ();
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto & node = *system.add_node (config);
 
 	// Must be smaller than optimistic scheduler `gap_threshold`
@@ -98,7 +98,7 @@ TEST (optimistic_scheduler, under_gap_threshold)
 	nano::test::confirm (node.ledger, blocks.at (55));
 
 	// Manually trigger backlog scan
-	node.backlog.trigger ();
+	node.backlog_scan.trigger ();
 
 	// Ensure unconfirmed account head block gets activated
 	auto const & block = blocks.back ();

nano/core_test/request_aggregator.cpp

@@ -21,7 +21,7 @@ TEST (request_aggregator, one)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node (*system.add_node (node_config));
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 	nano::block_builder builder;
@@ -71,7 +71,7 @@ TEST (request_aggregator, one_update)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node (*system.add_node (node_config));
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 	nano::keypair key1;
@@ -137,7 +137,7 @@ TEST (request_aggregator, two)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node (*system.add_node (node_config));
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 	nano::keypair key1;
@@ -208,7 +208,7 @@ TEST (request_aggregator, two_endpoints)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	nano::node_flags node_flags;
 	node_flags.disable_rep_crawler = true;
 	auto & node1 (*system.add_node (node_config, node_flags));
@@ -266,7 +266,7 @@ TEST (request_aggregator, split)
 	size_t max_vbh = nano::network::confirm_ack_hashes_max;
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node (*system.add_node (node_config));
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 	std::vector<std::pair<nano::block_hash, nano::root>> request;
@@ -319,7 +319,7 @@ TEST (request_aggregator, channel_max_queue)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	node_config.request_aggregator.max_queue = 0;
 	auto & node (*system.add_node (node_config));
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
@@ -349,7 +349,7 @@ TEST (request_aggregator, DISABLED_unique)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node (*system.add_node (node_config));
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 	nano::block_builder builder;

nano/core_test/toml.cpp

@@ -116,7 +116,7 @@ TEST (toml, daemon_config_deserialize_defaults)
 	std::stringstream ss;
 	ss << R"toml(
 	[node]
-	[node.backlog_population]
+	[node.backlog_scan]
 	[node.bootstrap]
 	[node.bootstrap_server]
 	[node.block_processor]
@@ -197,9 +197,9 @@ TEST (toml, daemon_config_deserialize_defaults)
 	ASSERT_EQ (conf.node.max_queued_requests, defaults.node.max_queued_requests);
 	ASSERT_EQ (conf.node.request_aggregator_threads, defaults.node.request_aggregator_threads);
 	ASSERT_EQ (conf.node.max_unchecked_blocks, defaults.node.max_unchecked_blocks);
-	ASSERT_EQ (conf.node.backlog_population.enable, defaults.node.backlog_population.enable);
+	ASSERT_EQ (conf.node.backlog_scan.enable, defaults.node.backlog_scan.enable);
-	ASSERT_EQ (conf.node.backlog_population.batch_size, defaults.node.backlog_population.batch_size);
+	ASSERT_EQ (conf.node.backlog_scan.batch_size, defaults.node.backlog_scan.batch_size);
-	ASSERT_EQ (conf.node.backlog_population.frequency, defaults.node.backlog_population.frequency);
+	ASSERT_EQ (conf.node.backlog_scan.rate_limit, defaults.node.backlog_scan.rate_limit);
 	ASSERT_EQ (conf.node.enable_upnp, defaults.node.enable_upnp);
 
 	ASSERT_EQ (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled);
@@ -465,10 +465,10 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	frontiers_confirmation = "always"
 	enable_upnp = false
 
-	[node.backlog_population]
+	[node.backlog_scan]
 	enable = false
 	batch_size = 999
-	frequency = 999
+	rate_limit = 999
 
 	[node.block_processor]
 	max_peer_queue = 999
@@ -704,9 +704,9 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	ASSERT_NE (conf.node.work_threads, defaults.node.work_threads);
 	ASSERT_NE (conf.node.max_queued_requests, defaults.node.max_queued_requests);
 	ASSERT_NE (conf.node.request_aggregator_threads, defaults.node.request_aggregator_threads);
-	ASSERT_NE (conf.node.backlog_population.enable, defaults.node.backlog_population.enable);
+	ASSERT_NE (conf.node.backlog_scan.enable, defaults.node.backlog_scan.enable);
-	ASSERT_NE (conf.node.backlog_population.batch_size, defaults.node.backlog_population.batch_size);
+	ASSERT_NE (conf.node.backlog_scan.batch_size, defaults.node.backlog_scan.batch_size);
-	ASSERT_NE (conf.node.backlog_population.frequency, defaults.node.backlog_population.frequency);
+	ASSERT_NE (conf.node.backlog_scan.rate_limit, defaults.node.backlog_scan.rate_limit);
 	ASSERT_NE (conf.node.enable_upnp, defaults.node.enable_upnp);
 
 	ASSERT_NE (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled);

nano/core_test/vote_processor.cpp

@@ -21,7 +21,7 @@ TEST (vote_processor, codes)
 	nano::test::system system;
 	auto node_config = system.default_config ();
 	// Disable all election schedulers
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	node_config.hinted_scheduler.enable = false;
 	node_config.optimistic_scheduler.enable = false;
 	auto & node = *system.add_node (node_config);
@@ -159,10 +159,10 @@ TEST (vote_processor, no_broadcast_local)
 	flags.disable_request_loop = true;
 	nano::node_config config1, config2;
 	config1.representative_vote_weight_minimum = 0;
-	config1.backlog_population.enable = false;
+	config1.backlog_scan.enable = false;
 	auto & node (*system.add_node (config1, flags));
 	config2.representative_vote_weight_minimum = 0;
-	config2.backlog_population.enable = false;
+	config2.backlog_scan.enable = false;
 	config2.peering_port = system.get_available_port ();
 	system.add_node (config2, flags);
 	nano::block_builder builder;
@@ -214,10 +214,10 @@ TEST (vote_processor, local_broadcast_without_a_representative)
 	flags.disable_request_loop = true;
 	nano::node_config config1, config2;
 	config1.representative_vote_weight_minimum = 0;
-	config1.backlog_population.enable = false;
+	config1.backlog_scan.enable = false;
 	auto & node (*system.add_node (config1, flags));
 	config2.representative_vote_weight_minimum = 0;
-	config2.backlog_population.enable = false;
+	config2.backlog_scan.enable = false;
 	config2.peering_port = system.get_available_port ();
 	system.add_node (config2, flags);
 	nano::block_builder builder;
@@ -263,9 +263,9 @@ TEST (vote_processor, no_broadcast_local_with_a_principal_representative)
 	nano::node_flags flags;
 	flags.disable_request_loop = true;
 	nano::node_config config1, config2;
-	config1.backlog_population.enable = false;
+	config1.backlog_scan.enable = false;
 	auto & node (*system.add_node (config1, flags));
-	config2.backlog_population.enable = false;
+	config2.backlog_scan.enable = false;
 	config2.peering_port = system.get_available_port ();
 	system.add_node (config2, flags);
 	nano::block_builder builder;

nano/core_test/voting.cpp

@@ -146,7 +146,7 @@ TEST (vote_spacing, prune)
 TEST (vote_spacing, vote_generator)
 {
 	nano::node_config config;
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	config.active_elections.hinted_limit_percentage = 0; // Disable election hinting
 	nano::test::system system;
 	nano::node_flags node_flags;
@@ -190,7 +190,7 @@ TEST (vote_spacing, vote_generator)
 TEST (vote_spacing, rapid)
 {
 	nano::node_config config;
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	config.active_elections.hinted_limit_percentage = 0; // Disable election hinting
 	nano::test::system system;
 	nano::node_flags node_flags;

nano/core_test/wallet.cpp

@@ -1150,7 +1150,7 @@ TEST (wallet, search_receivable)
 	nano::test::system system;
 	nano::node_config config = system.default_config ();
 	config.enable_voting = false;
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	nano::node_flags flags;
 	flags.disable_search_pending = true;
 	auto & node (*system.add_node (config, flags));

nano/core_test/wallets.cpp

@@ -195,7 +195,7 @@ TEST (wallets, search_receivable)
 		nano::test::system system;
 		nano::node_config config = system.default_config ();
 		config.enable_voting = false;
-		config.backlog_population.enable = false;
+		config.backlog_scan.enable = false;
 		nano::node_flags flags;
 		flags.disable_search_pending = true;
 		auto & node (*system.add_node (config, flags));

nano/lib/numbers.hpp

@@ -528,6 +528,33 @@ namespace difficulty
 	uint64_t from_multiplier (double const, uint64_t const);
 	double to_multiplier (uint64_t const, uint64_t const);
 }
+
+/**
+ * Add to or substract from a value without overflow
+ * TODO: C++26 replace with std::add_sat and std::sub_sat
+ */
+template <typename T>
+T add_sat (T const & value, T const & diff) noexcept
+{
+	static_assert (std::numeric_limits<T>::is_specialized, "std::numeric_limits<T> must be specialized");
+	return (value > std::numeric_limits<T>::max () - diff) ? std::numeric_limits<T>::max () : value + diff;
+}
+template <typename T>
+T sub_sat (T const & value, T const & diff) noexcept
+{
+	static_assert (std::numeric_limits<T>::is_specialized, "std::numeric_limits<T> must be specialized");
+	return (value < std::numeric_limits<T>::min () + diff) ? std::numeric_limits<T>::min () : value - diff;
+}
+template <typename T>
+T inc_sat (T const & value) noexcept
+{
+	return add_sat (value, static_cast<T> (1));
+}
+template <typename T>
+T dec_sat (T const & value) noexcept
+{
+	return sub_sat (value, static_cast<T> (1));
+}
 }
 
 /*

nano/lib/stats_enums.hpp

@@ -84,6 +84,7 @@ enum class type
 	active_elections_timeout,
 	active_elections_cancelled,
 	active_elections_cemented,
+	backlog_scan,
 	backlog,
 	unchecked,
 	election_scheduler,
@@ -418,6 +419,7 @@ enum class detail
 	activate_failed,
 	activate_skip,
 	activate_full,
+	scanned,
 
 	// active
 	insert,

nano/lib/thread_roles.cpp

@@ -94,8 +94,8 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
 		case nano::thread_role::name::unchecked:
 			thread_role_name_string = "Unchecked";
 			break;
-		case nano::thread_role::name::backlog_population:
+		case nano::thread_role::name::backlog_scan:
-			thread_role_name_string = "Backlog";
+			thread_role_name_string = "Backlog scan";
 			break;
 		case nano::thread_role::name::vote_generator_queue:
 			thread_role_name_string = "Voting que";

nano/lib/thread_roles.hpp

@@ -36,7 +36,7 @@ enum class name
 	epoch_upgrader,
 	db_parallel_traversal,
 	unchecked,
-	backlog_population,
+	backlog_scan,
 	vote_generator_queue,
 	telemetry,
 	bootstrap,

nano/node/CMakeLists.txt

@@ -16,8 +16,8 @@ add_library(
   ${platform_sources}
   active_elections.hpp
   active_elections.cpp
-  backlog_population.hpp
+  backlog_scan.hpp
-  backlog_population.cpp
+  backlog_scan.cpp
   bandwidth_limiter.hpp
   bandwidth_limiter.cpp
   blockprocessor.hpp

nano/node/backlog_population.cpp

@@ -1,165 +0,0 @@
-#include <nano/lib/thread_roles.hpp>
-#include <nano/lib/threading.hpp>
-#include <nano/node/backlog_population.hpp>
-#include <nano/node/nodeconfig.hpp>
-#include <nano/node/scheduler/priority.hpp>
-#include <nano/secure/ledger.hpp>
-#include <nano/store/account.hpp>
-#include <nano/store/component.hpp>
-#include <nano/store/confirmation_height.hpp>
-
-nano::backlog_population::backlog_population (backlog_population_config const & config_a, nano::scheduler::component & schedulers, nano::ledger & ledger, nano::stats & stats_a) :
-	config{ config_a },
-	schedulers{ schedulers },
-	ledger{ ledger },
-	stats{ stats_a }
-{
-}
-
-nano::backlog_population::~backlog_population ()
-{
-	// Thread must be stopped before destruction
-	debug_assert (!thread.joinable ());
-}
-
-void nano::backlog_population::start ()
-{
-	debug_assert (!thread.joinable ());
-
-	thread = std::thread{ [this] () {
-		nano::thread_role::set (nano::thread_role::name::backlog_population);
-		run ();
-	} };
-}
-
-void nano::backlog_population::stop ()
-{
-	{
-		nano::lock_guard<nano::mutex> lock{ mutex };
-		stopped = true;
-	}
-	notify ();
-	nano::join_or_pass (thread);
-}
-
-void nano::backlog_population::trigger ()
-{
-	{
-		nano::unique_lock<nano::mutex> lock{ mutex };
-		triggered = true;
-	}
-	notify ();
-}
-
-void nano::backlog_population::notify ()
-{
-	condition.notify_all ();
-}
-
-bool nano::backlog_population::predicate () const
-{
-	return triggered || config.enable;
-}
-
-void nano::backlog_population::run ()
-{
-	nano::unique_lock<nano::mutex> lock{ mutex };
-	while (!stopped)
-	{
-		if (predicate ())
-		{
-			stats.inc (nano::stat::type::backlog, nano::stat::detail::loop);
-
-			triggered = false;
-			populate_backlog (lock);
-		}
-
-		condition.wait (lock, [this] () {
-			return stopped || predicate ();
-		});
-	}
-}
-
-void nano::backlog_population::populate_backlog (nano::unique_lock<nano::mutex> & lock)
-{
-	debug_assert (config.frequency > 0);
-
-	const auto chunk_size = config.batch_size / config.frequency;
-	auto done = false;
-	nano::account next = 0;
-	uint64_t total = 0;
-	while (!stopped && !done)
-	{
-		lock.unlock ();
-
-		{
-			auto transaction = ledger.tx_begin_read ();
-
-			auto it = ledger.store.account.begin (transaction, next);
-			auto const end = ledger.store.account.end (transaction);
-
-			auto should_refresh = [&transaction] () {
-				auto cutoff = std::chrono::steady_clock::now () - 100ms; // TODO: Make this configurable
-				return transaction.timestamp () < cutoff;
-			};
-
-			for (size_t count = 0; it != end && count < chunk_size && !should_refresh (); ++it, ++count, ++total)
-			{
-				stats.inc (nano::stat::type::backlog, nano::stat::detail::total);
-
-				auto const & account = it->first;
-				auto const & account_info = it->second;
-
-				activate (transaction, account, account_info);
-
-				next = account.number () + 1;
-			}
-
-			done = ledger.store.account.begin (transaction, next) == end;
-		}
-
-		lock.lock ();
-
-		// Give the rest of the node time to progress without holding database lock
-		condition.wait_for (lock, std::chrono::milliseconds{ 1000 / config.frequency });
-	}
-}
-
-void nano::backlog_population::activate (secure::transaction const & transaction, nano::account const & account, nano::account_info const & account_info)
-{
-	auto const maybe_conf_info = ledger.store.confirmation_height.get (transaction, account);
-	auto const conf_info = maybe_conf_info.value_or (nano::confirmation_height_info{});
-
-	// If conf info is empty then it means then it means nothing is confirmed yet
-	if (conf_info.height < account_info.block_count)
-	{
-		stats.inc (nano::stat::type::backlog, nano::stat::detail::activated);
-
-		activate_callback.notify (transaction, account);
-
-		schedulers.optimistic.activate (account, account_info, conf_info);
-		schedulers.priority.activate (transaction, account, account_info, conf_info);
-	}
-}
-
-/*
- * backlog_population_config
- */
-
-nano::error nano::backlog_population_config::serialize (nano::tomlconfig & toml) const
-{
-	toml.put ("enable", enable, "Control if ongoing backlog population is enabled. If not, backlog population can still be triggered by RPC \ntype:bool");
-	toml.put ("batch_size", batch_size, "Number of accounts per second to process when doing backlog population scan. Increasing this value will help unconfirmed frontiers get into election prioritization queue faster, however it will also increase resource usage. \ntype:uint");
-	toml.put ("frequency", frequency, "Backlog scan divides the scan into smaller batches, number of which is controlled by this value. Higher frequency helps to utilize resources more uniformly, however it also introduces more overhead. The resulting number of accounts per single batch is `backlog_scan_batch_size / backlog_scan_frequency` \ntype:uint");
-
-	return toml.get_error ();
-}
-
-nano::error nano::backlog_population_config::deserialize (nano::tomlconfig & toml)
-{
-	toml.get ("enable", enable);
-	toml.get ("batch_size", batch_size);
-	toml.get ("frequency", frequency);
-
-	return toml.get_error ();
-}

nano/node/backlog_scan.cpp

@@ -0,0 +1,175 @@
+#include <nano/lib/thread_roles.hpp>
+#include <nano/lib/threading.hpp>
+#include <nano/node/backlog_scan.hpp>
+#include <nano/node/nodeconfig.hpp>
+#include <nano/node/scheduler/priority.hpp>
+#include <nano/secure/ledger.hpp>
+#include <nano/store/account.hpp>
+#include <nano/store/component.hpp>
+#include <nano/store/confirmation_height.hpp>
+
+nano::backlog_scan::backlog_scan (backlog_scan_config const & config_a, nano::ledger & ledger_a, nano::stats & stats_a) :
+	config{ config_a },
+	ledger{ ledger_a },
+	stats{ stats_a },
+	limiter{ config.rate_limit }
+{
+}
+
+nano::backlog_scan::~backlog_scan ()
+{
+	// Thread must be stopped before destruction
+	debug_assert (!thread.joinable ());
+}
+
+void nano::backlog_scan::start ()
+{
+	debug_assert (!thread.joinable ());
+
+	thread = std::thread{ [this] () {
+		nano::thread_role::set (nano::thread_role::name::backlog_scan);
+		run ();
+	} };
+}
+
+void nano::backlog_scan::stop ()
+{
+	{
+		nano::lock_guard<nano::mutex> lock{ mutex };
+		stopped = true;
+	}
+	notify ();
+	nano::join_or_pass (thread);
+}
+
+void nano::backlog_scan::trigger ()
+{
+	{
+		nano::unique_lock<nano::mutex> lock{ mutex };
+		triggered = true;
+	}
+	notify ();
+}
+
+void nano::backlog_scan::notify ()
+{
+	condition.notify_all ();
+}
+
+bool nano::backlog_scan::predicate () const
+{
+	return triggered || config.enable;
+}
+
+void nano::backlog_scan::run ()
+{
+	nano::unique_lock<nano::mutex> lock{ mutex };
+	while (!stopped)
+	{
+		if (predicate ())
+		{
+			stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::loop);
+			triggered = false;
+			populate_backlog (lock); // Does a single iteration over all accounts
+			debug_assert (lock.owns_lock ());
+		}
+		else
+		{
+			condition.wait (lock, [this] () {
+				return stopped || predicate ();
+			});
+		}
+	}
+}
+
+void nano::backlog_scan::populate_backlog (nano::unique_lock<nano::mutex> & lock)
+{
+	uint64_t total = 0;
+
+	nano::account next = 0;
+	bool done = false;
+	while (!stopped && !done)
+	{
+		// Wait for the rate limiter
+		while (!limiter.should_pass (config.batch_size))
+		{
+			std::chrono::milliseconds const wait_time{ 1000 / std::max ((config.rate_limit / config.batch_size), size_t{ 1 }) / 2 };
+			condition.wait_for (lock, std::max (wait_time, 10ms));
+			if (stopped)
+			{
+				return;
+			}
+		}
+
+		lock.unlock ();
+
+		std::deque<activated_info> scanned;
+		std::deque<activated_info> activated;
+		{
+			auto transaction = ledger.tx_begin_read ();
+
+			auto it = ledger.store.account.begin (transaction, next);
+			auto const end = ledger.store.account.end (transaction);
+
+			for (size_t count = 0; it != end && count < config.batch_size; ++it, ++count, ++total)
+			{
+				stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::total);
+
+				auto const [account, account_info] = *it;
+				auto const maybe_conf_info = ledger.store.confirmation_height.get (transaction, account);
+				auto const conf_info = maybe_conf_info.value_or (nano::confirmation_height_info{});
+
+				activated_info info{ account, account_info, conf_info };
+
+				scanned.push_back (info);
+				if (conf_info.height < account_info.block_count)
+				{
+					activated.push_back (info);
+				}
+
+				next = inc_sat (account.number ());
+			}
+
+			done = (it == end);
+		}
+
+		stats.add (nano::stat::type::backlog_scan, nano::stat::detail::scanned, scanned.size ());
+		stats.add (nano::stat::type::backlog_scan, nano::stat::detail::activated, activated.size ());
+
+		// Notify about scanned and activated accounts without holding database transaction
+		batch_scanned.notify (scanned);
+		batch_activated.notify (activated);
+
+		lock.lock ();
+	}
+}
+
+nano::container_info nano::backlog_scan::container_info () const
+{
+	nano::lock_guard<nano::mutex> guard{ mutex };
+	nano::container_info info;
+	info.put ("limiter", limiter.size ());
+	return info;
+}
+
+/*
+ * backlog_scan_config
+ */
+
+nano::error nano::backlog_scan_config::serialize (nano::tomlconfig & toml) const
+{
+	toml.put ("enable", enable, "Control if ongoing backlog population is enabled. If not, backlog population can still be triggered by RPC \ntype:bool");
+	toml.put ("batch_size", batch_size, "Size of a single batch. Larger batches reduce overhead, but may put more pressure on other node components. \ntype:uint");
+	toml.put ("rate_limit", rate_limit, "Number of accounts per second to process when doing backlog population scan. Increasing this value will help unconfirmed frontiers get into election prioritization queue faster. Use 0 to process as fast as possible, but be aware that it may consume a lot of resources. \ntype:uint");
+
+	return toml.get_error ();
+}
+
+nano::error nano::backlog_scan_config::deserialize (nano::tomlconfig & toml)
+{
+	toml.get ("enable", enable);
+	toml.get ("batch_size", batch_size);
+	toml.get ("rate_limit", rate_limit);
+
+	return toml.get_error ();
+}

nano/node/backlog_scan.hpp

@@ -3,24 +3,18 @@
 #include <nano/lib/locks.hpp>
 #include <nano/lib/numbers.hpp>
 #include <nano/lib/observer_set.hpp>
-#include <nano/node/scheduler/component.hpp>
+#include <nano/lib/rate_limiting.hpp>
+#include <nano/node/fwd.hpp>
+#include <nano/secure/account_info.hpp>
 #include <nano/secure/common.hpp>
 
 #include <condition_variable>
+#include <deque>
 #include <thread>
 
-namespace nano::secure
-{
-class transaction;
-}
 namespace nano
 {
-class account_info;
+class backlog_scan_config final
-class election_scheduler;
-class ledger;
-class stats;
-
-class backlog_population_config final
 {
 public:
 	nano::error deserialize (nano::tomlconfig &);
@@ -29,17 +23,17 @@ public:
 public:
 	/** Control if ongoing backlog population is enabled. If not, backlog population can still be triggered by RPC */
 	bool enable{ true };
-	/** Number of accounts per second to process. Number of accounts per single batch is this value divided by `frequency` */
+	/** Number of accounts to scan per second. */
-	unsigned batch_size{ 10 * 1000 };
+	size_t rate_limit{ 10000 };
-	/** Number of batches to run per second. Batches run in 1 second / `frequency` intervals */
+	/** Number of accounts per second to process. */
-	unsigned frequency{ 10 };
+	size_t batch_size{ 1000 };
 };
 
-class backlog_population final
+class backlog_scan final
 {
 public:
-	backlog_population (backlog_population_config const &, nano::scheduler::component &, nano::ledger &, nano::stats &);
+	backlog_scan (backlog_scan_config const &, nano::ledger &, nano::stats &);
-	~backlog_population ();
+	~backlog_scan ();
 
 	void start ();
 	void stop ();
@@ -50,16 +44,22 @@ public:
 	/** Notify about AEC vacancy */
 	void notify ();
 
+	nano::container_info container_info () const;
+
 public:
-	/**
+	struct activated_info
-	 * Callback called for each backlogged account
+	{
-	 */
+		nano::account account;
-	using callback_t = nano::observer_set<secure::transaction const &, nano::account const &>;
+		nano::account_info account_info;
-	callback_t activate_callback;
+		nano::confirmation_height_info conf_info;
+	};
+
+	using batch_event_t = nano::observer_set<std::deque<activated_info>>;
+	batch_event_t batch_scanned; // Accounts scanned but not activated
+	batch_event_t batch_activated; // Accounts activated
 
 private: // Dependencies
-	backlog_population_config const & config;
+	backlog_scan_config const & config;
-	nano::scheduler::component & schedulers;
 	nano::ledger & ledger;
 	nano::stats & stats;
 
@@ -67,9 +67,10 @@ private:
 	void run ();
 	bool predicate () const;
 	void populate_backlog (nano::unique_lock<nano::mutex> & lock);
-	void activate (secure::transaction const &, nano::account const &, nano::account_info const &);
 
 private:
+	nano::rate_limiter limiter;
+
 	/** This is a manual trigger, the ongoing backlog population does not use this.
 	 *  It can be triggered even when backlog population (frontiers confirmation) is disabled. */
 	bool triggered{ false };

nano/node/bootstrap/crawlers.hpp

@@ -134,7 +134,7 @@ struct pending_database_crawler
 		if (it != end)
 		{
 			// If that fails, perform a fresh lookup
-			seek (starting_account.number () + 1);
+			seek (inc_sat (starting_account.number ()));
 		}
 
 		update_current ();

nano/node/bootstrap/database_scan.cpp

@@ -78,7 +78,7 @@ std::deque<nano::account> nano::bootstrap::account_database_scanner::next_batch
 	{
 		auto const & [account, info] = crawler.current.value ();
 		result.push_back (account);
-		next = account.number () + 1; // TODO: Handle account number overflow
+		next = inc_sat (account.number ());
 	}
 
 	// Empty current value indicates the end of the table
@@ -106,7 +106,7 @@ std::deque<nano::account> nano::bootstrap::pending_database_scanner::next_batch
 	{
 		auto const & [key, info] = crawler.current.value ();
 		result.push_back (key.account);
-		next = key.account.number () + 1; // TODO: Handle account number overflow
+		next = inc_sat (key.account.number ());
 	}
 
 	// Empty current value indicates the end of the table

nano/node/json_handler.cpp

@@ -2173,7 +2173,7 @@ void nano::json_handler::delegators ()
 	{
 		auto transaction (node.ledger.tx_begin_read ());
 		boost::property_tree::ptree delegators;
-		for (auto i (node.store.account.begin (transaction, start_account.number () + 1)), n (node.store.account.end (transaction)); i != n && delegators.size () < count; ++i)
+		for (auto i (node.store.account.begin (transaction, inc_sat (start_account.number ()))), n (node.store.account.end (transaction)); i != n && delegators.size () < count; ++i)
 		{
 			nano::account_info const & info (i->second);
 			if (info.representative == representative)
@@ -4189,7 +4189,7 @@ void nano::json_handler::unopened ()
 					break;
 				}
 				// Skip existing accounts
-				iterator = node.store.pending.begin (transaction, nano::pending_key (account.number () + 1, 0));
+				iterator = node.store.pending.begin (transaction, nano::pending_key (inc_sat (account.number ()), 0));
 			}
 			else
 			{
@@ -5154,7 +5154,7 @@ void nano::json_handler::work_peers_clear ()
 
 void nano::json_handler::populate_backlog ()
 {
-	node.backlog.trigger ();
+	node.backlog_scan.trigger ();
 	response_l.put ("success", "");
 	response_errors ();
 }

nano/node/node.cpp

@@ -8,7 +8,7 @@
 #include <nano/lib/utility.hpp>
 #include <nano/lib/work_version.hpp>
 #include <nano/node/active_elections.hpp>
-#include <nano/node/backlog_population.hpp>
+#include <nano/node/backlog_scan.hpp>
 #include <nano/node/bandwidth_limiter.hpp>
 #include <nano/node/bootstrap/bootstrap_server.hpp>
 #include <nano/node/bootstrap/bootstrap_service.hpp>
@@ -154,8 +154,8 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
 	aggregator_impl{ std::make_unique<nano::request_aggregator> (config.request_aggregator, *this, stats, generator, final_generator, history, ledger, wallets, vote_router) },
 	aggregator{ *aggregator_impl },
 	wallets (wallets_store.init_error (), *this),
-	backlog_impl{ std::make_unique<nano::backlog_population> (config.backlog_population, scheduler, ledger, stats) },
+	backlog_scan_impl{ std::make_unique<nano::backlog_scan> (config.backlog_scan, ledger, stats) },
-	backlog{ *backlog_impl },
+	backlog_scan{ *backlog_scan_impl },
 	bootstrap_server_impl{ std::make_unique<nano::bootstrap_server> (config.bootstrap_server, store, ledger, network_params.network, stats) },
 	bootstrap_server{ *bootstrap_server_impl },
 	bootstrap_impl{ std::make_unique<nano::bootstrap_service> (config, block_processor, ledger, network, stats, logger) },
@@ -180,6 +180,16 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
 		return ledger.weight (rep);
 	};
 
+	// TODO: Hook this direclty in the schedulers
+	backlog_scan.batch_activated.add ([this] (auto const & batch) {
+		auto transaction = ledger.tx_begin_read ();
+		for (auto const & info : batch)
+		{
+			scheduler.optimistic.activate (info.account, info.account_info, info.conf_info);
+			scheduler.priority.activate (transaction, info.account, info.account_info, info.conf_info);
+		}
+	});
+
 	// Republish vote if it is new and the node does not host a principal representative (or close to)
 	vote_router.vote_processed.add ([this] (std::shared_ptr<nano::vote> const & vote, nano::vote_source source, std::unordered_map<nano::block_hash, nano::vote_code> const & results) {
 		bool processed = std::any_of (results.begin (), results.end (), [] (auto const & result) {
@@ -636,7 +646,7 @@ void nano::node::start ()
 	confirming_set.start ();
 	scheduler.start ();
 	aggregator.start ();
-	backlog.start ();
+	backlog_scan.start ();
 	bootstrap_server.start ();
 	bootstrap.start ();
 	websocket.start ();
@@ -667,7 +677,7 @@ void nano::node::stop ()
 	// Cancels ongoing work generation tasks, which may be blocking other threads
 	// No tasks may wait for work generation in I/O threads, or termination signal capturing will be unable to call node::stop()
 	distributed_work.stop ();
-	backlog.stop ();
+	backlog_scan.stop ();
 	bootstrap.stop ();
 	rep_crawler.stop ();
 	unchecked.stop ();
@@ -849,7 +859,7 @@ bool nano::node::collect_ledger_pruning_targets (std::deque<nano::block_hash> &
 		read_operations += depth;
 		if (read_operations >= batch_read_size_a)
 		{
-			last_account_a = account.number () + 1;
+			last_account_a = inc_sat (account.number ());
 			finish_transaction = true;
 		}
 		else
@@ -1196,6 +1206,7 @@ nano::container_info nano::node::container_info () const
 	info.add ("rep_tiers", rep_tiers.container_info ());
 	info.add ("message_processor", message_processor.container_info ());
 	info.add ("bandwidth", outbound_limiter.container_info ());
+	info.add ("backlog_scan", backlog_scan.container_info ());
 	return info;
 }
 

nano/node/node.hpp

@@ -33,7 +33,7 @@
 namespace nano
 {
 class active_elections;
-class backlog_population;
+class backlog_scan;
 class bandwidth_limiter;
 class confirming_set;
 class message_processor;
@@ -205,8 +205,8 @@ public:
 	std::unique_ptr<nano::request_aggregator> aggregator_impl;
 	nano::request_aggregator & aggregator;
 	nano::wallets wallets;
-	std::unique_ptr<nano::backlog_population> backlog_impl;
+	std::unique_ptr<nano::backlog_scan> backlog_scan_impl;
-	nano::backlog_population & backlog;
+	nano::backlog_scan & backlog_scan;
 	std::unique_ptr<nano::bootstrap_server> bootstrap_server_impl;
 	nano::bootstrap_server & bootstrap_server;
 	std::unique_ptr<nano::bootstrap_service> bootstrap_impl;

nano/node/nodeconfig.cpp

@@ -258,9 +258,9 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
 	monitor.serialize (monitor_l);
 	toml.put_child ("monitor", monitor_l);
 
-	nano::tomlconfig backlog_population_l;
+	nano::tomlconfig backlog_scan_l;
-	backlog_population.serialize (backlog_population_l);
+	backlog_scan.serialize (backlog_scan_l);
-	toml.put_child ("backlog_population", backlog_population_l);
+	toml.put_child ("backlog_scan", backlog_scan_l);
 
 	return toml.get_error ();
 }
@@ -395,10 +395,10 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 			monitor.deserialize (config_l);
 		}
 
-		if (toml.has_key ("backlog_population"))
+		if (toml.has_key ("backlog_scan"))
 		{
-			auto config_l = toml.get_required_child ("backlog_population");
+			auto config_l = toml.get_required_child ("backlog_scan");
-			backlog_population.deserialize (config_l);
+			backlog_scan.deserialize (config_l);
 		}
 
 		/*

nano/node/nodeconfig.hpp

@@ -9,7 +9,7 @@
 #include <nano/lib/rocksdbconfig.hpp>
 #include <nano/lib/stats.hpp>
 #include <nano/node/active_elections.hpp>
-#include <nano/node/backlog_population.hpp>
+#include <nano/node/backlog_scan.hpp>
 #include <nano/node/blockprocessor.hpp>
 #include <nano/node/bootstrap/bootstrap_config.hpp>
 #include <nano/node/bootstrap/bootstrap_server.hpp>
@@ -146,7 +146,7 @@ public:
 	nano::local_block_broadcaster_config local_block_broadcaster;
 	nano::confirming_set_config confirming_set;
 	nano::monitor_config monitor;
-	nano::backlog_population_config backlog_population;
+	nano::backlog_scan_config backlog_scan;
 
 public:
 	/** Entry is ignored if it cannot be parsed as a valid address:port */

nano/node/scheduler/priority.cpp

@@ -135,9 +135,12 @@ bool nano::scheduler::priority::activate (secure::transaction const & transactio
 {
 	debug_assert (conf_info.frontier != account_info.head);
 
-	auto hash = conf_info.height == 0 ? account_info.open_block : ledger.any.block_successor (transaction, conf_info.frontier).value ();
+	auto const hash = conf_info.height == 0 ? account_info.open_block : ledger.any.block_successor (transaction, conf_info.frontier).value_or (0);
-	auto block = ledger.any.block_get (transaction, hash);
+	auto const block = ledger.any.block_get (transaction, hash);
-	release_assert (block != nullptr);
+	if (!block)
+	{
+		return false; // Not activated
+	}
 
 	if (ledger.dependents_confirmed (transaction, *block))
 	{

nano/rpc_test/receivable.cpp

@@ -133,7 +133,7 @@ TEST (rpc, receivable_unconfirmed)
 {
 	nano::test::system system;
 	nano::node_config config;
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto node = add_ipc_enabled_node (system, config);
 	auto chain = nano::test::setup_chain (system, *node, 1, nano::dev::genesis_key, false);
 	auto block1 = chain[0];
@@ -530,7 +530,7 @@ TEST (rpc, accounts_receivable_confirmed)
 {
 	nano::test::system system;
 	nano::node_config config;
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto node = add_ipc_enabled_node (system, config);
 	auto chain = nano::test::setup_chain (system, *node, 1, nano::dev::genesis_key, false);
 	auto block1 = chain[0];

nano/rpc_test/rpc.cpp

@@ -2924,7 +2924,7 @@ TEST (rpc, accounts_balances_unopened_account_with_receivables)
 {
 	nano::test::system system;
 	nano::node_config config;
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto node = add_ipc_enabled_node (system, config);
 
 	// send a 1 raw to the unopened account which will have receivables
@@ -3228,7 +3228,7 @@ TEST (rpc, pending_exists)
 {
 	nano::test::system system;
 	nano::node_config config;
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto node = add_ipc_enabled_node (system, config);
 	nano::keypair key1;
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
@@ -3287,7 +3287,7 @@ TEST (rpc, wallet_receivable)
 {
 	nano::test::system system;
 	nano::node_config config;
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto node = add_ipc_enabled_node (system, config);
 	nano::keypair key1;
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
@@ -4351,7 +4351,7 @@ TEST (rpc, populate_backlog)
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
 	// Disable automatic backlog population
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = add_ipc_enabled_node (system, node_config);
 
 	// Create and process a block that won't get automatically scheduled for confirmation

nano/slow_test/node.cpp

@@ -643,7 +643,7 @@ TEST (confirmation_height, many_accounts_single_confirmation)
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
 	node_config.online_weight_minimum = 100;
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = system.add_node (node_config);
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 
@@ -726,7 +726,7 @@ TEST (confirmation_height, many_accounts_many_confirmations)
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
 	node_config.online_weight_minimum = 100;
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = system.add_node (node_config);
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 
@@ -799,7 +799,7 @@ TEST (confirmation_height, long_chains)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = system.add_node (node_config);
 	nano::keypair key1;
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
@@ -944,7 +944,7 @@ TEST (confirmation_height, dynamic_algorithm)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto node = system.add_node (node_config);
 	nano::keypair key;
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
@@ -992,7 +992,7 @@ TEST (confirmation_height, many_accounts_send_receive_self)
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
 	node_config.online_weight_minimum = 100;
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	node_config.active_elections.size = 400000;
 	nano::node_flags node_flags;
 	auto node = system.add_node (node_config);
@@ -1411,7 +1411,7 @@ TEST (telemetry, under_load)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	nano::node_flags node_flags;
 	auto node = system.add_node (node_config, node_flags);
 	node_config.peering_port = system.get_available_port ();
@@ -1772,7 +1772,7 @@ TEST (node, mass_block_new)
 {
 	nano::test::system system;
 	nano::node_config node_config = system.default_config ();
-	node_config.backlog_population.enable = false;
+	node_config.backlog_scan.enable = false;
 	auto & node = *system.add_node (node_config);
 	node.network_params.network.aec_loop_interval_ms = 500;
 
@@ -2035,7 +2035,7 @@ TEST (node, wallet_create_block_confirm_conflicts)
 		nano::test::system system;
 		nano::block_builder builder;
 		nano::node_config node_config (system.get_available_port ());
-		node_config.backlog_population.enable = false;
+		node_config.backlog_scan.enable = false;
 		auto node = system.add_node (node_config);
 		auto const num_blocks = 10000;
 
@@ -2106,7 +2106,7 @@ TEST (system, block_sequence)
 	config.peering_port = system.get_available_port ();
 	// config.bandwidth_limit = 16 * 1024;
 	config.enable_voting = true;
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	nano::node_flags flags;
 	flags.disable_max_peers_per_ip = true;
 	flags.disable_ongoing_bootstrap = true;

nano/slow_test/vote_cache.cpp

@@ -134,7 +134,7 @@ TEST (vote_cache, perf_singlethreaded)
 	nano::test::system system;
 	nano::node_flags flags;
 	nano::node_config config = system.default_config ();
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto & node = *system.add_node (config, flags);
 
 	const int rep_count = 50;
@@ -193,7 +193,7 @@ TEST (vote_cache, perf_multithreaded)
 	nano::test::system system;
 	nano::node_flags flags;
 	nano::node_config config = system.default_config ();
-	config.backlog_population.enable = false;
+	config.backlog_scan.enable = false;
 	auto & node = *system.add_node (config, flags);
 
 	const int thread_count = 12;