diff --git a/nano/core_test/active_elections.cpp b/nano/core_test/active_elections.cpp index 32d2a8964..52f409ccc 100644 --- a/nano/core_test/active_elections.cpp +++ b/nano/core_test/active_elections.cpp @@ -170,7 +170,7 @@ TEST (active_elections, DISABLED_keep_local) // Bound to 2, won't drop wallet created transactions, but good to test dropping remote node_config.active_elections.size = 2; // Disable frontier confirmation to allow the test to finish before - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); auto & wallet (*system.wallet (0)); @@ -330,7 +330,7 @@ TEST (inactive_votes_cache, existing_vote) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); nano::block_hash latest (node.latest (nano::dev::genesis_key.pub)); nano::keypair key; @@ -384,7 +384,7 @@ TEST (inactive_votes_cache, multiple_votes) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); nano::keypair key1; nano::block_builder builder; @@ -437,7 +437,7 @@ TEST (inactive_votes_cache, election_start) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.priority_scheduler.enable = false; node_config.optimistic_scheduler.enable = false; auto & node = *system.add_node (node_config); @@ -543,7 +543,7 @@ TEST (active_elections, vote_replays) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.enable_voting = false; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); nano::keypair key; nano::state_block_builder builder; @@ -699,7 +699,7 @@ TEST (active_elections, republish_winner) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); node_config.peering_port = system.get_available_port (); auto & node2 = *system.add_node (node_config); @@ -765,7 +765,7 @@ TEST (active_elections, fork_filter_cleanup) nano::test::system system{}; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); nano::keypair key{}; @@ -846,7 +846,7 @@ TEST (active_elections, fork_replacement_tally) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 (*system.add_node (node_config)); size_t const reps_count = 20; @@ -1003,7 +1003,7 @@ TEST (active_elections, confirmation_consistency) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); for (unsigned i = 0; i < 10; ++i) @@ -1095,7 +1095,7 @@ TEST (active_elections, activate_account_chain) nano::test::system system; nano::node_flags flags; nano::node_config config = system.default_config (); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config, flags); nano::keypair key; @@ -1187,7 +1187,7 @@ TEST (active_elections, activate_inactive) nano::test::system system; nano::node_flags flags; nano::node_config config = system.default_config (); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config, flags); nano::keypair key; @@ -1337,7 +1337,7 @@ TEST (active_elections, limit_vote_hinted_elections) nano::test::system system; nano::node_config config = system.default_config (); const int aec_limit = 10; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.optimistic_scheduler.enable = false; config.active_elections.size = aec_limit; config.active_elections.hinted_limit_percentage = 10; // Should give us a limit of 1 hinted election diff --git a/nano/core_test/backlog.cpp b/nano/core_test/backlog.cpp index 21a2e80eb..7e1cf5da1 100644 --- a/nano/core_test/backlog.cpp +++ b/nano/core_test/backlog.cpp @@ -22,10 +22,12 @@ TEST (backlog, population) nano::test::system system{}; auto & node = *system.add_node (); - node.backlog.activate_callback.add ([&] (nano::secure::transaction const & transaction, nano::account const & account) { + node.backlog_scan.batch_activated.add ([&] (auto const & batch) { nano::lock_guard lock{ mutex }; - - activated.insert (account); + for (auto const & info : batch) + { + activated.insert (info.account); + } }); auto blocks = nano::test::setup_independent_blocks (system, node, 256); diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp index 1073a83a3..ec30708eb 100644 --- a/nano/core_test/bootstrap.cpp +++ b/nano/core_test/bootstrap.cpp @@ -299,7 +299,7 @@ TEST (bootstrap, frontier_scan) config.bootstrap.enable_scan = false; config.bootstrap.enable_dependency_walker = false; // Disable election activation - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.priority_scheduler.enable = false; config.optimistic_scheduler.enable = false; config.hinted_scheduler.enable = false; @@ -395,7 +395,7 @@ TEST (bootstrap, frontier_scan_pending) config.bootstrap.enable_scan = false; config.bootstrap.enable_dependency_walker = false; // Disable election activation - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.priority_scheduler.enable = false; config.optimistic_scheduler.enable = false; config.hinted_scheduler.enable = false; @@ -477,7 +477,7 @@ TEST (bootstrap, frontier_scan_cannot_prioritize) config.bootstrap.enable_scan = false; config.bootstrap.enable_dependency_walker = false; // Disable election activation - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.priority_scheduler.enable = false; config.optimistic_scheduler.enable = false; config.hinted_scheduler.enable = false; diff --git a/nano/core_test/confirming_set.cpp b/nano/core_test/confirming_set.cpp index f412b0930..0b891ee0d 100644 --- a/nano/core_test/confirming_set.cpp +++ b/nano/core_test/confirming_set.cpp @@ -100,7 +100,7 @@ TEST (confirmation_callback, observer_callbacks) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -145,7 +145,7 @@ TEST (confirmation_callback, confirmed_history) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.bootstrap.enable = false; auto node = system.add_node (node_config); @@ -217,7 +217,7 @@ TEST (confirmation_callback, dependent_election) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); diff --git a/nano/core_test/election.cpp b/nano/core_test/election.cpp index 200f84ffb..6c386ce08 100644 --- a/nano/core_test/election.cpp +++ b/nano/core_test/election.cpp @@ -36,7 +36,7 @@ TEST (election, quorum_minimum_flip_success) nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = nano::dev::constants.genesis_amount; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); auto const latest_hash = nano::dev::genesis->hash (); @@ -86,7 +86,7 @@ TEST (election, quorum_minimum_flip_fail) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = nano::dev::constants.genesis_amount; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); nano::state_block_builder builder; @@ -137,7 +137,7 @@ TEST (election, quorum_minimum_confirm_success) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = nano::dev::constants.genesis_amount; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); nano::keypair key1; nano::block_builder builder; @@ -167,7 +167,7 @@ TEST (election, quorum_minimum_confirm_fail) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = nano::dev::constants.genesis_amount; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); nano::block_builder builder; @@ -205,7 +205,7 @@ TEST (election, quorum_minimum_update_weight_before_quorum_checks) nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); diff --git a/nano/core_test/election_scheduler.cpp b/nano/core_test/election_scheduler.cpp index 422df9971..12e3cc290 100644 --- a/nano/core_test/election_scheduler.cpp +++ b/nano/core_test/election_scheduler.cpp @@ -178,7 +178,7 @@ TEST (election_scheduler, no_vacancy) nano::node_config config = system.default_config (); config.active_elections.size = 1; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config); nano::state_block_builder builder{}; diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index 7693af460..359804592 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -1018,7 +1018,7 @@ TEST (votes, add_existing) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = nano::dev::constants.genesis_amount; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); nano::keypair key1; nano::block_builder builder; @@ -4267,7 +4267,7 @@ TEST (ledger, unchecked_epoch_invalid) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 (*system.add_node (node_config)); nano::keypair destination; nano::block_builder builder; diff --git a/nano/core_test/ledger_confirm.cpp b/nano/core_test/ledger_confirm.cpp index f9bfeff99..a4789f296 100644 --- a/nano/core_test/ledger_confirm.cpp +++ b/nano/core_test/ledger_confirm.cpp @@ -58,7 +58,7 @@ TEST (ledger_confirm, multiple_accounts) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); nano::keypair key1; nano::keypair key2; @@ -232,7 +232,7 @@ TEST (ledger_confirm, send_receive_between_2_accounts) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); nano::keypair key1; nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); @@ -361,7 +361,7 @@ TEST (ledger_confirm, send_receive_self) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); @@ -449,7 +449,7 @@ TEST (ledger_confirm, all_block_types) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); nano::keypair key1; diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index 2db65d6af..1d9118ec2 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -263,7 +263,7 @@ TEST (node, auto_bootstrap) { nano::test::system system; nano::node_config config (system.get_available_port ()); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; nano::node_flags node_flags; node_flags.disable_bootstrap_bulk_push_client = true; node_flags.disable_lazy_bootstrap = true; @@ -291,7 +291,7 @@ TEST (node, auto_bootstrap_reverse) { nano::test::system system; nano::node_config config (system.get_available_port ()); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; nano::node_flags node_flags; node_flags.disable_bootstrap_bulk_push_client = true; node_flags.disable_lazy_bootstrap = true; @@ -364,7 +364,7 @@ TEST (node, search_receivable_confirmed) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); nano::keypair key2; system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -393,7 +393,7 @@ TEST (node, search_receivable_pruned) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node1 = system.add_node (node_config); nano::node_flags node_flags; node_flags.enable_pruning = true; @@ -699,7 +699,7 @@ TEST (node, fork_multi_flip) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 (*system.add_node (node_config, node_flags, type)); node_config.peering_port = system.get_available_port (); node_config.bootstrap.account_sets.cooldown = 100ms; // Reduce cooldown to speed up fork resolution @@ -752,7 +752,7 @@ TEST (node, fork_bootstrap_flip) { nano::test::system system; nano::node_config config1{ system.get_available_port () }; - config1.backlog_population.enable = false; + config1.backlog_scan.enable = false; nano::node_flags node_flags; node_flags.disable_bootstrap_bulk_push_client = true; node_flags.disable_lazy_bootstrap = true; @@ -1159,7 +1159,7 @@ TEST (node, DISABLED_broadcast_elected) nano::node_flags node_flags; nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node0 = system.add_node (node_config, node_flags, type); node_config.peering_port = system.get_available_port (); auto node1 = system.add_node (node_config, node_flags, type); @@ -1286,7 +1286,7 @@ TEST (node, rep_self_vote) nano::test::system system; nano::node_config node_config (system.get_available_port ()); node_config.online_weight_minimum = std::numeric_limits::max (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node0 = system.add_node (node_config); nano::keypair rep_big; nano::block_builder builder; @@ -1689,7 +1689,7 @@ TEST (node, DISABLED_local_votes_cache) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.receive_minimum = nano::dev::constants.genesis_amount; auto & node (*system.add_node (node_config)); nano::state_block_builder builder; @@ -1773,7 +1773,7 @@ TEST (node, DISABLED_local_votes_cache_batch) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); ASSERT_GE (node.network_params.voting.max_cache, 2); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -1847,7 +1847,7 @@ TEST (node, DISABLED_local_votes_cache_generate_new_vote) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -1901,7 +1901,7 @@ TEST (node, DISABLED_local_votes_cache_fork) node_flags.disable_legacy_bootstrap = true; node_flags.disable_wallet_bootstrap = true; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 (*system.add_node (node_config, node_flags)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); auto send1 = nano::state_block_builder () @@ -2150,7 +2150,7 @@ TEST (node, epoch_conflict_confirm) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node0 = *system.add_node (node_config); node_config.peering_port = system.get_available_port (); auto & node1 = *system.add_node (node_config); @@ -2661,7 +2661,7 @@ TEST (node, bidirectional_tcp) node_flags.disable_lazy_bootstrap = true; node_flags.disable_wallet_bootstrap = true; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node1 = system.add_node (node_config, node_flags); node_config.peering_port = system.get_available_port (); node_config.tcp_incoming_connections_max = 0; // Disable incoming TCP connections for node 2 @@ -2854,7 +2854,7 @@ TEST (node, rollback_gap_source) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); nano::state_block_builder builder; nano::keypair key; @@ -2922,7 +2922,7 @@ TEST (node, dependency_graph) { nano::test::system system; nano::node_config config (system.get_available_port ()); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config); nano::state_block_builder builder; @@ -3120,10 +3120,10 @@ TEST (node, dependency_graph_frontier) { nano::test::system system; nano::node_config config (system.get_available_port ()); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node1 = *system.add_node (config); config.peering_port = system.get_available_port (); - config.backlog_population.enable = true; + config.backlog_scan.enable = true; auto & node2 = *system.add_node (config); nano::state_block_builder builder; @@ -3287,9 +3287,9 @@ TEST (node, deferred_dependent_elections) { nano::test::system system; nano::node_config node_config_1{ system.get_available_port () }; - node_config_1.backlog_population.enable = false; + node_config_1.backlog_scan.enable = false; nano::node_config node_config_2{ system.get_available_port () }; - node_config_2.backlog_population.enable = false; + node_config_2.backlog_scan.enable = false; nano::node_flags flags; flags.disable_request_loop = true; auto & node = *system.add_node (node_config_1, flags); diff --git a/nano/core_test/numbers.cpp b/nano/core_test/numbers.cpp index ec8bed3f4..7b6e77015 100644 --- a/nano/core_test/numbers.cpp +++ b/nano/core_test/numbers.cpp @@ -649,3 +649,98 @@ TEST (uint512_union, hash) } } } + +TEST (sat_math, add_sat) +{ + // Test uint128_t + { + nano::uint128_t max = std::numeric_limits::max (); + nano::uint128_t one = 1; + nano::uint128_t large_val = max - 100; + + // Normal addition + ASSERT_EQ (nano::add_sat (one, one), nano::uint128_t (2)); + + // Saturation at max + ASSERT_EQ (nano::add_sat (max, one), max); + ASSERT_EQ (nano::add_sat (large_val, nano::uint128_t (200)), max); + ASSERT_EQ (nano::add_sat (max, max), max); + } + // Test uint256_t + { + nano::uint256_t max = std::numeric_limits::max (); + nano::uint256_t one = 1; + nano::uint256_t large_val = max - 100; + + // Normal addition + ASSERT_EQ (nano::add_sat (one, one), nano::uint256_t (2)); + + // Saturation at max + ASSERT_EQ (nano::add_sat (max, one), max); + ASSERT_EQ (nano::add_sat (large_val, nano::uint256_t (200)), max); + ASSERT_EQ (nano::add_sat (max, max), max); + } + // Test uint512_t + { + nano::uint512_t max = std::numeric_limits::max (); + nano::uint512_t one = 1; + nano::uint512_t large_val = max - 100; + + // Normal addition + ASSERT_EQ (nano::add_sat (one, one), nano::uint512_t (2)); + + // Saturation at max + ASSERT_EQ (nano::add_sat (max, one), max); + ASSERT_EQ (nano::add_sat (large_val, nano::uint512_t (200)), max); + ASSERT_EQ (nano::add_sat (max, max), max); + } +} + +TEST (sat_math, sub_sat) +{ + // Test uint128_t + { + nano::uint128_t max = std::numeric_limits::max (); + nano::uint128_t min = std::numeric_limits::min (); + nano::uint128_t one = 1; + nano::uint128_t hundred (100); + + // Normal subtraction + ASSERT_EQ (nano::sub_sat (hundred, one), nano::uint128_t (99)); + + // Saturation at min + ASSERT_EQ (nano::sub_sat (min, one), min); + ASSERT_EQ (nano::sub_sat (hundred, nano::uint128_t (200)), min); + ASSERT_EQ (nano::sub_sat (min, max), min); + } + // Test uint256_t + { + nano::uint256_t max = std::numeric_limits::max (); + nano::uint256_t min = std::numeric_limits::min (); + nano::uint256_t one = 1; + nano::uint256_t hundred (100); + + // Normal subtraction + ASSERT_EQ (nano::sub_sat (hundred, one), nano::uint256_t (99)); + + // Saturation at min + ASSERT_EQ (nano::sub_sat (min, one), min); + ASSERT_EQ (nano::sub_sat (hundred, nano::uint256_t (200)), min); + ASSERT_EQ (nano::sub_sat (min, max), min); + } + // Test uint512_t + { + nano::uint512_t max = std::numeric_limits::max (); + nano::uint512_t min = std::numeric_limits::min (); + nano::uint512_t one = 1; + nano::uint512_t hundred (100); + + // Normal subtraction + ASSERT_EQ (nano::sub_sat (hundred, one), nano::uint512_t (99)); + + // Saturation at min + ASSERT_EQ (nano::sub_sat (min, one), min); + ASSERT_EQ (nano::sub_sat (hundred, nano::uint512_t (200)), min); + ASSERT_EQ (nano::sub_sat (min, max), min); + } +} \ No newline at end of file diff --git a/nano/core_test/optimistic_scheduler.cpp b/nano/core_test/optimistic_scheduler.cpp index 25d0771db..fa2107d78 100644 --- a/nano/core_test/optimistic_scheduler.cpp +++ b/nano/core_test/optimistic_scheduler.cpp @@ -85,7 +85,7 @@ TEST (optimistic_scheduler, under_gap_threshold) { nano::test::system system{}; nano::node_config config = system.default_config (); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config); // Must be smaller than optimistic scheduler `gap_threshold` @@ -98,7 +98,7 @@ TEST (optimistic_scheduler, under_gap_threshold) nano::test::confirm (node.ledger, blocks.at (55)); // Manually trigger backlog scan - node.backlog.trigger (); + node.backlog_scan.trigger (); // Ensure unconfirmed account head block gets activated auto const & block = blocks.back (); diff --git a/nano/core_test/request_aggregator.cpp b/nano/core_test/request_aggregator.cpp index 5e99210c2..33b96df5f 100644 --- a/nano/core_test/request_aggregator.cpp +++ b/nano/core_test/request_aggregator.cpp @@ -21,7 +21,7 @@ TEST (request_aggregator, one) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); nano::block_builder builder; @@ -71,7 +71,7 @@ TEST (request_aggregator, one_update) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); nano::keypair key1; @@ -137,7 +137,7 @@ TEST (request_aggregator, two) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); nano::keypair key1; @@ -208,7 +208,7 @@ TEST (request_aggregator, two_endpoints) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; nano::node_flags node_flags; node_flags.disable_rep_crawler = true; auto & node1 (*system.add_node (node_config, node_flags)); @@ -266,7 +266,7 @@ TEST (request_aggregator, split) size_t max_vbh = nano::network::confirm_ack_hashes_max; nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); std::vector> request; @@ -319,7 +319,7 @@ TEST (request_aggregator, channel_max_queue) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.request_aggregator.max_queue = 0; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -349,7 +349,7 @@ TEST (request_aggregator, DISABLED_unique) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); nano::block_builder builder; diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index 0da8a59f7..9d6e2124a 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -116,7 +116,7 @@ TEST (toml, daemon_config_deserialize_defaults) std::stringstream ss; ss << R"toml( [node] - [node.backlog_population] + [node.backlog_scan] [node.bootstrap] [node.bootstrap_server] [node.block_processor] @@ -197,9 +197,9 @@ TEST (toml, daemon_config_deserialize_defaults) ASSERT_EQ (conf.node.max_queued_requests, defaults.node.max_queued_requests); ASSERT_EQ (conf.node.request_aggregator_threads, defaults.node.request_aggregator_threads); ASSERT_EQ (conf.node.max_unchecked_blocks, defaults.node.max_unchecked_blocks); - ASSERT_EQ (conf.node.backlog_population.enable, defaults.node.backlog_population.enable); - ASSERT_EQ (conf.node.backlog_population.batch_size, defaults.node.backlog_population.batch_size); - ASSERT_EQ (conf.node.backlog_population.frequency, defaults.node.backlog_population.frequency); + ASSERT_EQ (conf.node.backlog_scan.enable, defaults.node.backlog_scan.enable); + ASSERT_EQ (conf.node.backlog_scan.batch_size, defaults.node.backlog_scan.batch_size); + ASSERT_EQ (conf.node.backlog_scan.rate_limit, defaults.node.backlog_scan.rate_limit); ASSERT_EQ (conf.node.enable_upnp, defaults.node.enable_upnp); ASSERT_EQ (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled); @@ -465,10 +465,10 @@ TEST (toml, daemon_config_deserialize_no_defaults) frontiers_confirmation = "always" enable_upnp = false - [node.backlog_population] + [node.backlog_scan] enable = false batch_size = 999 - frequency = 999 + rate_limit = 999 [node.block_processor] max_peer_queue = 999 @@ -704,9 +704,9 @@ TEST (toml, daemon_config_deserialize_no_defaults) ASSERT_NE (conf.node.work_threads, defaults.node.work_threads); ASSERT_NE (conf.node.max_queued_requests, defaults.node.max_queued_requests); ASSERT_NE (conf.node.request_aggregator_threads, defaults.node.request_aggregator_threads); - ASSERT_NE (conf.node.backlog_population.enable, defaults.node.backlog_population.enable); - ASSERT_NE (conf.node.backlog_population.batch_size, defaults.node.backlog_population.batch_size); - ASSERT_NE (conf.node.backlog_population.frequency, defaults.node.backlog_population.frequency); + ASSERT_NE (conf.node.backlog_scan.enable, defaults.node.backlog_scan.enable); + ASSERT_NE (conf.node.backlog_scan.batch_size, defaults.node.backlog_scan.batch_size); + ASSERT_NE (conf.node.backlog_scan.rate_limit, defaults.node.backlog_scan.rate_limit); ASSERT_NE (conf.node.enable_upnp, defaults.node.enable_upnp); ASSERT_NE (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled); diff --git a/nano/core_test/vote_processor.cpp b/nano/core_test/vote_processor.cpp index 4e237b34a..ea9ada780 100644 --- a/nano/core_test/vote_processor.cpp +++ b/nano/core_test/vote_processor.cpp @@ -21,7 +21,7 @@ TEST (vote_processor, codes) nano::test::system system; auto node_config = system.default_config (); // Disable all election schedulers - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.hinted_scheduler.enable = false; node_config.optimistic_scheduler.enable = false; auto & node = *system.add_node (node_config); @@ -159,10 +159,10 @@ TEST (vote_processor, no_broadcast_local) flags.disable_request_loop = true; nano::node_config config1, config2; config1.representative_vote_weight_minimum = 0; - config1.backlog_population.enable = false; + config1.backlog_scan.enable = false; auto & node (*system.add_node (config1, flags)); config2.representative_vote_weight_minimum = 0; - config2.backlog_population.enable = false; + config2.backlog_scan.enable = false; config2.peering_port = system.get_available_port (); system.add_node (config2, flags); nano::block_builder builder; @@ -214,10 +214,10 @@ TEST (vote_processor, local_broadcast_without_a_representative) flags.disable_request_loop = true; nano::node_config config1, config2; config1.representative_vote_weight_minimum = 0; - config1.backlog_population.enable = false; + config1.backlog_scan.enable = false; auto & node (*system.add_node (config1, flags)); config2.representative_vote_weight_minimum = 0; - config2.backlog_population.enable = false; + config2.backlog_scan.enable = false; config2.peering_port = system.get_available_port (); system.add_node (config2, flags); nano::block_builder builder; @@ -263,9 +263,9 @@ TEST (vote_processor, no_broadcast_local_with_a_principal_representative) nano::node_flags flags; flags.disable_request_loop = true; nano::node_config config1, config2; - config1.backlog_population.enable = false; + config1.backlog_scan.enable = false; auto & node (*system.add_node (config1, flags)); - config2.backlog_population.enable = false; + config2.backlog_scan.enable = false; config2.peering_port = system.get_available_port (); system.add_node (config2, flags); nano::block_builder builder; diff --git a/nano/core_test/voting.cpp b/nano/core_test/voting.cpp index 55d63259f..dd0487ddf 100644 --- a/nano/core_test/voting.cpp +++ b/nano/core_test/voting.cpp @@ -146,7 +146,7 @@ TEST (vote_spacing, prune) TEST (vote_spacing, vote_generator) { nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.active_elections.hinted_limit_percentage = 0; // Disable election hinting nano::test::system system; nano::node_flags node_flags; @@ -190,7 +190,7 @@ TEST (vote_spacing, vote_generator) TEST (vote_spacing, rapid) { nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.active_elections.hinted_limit_percentage = 0; // Disable election hinting nano::test::system system; nano::node_flags node_flags; diff --git a/nano/core_test/wallet.cpp b/nano/core_test/wallet.cpp index 79068966e..6a95ba34e 100644 --- a/nano/core_test/wallet.cpp +++ b/nano/core_test/wallet.cpp @@ -1150,7 +1150,7 @@ TEST (wallet, search_receivable) nano::test::system system; nano::node_config config = system.default_config (); config.enable_voting = false; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; nano::node_flags flags; flags.disable_search_pending = true; auto & node (*system.add_node (config, flags)); diff --git a/nano/core_test/wallets.cpp b/nano/core_test/wallets.cpp index 71caf12ad..4b6e2a09f 100644 --- a/nano/core_test/wallets.cpp +++ b/nano/core_test/wallets.cpp @@ -195,7 +195,7 @@ TEST (wallets, search_receivable) nano::test::system system; nano::node_config config = system.default_config (); config.enable_voting = false; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; nano::node_flags flags; flags.disable_search_pending = true; auto & node (*system.add_node (config, flags)); diff --git a/nano/lib/numbers.hpp b/nano/lib/numbers.hpp index 26b6f1444..20384190a 100644 --- a/nano/lib/numbers.hpp +++ b/nano/lib/numbers.hpp @@ -528,6 +528,33 @@ namespace difficulty uint64_t from_multiplier (double const, uint64_t const); double to_multiplier (uint64_t const, uint64_t const); } + +/** + * Add to or substract from a value without overflow + * TODO: C++26 replace with std::add_sat and std::sub_sat + */ +template +T add_sat (T const & value, T const & diff) noexcept +{ + static_assert (std::numeric_limits::is_specialized, "std::numeric_limits must be specialized"); + return (value > std::numeric_limits::max () - diff) ? std::numeric_limits::max () : value + diff; +} +template +T sub_sat (T const & value, T const & diff) noexcept +{ + static_assert (std::numeric_limits::is_specialized, "std::numeric_limits must be specialized"); + return (value < std::numeric_limits::min () + diff) ? std::numeric_limits::min () : value - diff; +} +template +T inc_sat (T const & value) noexcept +{ + return add_sat (value, static_cast (1)); +} +template +T dec_sat (T const & value) noexcept +{ + return sub_sat (value, static_cast (1)); +} } /* diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index b9ce858a1..828175401 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -84,6 +84,7 @@ enum class type active_elections_timeout, active_elections_cancelled, active_elections_cemented, + backlog_scan, backlog, unchecked, election_scheduler, @@ -418,6 +419,7 @@ enum class detail activate_failed, activate_skip, activate_full, + scanned, // active insert, diff --git a/nano/lib/thread_roles.cpp b/nano/lib/thread_roles.cpp index efc6aa232..89b2c5da0 100644 --- a/nano/lib/thread_roles.cpp +++ b/nano/lib/thread_roles.cpp @@ -94,8 +94,8 @@ std::string nano::thread_role::get_string (nano::thread_role::name role) case nano::thread_role::name::unchecked: thread_role_name_string = "Unchecked"; break; - case nano::thread_role::name::backlog_population: - thread_role_name_string = "Backlog"; + case nano::thread_role::name::backlog_scan: + thread_role_name_string = "Backlog scan"; break; case nano::thread_role::name::vote_generator_queue: thread_role_name_string = "Voting que"; diff --git a/nano/lib/thread_roles.hpp b/nano/lib/thread_roles.hpp index 04325743e..15b4a511c 100644 --- a/nano/lib/thread_roles.hpp +++ b/nano/lib/thread_roles.hpp @@ -36,7 +36,7 @@ enum class name epoch_upgrader, db_parallel_traversal, unchecked, - backlog_population, + backlog_scan, vote_generator_queue, telemetry, bootstrap, diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index f50649b04..27b95f0d2 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -16,8 +16,8 @@ add_library( ${platform_sources} active_elections.hpp active_elections.cpp - backlog_population.hpp - backlog_population.cpp + backlog_scan.hpp + backlog_scan.cpp bandwidth_limiter.hpp bandwidth_limiter.cpp blockprocessor.hpp diff --git a/nano/node/backlog_population.cpp b/nano/node/backlog_population.cpp deleted file mode 100644 index 284b7c599..000000000 --- a/nano/node/backlog_population.cpp +++ /dev/null @@ -1,165 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -nano::backlog_population::backlog_population (backlog_population_config const & config_a, nano::scheduler::component & schedulers, nano::ledger & ledger, nano::stats & stats_a) : - config{ config_a }, - schedulers{ schedulers }, - ledger{ ledger }, - stats{ stats_a } -{ -} - -nano::backlog_population::~backlog_population () -{ - // Thread must be stopped before destruction - debug_assert (!thread.joinable ()); -} - -void nano::backlog_population::start () -{ - debug_assert (!thread.joinable ()); - - thread = std::thread{ [this] () { - nano::thread_role::set (nano::thread_role::name::backlog_population); - run (); - } }; -} - -void nano::backlog_population::stop () -{ - { - nano::lock_guard lock{ mutex }; - stopped = true; - } - notify (); - nano::join_or_pass (thread); -} - -void nano::backlog_population::trigger () -{ - { - nano::unique_lock lock{ mutex }; - triggered = true; - } - notify (); -} - -void nano::backlog_population::notify () -{ - condition.notify_all (); -} - -bool nano::backlog_population::predicate () const -{ - return triggered || config.enable; -} - -void nano::backlog_population::run () -{ - nano::unique_lock lock{ mutex }; - while (!stopped) - { - if (predicate ()) - { - stats.inc (nano::stat::type::backlog, nano::stat::detail::loop); - - triggered = false; - populate_backlog (lock); - } - - condition.wait (lock, [this] () { - return stopped || predicate (); - }); - } -} - -void nano::backlog_population::populate_backlog (nano::unique_lock & lock) -{ - debug_assert (config.frequency > 0); - - const auto chunk_size = config.batch_size / config.frequency; - auto done = false; - nano::account next = 0; - uint64_t total = 0; - while (!stopped && !done) - { - lock.unlock (); - - { - auto transaction = ledger.tx_begin_read (); - - auto it = ledger.store.account.begin (transaction, next); - auto const end = ledger.store.account.end (transaction); - - auto should_refresh = [&transaction] () { - auto cutoff = std::chrono::steady_clock::now () - 100ms; // TODO: Make this configurable - return transaction.timestamp () < cutoff; - }; - - for (size_t count = 0; it != end && count < chunk_size && !should_refresh (); ++it, ++count, ++total) - { - stats.inc (nano::stat::type::backlog, nano::stat::detail::total); - - auto const & account = it->first; - auto const & account_info = it->second; - - activate (transaction, account, account_info); - - next = account.number () + 1; - } - - done = ledger.store.account.begin (transaction, next) == end; - } - - lock.lock (); - - // Give the rest of the node time to progress without holding database lock - condition.wait_for (lock, std::chrono::milliseconds{ 1000 / config.frequency }); - } -} - -void nano::backlog_population::activate (secure::transaction const & transaction, nano::account const & account, nano::account_info const & account_info) -{ - auto const maybe_conf_info = ledger.store.confirmation_height.get (transaction, account); - auto const conf_info = maybe_conf_info.value_or (nano::confirmation_height_info{}); - - // If conf info is empty then it means then it means nothing is confirmed yet - if (conf_info.height < account_info.block_count) - { - stats.inc (nano::stat::type::backlog, nano::stat::detail::activated); - - activate_callback.notify (transaction, account); - - schedulers.optimistic.activate (account, account_info, conf_info); - schedulers.priority.activate (transaction, account, account_info, conf_info); - } -} - -/* - * backlog_population_config - */ - -nano::error nano::backlog_population_config::serialize (nano::tomlconfig & toml) const -{ - toml.put ("enable", enable, "Control if ongoing backlog population is enabled. If not, backlog population can still be triggered by RPC \ntype:bool"); - toml.put ("batch_size", batch_size, "Number of accounts per second to process when doing backlog population scan. Increasing this value will help unconfirmed frontiers get into election prioritization queue faster, however it will also increase resource usage. \ntype:uint"); - toml.put ("frequency", frequency, "Backlog scan divides the scan into smaller batches, number of which is controlled by this value. Higher frequency helps to utilize resources more uniformly, however it also introduces more overhead. The resulting number of accounts per single batch is `backlog_scan_batch_size / backlog_scan_frequency` \ntype:uint"); - - return toml.get_error (); -} - -nano::error nano::backlog_population_config::deserialize (nano::tomlconfig & toml) -{ - toml.get ("enable", enable); - toml.get ("batch_size", batch_size); - toml.get ("frequency", frequency); - - return toml.get_error (); -} diff --git a/nano/node/backlog_scan.cpp b/nano/node/backlog_scan.cpp new file mode 100644 index 000000000..c725c04b1 --- /dev/null +++ b/nano/node/backlog_scan.cpp @@ -0,0 +1,175 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +nano::backlog_scan::backlog_scan (backlog_scan_config const & config_a, nano::ledger & ledger_a, nano::stats & stats_a) : + config{ config_a }, + ledger{ ledger_a }, + stats{ stats_a }, + limiter{ config.rate_limit } +{ +} + +nano::backlog_scan::~backlog_scan () +{ + // Thread must be stopped before destruction + debug_assert (!thread.joinable ()); +} + +void nano::backlog_scan::start () +{ + debug_assert (!thread.joinable ()); + + thread = std::thread{ [this] () { + nano::thread_role::set (nano::thread_role::name::backlog_scan); + run (); + } }; +} + +void nano::backlog_scan::stop () +{ + { + nano::lock_guard lock{ mutex }; + stopped = true; + } + notify (); + nano::join_or_pass (thread); +} + +void nano::backlog_scan::trigger () +{ + { + nano::unique_lock lock{ mutex }; + triggered = true; + } + notify (); +} + +void nano::backlog_scan::notify () +{ + condition.notify_all (); +} + +bool nano::backlog_scan::predicate () const +{ + return triggered || config.enable; +} + +void nano::backlog_scan::run () +{ + nano::unique_lock lock{ mutex }; + while (!stopped) + { + if (predicate ()) + { + stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::loop); + triggered = false; + populate_backlog (lock); // Does a single iteration over all accounts + debug_assert (lock.owns_lock ()); + } + else + { + condition.wait (lock, [this] () { + return stopped || predicate (); + }); + } + } +} + +void nano::backlog_scan::populate_backlog (nano::unique_lock & lock) +{ + uint64_t total = 0; + + nano::account next = 0; + bool done = false; + while (!stopped && !done) + { + // Wait for the rate limiter + while (!limiter.should_pass (config.batch_size)) + { + std::chrono::milliseconds const wait_time{ 1000 / std::max ((config.rate_limit / config.batch_size), size_t{ 1 }) / 2 }; + condition.wait_for (lock, std::max (wait_time, 10ms)); + if (stopped) + { + return; + } + } + + lock.unlock (); + + std::deque scanned; + std::deque activated; + { + auto transaction = ledger.tx_begin_read (); + + auto it = ledger.store.account.begin (transaction, next); + auto const end = ledger.store.account.end (transaction); + + for (size_t count = 0; it != end && count < config.batch_size; ++it, ++count, ++total) + { + stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::total); + + auto const [account, account_info] = *it; + auto const maybe_conf_info = ledger.store.confirmation_height.get (transaction, account); + auto const conf_info = maybe_conf_info.value_or (nano::confirmation_height_info{}); + + activated_info info{ account, account_info, conf_info }; + + scanned.push_back (info); + if (conf_info.height < account_info.block_count) + { + activated.push_back (info); + } + + next = inc_sat (account.number ()); + } + + done = (it == end); + } + + stats.add (nano::stat::type::backlog_scan, nano::stat::detail::scanned, scanned.size ()); + stats.add (nano::stat::type::backlog_scan, nano::stat::detail::activated, activated.size ()); + + // Notify about scanned and activated accounts without holding database transaction + batch_scanned.notify (scanned); + batch_activated.notify (activated); + + lock.lock (); + } +} + +nano::container_info nano::backlog_scan::container_info () const +{ + nano::lock_guard guard{ mutex }; + nano::container_info info; + info.put ("limiter", limiter.size ()); + return info; +} + +/* + * backlog_scan_config + */ + +nano::error nano::backlog_scan_config::serialize (nano::tomlconfig & toml) const +{ + toml.put ("enable", enable, "Control if ongoing backlog population is enabled. If not, backlog population can still be triggered by RPC \ntype:bool"); + toml.put ("batch_size", batch_size, "Size of a single batch. Larger batches reduce overhead, but may put more pressure on other node components. \ntype:uint"); + toml.put ("rate_limit", rate_limit, "Number of accounts per second to process when doing backlog population scan. Increasing this value will help unconfirmed frontiers get into election prioritization queue faster. Use 0 to process as fast as possible, but be aware that it may consume a lot of resources. \ntype:uint"); + + return toml.get_error (); +} + +nano::error nano::backlog_scan_config::deserialize (nano::tomlconfig & toml) +{ + toml.get ("enable", enable); + toml.get ("batch_size", batch_size); + toml.get ("rate_limit", rate_limit); + + return toml.get_error (); +} diff --git a/nano/node/backlog_population.hpp b/nano/node/backlog_scan.hpp similarity index 56% rename from nano/node/backlog_population.hpp rename to nano/node/backlog_scan.hpp index 37123f4ba..5ff41629a 100644 --- a/nano/node/backlog_population.hpp +++ b/nano/node/backlog_scan.hpp @@ -3,24 +3,18 @@ #include #include #include -#include +#include +#include +#include #include #include +#include #include -namespace nano::secure -{ -class transaction; -} namespace nano { -class account_info; -class election_scheduler; -class ledger; -class stats; - -class backlog_population_config final +class backlog_scan_config final { public: nano::error deserialize (nano::tomlconfig &); @@ -29,17 +23,17 @@ public: public: /** Control if ongoing backlog population is enabled. If not, backlog population can still be triggered by RPC */ bool enable{ true }; - /** Number of accounts per second to process. Number of accounts per single batch is this value divided by `frequency` */ - unsigned batch_size{ 10 * 1000 }; - /** Number of batches to run per second. Batches run in 1 second / `frequency` intervals */ - unsigned frequency{ 10 }; + /** Number of accounts to scan per second. */ + size_t rate_limit{ 10000 }; + /** Number of accounts per second to process. */ + size_t batch_size{ 1000 }; }; -class backlog_population final +class backlog_scan final { public: - backlog_population (backlog_population_config const &, nano::scheduler::component &, nano::ledger &, nano::stats &); - ~backlog_population (); + backlog_scan (backlog_scan_config const &, nano::ledger &, nano::stats &); + ~backlog_scan (); void start (); void stop (); @@ -50,16 +44,22 @@ public: /** Notify about AEC vacancy */ void notify (); + nano::container_info container_info () const; + public: - /** - * Callback called for each backlogged account - */ - using callback_t = nano::observer_set; - callback_t activate_callback; + struct activated_info + { + nano::account account; + nano::account_info account_info; + nano::confirmation_height_info conf_info; + }; + + using batch_event_t = nano::observer_set>; + batch_event_t batch_scanned; // Accounts scanned but not activated + batch_event_t batch_activated; // Accounts activated private: // Dependencies - backlog_population_config const & config; - nano::scheduler::component & schedulers; + backlog_scan_config const & config; nano::ledger & ledger; nano::stats & stats; @@ -67,9 +67,10 @@ private: void run (); bool predicate () const; void populate_backlog (nano::unique_lock & lock); - void activate (secure::transaction const &, nano::account const &, nano::account_info const &); private: + nano::rate_limiter limiter; + /** This is a manual trigger, the ongoing backlog population does not use this. * It can be triggered even when backlog population (frontiers confirmation) is disabled. */ bool triggered{ false }; diff --git a/nano/node/bootstrap/crawlers.hpp b/nano/node/bootstrap/crawlers.hpp index a29d53fa9..dec7ac8de 100644 --- a/nano/node/bootstrap/crawlers.hpp +++ b/nano/node/bootstrap/crawlers.hpp @@ -134,7 +134,7 @@ struct pending_database_crawler if (it != end) { // If that fails, perform a fresh lookup - seek (starting_account.number () + 1); + seek (inc_sat (starting_account.number ())); } update_current (); diff --git a/nano/node/bootstrap/database_scan.cpp b/nano/node/bootstrap/database_scan.cpp index dc88a9b30..013512939 100644 --- a/nano/node/bootstrap/database_scan.cpp +++ b/nano/node/bootstrap/database_scan.cpp @@ -78,7 +78,7 @@ std::deque nano::bootstrap::account_database_scanner::next_batch { auto const & [account, info] = crawler.current.value (); result.push_back (account); - next = account.number () + 1; // TODO: Handle account number overflow + next = inc_sat (account.number ()); } // Empty current value indicates the end of the table @@ -106,7 +106,7 @@ std::deque nano::bootstrap::pending_database_scanner::next_batch { auto const & [key, info] = crawler.current.value (); result.push_back (key.account); - next = key.account.number () + 1; // TODO: Handle account number overflow + next = inc_sat (key.account.number ()); } // Empty current value indicates the end of the table diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 91dca52cd..681fcaba8 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -2173,7 +2173,7 @@ void nano::json_handler::delegators () { auto transaction (node.ledger.tx_begin_read ()); boost::property_tree::ptree delegators; - for (auto i (node.store.account.begin (transaction, start_account.number () + 1)), n (node.store.account.end (transaction)); i != n && delegators.size () < count; ++i) + for (auto i (node.store.account.begin (transaction, inc_sat (start_account.number ()))), n (node.store.account.end (transaction)); i != n && delegators.size () < count; ++i) { nano::account_info const & info (i->second); if (info.representative == representative) @@ -4189,7 +4189,7 @@ void nano::json_handler::unopened () break; } // Skip existing accounts - iterator = node.store.pending.begin (transaction, nano::pending_key (account.number () + 1, 0)); + iterator = node.store.pending.begin (transaction, nano::pending_key (inc_sat (account.number ()), 0)); } else { @@ -5154,7 +5154,7 @@ void nano::json_handler::work_peers_clear () void nano::json_handler::populate_backlog () { - node.backlog.trigger (); + node.backlog_scan.trigger (); response_l.put ("success", ""); response_errors (); } diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 34c72739f..195525f46 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include @@ -154,8 +154,8 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy aggregator_impl{ std::make_unique (config.request_aggregator, *this, stats, generator, final_generator, history, ledger, wallets, vote_router) }, aggregator{ *aggregator_impl }, wallets (wallets_store.init_error (), *this), - backlog_impl{ std::make_unique (config.backlog_population, scheduler, ledger, stats) }, - backlog{ *backlog_impl }, + backlog_scan_impl{ std::make_unique (config.backlog_scan, ledger, stats) }, + backlog_scan{ *backlog_scan_impl }, bootstrap_server_impl{ std::make_unique (config.bootstrap_server, store, ledger, network_params.network, stats) }, bootstrap_server{ *bootstrap_server_impl }, bootstrap_impl{ std::make_unique (config, block_processor, ledger, network, stats, logger) }, @@ -180,6 +180,16 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy return ledger.weight (rep); }; + // TODO: Hook this direclty in the schedulers + backlog_scan.batch_activated.add ([this] (auto const & batch) { + auto transaction = ledger.tx_begin_read (); + for (auto const & info : batch) + { + scheduler.optimistic.activate (info.account, info.account_info, info.conf_info); + scheduler.priority.activate (transaction, info.account, info.account_info, info.conf_info); + } + }); + // Republish vote if it is new and the node does not host a principal representative (or close to) vote_router.vote_processed.add ([this] (std::shared_ptr const & vote, nano::vote_source source, std::unordered_map const & results) { bool processed = std::any_of (results.begin (), results.end (), [] (auto const & result) { @@ -636,7 +646,7 @@ void nano::node::start () confirming_set.start (); scheduler.start (); aggregator.start (); - backlog.start (); + backlog_scan.start (); bootstrap_server.start (); bootstrap.start (); websocket.start (); @@ -667,7 +677,7 @@ void nano::node::stop () // Cancels ongoing work generation tasks, which may be blocking other threads // No tasks may wait for work generation in I/O threads, or termination signal capturing will be unable to call node::stop() distributed_work.stop (); - backlog.stop (); + backlog_scan.stop (); bootstrap.stop (); rep_crawler.stop (); unchecked.stop (); @@ -849,7 +859,7 @@ bool nano::node::collect_ledger_pruning_targets (std::deque & read_operations += depth; if (read_operations >= batch_read_size_a) { - last_account_a = account.number () + 1; + last_account_a = inc_sat (account.number ()); finish_transaction = true; } else @@ -1196,6 +1206,7 @@ nano::container_info nano::node::container_info () const info.add ("rep_tiers", rep_tiers.container_info ()); info.add ("message_processor", message_processor.container_info ()); info.add ("bandwidth", outbound_limiter.container_info ()); + info.add ("backlog_scan", backlog_scan.container_info ()); return info; } diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 2d4f4a351..a47194237 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -33,7 +33,7 @@ namespace nano { class active_elections; -class backlog_population; +class backlog_scan; class bandwidth_limiter; class confirming_set; class message_processor; @@ -205,8 +205,8 @@ public: std::unique_ptr aggregator_impl; nano::request_aggregator & aggregator; nano::wallets wallets; - std::unique_ptr backlog_impl; - nano::backlog_population & backlog; + std::unique_ptr backlog_scan_impl; + nano::backlog_scan & backlog_scan; std::unique_ptr bootstrap_server_impl; nano::bootstrap_server & bootstrap_server; std::unique_ptr bootstrap_impl; diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index c7735299e..8fb095afd 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -258,9 +258,9 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const monitor.serialize (monitor_l); toml.put_child ("monitor", monitor_l); - nano::tomlconfig backlog_population_l; - backlog_population.serialize (backlog_population_l); - toml.put_child ("backlog_population", backlog_population_l); + nano::tomlconfig backlog_scan_l; + backlog_scan.serialize (backlog_scan_l); + toml.put_child ("backlog_scan", backlog_scan_l); return toml.get_error (); } @@ -395,10 +395,10 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml) monitor.deserialize (config_l); } - if (toml.has_key ("backlog_population")) + if (toml.has_key ("backlog_scan")) { - auto config_l = toml.get_required_child ("backlog_population"); - backlog_population.deserialize (config_l); + auto config_l = toml.get_required_child ("backlog_scan"); + backlog_scan.deserialize (config_l); } /* diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index aa5a13f8f..ed4ca6fa3 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include @@ -146,7 +146,7 @@ public: nano::local_block_broadcaster_config local_block_broadcaster; nano::confirming_set_config confirming_set; nano::monitor_config monitor; - nano::backlog_population_config backlog_population; + nano::backlog_scan_config backlog_scan; public: /** Entry is ignored if it cannot be parsed as a valid address:port */ diff --git a/nano/node/scheduler/priority.cpp b/nano/node/scheduler/priority.cpp index 6bc9091e6..4f63b43b9 100644 --- a/nano/node/scheduler/priority.cpp +++ b/nano/node/scheduler/priority.cpp @@ -135,9 +135,12 @@ bool nano::scheduler::priority::activate (secure::transaction const & transactio { debug_assert (conf_info.frontier != account_info.head); - auto hash = conf_info.height == 0 ? account_info.open_block : ledger.any.block_successor (transaction, conf_info.frontier).value (); - auto block = ledger.any.block_get (transaction, hash); - release_assert (block != nullptr); + auto const hash = conf_info.height == 0 ? account_info.open_block : ledger.any.block_successor (transaction, conf_info.frontier).value_or (0); + auto const block = ledger.any.block_get (transaction, hash); + if (!block) + { + return false; // Not activated + } if (ledger.dependents_confirmed (transaction, *block)) { diff --git a/nano/rpc_test/receivable.cpp b/nano/rpc_test/receivable.cpp index 9199a1461..672a8f258 100644 --- a/nano/rpc_test/receivable.cpp +++ b/nano/rpc_test/receivable.cpp @@ -133,7 +133,7 @@ TEST (rpc, receivable_unconfirmed) { nano::test::system system; nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, config); auto chain = nano::test::setup_chain (system, *node, 1, nano::dev::genesis_key, false); auto block1 = chain[0]; @@ -530,7 +530,7 @@ TEST (rpc, accounts_receivable_confirmed) { nano::test::system system; nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, config); auto chain = nano::test::setup_chain (system, *node, 1, nano::dev::genesis_key, false); auto block1 = chain[0]; diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index cfc63f0b3..4969605f7 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -2924,7 +2924,7 @@ TEST (rpc, accounts_balances_unopened_account_with_receivables) { nano::test::system system; nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, config); // send a 1 raw to the unopened account which will have receivables @@ -3228,7 +3228,7 @@ TEST (rpc, pending_exists) { nano::test::system system; nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, config); nano::keypair key1; system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -3287,7 +3287,7 @@ TEST (rpc, wallet_receivable) { nano::test::system system; nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, config); nano::keypair key1; system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -4351,7 +4351,7 @@ TEST (rpc, populate_backlog) nano::test::system system; nano::node_config node_config = system.default_config (); // Disable automatic backlog population - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, node_config); // Create and process a block that won't get automatically scheduled for confirmation diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index 2736d7dd0..75bf0ea2f 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -643,7 +643,7 @@ TEST (confirmation_height, many_accounts_single_confirmation) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = 100; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -726,7 +726,7 @@ TEST (confirmation_height, many_accounts_many_confirmations) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = 100; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -799,7 +799,7 @@ TEST (confirmation_height, long_chains) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); nano::keypair key1; system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -944,7 +944,7 @@ TEST (confirmation_height, dynamic_algorithm) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); nano::keypair key; system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -992,7 +992,7 @@ TEST (confirmation_height, many_accounts_send_receive_self) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = 100; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.active_elections.size = 400000; nano::node_flags node_flags; auto node = system.add_node (node_config); @@ -1411,7 +1411,7 @@ TEST (telemetry, under_load) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; nano::node_flags node_flags; auto node = system.add_node (node_config, node_flags); node_config.peering_port = system.get_available_port (); @@ -1772,7 +1772,7 @@ TEST (node, mass_block_new) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); node.network_params.network.aec_loop_interval_ms = 500; @@ -2035,7 +2035,7 @@ TEST (node, wallet_create_block_confirm_conflicts) nano::test::system system; nano::block_builder builder; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); auto const num_blocks = 10000; @@ -2106,7 +2106,7 @@ TEST (system, block_sequence) config.peering_port = system.get_available_port (); // config.bandwidth_limit = 16 * 1024; config.enable_voting = true; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; nano::node_flags flags; flags.disable_max_peers_per_ip = true; flags.disable_ongoing_bootstrap = true; diff --git a/nano/slow_test/vote_cache.cpp b/nano/slow_test/vote_cache.cpp index f7d8e61bb..9f70affa9 100644 --- a/nano/slow_test/vote_cache.cpp +++ b/nano/slow_test/vote_cache.cpp @@ -134,7 +134,7 @@ TEST (vote_cache, perf_singlethreaded) nano::test::system system; nano::node_flags flags; nano::node_config config = system.default_config (); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config, flags); const int rep_count = 50; @@ -193,7 +193,7 @@ TEST (vote_cache, perf_multithreaded) nano::test::system system; nano::node_flags flags; nano::node_config config = system.default_config (); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config, flags); const int thread_count = 12;