From ddb3cf55623b0be9c99b94052de353497dc1c22b Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Sat, 23 Mar 2024 13:52:55 +0000 Subject: [PATCH] Replacing confirmation_height_processor with confirming_set --- nano/core_test/CMakeLists.txt | 1 - nano/core_test/active_transactions.cpp | 4 +- nano/core_test/confirmation_height.cpp | 275 ------------------------- nano/core_test/confirming_set.cpp | 212 +++++++++++++++++++ nano/core_test/node.cpp | 4 +- nano/core_test/request_aggregator.cpp | 6 +- nano/core_test/toml.cpp | 8 +- nano/nano_node/entry.cpp | 4 +- nano/node/active_transactions.cpp | 7 +- nano/node/active_transactions.hpp | 6 +- nano/node/json_handler.cpp | 2 +- nano/node/node.cpp | 13 +- nano/node/node.hpp | 6 +- nano/node/nodeconfig.cpp | 8 +- nano/node/nodeconfig.hpp | 3 +- nano/node/wallet.cpp | 2 +- nano/rpc_test/rpc.cpp | 2 +- nano/slow_test/node.cpp | 115 +---------- 18 files changed, 260 insertions(+), 418 deletions(-) delete mode 100644 nano/core_test/confirmation_height.cpp diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index 74410c67..b4111eea 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -12,7 +12,6 @@ add_executable( bootstrap_ascending.cpp bootstrap_server.cpp cli.cpp - confirmation_height.cpp confirmation_solicitor.cpp confirming_set.cpp conflicts.cpp diff --git a/nano/core_test/active_transactions.cpp b/nano/core_test/active_transactions.cpp index 196d65bf..4360d1f2 100644 --- a/nano/core_test/active_transactions.cpp +++ b/nano/core_test/active_transactions.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include #include @@ -1244,7 +1244,7 @@ TEST (active_transactions, activate_inactive) ASSERT_NE (nullptr, election); election->force_confirm (); - ASSERT_TIMELY (5s, !node.confirmation_height_processor.is_processing_added_block (send2->hash ())); + ASSERT_TIMELY (5s, !node.confirmation_height_processor.exists (send2->hash ())); ASSERT_TIMELY (5s, node.block_confirmed (send2->hash ())); ASSERT_TIMELY (5s, node.block_confirmed (send->hash ())); diff --git a/nano/core_test/confirmation_height.cpp b/nano/core_test/confirmation_height.cpp deleted file mode 100644 index 34c4ac9e..00000000 --- a/nano/core_test/confirmation_height.cpp +++ /dev/null @@ -1,275 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -using namespace std::chrono_literals; - -namespace -{ -void add_callback_stats (nano::node & node, std::vector * observer_order = nullptr, nano::mutex * mutex = nullptr) -{ - node.observers.blocks.add ([&stats = node.stats, observer_order, mutex] (nano::election_status const & status_a, std::vector const &, nano::account const &, nano::amount const &, bool, bool) { - stats.inc (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out); - if (mutex) - { - nano::lock_guard guard (*mutex); - debug_assert (observer_order); - observer_order->push_back (status_a.winner->hash ()); - } - }); -} -nano::stat::detail get_stats_detail (nano::confirmation_height_mode mode_a) -{ - debug_assert (mode_a == nano::confirmation_height_mode::bounded || mode_a == nano::confirmation_height_mode::unbounded); - return (mode_a == nano::confirmation_height_mode::bounded) ? nano::stat::detail::blocks_confirmed_bounded : nano::stat::detail::blocks_confirmed_unbounded; -} -} - -TEST (confirmation_height, pending_observer_callbacks) -{ - auto test_mode = [] (nano::confirmation_height_mode mode_a) { - nano::test::system system; - nano::node_flags node_flags; - node_flags.confirmation_height_processor_mode = mode_a; - nano::node_config node_config = system.default_config (); - node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; - auto node = system.add_node (node_config, node_flags); - - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); - - nano::keypair key1; - nano::block_builder builder; - auto send = builder - .send () - .previous (latest) - .destination (key1.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (latest)) - .build (); - auto send1 = builder - .send () - .previous (send->hash ()) - .destination (key1.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio * 2) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (send->hash ())) - .build (); - - { - auto transaction = node->store.tx_begin_write (); - ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send)); - ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send1)); - } - - add_callback_stats (*node); - - node->confirmation_height_processor.add (send1); - - // Callback is performed for all blocks that are confirmed - ASSERT_TIMELY_EQ (5s, 2, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out)) - ASSERT_TIMELY_EQ (5s, 2, node->ledger.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::all, nano::stat::dir::out)); - - ASSERT_EQ (2, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); - ASSERT_EQ (2, node->stats.count (nano::stat::type::confirmation_height, get_stats_detail (mode_a), nano::stat::dir::in)); - ASSERT_EQ (3, node->ledger.cache.cemented_count); - ASSERT_EQ (0, node->active.election_winner_details_size ()); - }; - - test_mode (nano::confirmation_height_mode::bounded); - test_mode (nano::confirmation_height_mode::unbounded); -} - -// The callback and confirmation history should only be updated after confirmation height is set (and not just after voting) -TEST (confirmation_height, callback_confirmed_history) -{ - auto test_mode = [] (nano::confirmation_height_mode mode_a) { - nano::test::system system; - nano::node_flags node_flags; - node_flags.force_use_write_database_queue = true; - node_flags.confirmation_height_processor_mode = mode_a; - nano::node_config node_config = system.default_config (); - node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; - auto node = system.add_node (node_config, node_flags); - - nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); - - nano::keypair key1; - nano::block_builder builder; - auto send = builder - .send () - .previous (latest) - .destination (key1.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (latest)) - .build (); - { - auto transaction = node->store.tx_begin_write (); - ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send)); - } - - auto send1 = builder - .send () - .previous (send->hash ()) - .destination (key1.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio * 2) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (send->hash ())) - .build (); - - add_callback_stats (*node); - - node->process_active (send1); - std::shared_ptr election; - ASSERT_TIMELY (5s, election = nano::test::start_election (system, *node, send1->hash ())); - { - // The write guard prevents the confirmation height processor doing any writes - auto write_guard = node->write_database_queue.wait (nano::writer::testing); - - // Confirm send1 - election->force_confirm (); - ASSERT_TIMELY_EQ (10s, node->active.size (), 0); - ASSERT_EQ (0, node->active.recently_cemented.list ().size ()); - ASSERT_TRUE (node->active.empty ()); - - auto transaction = node->store.tx_begin_read (); - ASSERT_FALSE (node->ledger.block_confirmed (transaction, send->hash ())); - - ASSERT_TIMELY (10s, node->write_database_queue.contains (nano::writer::confirmation_height)); - - // Confirm that no inactive callbacks have been called when the confirmation height processor has already iterated over it, waiting to write - ASSERT_EQ (0, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out)); - } - - ASSERT_TIMELY (10s, !node->write_database_queue.contains (nano::writer::confirmation_height)); - - auto transaction = node->store.tx_begin_read (); - ASSERT_TRUE (node->ledger.block_confirmed (transaction, send->hash ())); - - ASSERT_TIMELY_EQ (10s, node->active.size (), 0); - ASSERT_TIMELY_EQ (10s, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out), 1); - - // Each block that's confirmed is in the recently_cemented history - ASSERT_EQ (2, node->active.recently_cemented.list ().size ()); - ASSERT_TRUE (node->active.empty ()); - - // Confirm the callback is not called under this circumstance - ASSERT_EQ (2, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out)); - ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out)); - ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out)); - ASSERT_EQ (2, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); - ASSERT_EQ (2, node->stats.count (nano::stat::type::confirmation_height, get_stats_detail (mode_a), nano::stat::dir::in)); - ASSERT_EQ (3, node->ledger.cache.cemented_count); - ASSERT_EQ (0, node->active.election_winner_details_size ()); - }; - - test_mode (nano::confirmation_height_mode::bounded); - test_mode (nano::confirmation_height_mode::unbounded); -} - -namespace nano -{ -TEST (confirmation_height, dependent_election) -{ - auto test_mode = [] (nano::confirmation_height_mode mode_a) { - nano::test::system system; - nano::node_flags node_flags; - node_flags.confirmation_height_processor_mode = mode_a; - node_flags.force_use_write_database_queue = true; - nano::node_config node_config = system.default_config (); - node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; - auto node = system.add_node (node_config, node_flags); - - nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); - - nano::keypair key1; - nano::block_builder builder; - auto send = builder - .send () - .previous (latest) - .destination (key1.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (latest)) - .build (); - auto send1 = builder - .send () - .previous (send->hash ()) - .destination (key1.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio * 2) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (send->hash ())) - .build (); - auto send2 = builder - .send () - .previous (send1->hash ()) - .destination (key1.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio * 3) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (send1->hash ())) - .build (); - { - auto transaction = node->store.tx_begin_write (); - ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send)); - ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send1)); - ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send2)); - } - - add_callback_stats (*node); - - // This election should be confirmed as active_conf_height - ASSERT_TRUE (nano::test::start_election (system, *node, send1->hash ())); - // Start an election and confirm it - auto election = nano::test::start_election (system, *node, send2->hash ()); - ASSERT_NE (nullptr, election); - election->force_confirm (); - - ASSERT_TIMELY_EQ (5s, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out), 3); - - ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out)); - ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out)); - ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out)); - ASSERT_EQ (3, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); - ASSERT_EQ (3, node->stats.count (nano::stat::type::confirmation_height, get_stats_detail (mode_a), nano::stat::dir::in)); - ASSERT_EQ (4, node->ledger.cache.cemented_count); - - ASSERT_EQ (0, node->active.election_winner_details_size ()); - }; - - test_mode (nano::confirmation_height_mode::bounded); - test_mode (nano::confirmation_height_mode::unbounded); -} - -TEST (confirmation_height, election_winner_details_clearing_node_process_confirmed) -{ - // Make sure election_winner_details is also cleared if the block never enters the confirmation height processor from node::process_confirmed - nano::test::system system (1); - auto node = system.nodes.front (); - - nano::block_builder builder; - auto send = builder - .send () - .previous (nano::dev::genesis->hash ()) - .destination (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (nano::dev::genesis->hash ())) - .build (); - // Add to election_winner_details. Use an unrealistic iteration so that it should fall into the else case and do a cleanup - node->active.add_election_winner_details (send->hash (), nullptr); - nano::election_status election; - election.winner = send; - node->process_confirmed (election, 1000000); - ASSERT_EQ (0, node->active.election_winner_details_size ()); -} -} diff --git a/nano/core_test/confirming_set.cpp b/nano/core_test/confirming_set.cpp index 44df1ea1..6050d437 100644 --- a/nano/core_test/confirming_set.cpp +++ b/nano/core_test/confirming_set.cpp @@ -66,3 +66,215 @@ TEST (confirming_set, process_multiple) ASSERT_EQ (2, ctx.stats ().count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); ASSERT_EQ (3, ctx.ledger ().cache.cemented_count); } + +TEST (confirmation_callback, observer_callbacks) +{ + nano::test::system system; + nano::node_flags node_flags; + nano::node_config node_config = system.default_config (); + node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; + auto node = system.add_node (node_config, node_flags); + + system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); + nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); + + nano::keypair key1; + nano::block_builder builder; + auto send = builder + .send () + .previous (latest) + .destination (key1.pub) + .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*system.work.generate (latest)) + .build (); + auto send1 = builder + .send () + .previous (send->hash ()) + .destination (key1.pub) + .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio * 2) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*system.work.generate (send->hash ())) + .build (); + + { + auto transaction = node->store.tx_begin_write (); + ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send)); + ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send1)); + } + + node->confirmation_height_processor.add (send1->hash ()); + + // Callback is performed for all blocks that are confirmed + ASSERT_TIMELY_EQ (5s, 2, node->ledger.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::all, nano::stat::dir::out)); + + ASSERT_EQ (2, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); + ASSERT_EQ (3, node->ledger.cache.cemented_count); + ASSERT_EQ (0, node->active.election_winner_details_size ()); +} + +// The callback and confirmation history should only be updated after confirmation height is set (and not just after voting) +TEST (confirmation_callback, confirmed_history) +{ + nano::test::system system; + nano::node_flags node_flags; + node_flags.force_use_write_database_queue = true; + node_flags.disable_ascending_bootstrap = true; + nano::node_config node_config = system.default_config (); + node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; + auto node = system.add_node (node_config, node_flags); + + nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); + + nano::keypair key1; + nano::block_builder builder; + auto send = builder + .send () + .previous (latest) + .destination (key1.pub) + .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*system.work.generate (latest)) + .build (); + { + auto transaction = node->store.tx_begin_write (); + ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send)); + } + + auto send1 = builder + .send () + .previous (send->hash ()) + .destination (key1.pub) + .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio * 2) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*system.work.generate (send->hash ())) + .build (); + + node->process_active (send1); + std::shared_ptr election; + ASSERT_TIMELY (5s, election = nano::test::start_election (system, *node, send1->hash ())); + { + // The write guard prevents the confirmation height processor doing any writes + auto write_guard = node->write_database_queue.wait (nano::writer::testing); + + // Confirm send1 + election->force_confirm (); + ASSERT_TIMELY_EQ (10s, node->active.size (), 0); + ASSERT_EQ (0, node->active.recently_cemented.list ().size ()); + ASSERT_TRUE (node->active.empty ()); + + auto transaction = node->store.tx_begin_read (); + ASSERT_FALSE (node->ledger.block_confirmed (transaction, send->hash ())); + + ASSERT_TIMELY (10s, node->write_database_queue.contains (nano::writer::confirmation_height)); + + // Confirm that no inactive callbacks have been called when the confirmation height processor has already iterated over it, waiting to write + ASSERT_EQ (0, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out)); + } + + ASSERT_TIMELY (10s, !node->write_database_queue.contains (nano::writer::confirmation_height)); + + auto transaction = node->store.tx_begin_read (); + ASSERT_TRUE (node->ledger.block_confirmed (transaction, send->hash ())); + + ASSERT_TIMELY_EQ (10s, node->active.size (), 0); + ASSERT_TIMELY_EQ (10s, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out), 1); + + // Each block that's confirmed is in the recently_cemented history + ASSERT_EQ (2, node->active.recently_cemented.list ().size ()); + ASSERT_TRUE (node->active.empty ()); + + // Confirm the callback is not called under this circumstance + ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out)); + ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out)); + ASSERT_EQ (2, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); + ASSERT_EQ (3, node->ledger.cache.cemented_count); + ASSERT_EQ (0, node->active.election_winner_details_size ()); +} + +TEST (confirmation_callback, dependent_election) +{ + nano::test::system system; + nano::node_flags node_flags; + node_flags.force_use_write_database_queue = true; + nano::node_config node_config = system.default_config (); + node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; + auto node = system.add_node (node_config, node_flags); + + nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); + + nano::keypair key1; + nano::block_builder builder; + auto send = builder + .send () + .previous (latest) + .destination (key1.pub) + .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*system.work.generate (latest)) + .build (); + auto send1 = builder + .send () + .previous (send->hash ()) + .destination (key1.pub) + .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio * 2) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*system.work.generate (send->hash ())) + .build (); + auto send2 = builder + .send () + .previous (send1->hash ()) + .destination (key1.pub) + .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio * 3) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*system.work.generate (send1->hash ())) + .build (); + { + auto transaction = node->store.tx_begin_write (); + ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send)); + ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send1)); + ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send2)); + } + + // This election should be confirmed as active_conf_height + ASSERT_TRUE (nano::test::start_election (system, *node, send1->hash ())); + // Start an election and confirm it + auto election = nano::test::start_election (system, *node, send2->hash ()); + ASSERT_NE (nullptr, election); + election->force_confirm (); + + // Wait for blocks to be confirmed in ledger, callbacks will happen after + ASSERT_TIMELY_EQ (5s, 3, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); + // Once the item added to the confirming set no longer exists, callbacks have completed + ASSERT_TIMELY (5s, !node->confirmation_height_processor.exists (send2->hash ())); + + ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out)); + ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out)); + ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out)); + ASSERT_EQ (4, node->ledger.cache.cemented_count); + + ASSERT_EQ (0, node->active.election_winner_details_size ()); +} + +TEST (confirmation_callback, election_winner_details_clearing_node_process_confirmed) +{ + // Make sure election_winner_details is also cleared if the block never enters the confirmation height processor from node::process_confirmed + nano::test::system system (1); + auto node = system.nodes.front (); + + nano::block_builder builder; + auto send = builder + .send () + .previous (nano::dev::genesis->hash ()) + .destination (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*system.work.generate (nano::dev::genesis->hash ())) + .build (); + // Add to election_winner_details. Use an unrealistic iteration so that it should fall into the else case and do a cleanup + node->active.add_election_winner_details (send->hash (), nullptr); + nano::election_status election; + election.winner = send; + node->process_confirmed (election, 1000000); + ASSERT_EQ (0, node->active.election_winner_details_size ()); +} diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index 9ceff444..988008f2 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include @@ -1984,7 +1984,7 @@ TEST (node, DISABLED_local_votes_cache_batch) .work (*node.work_generate_blocking (nano::dev::genesis->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, node.ledger.process (node.store.tx_begin_write (), send1)); - node.confirmation_height_processor.add (send1); + node.confirmation_height_processor.add (send1->hash ()); ASSERT_TIMELY (5s, node.ledger.block_confirmed (node.store.tx_begin_read (), send1->hash ())); auto send2 = nano::state_block_builder () .account (nano::dev::genesis_key.pub) diff --git a/nano/core_test/request_aggregator.cpp b/nano/core_test/request_aggregator.cpp index da55fd01..ebaa7553 100644 --- a/nano/core_test/request_aggregator.cpp +++ b/nano/core_test/request_aggregator.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include #include @@ -80,7 +80,7 @@ TEST (request_aggregator, one_update) .work (*node.work_generate_blocking (nano::dev::genesis->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, node.ledger.process (node.store.tx_begin_write (), send1)); - node.confirmation_height_processor.add (send1); + node.confirmation_height_processor.add (send1->hash ()); ASSERT_TIMELY (5s, node.ledger.block_confirmed (node.store.tx_begin_read (), send1->hash ())); auto send2 = nano::state_block_builder () .account (nano::dev::genesis_key.pub) @@ -146,7 +146,7 @@ TEST (request_aggregator, two) .work (*node.work_generate_blocking (nano::dev::genesis->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, node.ledger.process (node.store.tx_begin_write (), send1)); - node.confirmation_height_processor.add (send1); + node.confirmation_height_processor.add (send1->hash ()); ASSERT_TIMELY (5s, node.ledger.block_confirmed (node.store.tx_begin_read (), send1->hash ())); auto send2 = builder.make_block () .account (nano::dev::genesis_key.pub) diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index c57c301e..ae6f432a 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -164,7 +164,7 @@ TEST (toml, daemon_config_deserialize_defaults) ASSERT_EQ (conf.node.bootstrap_serving_threads, defaults.node.bootstrap_serving_threads); ASSERT_EQ (conf.node.bootstrap_frontier_request_count, defaults.node.bootstrap_frontier_request_count); ASSERT_EQ (conf.node.bootstrap_fraction_numerator, defaults.node.bootstrap_fraction_numerator); - ASSERT_EQ (conf.node.conf_height_processor_batch_min_time, defaults.node.conf_height_processor_batch_min_time); + ASSERT_EQ (conf.node.confirming_set_batch_time, defaults.node.confirming_set_batch_time); ASSERT_EQ (conf.node.confirmation_history_size, defaults.node.confirmation_history_size); ASSERT_EQ (conf.node.enable_voting, defaults.node.enable_voting); ASSERT_EQ (conf.node.external_address, defaults.node.external_address); @@ -396,7 +396,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) bootstrap_serving_threads = 999 bootstrap_frontier_request_count = 9999 bootstrap_fraction_numerator = 999 - conf_height_processor_batch_min_time = 999 + confirming_set_batch_time = 999 confirmation_history_size = 999 enable_voting = false external_address = "0:0:0:0:0:ffff:7f01:101" @@ -586,7 +586,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) ASSERT_NE (conf.node.bootstrap_serving_threads, defaults.node.bootstrap_serving_threads); ASSERT_NE (conf.node.bootstrap_frontier_request_count, defaults.node.bootstrap_frontier_request_count); ASSERT_NE (conf.node.bootstrap_fraction_numerator, defaults.node.bootstrap_fraction_numerator); - ASSERT_NE (conf.node.conf_height_processor_batch_min_time, defaults.node.conf_height_processor_batch_min_time); + ASSERT_NE (conf.node.confirming_set_batch_time, defaults.node.confirming_set_batch_time); ASSERT_NE (conf.node.confirmation_history_size, defaults.node.confirmation_history_size); ASSERT_NE (conf.node.enable_voting, defaults.node.enable_voting); ASSERT_NE (conf.node.external_address, defaults.node.external_address); @@ -1017,4 +1017,4 @@ TEST (toml, log_config_no_required) confg.deserialize_toml (toml); ASSERT_FALSE (toml.get_error ()) << toml.get_error ().get_message (); -} \ No newline at end of file +} diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index 0e5e4857..9b8486ed 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include @@ -1213,7 +1213,7 @@ int main (int argc, char * const * argv) // Confirm blocks for node1 for (auto & block : blocks) { - node1->confirmation_height_processor.add (block); + node1->confirmation_height_processor.add (block->hash ()); } while (node1->ledger.cache.cemented_count != node1->ledger.cache.block_count) { diff --git a/nano/node/active_transactions.cpp b/nano/node/active_transactions.cpp index c8bdeb5f..4393f097 100644 --- a/nano/node/active_transactions.cpp +++ b/nano/node/active_transactions.cpp @@ -1,8 +1,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -15,7 +15,7 @@ using namespace std::chrono; -nano::active_transactions::active_transactions (nano::node & node_a, nano::confirmation_height_processor & confirmation_height_processor_a, nano::block_processor & block_processor_a) : +nano::active_transactions::active_transactions (nano::node & node_a, nano::confirming_set & confirmation_height_processor_a, nano::block_processor & block_processor_a) : node{ node_a }, confirmation_height_processor{ confirmation_height_processor_a }, block_processor{ block_processor_a }, @@ -82,6 +82,7 @@ void nano::active_transactions::stop () void nano::active_transactions::block_cemented_callback (std::shared_ptr const & block) { + debug_assert (node.block_confirmed (block->hash ())); if (auto election_l = election (block->qualified_root ())) { election_l->try_confirm (block->hash ()); @@ -95,7 +96,7 @@ void nano::active_transactions::block_cemented_callback (std::shared_ptrget_status (); votes = election->votes_with_weight (); } - if (confirmation_height_processor.is_processing_added_block (block->hash ())) + if (confirmation_height_processor.exists (block->hash ())) { status.type = nano::election_status_type::active_confirmed_quorum; } diff --git a/nano/node/active_transactions.hpp b/nano/node/active_transactions.hpp index 5c2a07d9..08885480 100644 --- a/nano/node/active_transactions.hpp +++ b/nano/node/active_transactions.hpp @@ -28,9 +28,9 @@ class active_transactions; class block; class block_sideband; class block_processor; +class confirming_set; class election; class vote; -class confirmation_height_processor; class stats; } namespace nano::store @@ -141,7 +141,7 @@ private: // Elections std::unordered_map> blocks; public: - active_transactions (nano::node &, nano::confirmation_height_processor &, nano::block_processor &); + active_transactions (nano::node &, nano::confirming_set &, nano::block_processor &); ~active_transactions (); void start (); @@ -209,7 +209,7 @@ private: private: // Dependencies nano::node & node; - nano::confirmation_height_processor & confirmation_height_processor; + nano::confirming_set & confirmation_height_processor; nano::block_processor & block_processor; public: diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index c54baa7a..fe217b75 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 3a6634ad..a032e60b 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -172,9 +172,9 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy application_path (application_path_a), port_mapping (*this), block_processor (*this, write_database_queue), - confirmation_height_processor_impl{ std::make_unique (ledger, write_database_queue, config.conf_height_processor_batch_min_time, logger, node_initialized_latch, flags.confirmation_height_processor_mode) }, - confirmation_height_processor{ *confirmation_height_processor_impl }, - active_impl{ std::make_unique (*this, confirmation_height_processor, block_processor) }, + confirming_set_impl{ std::make_unique (ledger, write_database_queue, config.confirming_set_batch_time) }, + confirming_set{ *confirming_set_impl }, + active_impl{ std::make_unique (*this, confirming_set, block_processor) }, active{ *active_impl }, rep_crawler (config.rep_crawler, *this), rep_tiers{ ledger, network_params, online_reps, stats, logger }, @@ -570,7 +570,7 @@ std::unique_ptr nano::collect_container_info (no composite->add_component (node.history.collect_container_info ("history")); composite->add_component (node.block_uniquer.collect_container_info ("block_uniquer")); composite->add_component (node.vote_uniquer.collect_container_info ("vote_uniquer")); - composite->add_component (collect_container_info (node.confirmation_height_processor, "confirmation_height_processor")); + composite->add_component (node.confirmation_height_processor.collect_container_info ("confirmation_queue")); composite->add_component (collect_container_info (node.distributed_work, "distributed_work")); composite->add_component (collect_container_info (node.aggregator, "request_aggregator")); composite->add_component (node.scheduler.collect_container_info ("election_scheduler")); @@ -681,6 +681,7 @@ void nano::node::start () active.start (); generator.start (); final_generator.start (); + confirmation_height_processor.start (); scheduler.start (); backlog.start (); bootstrap_server.start (); @@ -1259,7 +1260,7 @@ void nano::node::process_confirmed (nano::election_status const & status_a, uint { logger.trace (nano::log::type::node, nano::log::detail::process_confirmed, nano::log::arg{ "block", block_l }); - confirmation_height_processor.add (block_l); + confirmation_height_processor.add (block_l->hash ()); } else if (iteration_a < num_iters) { diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 2254c2f6..8c1a0476 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -46,7 +46,7 @@ namespace nano { class active_transactions; -class confirmation_height_processor; +class confirming_set; class node; class work_pool; @@ -168,8 +168,8 @@ public: nano::node_observers observers; nano::port_mapping port_mapping; nano::block_processor block_processor; - std::unique_ptr confirmation_height_processor_impl; - nano::confirmation_height_processor & confirmation_height_processor; + std::unique_ptr confirmation_height_processor_impl; + nano::confirming_set & confirmation_height_processor; std::unique_ptr active_impl; nano::active_transactions & active; nano::online_reps online_reps; diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index c240c583..ad597389 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -126,7 +126,7 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const toml.put ("bootstrap_bandwidth_limit", bootstrap_bandwidth_limit, "Outbound bootstrap traffic limit in bytes/sec after which messages will be dropped.\nNote: changing to unlimited bandwidth (0) is not recommended for limited connections.\ntype:uint64"); toml.put ("bootstrap_bandwidth_burst_ratio", bootstrap_bandwidth_burst_ratio, "Burst ratio for outbound bootstrap traffic.\ntype:double"); - toml.put ("conf_height_processor_batch_min_time", conf_height_processor_batch_min_time.count (), "Minimum write batching time when there are blocks pending confirmation height.\ntype:milliseconds"); + toml.put ("confirming_set_batch_time", confirming_set_batch_time.count (), "Maximum time the confirming set will hold the database write transaction.\ntype:milliseconds"); toml.put ("backup_before_upgrade", backup_before_upgrade, "Backup the ledger database before performing upgrades.\nWarning: uses more disk storage and increases startup time when upgrading.\ntype:bool"); toml.put ("max_work_generate_multiplier", max_work_generate_multiplier, "Maximum allowed difficulty multiplier for work generation.\ntype:double,[1..]"); toml.put ("frontiers_confirmation", serialize_frontiers_confirmation (frontiers_confirmation), "Mode controlling frontier confirmation rate.\ntype:string,{auto,always,disabled}"); @@ -432,9 +432,9 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml) toml.get ("backup_before_upgrade", backup_before_upgrade); - auto conf_height_processor_batch_min_time_l (conf_height_processor_batch_min_time.count ()); - toml.get ("conf_height_processor_batch_min_time", conf_height_processor_batch_min_time_l); - conf_height_processor_batch_min_time = std::chrono::milliseconds (conf_height_processor_batch_min_time_l); + auto confirming_set_batch_time_l (confirming_set_batch_time.count ()); + toml.get ("confirming_set_batch_time", confirming_set_batch_time_l); + confirming_set_batch_time = std::chrono::milliseconds (confirming_set_batch_time_l); toml.get ("max_work_generate_multiplier", max_work_generate_multiplier); diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index b93611ac..2b8f4535 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -119,7 +119,7 @@ public: /** Bootstrap traffic does not need bursts */ double bootstrap_bandwidth_burst_ratio{ 1. }; nano::bootstrap_ascending_config bootstrap_ascending; - std::chrono::milliseconds conf_height_processor_batch_min_time{ 50 }; + std::chrono::milliseconds confirming_set_batch_time{ 250 }; bool backup_before_upgrade{ false }; double max_work_generate_multiplier{ 64. }; uint32_t max_queued_requests{ 512 }; @@ -175,7 +175,6 @@ public: bool fast_bootstrap{ false }; bool read_only{ false }; bool disable_connection_cleanup{ false }; - nano::confirmation_height_mode confirmation_height_processor_mode{ nano::confirmation_height_mode::automatic }; nano::generate_cache_flags generate_cache; bool inactive_node{ false }; std::size_t block_processor_batch_size{ 0 }; diff --git a/nano/node/wallet.cpp b/nano/node/wallet.cpp index 28a3586d..a322b003 100644 --- a/nano/node/wallet.cpp +++ b/nano/node/wallet.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index 44d72c7e..66c10fac 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index 28eee657..2c1ff3c9 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -644,7 +644,6 @@ TEST (confirmation_height, many_accounts_single_confirmation) system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); // The number of frontiers should be more than the nano::confirmation_height::unbounded_cutoff to test the amount of blocks confirmed is correct. - node->confirmation_height_processor.batch_write_size = 500; auto const num_accounts = nano::confirmation_height::unbounded_cutoff * 2 + 50; nano::keypair last_keypair = nano::dev::genesis_key; nano::block_builder builder; @@ -728,7 +727,6 @@ TEST (confirmation_height, many_accounts_many_confirmations) auto node = system.add_node (node_config); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - node->confirmation_height_processor.batch_write_size = 500; auto const num_accounts = nano::confirmation_height::unbounded_cutoff * 2 + 50; auto latest_genesis = node->latest (nano::dev::genesis_key.pub); nano::block_builder builder; @@ -807,7 +805,6 @@ TEST (confirmation_height, long_chains) nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); system.wallet (0)->insert_adhoc (key1.prv); - node->confirmation_height_processor.batch_write_size = 500; auto const num_blocks = nano::confirmation_height::unbounded_cutoff * 2 + 50; nano::block_builder builder; @@ -978,10 +975,10 @@ TEST (confirmation_height, dynamic_algorithm) } } - node->confirmation_height_processor.add (state_blocks.front ()); + node->confirmation_height_processor.add (state_blocks.front ()->hash ()); ASSERT_TIMELY_EQ (20s, node->ledger.cache.cemented_count, 2); - node->confirmation_height_processor.add (latest_genesis); + node->confirmation_height_processor.add (latest_genesis->hash ()); ASSERT_TIMELY_EQ (20s, node->ledger.cache.cemented_count, num_blocks + 1); @@ -991,98 +988,6 @@ TEST (confirmation_height, dynamic_algorithm) ASSERT_TIMELY_EQ (10s, node->active.election_winner_details_size (), 0); } -/* - * This tests an issue of incorrect cemented block counts during the transition of conf height algorithms - * The scenario was as follows: - * - There is at least 1 pending write entry in the unbounded conf height processor - * - 0 blocks currently awaiting processing in the main conf height processor class - * - A block was confirmed when hit the chain in the pending write above but was not a block higher than it. - * - It must be in `confirmation_height_processor::pause ()` function so that `pause` is set (and the difference between the number - * of blocks uncemented is > unbounded_cutoff so that it hits the bounded processor), the main `run` loop on the conf height processor is iterated. - * - * This cause unbounded pending entries not to be written, and then the bounded processor would write them, causing some inconsistencies. - */ -TEST (confirmation_height, dynamic_algorithm_no_transition_while_pending) -{ - // Repeat in case of intermittent issues not replicating the issue talked about above. - for (auto _ = 0; _ < 3; ++_) - { - nano::test::system system; - nano::node_config node_config = system.default_config (); - node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; - nano::node_flags node_flags; - node_flags.force_use_write_database_queue = true; - auto node = system.add_node (node_config, node_flags); - nano::keypair key; - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - - auto latest_genesis = node->latest (nano::dev::genesis_key.pub); - std::vector> state_blocks; - auto const num_blocks = nano::confirmation_height::unbounded_cutoff - 2; - - auto add_block_to_genesis_chain = [&] (store::write_transaction & transaction) { - static int num = 0; - nano::block_builder builder; - auto send = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (latest_genesis) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - num - 1) - .link (key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (latest_genesis)) - .build (); - latest_genesis = send->hash (); - state_blocks.push_back (send); - ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send)); - ++num; - }; - - for (auto i = 0; i < num_blocks; ++i) - { - auto transaction = node->store.tx_begin_write (); - add_block_to_genesis_chain (transaction); - } - - { - auto write_guard = node->write_database_queue.wait (nano::writer::testing); - // To limit any data races we are not calling node.block_confirm - node->confirmation_height_processor.add (state_blocks.back ()); - - nano::timer<> timer; - timer.start (); - while (node->confirmation_height_processor.current ().is_zero ()) - { - ASSERT_LT (timer.since_start (), 2s); - } - - // Pausing prevents any writes in the outer while loop in the confirmation height processor (implementation detail) - node->confirmation_height_processor.pause (); - - timer.restart (); - ASSERT_TIMELY (10s, node->confirmation_height_processor.unbounded_processor.pending_writes_size != 0); - - { - // Make it so that the number of blocks exceed the unbounded cutoff would go into the bounded processor (but shouldn't due to unbounded pending writes) - auto transaction = node->store.tx_begin_write (); - add_block_to_genesis_chain (transaction); - add_block_to_genesis_chain (transaction); - } - // Make sure this is at a height lower than the block in the add () call above - node->confirmation_height_processor.add (state_blocks.front ()); - node->confirmation_height_processor.unpause (); - } - - ASSERT_TIMELY_EQ (10s, node->ledger.cache.cemented_count, num_blocks + 1); - - ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in), num_blocks); - ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_bounded, nano::stat::dir::in), 0); - ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_unbounded, nano::stat::dir::in), num_blocks); - ASSERT_TIMELY_EQ (10s, node->active.election_winner_details_size (), 0); - } -} - TEST (confirmation_height, many_accounts_send_receive_self) { nano::test::system system; @@ -1091,7 +996,6 @@ TEST (confirmation_height, many_accounts_send_receive_self) node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; node_config.active_elections_size = 400000; nano::node_flags node_flags; - node_flags.confirmation_height_processor_mode = nano::confirmation_height_mode::unbounded; auto node = system.add_node (node_config); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -1239,7 +1143,8 @@ TEST (confirmation_height, many_accounts_send_receive_self_no_elections) boost::latch initialized_latch{ 0 }; nano::block_hash block_hash_being_processed{ 0 }; - nano::confirmation_height_processor confirmation_height_processor{ ledger, write_database_queue, 10ms, logger, initialized_latch, confirmation_height_mode::automatic }; + nano::write_database_queue write_queue{ false }; + nano::confirming_set confirmation_height_processor{ ledger, write_queue }; auto const num_accounts = 100000; @@ -1284,7 +1189,7 @@ TEST (confirmation_height, many_accounts_send_receive_self_no_elections) for (auto & open_block : open_blocks) { - confirmation_height_processor.add (open_block); + confirmation_height_processor.add (open_block->hash ()); } system.deadline_set (1000s); @@ -1335,8 +1240,8 @@ TEST (confirmation_height, many_accounts_send_receive_self_no_elections) // Now send and receive to self for (int i = 0; i < open_blocks.size (); ++i) { - confirmation_height_processor.add (send_blocks[i]); - confirmation_height_processor.add (receive_blocks[i]); + confirmation_height_processor.add (send_blocks[i]->hash ()); + confirmation_height_processor.add (receive_blocks[i]->hash ()); } system.deadline_set (1000s); @@ -1346,7 +1251,7 @@ TEST (confirmation_height, many_accounts_send_receive_self_no_elections) ASSERT_NO_ERROR (system.poll ()); } - while (!confirmation_height_processor.current ().is_zero ()) + while (confirmation_height_processor.size () > 0) { ASSERT_NO_ERROR (system.poll ()); } @@ -2173,7 +2078,7 @@ TEST (node, wallet_create_block_confirm_conflicts) election->force_confirm (); } - ASSERT_TIMELY (120s, node->ledger.block_confirmed (node->store.tx_begin_read (), latest) && node->confirmation_height_processor.current () == 0); + ASSERT_TIMELY (120s, node->ledger.block_confirmed (node->store.tx_begin_read (), latest) && node->confirmation_height_processor.size () == 0); done = true; t.join (); }