Some confirmed block observer callbacks being missed (#2723)

* Observer callbacks being missed

* Small optimization to potentially save read disk io

* Also reset variables after cementinting all

* (Unrelated) Remove old comment

* Add check for election_winner_details in case of duplicate elections for same block

* Only check for inconsistency between blocks_confirmed and observer stats on beta/live

* Revert test changes which can go back to using confirmation_height_processor::add directly

* Update observer (Gui comment)
This commit is contained in:
Wesley Shillingford 2020-04-23 12:17:51 +01:00 committed by GitHub
commit ed3771a7e4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 130 additions and 92 deletions

View file

@ -871,7 +871,7 @@ TEST (confirmation_height, pending_observer_callbacks)
system.deadline_set (10s);
// Confirm the callback is not called under this circumstance because there is no election information
while (node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out) != 1 || node->ledger.stats.count (nano::stat::type::observer, nano::stat::detail::all, nano::stat::dir::out) != 1)
while (node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out) != 1 || node->ledger.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::all, nano::stat::dir::out) != 1)
{
ASSERT_NO_ERROR (system.poll ());
}
@ -1147,7 +1147,7 @@ TEST (confirmation_height, callback_confirmed_history)
}
// Confirm that no inactive callbacks have been called when the confirmation height processor has already iterated over it, waiting to write
ASSERT_EQ (0, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_inactive, nano::stat::dir::out));
ASSERT_EQ (0, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
}
system.deadline_set (10s);
@ -1166,7 +1166,7 @@ TEST (confirmation_height, callback_confirmed_history)
}
system.deadline_set (10s);
while (node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_active_quorum, nano::stat::dir::out) != 1)
while (node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out) != 1)
{
ASSERT_NO_ERROR (system.poll ());
}
@ -1176,8 +1176,8 @@ TEST (confirmation_height, callback_confirmed_history)
// Confirm the callback is not called under this circumstance
ASSERT_EQ (2, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_active_quorum, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_inactive, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
ASSERT_EQ (2, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
ASSERT_EQ (2, node->stats.count (nano::stat::type::confirmation_height, get_stats_detail (mode_a), nano::stat::dir::in));
ASSERT_EQ (3, node->ledger.cache.cemented_count);
@ -1236,9 +1236,9 @@ TEST (confirmation_height, dependent_election)
ASSERT_NO_ERROR (system.poll ());
}
ASSERT_EQ (1, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_active_quorum, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_active_conf_height, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_inactive, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
ASSERT_EQ (3, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
ASSERT_EQ (3, node->stats.count (nano::stat::type::confirmation_height, get_stats_detail (mode_a), nano::stat::dir::in));
ASSERT_EQ (4, node->ledger.cache.cemented_count);
@ -1318,9 +1318,9 @@ TEST (confirmation_height, cemented_gap_below_receive)
auto transaction = node->store.tx_begin_read ();
ASSERT_TRUE (node->ledger.block_confirmed (transaction, open1->hash ()));
ASSERT_EQ (1, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_active_quorum, nano::stat::dir::out));
ASSERT_EQ (0, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_active_conf_height, nano::stat::dir::out));
ASSERT_EQ (9, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_inactive, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
ASSERT_EQ (0, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out));
ASSERT_EQ (9, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
ASSERT_EQ (10, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
ASSERT_EQ (10, node->stats.count (nano::stat::type::confirmation_height, get_stats_detail (mode_a), nano::stat::dir::in));
ASSERT_EQ (11, node->ledger.cache.cemented_count);
@ -1411,9 +1411,9 @@ TEST (confirmation_height, cemented_gap_below_no_cache)
auto transaction = node->store.tx_begin_read ();
ASSERT_TRUE (node->ledger.block_confirmed (transaction, open1->hash ()));
ASSERT_EQ (node->active.election_winner_details_size (), 0);
ASSERT_EQ (1, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_active_quorum, nano::stat::dir::out));
ASSERT_EQ (0, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_active_conf_height, nano::stat::dir::out));
ASSERT_EQ (5, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_inactive, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
ASSERT_EQ (0, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out));
ASSERT_EQ (5, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
ASSERT_EQ (6, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
ASSERT_EQ (6, node->stats.count (nano::stat::type::confirmation_height, get_stats_detail (mode_a), nano::stat::dir::in));
ASSERT_EQ (7, node->ledger.cache.cemented_count);
@ -1473,7 +1473,7 @@ TEST (confirmation_height, election_winner_details_clearing)
ASSERT_NO_ERROR (system.poll ());
}
ASSERT_EQ (1, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_inactive, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
node->block_confirm (send2);
system.deadline_set (10s);
@ -1492,9 +1492,9 @@ TEST (confirmation_height, election_winner_details_clearing)
ASSERT_NO_ERROR (system.poll ());
}
ASSERT_EQ (1, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_inactive, nano::stat::dir::out));
ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
ASSERT_EQ (3, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out));
ASSERT_EQ (2, node->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_active_quorum, nano::stat::dir::out));
ASSERT_EQ (2, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
ASSERT_EQ (3, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
ASSERT_EQ (3, node->stats.count (nano::stat::type::confirmation_height, get_stats_detail (mode_a), nano::stat::dir::in));
ASSERT_EQ (4, node->ledger.cache.cemented_count);

View file

@ -1853,7 +1853,7 @@ TEST (node, broadcast_elected)
ASSERT_NO_ERROR (ec);
}
system.deadline_set (5s);
while (node1->stats.count (nano::stat::type::observer, nano::stat::detail::observer_confirmation_inactive, nano::stat::dir::out) == 0)
while (node1->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out) == 0)
{
ASSERT_NO_ERROR (system.poll ());
}

View file

@ -429,7 +429,7 @@ std::string nano::stat::type_to_string (uint32_t key)
case nano::stat::type::message:
res = "message";
break;
case nano::stat::type::observer:
case nano::stat::type::confirmation_observer:
res = "observer";
break;
case nano::stat::type::confirmation_height:
@ -490,13 +490,13 @@ std::string nano::stat::detail_to_string (uint32_t key)
case nano::stat::detail::bulk_push:
res = "bulk_push";
break;
case nano::stat::detail::observer_confirmation_active_quorum:
case nano::stat::detail::active_quorum:
res = "observer_confirmation_active_quorum";
break;
case nano::stat::detail::observer_confirmation_active_conf_height:
case nano::stat::detail::active_conf_height:
res = "observer_confirmation_active_conf_height";
break;
case nano::stat::detail::observer_confirmation_inactive:
case nano::stat::detail::inactive_conf_height:
res = "observer_confirmation_inactive";
break;
case nano::stat::detail::error_socket_close:

View file

@ -195,8 +195,8 @@ public:
ipc,
tcp,
udp,
observer,
confirmation_height,
confirmation_observer,
drop,
aggregator,
requests,
@ -215,10 +215,10 @@ public:
http_callback,
unreachable_host,
// observer specific
observer_confirmation_active_quorum,
observer_confirmation_active_conf_height,
observer_confirmation_inactive,
// confirmation_observer specific
active_quorum,
active_conf_height,
inactive_conf_height,
// ledger, block, bootstrap
send,

View file

@ -987,22 +987,36 @@ boost::optional<nano::election_status_type> nano::active_transactions::confirm_b
auto hash (block_a->hash ());
nano::unique_lock<std::mutex> lock (mutex);
auto existing (blocks.find (hash));
boost::optional<nano::election_status_type> status_type;
if (existing != blocks.end ())
{
if (!existing->second->confirmed () && existing->second->status.winner->hash () == hash)
if (existing->second->status.winner && existing->second->status.winner->hash () == hash)
{
existing->second->confirm_once (nano::election_status_type::active_confirmation_height);
return nano::election_status_type::active_confirmation_height;
if (!existing->second->confirmed ())
{
existing->second->confirm_once (nano::election_status_type::active_confirmation_height);
status_type = nano::election_status_type::active_confirmation_height;
}
else
{
#ifndef NDEBUG
nano::unique_lock<std::mutex> election_winners_lk (election_winner_details_mutex);
debug_assert (election_winner_details.find (hash) != election_winner_details.cend ());
#endif
status_type = nano::election_status_type::active_confirmed_quorum;
}
}
else
{
return boost::optional<nano::election_status_type>{};
status_type = boost::optional<nano::election_status_type>{};
}
}
else
{
return nano::election_status_type::inactive_confirmation_height;
status_type = nano::election_status_type::inactive_confirmation_height;
}
return status_type;
}
size_t nano::active_transactions::priority_cementable_frontiers_size ()

View file

@ -491,6 +491,13 @@ bool nano::confirmation_height_bounded::cement_blocks (nano::write_guard & scope
scoped_write_guard_a.release ();
notify_observers_callback (cemented_blocks);
}
// Tests should check this already at the end, but not all blocks may have elections (e.g from manual calls to confirmation_height_processor::add), this should catch any inconsistencies on live/beta though
if (!network_params.network.is_test_network ())
{
auto blocks_confirmed_stats = ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed);
auto observer_stats = ledger.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::all, nano::stat::dir::out);
debug_assert (blocks_confirmed_stats == observer_stats);
}
debug_assert (pending_writes.empty ());
debug_assert (pending_writes_size == 0);
@ -502,7 +509,7 @@ bool nano::confirmation_height_bounded::pending_empty () const
return pending_writes.empty ();
}
void nano::confirmation_height_bounded::prepare_new ()
void nano::confirmation_height_bounded::reset ()
{
accounts_confirmed_info.clear ();
accounts_confirmed_info_size = 0;

View file

@ -19,7 +19,7 @@ class confirmation_height_bounded final
public:
confirmation_height_bounded (nano::ledger &, nano::write_database_queue &, std::chrono::milliseconds, nano::logger_mt &, std::atomic<bool> &, nano::block_hash const &, uint64_t &, std::function<void(std::vector<std::shared_ptr<nano::block>> const &)> const &, std::function<void(nano::block_hash const &)> const &, std::function<uint64_t ()> const &);
bool pending_empty () const;
void prepare_new ();
void reset ();
void process ();
bool cement_blocks (nano::write_guard & scoped_write_guard_a);

View file

@ -73,7 +73,7 @@ void nano::confirmation_height_processor::run (confirmation_height_mode mode_a)
debug_assert (confirmation_height_bounded_processor.pending_empty ());
if (confirmation_height_unbounded_processor.pending_empty ())
{
confirmation_height_unbounded_processor.prepare_new ();
confirmation_height_unbounded_processor.reset ();
}
confirmation_height_unbounded_processor.process ();
}
@ -83,7 +83,7 @@ void nano::confirmation_height_processor::run (confirmation_height_mode mode_a)
debug_assert (confirmation_height_unbounded_processor.pending_empty ());
if (confirmation_height_bounded_processor.pending_empty ())
{
confirmation_height_bounded_processor.prepare_new ();
confirmation_height_bounded_processor.reset ();
}
confirmation_height_bounded_processor.process ();
}
@ -111,6 +111,7 @@ void nano::confirmation_height_processor::run (confirmation_height_mode mode_a)
confirmation_height_bounded_processor.cement_blocks (scoped_write_guard);
}
lock_and_cleanup ();
confirmation_height_bounded_processor.reset ();
}
else if (!confirmation_height_unbounded_processor.pending_empty ())
{
@ -120,6 +121,7 @@ void nano::confirmation_height_processor::run (confirmation_height_mode mode_a)
confirmation_height_unbounded_processor.cement_blocks (scoped_write_guard);
}
lock_and_cleanup ();
confirmation_height_unbounded_processor.reset ();
}
else
{

View file

@ -381,6 +381,13 @@ bool nano::confirmation_height_unbounded::cement_blocks (nano::write_guard & sco
scoped_write_guard_a.release ();
notify_observers_callback (cemented_blocks);
// Tests should check this already at the end, but not all blocks may have elections (e.g from manual calls to confirmation_height_processor::add), this should catch any inconsistencies on live/beta though
if (!network_params.network.is_test_network ())
{
auto blocks_confirmed_stats = ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed);
auto observer_stats = ledger.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::all, nano::stat::dir::out);
debug_assert (blocks_confirmed_stats == observer_stats);
}
debug_assert (pending_writes.empty ());
return false;
}
@ -406,7 +413,7 @@ bool nano::confirmation_height_unbounded::pending_empty () const
return pending_writes.empty ();
}
void nano::confirmation_height_unbounded::prepare_new ()
void nano::confirmation_height_unbounded::reset ()
{
// Separate blocks which are pending confirmation height can be batched by a minimum processing time (to improve lmdb disk write performance),
// so make sure the slate is clean when a new batch is starting.

View file

@ -20,7 +20,7 @@ class confirmation_height_unbounded final
public:
confirmation_height_unbounded (nano::ledger &, nano::write_database_queue &, std::chrono::milliseconds, nano::logger_mt &, std::atomic<bool> &, nano::block_hash const &, uint64_t &, std::function<void(std::vector<std::shared_ptr<nano::block>> const &)> const &, std::function<void(nano::block_hash const &)> const &, std::function<uint64_t ()> const &);
bool pending_empty () const;
void prepare_new ();
void reset ();
void process ();
bool cement_blocks (nano::write_guard &);
@ -91,6 +91,7 @@ private:
void collect_unconfirmed_receive_and_sources_for_account (uint64_t, uint64_t, nano::block_hash const &, nano::account const &, nano::read_transaction const &, std::vector<receive_source_pair> &, std::vector<nano::block_hash> &);
void prepare_iterated_blocks_for_cementing (preparation_data &);
nano::network_params network_params;
nano::ledger & ledger;
nano::write_database_queue & write_database_queue;
std::chrono::milliseconds batch_separate_pending_min_time;

View file

@ -42,7 +42,9 @@ prioritized_m (prioritized_a)
void nano::election::confirm_once (nano::election_status_type type_a)
{
debug_assert (!node.active.mutex.try_lock ());
if (state_m.exchange (nano::election::state_t::confirmed) != nano::election::state_t::confirmed)
// This must be kept above the setting of election state, as dependent confirmed elections require up to date changes to election_winner_details
nano::unique_lock<std::mutex> election_winners_lk (node.active.election_winner_details_mutex);
if (state_m.exchange (nano::election::state_t::confirmed) != nano::election::state_t::confirmed && node.active.election_winner_details.find (status.winner->hash ()) == node.active.election_winner_details.cend ())
{
status.election_end = std::chrono::duration_cast<std::chrono::milliseconds> (std::chrono::system_clock::now ().time_since_epoch ());
status.election_duration = std::chrono::duration_cast<std::chrono::milliseconds> (std::chrono::steady_clock::now () - election_start);
@ -54,15 +56,10 @@ void nano::election::confirm_once (nano::election_status_type type_a)
auto node_l (node.shared ());
auto confirmation_action_l (confirmation_action);
auto this_l = shared_from_this ();
if (status_l.type == nano::election_status_type::active_confirmation_height)
{
// Need to add dependent election results here (and not in process_confirmed which is called asynchronously) so that
// election winner details can be cleared consistently sucessfully in active_transactions::block_cemented_callback
node.active.add_election_winner_details (status_l.winner->hash (), this_l);
}
node.active.election_winner_details.emplace (status.winner->hash (), this_l);
node.active.add_recently_confirmed (status_l.winner->qualified_root (), status_l.winner->hash ());
node_l->process_confirmed (status_l, this_l);
node.background ([node_l, status_l, confirmation_action_l, this_l]() {
node_l->process_confirmed (status_l, this_l);
confirmation_action_l (status_l.winner);
});
adjust_dependent_difficulty ();

View file

@ -279,13 +279,13 @@ node_seq (seq)
switch (status_a.type)
{
case nano::election_status_type::active_confirmed_quorum:
this->stats.inc (nano::stat::type::observer, nano::stat::detail::observer_confirmation_active_quorum, nano::stat::dir::out);
this->stats.inc (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out);
break;
case nano::election_status_type::active_confirmation_height:
this->stats.inc (nano::stat::type::observer, nano::stat::detail::observer_confirmation_active_conf_height, nano::stat::dir::out);
this->stats.inc (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out);
break;
case nano::election_status_type::inactive_confirmation_height:
this->stats.inc (nano::stat::type::observer, nano::stat::detail::observer_confirmation_inactive, nano::stat::dir::out);
this->stats.inc (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out);
break;
default:
break;
@ -1127,7 +1127,7 @@ void nano::node::block_confirm (std::shared_ptr<nano::block> block_a)
bool nano::node::block_confirmed_or_being_confirmed (nano::transaction const & transaction_a, nano::block_hash const & hash_a)
{
return ledger.block_confirmed (transaction_a, hash_a) || confirmation_height_processor.is_processing_block (hash_a);
return confirmation_height_processor.is_processing_block (hash_a) || ledger.block_confirmed (transaction_a, hash_a);
}
nano::uint128_t nano::node::delta () const
@ -1272,32 +1272,23 @@ void nano::node::process_confirmed_data (nano::transaction const & transaction_a
void nano::node::process_confirmed (nano::election_status const & status_a, std::shared_ptr<nano::election> const & election_a, uint8_t iteration_a)
{
if (status_a.type == nano::election_status_type::active_confirmed_quorum)
auto block_a (status_a.winner);
auto hash (block_a->hash ());
if (ledger.block_exists (block_a->type (), hash))
{
auto block_a (status_a.winner);
auto hash (block_a->hash ());
if (ledger.block_exists (block_a->type (), hash))
{
// Pausing to prevent this block being processed before adding to election winner details.
confirmation_height_processor.pause ();
confirmation_height_processor.add (hash);
confirmation_height_processor.add (hash);
}
// Limit to 0.5 * 20 = 10 seconds (more than max block_processor::process_batch finish time)
else if (iteration_a < 20)
{
iteration_a++;
std::weak_ptr<nano::node> node_w (shared ());
alarm.add (std::chrono::steady_clock::now () + network_params.node.process_confirmed_interval, [node_w, status_a, iteration_a, election_a]() {
if (auto node_l = node_w.lock ())
{
active.add_election_winner_details (hash, election_a);
node_l->process_confirmed (status_a, election_a, iteration_a);
}
confirmation_height_processor.unpause ();
}
// Limit to 0.5 * 20 = 10 seconds (more than max block_processor::process_batch finish time)
else if (iteration_a < 20)
{
iteration_a++;
std::weak_ptr<nano::node> node_w (shared ());
alarm.add (std::chrono::steady_clock::now () + network_params.node.process_confirmed_interval, [node_w, status_a, iteration_a, election_a]() {
if (auto node_l = node_w.lock ())
{
node_l->process_confirmed (status_a, election_a, iteration_a);
}
});
}
});
}
}

View file

@ -165,7 +165,6 @@ void nano::vote_processor::verify_votes (decltype (votes) const & votes_a)
}
}
// node.active.mutex lock required
nano::vote_code nano::vote_processor::vote_blocking (std::shared_ptr<nano::vote> vote_a, std::shared_ptr<nano::transport::channel> channel_a, bool validated)
{
auto result (nano::vote_code::invalid);

View file

@ -527,11 +527,15 @@ TEST (confirmation_height, many_accounts_single_confirmation)
ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_unbounded, nano::stat::dir::in), 0);
system.deadline_set (40s);
while ((node->ledger.cache.cemented_count - 1) != node->stats.count (nano::stat::type::observer, nano::stat::detail::all, nano::stat::dir::out))
while ((node->ledger.cache.cemented_count - 1) != node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::all, nano::stat::dir::out))
{
ASSERT_NO_ERROR (system.poll ());
}
system.deadline_set (10s);
while (node->active.election_winner_details_size () > 0)
{
ASSERT_NO_ERROR (system.poll ());
}
ASSERT_EQ (node->active.election_winner_details_size (), 0);
}
TEST (confirmation_height, many_accounts_many_confirmations)
@ -581,7 +585,7 @@ TEST (confirmation_height, many_accounts_many_confirmations)
ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_unbounded, nano::stat::dir::in), num_blocks_to_confirm - num_confirmed_bounded);
system.deadline_set (60s);
while ((node->ledger.cache.cemented_count - 1) != node->stats.count (nano::stat::type::observer, nano::stat::detail::all, nano::stat::dir::out))
while ((node->ledger.cache.cemented_count - 1) != node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::all, nano::stat::dir::out))
{
ASSERT_NO_ERROR (system.poll ());
}
@ -595,7 +599,18 @@ TEST (confirmation_height, many_accounts_many_confirmations)
ASSERT_EQ (num_blocks_to_confirm + 1, cemented_count);
ASSERT_EQ (cemented_count, node->ledger.cache.cemented_count);
ASSERT_EQ (node->active.election_winner_details_size (), 0);
system.deadline_set (20s);
while ((node->ledger.cache.cemented_count - 1) != node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::all, nano::stat::dir::out))
{
ASSERT_NO_ERROR (system.poll ());
}
system.deadline_set (10s);
while (node->active.election_winner_details_size () > 0)
{
ASSERT_NO_ERROR (system.poll ());
}
}
TEST (confirmation_height, long_chains)
@ -693,12 +708,15 @@ TEST (confirmation_height, long_chains)
ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_unbounded, nano::stat::dir::in), 0);
system.deadline_set (40s);
while ((node->ledger.cache.cemented_count - 1) != node->stats.count (nano::stat::type::observer, nano::stat::detail::all, nano::stat::dir::out))
while ((node->ledger.cache.cemented_count - 1) != node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::all, nano::stat::dir::out))
{
ASSERT_NO_ERROR (system.poll ());
}
system.deadline_set (10s);
while (node->active.election_winner_details_size () > 0)
{
ASSERT_NO_ERROR (system.poll ());
}
ASSERT_EQ (node->active.election_winner_details_size (), 0);
}
}
TEST (confirmation_height, dynamic_algorithm)
@ -745,11 +763,13 @@ TEST (confirmation_height, dynamic_algorithm)
ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in), num_blocks);
ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_bounded, nano::stat::dir::in), 1);
ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_unbounded, nano::stat::dir::in), num_blocks - 1);
ASSERT_EQ (node->active.election_winner_details_size (), 0);
system.deadline_set (10s);
while (node->active.election_winner_details_size () > 0)
{
ASSERT_NO_ERROR (system.poll ());
}
}
namespace nano
{
/*
* This tests an issue of incorrect cemented block counts during the transition of conf height algorithms
* The scenario was as follows:
@ -794,11 +814,8 @@ TEST (confirmation_height, dynamic_algorithm_no_transition_while_pending)
{
auto write_guard = node->write_database_queue.wait (nano::writer::testing);
// To limit any data races we are not calling node.block_confirm,
// the relevant code is replicated here (implementation detail):
node->confirmation_height_processor.pause ();
// To limit any data races we are not calling node.block_confirm
node->confirmation_height_processor.add (state_blocks.back ()->hash ());
node->confirmation_height_processor.unpause ();
nano::timer<> timer;
timer.start ();
@ -807,7 +824,7 @@ TEST (confirmation_height, dynamic_algorithm_no_transition_while_pending)
ASSERT_LT (timer.since_start (), 2s);
}
// Pausing prevents any writes in the outer while loop in the confirmaiton height processor (implementation detail)
// Pausing prevents any writes in the outer while loop in the confirmation height processor (implementation detail)
node->confirmation_height_processor.pause ();
timer.restart ();
@ -828,16 +845,19 @@ TEST (confirmation_height, dynamic_algorithm_no_transition_while_pending)
}
system.deadline_set (10s);
while (node->confirmation_height_processor.awaiting_processing_size () != 0 || !node->confirmation_height_processor.current ().is_zero ())
while (node->ledger.cache.cemented_count != num_blocks + 1)
{
ASSERT_NO_ERROR (system.poll ());
}
ASSERT_EQ (node->ledger.cache.cemented_count, num_blocks + 1);
ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in), num_blocks);
ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_bounded, nano::stat::dir::in), 0);
ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_unbounded, nano::stat::dir::in), num_blocks);
ASSERT_EQ (node->active.election_winner_details_size (), 0);
system.deadline_set (10s);
while (node->active.election_winner_details_size () > 0)
{
ASSERT_NO_ERROR (system.poll ());
}
}
}