Use cemented cache to prevent restarting cemented elections

nano/core_test/cementing_set.cpp

@@ -180,7 +180,7 @@ TEST (confirmation_callback, confirmed_history)
 		// Confirm send1
 		election->force_confirm ();
 		ASSERT_TIMELY_EQ (10s, node->active.size (), 0);
-		ASSERT_EQ (0, node->active.recently_cemented.list ().size ());
+		ASSERT_EQ (0, node->active.recently_cemented.size ());
 		ASSERT_TRUE (node->active.empty ());
 
 		auto transaction = node->ledger.tx_begin_read ();
@@ -200,7 +200,7 @@ TEST (confirmation_callback, confirmed_history)
 	ASSERT_TIMELY_EQ (10s, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out), 1);
 
 	// Each block that's confirmed is in the recently_cemented history
-	ASSERT_EQ (2, node->active.recently_cemented.list ().size ());
+	ASSERT_EQ (2, node->active.recently_cemented.size ());
 	ASSERT_TRUE (node->active.empty ());
 
 	// Confirm the callback is not called under this circumstance

nano/core_test/node.cpp

@@ -1605,7 +1605,7 @@ TEST (node, block_confirm)
 	ASSERT_TIMELY (5s, election = node2.active.election (send1_copy->qualified_root ()));
 	// Make node2 genesis representative so it can vote
 	system.wallet (1)->insert_adhoc (nano::dev::genesis_key.prv);
-	ASSERT_TIMELY_EQ (10s, node1.active.recently_cemented.list ().size (), 1);
+	ASSERT_TIMELY_EQ (10s, node1.active.recently_cemented.size (), 1);
 }
 
 TEST (node, confirm_quorum)

nano/node/active_elections.cpp

@@ -30,7 +30,7 @@ nano::active_elections::active_elections (nano::node & node_a, nano::ledger_noti
 	ledger_notifications{ ledger_notifications_a },
 	cementing_set{ cementing_set_a },
 	recently_confirmed{ config.confirmation_cache },
-	recently_cemented{ config.confirmation_history_size },
+	recently_cemented{ config.confirmation_cache },
 	workers{ 1, nano::thread_role::name::aec_notifications }
 {
 	// Cementing blocks might implicitly confirm dependent elections
@@ -157,7 +157,7 @@ auto nano::active_elections::insert (std::shared_ptr<nano::block> const & block,
 
 	if (!index.exists (root))
 	{
-		if (!recently_confirmed.exists (root))
+		if (!recently_confirmed.contains (root) && !recently_cemented.contains (root))
 		{
 			result.inserted = true;
 
@@ -307,7 +307,7 @@ void nano::active_elections::erase_election (nano::unique_lock<nano::mutex> & lo
 {
 	debug_assert (!mutex.try_lock ());
 	debug_assert (lock.owns_lock ());
-	debug_assert (!election->confirmed () || recently_confirmed.exists (election->qualified_root));
+	debug_assert (!election->confirmed () || recently_confirmed.contains (election->qualified_root));
 
 	auto blocks_l = election->blocks ();
 	node.vote_router.disconnect (*election);
@@ -814,7 +814,6 @@ nano::error nano::active_elections_config::serialize (nano::tomlconfig & toml) c
 	toml.put ("size", size, "Number of active elections. Elections beyond this limit have limited survival time.\nWarning: modifying this value may result in a lower confirmation rate. \ntype:uint64,[250..]");
 	toml.put ("hinted_limit_percentage", hinted_limit_percentage, "Limit of hinted elections as percentage of `active_elections_size` \ntype:uint64");
 	toml.put ("optimistic_limit_percentage", optimistic_limit_percentage, "Limit of optimistic elections as percentage of `active_elections_size`. \ntype:uint64");
-	toml.put ("confirmation_history_size", confirmation_history_size, "Maximum confirmation history size. If tracking the rate of block confirmations, the websocket feature is recommended instead. \ntype:uint64");
 	toml.put ("confirmation_cache", confirmation_cache, "Maximum number of confirmed elections kept in cache to prevent restarting an election. \ntype:uint64");
 	toml.put ("max_election_winners", max_election_winners, "Maximum size of election winner details set. \ntype:uint64");
 	toml.put ("stale_threshold", stale_threshold.count (), "Time after which additional bootstrap attempts are made to find missing blocks for an election. \ntype:seconds");
@@ -826,7 +825,6 @@ nano::error nano::active_elections_config::deserialize (nano::tomlconfig & toml)
 	toml.get ("size", size);
 	toml.get ("hinted_limit_percentage", hinted_limit_percentage);
 	toml.get ("optimistic_limit_percentage", optimistic_limit_percentage);
-	toml.get ("confirmation_history_size", confirmation_history_size);
 	toml.get ("confirmation_cache", confirmation_cache);
 	toml.get ("max_election_winners", max_election_winners);
 	toml.get_duration ("stale_threshold", stale_threshold);

nano/node/active_elections.hpp

@@ -37,10 +37,8 @@ public:
 	std::size_t hinted_limit_percentage{ 20 };
 	// Limit of optimistic elections as percentage of `active_elections_size`
 	std::size_t optimistic_limit_percentage{ 10 };
-	// Maximum confirmation history size
-	std::size_t confirmation_history_size{ 2048 };
 	// Maximum cache size for recently_confirmed
-	std::size_t confirmation_cache{ 65536 };
+	std::size_t confirmation_cache{ 1024 * 64 };
 	// Maximum size of election winner details set
 	std::size_t max_election_winners{ 1024 * 16 };
 

nano/node/bounded_backlog.cpp

@@ -274,7 +274,7 @@ bool nano::bounded_backlog::should_rollback (nano::block_hash const & hash) cons
 	{
 		return false;
 	}
-	if (node.active.recently_confirmed.exists (hash))
+	if (node.active.recently_confirmed.contains (hash))
 	{
 		return false;
 	}

nano/node/json_handler.cpp

@@ -2062,7 +2062,9 @@ void nano::json_handler::confirmation_history ()
 	}
 	if (!ec)
 	{
-		for (auto const & status : node.active.recently_cemented.list ())
+		// TODO: Allow passing a count parameter to limit the number of results
+		// Default to 2000 for now since it was the previous limit
+		for (auto const & status : node.active.recently_cemented.list (2000))
 		{
 			if (hash.is_zero () || status.winner->hash () == hash)
 			{
@@ -4919,7 +4921,7 @@ void nano::json_handler::wallet_representative_set ()
 				for (auto & account : accounts)
 				{
 					wallet->change_async (
-					account, representative, [] (std::shared_ptr<nano::block> const &) {}, 0, false);
+					account, representative, [] (std::shared_ptr<nano::block> const &) { }, 0, false);
 				}
 			}
 		}

nano/node/recently_cemented_cache.cpp

@@ -1,9 +1,8 @@
+#include <nano/lib/blocks.hpp>
 #include <nano/lib/utility.hpp>
 #include <nano/node/recently_cemented_cache.hpp>
 
-/*
- * class recently_cemented
- */
+#include <ranges>
 
 nano::recently_cemented_cache::recently_cemented_cache (std::size_t max_size_a) :
 	max_size{ max_size_a }
@@ -13,23 +12,53 @@ nano::recently_cemented_cache::recently_cemented_cache (std::size_t max_size_a)
 void nano::recently_cemented_cache::put (const nano::election_status & status)
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };
-	cemented.push_back (status);
-	if (cemented.size () > max_size)
+	entries.emplace_back (entry{ status.winner->qualified_root (), status.winner->hash (), status });
+	if (entries.size () > max_size)
 	{
-		cemented.pop_front ();
+		entries.pop_front (); // Remove oldest
 	}
 }
 
-nano::recently_cemented_cache::queue_t nano::recently_cemented_cache::list () const
+void nano::recently_cemented_cache::erase (const nano::block_hash & hash)
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };
-	return cemented;
+	entries.get<tag_hash> ().erase (hash);
+}
+
+void nano::recently_cemented_cache::clear ()
+{
+	nano::lock_guard<nano::mutex> guard{ mutex };
+	entries.clear ();
+}
+
+auto nano::recently_cemented_cache::list (size_t max_count) const -> std::deque<nano::election_status>
+{
+	nano::lock_guard<nano::mutex> guard{ mutex };
+	std::deque<nano::election_status> result;
+	auto it = entries.rbegin ();
+	for (size_t i = 0; i < max_count && it != entries.rend (); ++i, ++it)
+	{
+		result.push_back (it->status);
+	}
+	return result;
 }
 
 std::size_t nano::recently_cemented_cache::size () const
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };
-	return cemented.size ();
+	return entries.size ();
+}
+
+bool nano::recently_cemented_cache::contains (const nano::qualified_root & root) const
+{
+	nano::lock_guard<nano::mutex> guard{ mutex };
+	return entries.get<tag_root> ().contains (root);
+}
+
+bool nano::recently_cemented_cache::contains (const nano::block_hash & hash) const
+{
+	nano::lock_guard<nano::mutex> guard{ mutex };
+	return entries.get<tag_hash> ().contains (hash);
 }
 
 nano::container_info nano::recently_cemented_cache::container_info () const
@@ -37,6 +66,6 @@ nano::container_info nano::recently_cemented_cache::container_info () const
 	nano::lock_guard<nano::mutex> guard{ mutex };
 
 	nano::container_info info;
-	info.put ("cemented", cemented);
+	info.put ("entries", entries);
 	return info;
 }

nano/node/recently_cemented_cache.hpp

@@ -1,14 +1,20 @@
 #pragma once
 
 #include <nano/lib/locks.hpp>
+#include <nano/lib/numbers.hpp>
+#include <nano/lib/numbers_templ.hpp>
 #include <nano/node/election_status.hpp>
+#include <nano/secure/common.hpp>
+
+#include <boost/multi_index/hashed_index.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/random_access_index.hpp>
+#include <boost/multi_index/sequenced_index.hpp>
+#include <boost/multi_index_container.hpp>
 
 #include <deque>
 
-namespace nano
-{
-class container_info_component;
-}
+namespace mi = boost::multi_index;
 
 namespace nano
 {
@@ -18,18 +24,44 @@ namespace nano
 class recently_cemented_cache final
 {
 public:
-	using queue_t = std::deque<nano::election_status>;
-
-	explicit recently_cemented_cache (std::size_t max_size);
+	explicit recently_cemented_cache (size_t max_size);
 
 	void put (nano::election_status const &);
-	queue_t list () const;
+	void erase (nano::block_hash const &);
+	void clear ();
 	std::size_t size () const;
 
+	// Returns up to max_count most recent entries
+	std::deque<nano::election_status> list (size_t max_count = std::numeric_limits<size_t>::max ()) const;
+
+	bool contains (nano::qualified_root const &) const;
+	bool contains (nano::block_hash const &) const;
+
 	nano::container_info container_info () const;
 
 private:
-	queue_t cemented;
+	struct entry
+	{
+		nano::qualified_root root;
+		nano::block_hash hash;
+		nano::election_status status;
+	};
+
+	// clang-format off
+	class tag_hash {};
+	class tag_root {};
+	class tag_sequenced {};
+
+	using ordered_entries = boost::multi_index_container<entry,
+	mi::indexed_by<
+		mi::sequenced<mi::tag<tag_sequenced>>,
+		mi::hashed_unique<mi::tag<tag_root>,
+			mi::member<entry, nano::qualified_root, &entry::root>>,
+		mi::hashed_unique<mi::tag<tag_hash>,
+			mi::member<entry, nano::block_hash, &entry::hash>>>>;
+	// clang-format on
+	ordered_entries entries;
+
 	std::size_t const max_size;
 
 	mutable nano::mutex mutex;

nano/node/recently_confirmed_cache.cpp

@@ -1,10 +1,6 @@
 #include <nano/lib/utility.hpp>
 #include <nano/node/recently_confirmed_cache.hpp>
 
-/*
- * class recently_confirmed
- */
-
 nano::recently_confirmed_cache::recently_confirmed_cache (std::size_t max_size_a) :
 	max_size{ max_size_a }
 {
@@ -13,47 +9,47 @@ nano::recently_confirmed_cache::recently_confirmed_cache (std::size_t max_size_a
 void nano::recently_confirmed_cache::put (const nano::qualified_root & root, const nano::block_hash & hash)
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };
-	confirmed.get<tag_sequence> ().emplace_back (root, hash);
-	if (confirmed.size () > max_size)
+	entries.emplace_back (root, hash);
+	if (entries.size () > max_size)
 	{
-		confirmed.get<tag_sequence> ().pop_front ();
+		entries.pop_front ();
 	}
 }
 
 void nano::recently_confirmed_cache::erase (const nano::block_hash & hash)
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };
-	confirmed.get<tag_hash> ().erase (hash);
+	entries.get<tag_hash> ().erase (hash);
 }
 
 void nano::recently_confirmed_cache::clear ()
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };
-	confirmed.clear ();
+	entries.clear ();
 }
 
-bool nano::recently_confirmed_cache::exists (const nano::block_hash & hash) const
+bool nano::recently_confirmed_cache::contains (const nano::block_hash & hash) const
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };
-	return confirmed.get<tag_hash> ().find (hash) != confirmed.get<tag_hash> ().end ();
+	return entries.get<tag_hash> ().contains (hash);
 }
 
-bool nano::recently_confirmed_cache::exists (const nano::qualified_root & root) const
+bool nano::recently_confirmed_cache::contains (const nano::qualified_root & root) const
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };
-	return confirmed.get<tag_root> ().find (root) != confirmed.get<tag_root> ().end ();
+	return entries.get<tag_root> ().contains (root);
 }
 
 std::size_t nano::recently_confirmed_cache::size () const
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };
-	return confirmed.size ();
+	return entries.size ();
 }
 
 nano::recently_confirmed_cache::entry_t nano::recently_confirmed_cache::back () const
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };
-	return confirmed.back ();
+	return entries.back ();
 }
 
 nano::container_info nano::recently_confirmed_cache::container_info () const
@@ -61,6 +57,6 @@ nano::container_info nano::recently_confirmed_cache::container_info () const
 	nano::lock_guard<nano::mutex> guard{ mutex };
 
 	nano::container_info info;
-	info.put ("confirmed", confirmed);
+	info.put ("entries", entries);
 	return info;
 }

nano/node/recently_confirmed_cache.hpp

@@ -13,11 +13,6 @@
 
 namespace mi = boost::multi_index;
 
-namespace nano
-{
-class container_info_component;
-}
-
 namespace nano
 {
 class recently_confirmed_cache final
@@ -32,8 +27,8 @@ public:
 	void clear ();
 	std::size_t size () const;
 
-	bool exists (nano::qualified_root const &) const;
-	bool exists (nano::block_hash const &) const;
+	bool contains (nano::qualified_root const &) const;
+	bool contains (nano::block_hash const &) const;
 
 	nano::container_info container_info () const;
 
@@ -44,17 +39,17 @@ private:
 	// clang-format off
 	class tag_hash {};
 	class tag_root {};
-	class tag_sequence {};
+	class tag_sequenced {};
 
-	using ordered_recent_confirmations = boost::multi_index_container<entry_t,
+	using ordered_entries = boost::multi_index_container<entry_t,
 	mi::indexed_by<
-		mi::sequenced<mi::tag<tag_sequence>>,
+		mi::sequenced<mi::tag<tag_sequenced>>,
 		mi::hashed_unique<mi::tag<tag_root>,
 			mi::member<entry_t, nano::qualified_root, &entry_t::first>>,
 		mi::hashed_unique<mi::tag<tag_hash>,
 			mi::member<entry_t, nano::block_hash, &entry_t::second>>>>;
 	// clang-format on
-	ordered_recent_confirmations confirmed;
+	ordered_entries entries;
 
 	std::size_t const max_size;
 

nano/node/repcrawler.cpp

@@ -301,7 +301,7 @@ auto nano::rep_crawler::prepare_query_target () const -> hash_root_t
 	for (auto const & block : random_blocks)
 	{
 		// Avoid blocks that could still have live votes coming in
-		if (active.recently_confirmed.exists (block->hash ()))
+		if (active.recently_confirmed.contains (block->hash ()))
 		{
 			continue;
 		}

nano/node/vote_router.cpp

@@ -86,7 +86,7 @@ std::unordered_map<nano::block_hash, nano::vote_code> nano::vote_router::vote (s
 			}
 			else
 			{
-				if (recently_confirmed.exists (hash))
+				if (recently_confirmed.contains (hash))
 				{
 					results[hash] = nano::vote_code::late;
 				}