Benchmark inactive_vote_cache in slow_tests (#3885)

* Provide stat type & dir to string methods

* Add `rate_observer` test utility

* Add vote cache stat tracking

* Convenience test utility functions

* Add vote_cache performance test

* Formatting & Comments

* Stat naming

nano/lib/stats.cpp

@@ -462,6 +462,11 @@ void nano::stat::clear ()
 std::string nano::stat::type_to_string (uint32_t key)
 {
 	auto type = static_cast<stat::type> (key >> 16 & 0x000000ff);
+	return type_to_string (type);
+}
+
+std::string nano::stat::type_to_string (stat::type type)
+{
 	std::string res;
 	switch (type)
 	{
@@ -537,6 +542,9 @@ std::string nano::stat::type_to_string (uint32_t key)
 		case nano::stat::type::vote_generator:
 			res = "vote_generator";
 			break;
+		case nano::stat::type::vote_cache:
+			res = "vote_cache";
+			break;
 	}
 	return res;
 }
@@ -705,6 +713,9 @@ std::string nano::stat::detail_to_string (stat::detail detail)
 		case nano::stat::detail::vote_new:
 			res = "vote_new";
 			break;
+		case nano::stat::detail::vote_processed:
+			res = "vote_processed";
+			break;
 		case nano::stat::detail::vote_cached:
 			res = "vote_cached";
 			break;
@@ -939,6 +950,11 @@ std::string nano::stat::detail_to_string (uint32_t key)
 std::string nano::stat::dir_to_string (uint32_t key)
 {
 	auto dir = static_cast<stat::dir> (key & 0x000000ff);
+	return dir_to_string (dir);
+}
+
+std::string nano::stat::dir_to_string (dir dir)
+{
 	std::string res;
 	switch (dir)
 	{

nano/lib/stats.hpp

@@ -242,7 +242,8 @@ public:
 		requests,
 		filter,
 		telemetry,
-		vote_generator
+		vote_generator,
+		vote_cache
 	};
 
 	/** Optional detail type */
@@ -317,6 +318,7 @@ public:
 
 		// election specific
 		vote_new,
+		vote_processed,
 		vote_cached,
 		late_block,
 		late_block_seconds,
@@ -595,9 +597,15 @@ public:
 	/** Returns a new JSON log sink */
 	std::unique_ptr<stat_log_sink> log_sink_json () const;
 
+	/** Returns string representation of type */
+	static std::string type_to_string (stat::type type);
+
 	/** Returns string representation of detail */
 	static std::string detail_to_string (stat::detail detail);
 
+	/** Returns string representation of dir */
+	static std::string dir_to_string (stat::dir detail);
+
 	/** Stop stats being output */
 	void stop ();
 

nano/node/active_transactions.cpp

@@ -716,6 +716,8 @@ void nano::active_transactions::add_inactive_votes_cache (nano::unique_lock<nano
 				inactive_by_arrival.erase (inactive_by_arrival.begin ());
 			}
 		}
+
+		node.stats.inc (nano::stat::type::vote_cache, nano::stat::detail::vote_processed);
 	}
 }
 
@@ -838,6 +840,13 @@ std::size_t nano::active_transactions::election_winner_details_size ()
 	return election_winner_details.size ();
 }
 
+void nano::active_transactions::clear ()
+{
+	nano::lock_guard<nano::mutex> guard{ mutex };
+	blocks.clear ();
+	roots.clear ();
+}
+
 std::unique_ptr<nano::container_info_component> nano::collect_container_info (active_transactions & active_transactions, std::string const & name)
 {
 	std::size_t roots_count;
@@ -948,4 +957,4 @@ std::size_t nano::recently_cemented_cache::size () const
 {
 	nano::lock_guard<nano::mutex> guard{ mutex };
 	return cemented.size ();
-}
+}

nano/node/active_transactions.hpp

@@ -245,6 +245,9 @@ private:
 	friend class election_scheduler;
 	friend std::unique_ptr<container_info_component> collect_container_info (active_transactions &, std::string const &);
 
+public: // Tests
+	void clear ();
+
 	friend class active_transactions_vote_replays_Test;
 	friend class frontiers_confirmation_prioritize_frontiers_Test;
 	friend class frontiers_confirmation_prioritize_frontiers_max_optimistic_elections_Test;

nano/node/node.cpp

@@ -1270,7 +1270,7 @@ void nano::node::add_initial_peers ()
 	}
 }
 
-void nano::node::block_confirm (std::shared_ptr<nano::block> const & block_a)
+std::shared_ptr<nano::election> nano::node::block_confirm (std::shared_ptr<nano::block> const & block_a)
 {
 	scheduler.manual (block_a);
 	scheduler.flush ();
@@ -1278,7 +1278,9 @@ void nano::node::block_confirm (std::shared_ptr<nano::block> const & block_a)
 	if (election != nullptr)
 	{
 		election->transition_active ();
+		return election;
 	}
+	return {};
 }
 
 bool nano::node::block_confirmed (nano::block_hash const & hash_a)

nano/node/node.hpp

@@ -108,7 +108,11 @@ public:
 	boost::optional<uint64_t> work_generate_blocking (nano::work_version const, nano::root const &, uint64_t, boost::optional<nano::account> const & = boost::none);
 	void work_generate (nano::work_version const, nano::root const &, uint64_t, std::function<void (boost::optional<uint64_t>)>, boost::optional<nano::account> const & = boost::none, bool const = false);
 	void add_initial_peers ();
-	void block_confirm (std::shared_ptr<nano::block> const &);
+	/*
+	 * Starts an election for the block, DOES NOT confirm it
+	 * TODO: Rename to `start_election`
+	 */
+	std::shared_ptr<nano::election> block_confirm (std::shared_ptr<nano::block> const &);
 	bool block_confirmed (nano::block_hash const &);
 	bool block_confirmed_or_being_confirmed (nano::transaction const &, nano::block_hash const &);
 	void do_rpc_callback (boost::asio::ip::tcp::resolver::iterator i_a, std::string const &, uint16_t, std::shared_ptr<std::string> const &, std::shared_ptr<std::string> const &, std::shared_ptr<boost::asio::ip::tcp::resolver> const &);

nano/slow_test/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_executable(slow_test entry.cpp node.cpp vote_processor.cpp)
+add_executable(slow_test entry.cpp node.cpp vote_cache.cpp vote_processor.cpp)
 
 target_link_libraries(
   slow_test

nano/slow_test/vote_cache.cpp

@@ -0,0 +1,257 @@
+#include <nano/test_common/rate_observer.hpp>
+#include <nano/test_common/system.hpp>
+#include <nano/test_common/testutil.hpp>
+
+#include <gtest/gtest.h>
+
+#include <functional>
+#include <thread>
+
+using namespace std::chrono_literals;
+
+namespace
+{
+nano::keypair setup_rep (nano::test::system & system, nano::node & node, nano::uint128_t amount)
+{
+	auto latest = node.latest (nano::dev::genesis_key.pub);
+	auto balance = node.balance (nano::dev::genesis_key.pub);
+
+	nano::keypair key;
+	nano::block_builder builder;
+
+	auto send = builder
+				.send ()
+				.previous (latest)
+				.destination (key.pub)
+				.balance (balance - amount)
+				.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
+				.work (*system.work.generate (latest))
+				.build_shared ();
+
+	auto open = builder
+				.open ()
+				.source (send->hash ())
+				.representative (key.pub)
+				.account (key.pub)
+				.sign (key.prv, key.pub)
+				.work (*system.work.generate (key.pub))
+				.build_shared ();
+
+	EXPECT_TRUE (nano::test::process (node, { send, open }));
+	EXPECT_TRUE (nano::test::confirm (node, { send, open }));
+	// TODO: Create `EXPECT_TIMELY` macro to remove this boilerplate
+	system.poll_until_true (3s, [&node, &send, &open] () {
+		return nano::test::confirmed (node, { send, open });
+	});
+	EXPECT_TRUE (nano::test::confirmed (node, { send, open }));
+
+	return key;
+}
+
+std::vector<nano::keypair> setup_reps (nano::test::system & system, nano::node & node, int count)
+{
+	const nano::uint128_t weight = nano::Gxrb_ratio * 1000;
+	std::vector<nano::keypair> reps;
+	for (int n = 0; n < count; ++n)
+	{
+		reps.push_back (setup_rep (system, node, weight));
+	}
+	return reps;
+}
+
+/*
+ * Creates `count` number of unconfirmed blocks with their dependencies confirmed, each directly sent from genesis
+ */
+std::vector<std::shared_ptr<nano::block>> setup_blocks (nano::test::system & system, nano::node & node, int count)
+{
+	auto latest = node.latest (nano::dev::genesis_key.pub);
+	auto balance = node.balance (nano::dev::genesis_key.pub);
+
+	std::vector<std::shared_ptr<nano::block>> sends;
+	std::vector<std::shared_ptr<nano::block>> receives;
+	for (int n = 0; n < count; ++n)
+	{
+		if (n % 10000 == 0)
+			std::cout << "setup_blocks: " << n << std::endl;
+
+		nano::keypair key;
+		nano::block_builder builder;
+
+		balance -= 1;
+		auto send = builder
+					.send ()
+					.previous (latest)
+					.destination (key.pub)
+					.balance (balance)
+					.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
+					.work (*system.work.generate (latest))
+					.build_shared ();
+
+		auto open = builder
+					.open ()
+					.source (send->hash ())
+					.representative (key.pub)
+					.account (key.pub)
+					.sign (key.prv, key.pub)
+					.work (*system.work.generate (key.pub))
+					.build_shared ();
+
+		latest = send->hash ();
+
+		sends.push_back (send);
+		receives.push_back (open);
+
+		EXPECT_TRUE (nano::test::process (node, { send, open }));
+	}
+
+	// Confirm whole genesis chain at once
+	EXPECT_TRUE (nano::test::confirm (node, { sends.back () }));
+
+	// TODO: Create `EXPECT_TIMELY` macro to remove this boilerplate
+	system.poll_until_true (60s, [&node, &sends] () {
+		return nano::test::confirmed (node, { sends });
+	});
+	EXPECT_TRUE (nano::test::confirmed (node, { sends }));
+
+	return receives;
+}
+
+void run_parallel (int thread_count, std::function<void (int)> func)
+{
+	std::vector<std::thread> threads;
+	for (int n = 0; n < thread_count; ++n)
+	{
+		threads.emplace_back ([func, n] () {
+			func (n);
+		});
+	}
+	for (auto & thread : threads)
+	{
+		thread.join ();
+	}
+}
+}
+
+TEST (vote_cache, perf_singlethreaded)
+{
+	nano::test::system system;
+	nano::node_flags flags;
+	flags.inactive_votes_cache_size = 5000; // Keep it below block count size so it is forced to constantly evict stale entries
+	nano::node_config config = system.default_config ();
+	config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
+	auto & node = *system.add_node (config, flags);
+
+	const int rep_count = 50;
+	const int block_count = 20000;
+	const int vote_count = 100000;
+	const int single_vote_size = 7;
+	const int single_vote_reps = 7;
+
+	auto reps = setup_reps (system, node, rep_count);
+	auto blocks = setup_blocks (system, node, block_count);
+
+	std::cout << "preparation done" << std::endl;
+
+	// Start monitoring rate of blocks processed by vote cache
+	nano::test::rate_observer rate;
+	rate.observe (node, nano::stat::type::vote_cache, nano::stat::detail::vote_processed, nano::stat::dir::in);
+	rate.background_print (3s);
+
+	// Ensure votes are not inserted into active elections
+	node.active.clear ();
+
+	int block_idx = 0;
+	int rep_idx = 0;
+	std::vector<nano::block_hash> hashes;
+	for (int n = 0; n < vote_count; ++n)
+	{
+		// Fill block hashes for this vote
+		hashes.clear ();
+		for (int i = 0; i < single_vote_size; ++i)
+		{
+			block_idx = (block_idx + 1151) % blocks.size ();
+			hashes.push_back (blocks[block_idx]->hash ());
+		}
+
+		for (int i = 0; i < single_vote_reps; ++i)
+		{
+			rep_idx = (rep_idx + 13) % reps.size ();
+			auto vote = nano::test::make_vote (reps[rep_idx], hashes);
+
+			// Process the vote
+			node.active.vote (vote);
+		}
+	}
+
+	std::cout << "total votes processed: " << node.stats.count (nano::stat::type::vote_cache, nano::stat::detail::vote_processed, nano::stat::dir::in) << std::endl;
+
+	// Ensure we processed all the votes
+	ASSERT_EQ (node.stats.count (nano::stat::type::vote_cache, nano::stat::detail::vote_processed, nano::stat::dir::in), vote_count * single_vote_size * single_vote_reps);
+
+	// Ensure vote cache size is at max capacity
+	ASSERT_EQ (node.active.inactive_votes_cache_size (), flags.inactive_votes_cache_size);
+}
+
+TEST (vote_cache, perf_multithreaded)
+{
+	nano::test::system system;
+	nano::node_flags flags;
+	flags.inactive_votes_cache_size = 5000; // Keep it below block count size so it is forced to constantly evict stale entries
+	nano::node_config config = system.default_config ();
+	config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
+	auto & node = *system.add_node (config, flags);
+
+	const int thread_count = 12;
+	const int rep_count = 50;
+	const int block_count = 20000;
+	const int vote_count = 200000 / thread_count;
+	const int single_vote_size = 7;
+	const int single_vote_reps = 7;
+
+	auto reps = setup_reps (system, node, rep_count);
+	auto blocks = setup_blocks (system, node, block_count);
+
+	std::cout << "preparation done" << std::endl;
+
+	// Start monitoring rate of blocks processed by vote cache
+	nano::test::rate_observer rate;
+	rate.observe (node, nano::stat::type::vote_cache, nano::stat::detail::vote_processed, nano::stat::dir::in);
+	rate.background_print (3s);
+
+	// Ensure our generated votes go to inactive vote cache instead of active elections
+	node.active.clear ();
+
+	run_parallel (thread_count, [&node, &reps, &blocks] (int index) {
+		int block_idx = index;
+		int rep_idx = index;
+		std::vector<nano::block_hash> hashes;
+
+		// Each iteration generates vote with `single_vote_size` hashes in it
+		// and that vote is then independently signed by `single_vote_reps` representatives
+		// So total votes per thread is `vote_count` * `single_vote_reps`
+		for (int n = 0; n < vote_count; ++n)
+		{
+			// Fill block hashes for this vote
+			hashes.clear ();
+			for (int i = 0; i < single_vote_size; ++i)
+			{
+				block_idx = (block_idx + 1151) % blocks.size ();
+				hashes.push_back (blocks[block_idx]->hash ());
+			}
+
+			for (int i = 0; i < single_vote_reps; ++i)
+			{
+				rep_idx = (rep_idx + 13) % reps.size ();
+				auto vote = nano::test::make_vote (reps[rep_idx], hashes);
+
+				// Process the vote
+				node.active.vote (vote);
+			}
+		}
+	});
+
+	std::cout << "total votes processed: " << node.stats.count (nano::stat::type::vote_cache, nano::stat::detail::vote_processed, nano::stat::dir::in) << std::endl;
+
+	// Ensure vote cache size is at max capacity
+	ASSERT_EQ (node.active.inactive_votes_cache_size (), flags.inactive_votes_cache_size);
+}

nano/test_common/CMakeLists.txt

@@ -4,6 +4,8 @@ add_library(
   ledger.cpp
   network.hpp
   network.cpp
+  rate_observer.cpp
+  rate_observer.hpp
   system.hpp
   system.cpp
   telemetry.hpp

nano/test_common/rate_observer.cpp

@@ -0,0 +1,99 @@
+#include <nano/lib/stats.hpp>
+#include <nano/node/node.hpp>
+#include <nano/test_common/rate_observer.hpp>
+/*
+ * rate_observer::counter
+ */
+
+std::pair<uint64_t, std::chrono::milliseconds> nano::test::rate_observer::counter::observe ()
+{
+	auto now = std::chrono::system_clock::now ();
+	if (last_observation.time_since_epoch ().count () > 0)
+	{
+		auto time_delta = std::chrono::duration_cast<std::chrono::milliseconds> (now - last_observation);
+		last_observation = now;
+		return { count (), time_delta };
+	}
+	else
+	{
+		last_observation = now;
+		return { 0, std::chrono::milliseconds{ 0 } };
+	}
+}
+
+nano::test::rate_observer::stat_counter::stat_counter (nano::stat & stats_a, nano::stat::type type_a, nano::stat::detail detail_a, nano::stat::dir dir_a) :
+	stats{ stats_a },
+	type{ type_a },
+	detail{ detail_a },
+	dir{ dir_a }
+{
+}
+
+/*
+ * rate_observer::stat_counter
+ */
+
+uint64_t nano::test::rate_observer::stat_counter::count ()
+{
+	uint64_t cnt = stats.count (type, detail, dir);
+	uint64_t delta = cnt - last_count;
+	last_count = cnt;
+	return delta;
+}
+
+std::string nano::test::rate_observer::stat_counter::name ()
+{
+	return nano::stat::type_to_string (type) + "::" + nano::stat::detail_to_string (detail) + "::" + nano::stat::dir_to_string (dir);
+}
+
+/*
+ * rate_observer
+ */
+
+nano::test::rate_observer::~rate_observer ()
+{
+	if (!stopped.exchange (true))
+	{
+		if (thread.joinable ())
+		{
+			thread.join ();
+		}
+	}
+}
+
+void nano::test::rate_observer::background_print (std::chrono::seconds interval)
+{
+	release_assert (!thread.joinable ());
+	thread = std::thread{ [this, interval] () { background_print_impl (interval); } };
+}
+
+void nano::test::rate_observer::background_print_impl (std::chrono::seconds interval)
+{
+	while (!stopped)
+	{
+		print_once ();
+
+		std::this_thread::sleep_for (interval);
+	}
+}
+
+void nano::test::rate_observer::print_once ()
+{
+	for (auto & counter : counters)
+	{
+		const auto observation = counter->observe ();
+
+		// Convert delta milliseconds to seconds (double precision) and then divide the counter delta to get rate per second
+		auto per_sec = observation.first / (observation.second.count () / 1000.0);
+
+		std::cout << "rate of '" << counter->name () << "': "
+				  << std::setw (12) << std::setprecision (2) << std::fixed << per_sec << " /s"
+				  << std::endl;
+	}
+}
+
+void nano::test::rate_observer::observe (nano::node & node, nano::stat::type type, nano::stat::detail detail, nano::stat::dir dir)
+{
+	auto counter = std::make_shared<stat_counter> (node.stats, type, detail, dir);
+	counters.push_back (counter);
+}

nano/test_common/rate_observer.hpp

@@ -0,0 +1,78 @@
+#pragma once
+
+#include <nano/lib/stats.hpp>
+
+#include <chrono>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+namespace nano::test
+{
+class rate_observer
+{
+public:
+	/*
+	 * Base class used as a base to build counters
+	 */
+	class counter
+	{
+	public:
+		/*
+		 * Calculate count and time delta since last call
+		 */
+		std::pair<uint64_t, std::chrono::milliseconds> observe ();
+
+		virtual uint64_t count () = 0;
+		virtual std::string name () = 0;
+
+	private:
+		std::chrono::system_clock::time_point last_observation{};
+	};
+
+	/*
+	 * Counter that uses node stat container to provide info about rate
+	 */
+	class stat_counter final : public counter
+	{
+	public:
+		explicit stat_counter (nano::stat & stats, nano::stat::type type, nano::stat::detail detail, nano::stat::dir dir);
+
+		uint64_t count () override;
+		std::string name () override;
+
+	private:
+		const nano::stat::type type;
+		const nano::stat::detail detail;
+		const nano::stat::dir dir;
+
+		uint64_t last_count{ 0 };
+
+		nano::stat & stats;
+	};
+
+public:
+	rate_observer () = default;
+	~rate_observer ();
+
+	/*
+	 * Periodically prints all observed rates onto the standard output
+	 */
+	void background_print (std::chrono::seconds interval);
+
+	/*
+	 * Starts observing a particular node stat from stat container
+	 */
+	void observe (nano::node &, nano::stat::type type, nano::stat::detail detail, nano::stat::dir dir);
+
+private:
+	void background_print_impl (std::chrono::seconds interval);
+	void print_once ();
+
+	std::vector<std::shared_ptr<counter>> counters;
+
+	std::atomic<bool> stopped{ false };
+	std::thread thread;
+};
+}

nano/test_common/system.cpp

@@ -23,9 +23,15 @@ std::string nano::error_system_messages::message (int ev) const
 	return "Invalid error code";
 }
 
+nano::node & nano::test::system::node (std::size_t index) const
+{
+	debug_assert (index < nodes.size ());
+	return *nodes[index];
+}
+
 std::shared_ptr<nano::node> nano::test::system::add_node (nano::node_flags node_flags_a, nano::transport::transport_type type_a)
 {
-	return add_node (nano::node_config (nano::test::get_available_port (), logging), node_flags_a, type_a);
+	return add_node (default_config (), node_flags_a, type_a);
 }
 
 /** Returns the node added. */
@@ -138,8 +144,7 @@ nano::test::system::system (uint16_t count_a, nano::transport::transport_type ty
 	nodes.reserve (count_a);
 	for (uint16_t i (0); i < count_a; ++i)
 	{
-		nano::node_config config (nano::test::get_available_port (), logging);
+		add_node (default_config (), flags_a, type_a);
-		add_node (config, flags_a, type_a);
 	}
 }
 
@@ -583,6 +588,12 @@ void nano::test::system::stop ()
 	work.stop ();
 }
 
+nano::node_config nano::test::system::default_config ()
+{
+	nano::node_config config{ nano::test::get_available_port (), logging };
+	return config;
+}
+
 uint16_t nano::test::get_available_port ()
 {
 	// Maximum possible sockets which may feasibly be used in 1 test

nano/test_common/system.hpp

@@ -41,16 +41,25 @@ namespace test
 		/** Generate work with difficulty between \p min_difficulty_a (inclusive) and \p max_difficulty_a (exclusive) */
 		uint64_t work_generate_limited (nano::block_hash const & root_a, uint64_t min_difficulty_a, uint64_t max_difficulty_a);
 		/**
-	 * Polls, sleep if there's no work to be done (default 50ms), then check the deadline
+		 * Polls, sleep if there's no work to be done (default 50ms), then check the deadline
-	 * @returns 0 or nano::deadline_expired
+		 * @returns 0 or nano::deadline_expired
 		 */
 		std::error_code poll (std::chrono::nanoseconds const & sleep_time = std::chrono::milliseconds (50));
 		std::error_code poll_until_true (std::chrono::nanoseconds deadline, std::function<bool ()>);
 		void delay_ms (std::chrono::milliseconds const & delay);
 		void stop ();
 		void deadline_set (std::chrono::duration<double, std::nano> const & delta);
+		/*
+		 * Convenience function to get a reference to a node at given index. Does bound checking.
+		 */
+		nano::node & node (std::size_t index) const;
 		std::shared_ptr<nano::node> add_node (nano::node_flags = nano::node_flags (), nano::transport::transport_type = nano::transport::transport_type::tcp);
 		std::shared_ptr<nano::node> add_node (nano::node_config const &, nano::node_flags = nano::node_flags (), nano::transport::transport_type = nano::transport::transport_type::tcp);
+		/*
+		 * Returns default config for node running in test environment
+		 */
+		nano::node_config default_config ();
+
 		boost::asio::io_context io_ctx;
 		std::vector<std::shared_ptr<nano::node>> nodes;
 		nano::logging logging;

nano/test_common/testutil.cpp

@@ -36,3 +36,81 @@ void nano::test::wait_peer_connections (nano::test::system & system_a)
 	wait_peer_count (true);
 	wait_peer_count (false);
 }
+
+bool nano::test::process (nano::node & node, std::vector<std::shared_ptr<nano::block>> blocks)
+{
+	for (auto & block : blocks)
+	{
+		auto result = node.process (*block);
+		if (result.code != nano::process_result::progress)
+		{
+			return false;
+		}
+	}
+	return true;
+}
+
+bool nano::test::confirm (nano::node & node, std::vector<nano::block_hash> hashes)
+{
+	// Finish processing all blocks
+	node.block_processor.flush ();
+	for (auto & hash : hashes)
+	{
+		if (node.block_confirmed (hash))
+		{
+			continue;
+		}
+
+		auto disk_block = node.block (hash);
+		// A sideband is required to start an election
+		release_assert (disk_block != nullptr);
+		release_assert (disk_block->has_sideband ());
+		// This only starts election
+		auto election = node.block_confirm (disk_block);
+		if (election == nullptr)
+		{
+			return false;
+		}
+		// Here we actually confirm the block
+		election->force_confirm ();
+	}
+	return true;
+}
+
+bool nano::test::confirm (nano::node & node, std::vector<std::shared_ptr<nano::block>> blocks)
+{
+	std::vector<nano::block_hash> hashes;
+	std::transform (blocks.begin (), blocks.end (), std::back_inserter (hashes), [] (auto & block) { return block->hash (); });
+	return confirm (node, hashes);
+}
+
+bool nano::test::confirmed (nano::node & node, std::vector<nano::block_hash> hashes)
+{
+	for (auto & hash : hashes)
+	{
+		if (!node.block_confirmed (hash))
+		{
+			return false;
+		}
+	}
+	return true;
+}
+
+bool nano::test::confirmed (nano::node & node, std::vector<std::shared_ptr<nano::block>> blocks)
+{
+	std::vector<nano::block_hash> hashes;
+	std::transform (blocks.begin (), blocks.end (), std::back_inserter (hashes), [] (auto & block) { return block->hash (); });
+	return confirmed (node, hashes);
+}
+
+std::shared_ptr<nano::vote> nano::test::make_vote (nano::keypair key, std::vector<nano::block_hash> hashes, uint64_t timestamp, uint8_t duration)
+{
+	return std::make_shared<nano::vote> (key.pub, key.prv, timestamp, duration, hashes);
+}
+
+std::shared_ptr<nano::vote> nano::test::make_vote (nano::keypair key, std::vector<std::shared_ptr<nano::block>> blocks, uint64_t timestamp, uint8_t duration)
+{
+	std::vector<nano::block_hash> hashes;
+	std::transform (blocks.begin (), blocks.end (), std::back_inserter (hashes), [] (auto & block) { return block->hash (); });
+	return make_vote (key, hashes, timestamp, duration);
+}

nano/test_common/testutil.hpp

@@ -45,12 +45,15 @@
 /* Convenience globals for gtest projects */
 namespace nano
 {
+class node;
 using uint128_t = boost::multiprecision::uint128_t;
 class keypair;
 class public_key;
 class block_hash;
 class telemetry_data;
 class network_params;
+class vote;
+class block;
 
 extern nano::uint128_t const & genesis_amount;
 
@@ -205,5 +208,41 @@ namespace test
 	};
 
 	void wait_peer_connections (nano::test::system &);
+
+	/**
+		Convenience function to call `node::process` function for multiple blocks at once.
+		@return true if all blocks were successfully processed and inserted into ledger
+	 */
+	bool process (nano::node & node, std::vector<std::shared_ptr<nano::block>> blocks);
+	/*
+	 * Convenience function to confirm a list of blocks
+	 * The actual confirmation will happen asynchronously, check for that with `nano::test::confirmed (..)` function
+	 * @return true if successfully scheduled blocks to be confirmed
+	 */
+	bool confirm (nano::node & node, std::vector<std::shared_ptr<nano::block>> blocks);
+	/*
+	 * Convenience function to confirm a list of hashes
+	 * The actual confirmation will happen asynchronously, check for that with `nano::test::confirmed (..)` function
+	 * @return true if successfully scheduled blocks to be confirmed
+	 */
+	bool confirm (nano::node & node, std::vector<nano::block_hash> hashes);
+	/*
+	 * Convenience function to check whether a list of blocks is confirmed.
+	 * @return true if all blocks are confirmed, false otherwise
+	 */
+	bool confirmed (nano::node & node, std::vector<std::shared_ptr<nano::block>> blocks);
+	/*
+	 * Convenience function to check whether a list of hashes is confirmed.
+	 * @return true if all blocks are confirmed, false otherwise
+	 */
+	bool confirmed (nano::node & node, std::vector<nano::block_hash> hashes);
+	/*
+	 * Convenience function to create a new vote from list of blocks
+	 */
+	std::shared_ptr<nano::vote> make_vote (nano::keypair key, std::vector<std::shared_ptr<nano::block>> blocks, uint64_t timestamp = 0, uint8_t duration = 0);
+	/*
+	 * Convenience function to create a new vote from list of block hashes
+	 */
+	std::shared_ptr<nano::vote> make_vote (nano::keypair key, std::vector<nano::block_hash> hashes, uint64_t timestamp = 0, uint8_t duration = 0);
 }
-}
+}