Merge pull request #4689 from pwojcikdev/benchmarking

Benchmarking suite

.gitmodules

@@ -37,3 +37,6 @@
 [submodule "submodules/fmt"]
 	path = submodules/fmt
 	url = https://github.com/fmtlib/fmt.git
+[submodule "submodules/benchmark"]
+	path = submodules/benchmark
+	url = https://github.com/google/benchmark.git

CMakeLists.txt

@@ -763,10 +763,15 @@ if(NANO_TEST OR RAIBLOCKS_TEST)
     gtest PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
                      "${CMAKE_SOURCE_DIR}/submodules/gtest/googletest/include")
 
+  set(BENCHMARK_ENABLE_TESTING OFF)
+  add_subdirectory(submodules/benchmark)
+
   add_subdirectory(nano/test_common)
   add_subdirectory(nano/core_test)
   add_subdirectory(nano/rpc_test)
   add_subdirectory(nano/slow_test)
+  add_subdirectory(nano/benchmarks)
+
   add_custom_target(
     all_tests
     COMMAND echo "BATCH BUILDING TESTS"

nano/benchmarks/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_executable(benchmarks entry.cpp ledger.cpp stats.cpp)
+
+target_link_libraries(benchmarks test_common benchmark::benchmark)
+
+include_directories(${CMAKE_SOURCE_DIR}/submodules)

nano/benchmarks/entry.cpp

@@ -0,0 +1,41 @@
+#include <nano/lib/logging.hpp>
+
+#include <benchmark/benchmark.h>
+
+// Customized main, based on BENCHMARK_MAIN macro
+int main (int argc, char ** argv)
+{
+	nano::logger::initialize_dummy ();
+
+	benchmark::MaybeReenterWithoutASLR (argc, argv);
+	char arg0_default[] = "benchmark";
+	char * args_default = reinterpret_cast<char *> (arg0_default);
+	if (!argv)
+	{
+		argc = 1;
+		argv = &args_default;
+	}
+	::benchmark::Initialize (&argc, argv);
+	if (::benchmark::ReportUnrecognizedArguments (argc, argv))
+		return 1;
+	::benchmark::RunSpecifiedBenchmarks ();
+	::benchmark::Shutdown ();
+	return 0;
+}
+
+static void BM_StringCreation (benchmark::State & state)
+{
+	for (auto _ : state)
+		std::string empty_string;
+}
+// Register the function as a benchmark
+BENCHMARK (BM_StringCreation);
+
+// Define another benchmark
+static void BM_StringCopy (benchmark::State & state)
+{
+	std::string x = "hello";
+	for (auto _ : state)
+		std::string copy (x);
+}
+BENCHMARK (BM_StringCopy);

nano/benchmarks/ledger.cpp

@@ -0,0 +1,101 @@
+#include <nano/lib/logging.hpp>
+#include <nano/node/make_store.hpp>
+#include <nano/node/nodeconfig.hpp>
+#include <nano/secure/ledger.hpp>
+#include <nano/secure/ledger_set_any.hpp>
+#include <nano/secure/utility.hpp>
+#include <nano/store/account.hpp>
+
+#include <benchmark/benchmark.h>
+
+// Expects live ledger in default location
+// PLEASE NOTE: Make sure to purge disk cache between runs (`purge` command on macOS)
+static void BM_ledger_iterate_accounts (benchmark::State & state)
+{
+	nano::logger logger;
+	nano::stats stats{ logger };
+
+	// Use live ledger
+	nano::networks network = nano::networks::nano_live_network;
+	nano::network_params network_params{ network };
+	auto application_path = nano::working_path (network);
+
+	auto store_impl{ nano::make_store (logger, application_path, network_params.ledger) };
+	auto & store{ *store_impl };
+
+	if (store.init_error ())
+	{
+		state.SkipWithError ("Store initialization failed. Make sure ledger files are present in the default location.");
+	}
+
+	auto ledger_impl{ std::make_unique<nano::ledger> (store, network_params.ledger, stats, logger, nano::generate_cache_flags::all_disabled ()) };
+	auto & ledger{ *ledger_impl };
+
+	auto transaction = ledger.tx_begin_read ();
+	nano::account current{ 0 };
+	nano::account_info current_info;
+	auto it = ledger.any.account_begin (transaction);
+	auto end = ledger.any.account_end ();
+	for (auto _ : state)
+	{
+		if (it != end)
+		{
+			current = it->first;
+			current_info = it->second;
+			benchmark::DoNotOptimize (current);
+			benchmark::DoNotOptimize (current_info);
+
+			++it;
+		}
+		else
+		{
+			break;
+		}
+	}
+}
+BENCHMARK (BM_ledger_iterate_accounts);
+
+// Expects live ledger in default location
+// PLEASE NOTE: Make sure to purge disk cache between runs (`purge` command on macOS)
+static void BM_store_iterate_accounts (benchmark::State & state)
+{
+	nano::logger logger;
+	nano::stats stats{ logger };
+
+	// Use live ledger
+	nano::networks network = nano::networks::nano_live_network;
+	nano::network_params network_params{ network };
+	nano::node_flags flags;
+	auto application_path = nano::working_path (network);
+
+	auto store_impl{ nano::make_store (logger, application_path, network_params.ledger) };
+	auto & store{ *store_impl };
+
+	if (store.init_error ())
+	{
+		state.SkipWithError ("Store initialization failed. Make sure ledger files are present in the default location.");
+	}
+
+	auto transaction = store.tx_begin_read ();
+	nano::account current{ 0 };
+	nano::account_info current_info;
+	auto it = store.account.begin (transaction);
+	auto end = store.account.end (transaction);
+	for (auto _ : state)
+	{
+		if (it != end)
+		{
+			current = it->first;
+			current_info = it->second;
+			benchmark::DoNotOptimize (current);
+			benchmark::DoNotOptimize (current_info);
+
+			++it;
+		}
+		else
+		{
+			break;
+		}
+	}
+}
+BENCHMARK (BM_store_iterate_accounts);

nano/benchmarks/stats.cpp

@@ -0,0 +1,49 @@
+#include <nano/lib/enum_util.hpp>
+#include <nano/lib/logging.hpp>
+#include <nano/lib/stats.hpp>
+
+#include <random>
+
+#include <benchmark/benchmark.h>
+
+static void BM_stats_inc_single (benchmark::State & state)
+{
+	nano::logger logger;
+	nano::stats stats{ logger };
+
+	for (auto _ : state)
+	{
+		stats.inc (nano::stat::type::ledger, nano::stat::detail::open);
+	}
+}
+
+BENCHMARK (BM_stats_inc_single);
+BENCHMARK (BM_stats_inc_single)->Threads (10);
+
+static void BM_stats_inc_random (benchmark::State & state)
+{
+	nano::logger logger;
+	nano::stats stats{ logger };
+
+	auto random_subset = [] (auto elements, size_t count) -> std::vector<typename decltype (elements)::value_type> {
+		std::shuffle (elements.begin (), elements.end (), std::mt19937 (std::random_device () ()));
+		return { elements.begin (), elements.begin () + std::min (count, elements.size ()) };
+	};
+
+	auto stat_types = random_subset (nano::enum_util::values<nano::stat::type> (), state.range (0));
+	auto stat_details = random_subset (nano::enum_util::values<nano::stat::detail> (), state.range (1));
+
+	size_t type_index = 0;
+	size_t detail_index = 0;
+
+	for (auto _ : state)
+	{
+		stats.inc (stat_types[type_index], stat_details[detail_index]);
+
+		type_index = (type_index + 1) % stat_types.size ();
+		detail_index = (detail_index + 1) % stat_details.size ();
+	}
+}
+
+BENCHMARK (BM_stats_inc_random)->Args ({ 32, 32 });
+BENCHMARK (BM_stats_inc_random)->Args ({ 32, 32 })->Threads (10);

nano/lib/logging.cpp

@@ -117,6 +117,12 @@ void nano::logger::initialize_for_tests (nano::log_config fallback)
 	global_initialized = true;
 }
 
+void nano::logger::initialize_dummy ()
+{
+	initialize_common (nano::log_config::dummy_default (), std::nullopt);
+	global_initialized = true;
+}
+
 // Using std::cerr here, since logging may not be initialized yet
 void nano::logger::initialize_common (nano::log_config const & config, std::optional<std::filesystem::path> data_path)
 {
@@ -310,7 +316,7 @@ nano::log_config nano::log_config::cli_default ()
 {
 	log_config config{};
 	config.default_level = nano::log::level::critical;
-	config.console.colors = false; // to avoid printing warning about cerr and colors
+	config.console.colors = false;
 	config.console.to_cerr = true; // Use cerr to avoid interference with CLI output that goes to stdout
 	config.file.enable = false;
 	return config;
@@ -331,6 +337,16 @@ nano::log_config nano::log_config::tests_default ()
 	return config;
 }
 
+nano::log_config nano::log_config::dummy_default ()
+{
+	log_config config{};
+	config.default_level = nano::log::level::off;
+	config.flush_level = nano::log::level::off;
+	config.console.enable = false;
+	config.file.enable = false;
+	return config;
+}
+
 nano::log_config nano::log_config::sample_config ()
 {
 	log_config config{};

nano/lib/logging.hpp

@@ -137,6 +137,7 @@ public: // Predefined defaults
 	static log_config cli_default ();
 	static log_config daemon_default ();
 	static log_config tests_default ();
+	static log_config dummy_default (); // For empty logger
 	static log_config sample_config (); // For auto-generated sample config files
 
 private:
@@ -158,6 +159,7 @@ public:
 public:
 	static void initialize (nano::log_config fallback, std::optional<std::filesystem::path> data_path = std::nullopt, std::vector<std::string> const & config_overrides = {});
 	static void initialize_for_tests (nano::log_config fallback);
+	static void initialize_dummy (); // TODO: This is less than ideal, provide `nano::dummy_logger ()` instead
 	static void flush ();
 
 private:

nano/secure/generate_cache_flags.hpp

@@ -14,5 +14,17 @@ public:
 	bool block_count = true;
 
 	void enable_all ();
+
+public:
+	static generate_cache_flags all_disabled ()
+	{
+		generate_cache_flags flags;
+		flags.reps = false;
+		flags.cemented_count = false;
+		flags.unchecked_count = false;
+		flags.account_count = false;
+		flags.block_count = false;
+		return flags;
+	}
 };
 }

submodules/benchmark

@@ -0,0 +1 @@
+Subproject commit eddb0241389718a23a42db6af5f0164b6e0139af