Set maximum unchecked blocks in config (#4342)

Set maximum unchecked blocks in config

nano/core_test/block_store.cpp

@@ -436,7 +436,8 @@ TEST (block_store, empty_bootstrap)
 {
 	nano::test::system system{};
 	nano::logger logger;
-	nano::unchecked_map unchecked{ system.stats, false };
+	unsigned max_unchecked_blocks = 65536;
+	nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false };
 	size_t count = 0;
 	unchecked.for_each ([&count] (nano::unchecked_key const & key, nano::unchecked_info const & info) {
 		++count;

nano/core_test/toml.cpp

@@ -193,6 +193,7 @@ TEST (toml, daemon_config_deserialize_defaults)
 	ASSERT_EQ (conf.node.work_peers, defaults.node.work_peers);
 	ASSERT_EQ (conf.node.work_threads, defaults.node.work_threads);
 	ASSERT_EQ (conf.node.max_queued_requests, defaults.node.max_queued_requests);
+	ASSERT_EQ (conf.node.max_unchecked_blocks, defaults.node.max_unchecked_blocks);
 	ASSERT_EQ (conf.node.backlog_scan_batch_size, defaults.node.backlog_scan_batch_size);
 	ASSERT_EQ (conf.node.backlog_scan_frequency, defaults.node.backlog_scan_frequency);
 
@@ -421,6 +422,7 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	work_threads = 999
 	max_work_generate_multiplier = 1.0
 	max_queued_requests = 999
+	max_unchecked_blocks = 999
 	frontiers_confirmation = "always"
 	backlog_scan_batch_size = 999
 	backlog_scan_frequency = 999
@@ -585,6 +587,7 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	ASSERT_NE (conf.node.external_port, defaults.node.external_port);
 	ASSERT_NE (conf.node.io_threads, defaults.node.io_threads);
 	ASSERT_NE (conf.node.max_work_generate_multiplier, defaults.node.max_work_generate_multiplier);
+	ASSERT_NE (conf.node.max_unchecked_blocks, defaults.node.max_unchecked_blocks);
 	ASSERT_NE (conf.node.frontiers_confirmation, defaults.node.frontiers_confirmation);
 	ASSERT_NE (conf.node.network_threads, defaults.node.network_threads);
 	ASSERT_NE (conf.node.background_threads, defaults.node.background_threads);

nano/core_test/unchecked_map.cpp

@@ -14,11 +14,13 @@ using namespace std::chrono_literals;
 
 namespace
 {
+unsigned max_unchecked_blocks = 65536;
+
 class context
 {
 public:
 	context () :
-		unchecked{ stats, false }
+		unchecked{ max_unchecked_blocks, stats, false }
 	{
 	}
 	nano::stats stats;
@@ -54,7 +56,7 @@ TEST (unchecked_map, put_one)
 TEST (block_store, one_bootstrap)
 {
 	nano::test::system system{};
-	nano::unchecked_map unchecked{ system.stats, false };
+	nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false };
 	nano::block_builder builder;
 	auto block1 = builder
 				  .send ()
@@ -87,7 +89,7 @@ TEST (block_store, one_bootstrap)
 TEST (unchecked, simple)
 {
 	nano::test::system system{};
-	nano::unchecked_map unchecked{ system.stats, false };
+	nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false };
 	nano::block_builder builder;
 	auto block = builder
 				 .send ()
@@ -128,7 +130,7 @@ TEST (unchecked, multiple)
 		// Don't test this in rocksdb mode
 		GTEST_SKIP ();
 	}
-	nano::unchecked_map unchecked{ system.stats, false };
+	nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false };
 	nano::block_builder builder;
 	auto block = builder
 				 .send ()
@@ -158,7 +160,7 @@ TEST (unchecked, multiple)
 TEST (unchecked, double_put)
 {
 	nano::test::system system{};
-	nano::unchecked_map unchecked{ system.stats, false };
+	nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false };
 	nano::block_builder builder;
 	auto block = builder
 				 .send ()
@@ -189,7 +191,7 @@ TEST (unchecked, double_put)
 TEST (unchecked, multiple_get)
 {
 	nano::test::system system{};
-	nano::unchecked_map unchecked{ system.stats, false };
+	nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false };
 	// Instantiates three blocks
 	nano::block_builder builder;
 	auto block1 = builder

nano/node/node.cpp

@@ -153,7 +153,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, std::filesystem::path cons
 	distributed_work (*this),
 	store_impl (nano::make_store (logger, application_path_a, network_params.ledger, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_config, config_a.backup_before_upgrade)),
 	store (*store_impl),
-	unchecked{ stats, flags.disable_block_processor_unchecked_deletion },
+	unchecked{ config.max_unchecked_blocks, stats, flags.disable_block_processor_unchecked_deletion },
 	wallets_store_impl (std::make_unique<nano::mdb_wallets_store> (application_path_a / "wallets.ldb", config_a.lmdb_config)),
 	wallets_store (*wallets_store_impl),
 	gap_cache (*this),

nano/node/nodeconfig.cpp

@@ -128,6 +128,7 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
 	toml.put ("max_work_generate_multiplier", max_work_generate_multiplier, "Maximum allowed difficulty multiplier for work generation.\ntype:double,[1..]");
 	toml.put ("frontiers_confirmation", serialize_frontiers_confirmation (frontiers_confirmation), "Mode controlling frontier confirmation rate.\ntype:string,{auto,always,disabled}");
 	toml.put ("max_queued_requests", max_queued_requests, "Limit for number of queued confirmation requests for one channel, after which new requests are dropped until the queue drops below this value.\ntype:uint32");
+	toml.put ("max_unchecked_blocks", max_unchecked_blocks, "Maximum number of unchecked blocks to store in memory. Defaults to 65536. \ntype:uint64,[0..]");
 	toml.put ("rep_crawler_weight_minimum", rep_crawler_weight_minimum.to_string_dec (), "Rep crawler minimum weight, if this is less than minimum principal weight then this is taken as the minimum weight a rep must have to be tracked. If you want to track all reps set this to 0. If you do not want this to influence anything then set it to max value. This is only useful for debugging or for people who really know what they are doing.\ntype:string,amount,raw");
 	toml.put ("backlog_scan_batch_size", backlog_scan_batch_size, "Number of accounts per second to process when doing backlog population scan. Increasing this value will help unconfirmed frontiers get into election prioritization queue faster, however it will also increase resource usage. \ntype:uint");
 	toml.put ("backlog_scan_frequency", backlog_scan_frequency, "Backlog scan divides the scan into smaller batches, number of which is controlled by this value. Higher frequency helps to utilize resources more uniformly, however it also introduces more overhead. The resulting number of accounts per single batch is `backlog_scan_batch_size / backlog_scan_frequency` \ntype:uint");
@@ -415,6 +416,8 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 
 		toml.get<uint32_t> ("max_queued_requests", max_queued_requests);
 
+		toml.get<unsigned> ("max_unchecked_blocks", max_unchecked_blocks);
+
 		auto rep_crawler_weight_minimum_l (rep_crawler_weight_minimum.to_string_dec ());
 		if (toml.has_key ("rep_crawler_weight_minimum"))
 		{

nano/node/nodeconfig.hpp

@@ -116,6 +116,7 @@ public:
 	bool backup_before_upgrade{ false };
 	double max_work_generate_multiplier{ 64. };
 	uint32_t max_queued_requests{ 512 };
+	unsigned max_unchecked_blocks{ 65536 };
 	std::chrono::seconds max_pruning_age{ !network_params.network.is_beta_network () ? std::chrono::seconds (24 * 60 * 60) : std::chrono::seconds (5 * 60) }; // 1 day; 5 minutes for beta network
 	uint64_t max_pruning_depth{ 0 };
 	nano::rocksdb_config rocksdb_config;

nano/node/unchecked_map.cpp

@@ -5,7 +5,8 @@
 #include <nano/lib/timer.hpp>
 #include <nano/node/unchecked_map.hpp>
 
-nano::unchecked_map::unchecked_map (nano::stats & stats, bool const & disable_delete) :
+nano::unchecked_map::unchecked_map (unsigned const max_unchecked_blocks, nano::stats & stats, bool const & disable_delete) :
+	max_unchecked_blocks{ max_unchecked_blocks },
 	stats{ stats },
 	disable_delete{ disable_delete },
 	thread{ [this] () { run (); } }
@@ -23,7 +24,8 @@ void nano::unchecked_map::put (nano::hash_or_account const & dependency, nano::u
 	nano::lock_guard<std::recursive_mutex> lock{ entries_mutex };
 	nano::unchecked_key key{ dependency, info.block->hash () };
 	entries.get<tag_root> ().insert ({ key, info });
-	if (entries.size () > mem_block_count_max)
+
+	if (entries.size () > max_unchecked_blocks)
 	{
 		entries.get<tag_sequenced> ().pop_front ();
 	}

nano/node/unchecked_map.hpp

@@ -22,7 +22,7 @@ class stats;
 class unchecked_map
 {
 public:
-	unchecked_map (nano::stats &, bool const & do_delete);
+	unchecked_map (unsigned const max_unchecked_blocks, nano::stats &, bool const & do_delete);
 	~unchecked_map ();
 
 	void put (nano::hash_or_account const & dependency, nano::unchecked_info const & info);
@@ -62,11 +62,10 @@ private:
 	nano::condition_variable condition;
 	nano::mutex mutex;
 	std::thread thread;
+	unsigned const max_unchecked_blocks;
 
 	void process_queries (decltype (buffer) const & back_buffer);
 
-	static std::size_t constexpr mem_block_count_max = 64 * 1024;
-
 private:
 	struct entry
 	{