Cleanup unchecked table after each bootstrap attempt (#1724)

* Cleanup unchecked table after each bootstrap attempt

* Add default unchecked_cutoff_time to deserialize_json

* Correct 4 hours

nano/core_test/node.cpp

@@ -2249,7 +2249,7 @@ TEST (node, peers)
 	node->stop ();
 }
 
-TEST (node, unchecked_cleaning)
+TEST (node, unchecked_cleanup)
 {
 	nano::system system (24000, 1);
 	nano::keypair key;
@@ -2257,21 +2257,21 @@ TEST (node, unchecked_cleaning)
 	auto open (std::make_shared<nano::state_block> (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, system.work.generate (key.pub)));
 	node.process_active (open);
 	node.block_processor.flush ();
-	node.unchecked_cutoff = std::chrono::seconds (2);
+	node.config.unchecked_cutoff_time = std::chrono::seconds (2);
 	{
 		auto transaction (node.store.tx_begin ());
 		auto unchecked_count (node.store.unchecked_count (transaction));
 		ASSERT_EQ (unchecked_count, 1);
 	}
 	std::this_thread::sleep_for (std::chrono::seconds (1));
-	node.unchecked_cleaning ();
+	node.unchecked_cleanup ();
 	{
 		auto transaction (node.store.tx_begin ());
 		auto unchecked_count (node.store.unchecked_count (transaction));
 		ASSERT_EQ (unchecked_count, 1);
 	}
 	std::this_thread::sleep_for (std::chrono::seconds (2));
-	node.unchecked_cleaning ();
+	node.unchecked_cleanup ();
 	{
 		auto transaction (node.store.tx_begin ());
 		auto unchecked_count (node.store.unchecked_count (transaction));

nano/nano_node/entry.cpp

@@ -28,7 +28,7 @@ int main (int argc, char * const * argv)
 		("disable_legacy_bootstrap", "Disables legacy bootstrap")
 		("disable_wallet_bootstrap", "Disables wallet lazy bootstrap")
 		("disable_bootstrap_listener", "Disables bootstrap listener (incoming connections)")
-		("disable_unchecked_cleaning", "Disables periodic cleaning of old records from unchecked table")
+		("disable_unchecked_cleanup", "Disables periodic cleanup of old records from unchecked table")
 		("disable_unchecked_drop", "Disables drop of unchecked table at startup")
 		("fast_bootstrap", "Increase bootstrap speed for high end nodes with higher limits")
 		("batch_size",boost::program_options::value<std::size_t> (), "Increase sideband batch size, default 512")
@@ -98,7 +98,7 @@ int main (int argc, char * const * argv)
 			flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0);
 			flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0);
 			flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0);
-			flags.disable_unchecked_cleaning = (vm.count ("disable_unchecked_cleaning") > 0);
+			flags.disable_unchecked_cleanup = (vm.count ("disable_unchecked_cleanup") > 0);
 			flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
 			flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
 			daemon.run (data_path, flags);

nano/nano_wallet/entry.cpp

@@ -369,7 +369,7 @@ int main (int argc, char * const * argv)
 					flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0);
 					flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0);
 					flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0);
-					flags.disable_unchecked_cleaning = (vm.count ("disable_unchecked_cleaning") > 0);
+					flags.disable_unchecked_cleanup = (vm.count ("disable_unchecked_cleanup") > 0);
 					flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
 					flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
 					result = run_wallet (application, argc, argv, data_path, flags);

nano/node/bootstrap.cpp

@@ -1110,6 +1110,10 @@ void nano::bootstrap_attempt::run ()
 			lazy_run ();
 			lock.lock ();
 		}
+		if (!node->flags.disable_unchecked_cleanup)
+		{
+			node->unchecked_cleanup ();
+		}
 	}
 	stopped = true;
 	condition.notify_all ();

nano/node/node.cpp

@@ -22,7 +22,7 @@ std::chrono::seconds constexpr nano::node::syn_cookie_cutoff;
 std::chrono::minutes constexpr nano::node::backup_interval;
 std::chrono::seconds constexpr nano::node::search_pending_interval;
 std::chrono::seconds constexpr nano::node::peer_interval;
-std::chrono::hours constexpr nano::node::unchecked_cleaning_interval;
+std::chrono::hours constexpr nano::node::unchecked_cleanup_interval;
 std::chrono::milliseconds constexpr nano::node::process_confirmed_interval;
 
 int constexpr nano::port_mapping::mapping_timeout;
@@ -1895,6 +1895,10 @@ void nano::node::start ()
 	{
 		ongoing_bootstrap ();
 	}
+	else if (!flags.disable_unchecked_cleanup)
+	{
+		ongoing_unchecked_cleanup ();
+	}
 	ongoing_store_flush ();
 	ongoing_rep_crawl ();
 	ongoing_rep_calculation ();
@@ -1918,10 +1922,6 @@ void nano::node::start ()
 		});
 	}
 	port_mapping.start ();
-	if (!flags.disable_unchecked_cleaning)
-	{
-		unchecked_cleaning ();
-	}
 }
 
 void nano::node::stop ()
@@ -2165,41 +2165,47 @@ void nano::node::bootstrap_wallet ()
 	bootstrap_initiator.bootstrap_wallet (accounts);
 }
 
-void nano::node::unchecked_cleaning ()
+void nano::node::unchecked_cleanup ()
 {
-	std::deque<nano::unchecked_key> cleaning;
+	std::deque<nano::unchecked_key> cleaning_list;
 	// Collect old unchecked keys
-	if (!bootstrap_initiator.in_progress ())
 	{
 		auto now (nano::seconds_since_epoch ());
 		auto transaction (store.tx_begin_read ());
-		// Max 32k records to clean, max 60 seconds reading to prevent slow i/o systems start issues
+		// Max 128k records to clean, max 2 minutes reading to prevent slow i/o systems start issues
-		for (auto i (store.unchecked_begin (transaction)), n (store.unchecked_end ()); i != n && cleaning.size () < 64 * 1024 && nano::seconds_since_epoch () - now < 60; ++i)
+		for (auto i (store.unchecked_begin (transaction)), n (store.unchecked_end ()); i != n && cleaning_list.size () < 128 * 1024 && nano::seconds_since_epoch () - now < 120; ++i)
 		{
 			nano::unchecked_key key (i->first);
 			nano::unchecked_info info (i->second);
-			if ((now - info.modified) > unchecked_cutoff.count ())
+			if ((now - info.modified) > config.unchecked_cutoff_time.count ())
 			{
-				cleaning.push_back (key);
+				cleaning_list.push_back (key);
 			}
 		}
 	}
 	// Delete old unchecked keys in batches
-	while (!cleaning.empty () && !bootstrap_initiator.in_progress ())
+	while (!cleaning_list.empty ())
 	{
 		size_t deleted_count (0);
 		auto transaction (store.tx_begin_write ());
-		while (deleted_count++ < 2 * 1024 && !cleaning.empty ())
+		while (deleted_count++ < 2 * 1024 && !cleaning_list.empty ())
 		{
-			auto key (cleaning.front ());
+			auto key (cleaning_list.front ());
-			cleaning.pop_front ();
+			cleaning_list.pop_front ();
 			store.unchecked_del (transaction, key);
 		}
 	}
-	cleaning.clear ();
+}
+
+void nano::node::ongoing_unchecked_cleanup ()
+{
+	if (!bootstrap_initiator.in_progress ())
+	{
+		unchecked_cleanup ();
+	}
 	auto this_l (shared ());
-	alarm.add (std::chrono::steady_clock::now () + unchecked_cleaning_interval, [this_l]() {
+	alarm.add (std::chrono::steady_clock::now () + unchecked_cleanup_interval, [this_l]() {
-		this_l->unchecked_cleaning ();
+		this_l->ongoing_unchecked_cleanup ();
 	});
 }
 

nano/node/node.hpp

@@ -462,10 +462,11 @@ public:
 	void ongoing_bootstrap ();
 	void ongoing_store_flush ();
 	void ongoing_peer_store ();
+	void ongoing_unchecked_cleanup ();
 	void backup_wallet ();
 	void search_pending ();
 	void bootstrap_wallet ();
-	void unchecked_cleaning ();
+	void unchecked_cleanup ();
 	int price (nano::uint128_t const &, int);
 	void work_generate_blocking (nano::block &, uint64_t = nano::work_pool::publish_threshold);
 	uint64_t work_generate_blocking (nano::uint256_union const &, uint64_t = nano::work_pool::publish_threshold);
@@ -521,8 +522,7 @@ public:
 	static std::chrono::minutes constexpr backup_interval = std::chrono::minutes (5);
 	static std::chrono::seconds constexpr search_pending_interval = nano::is_test_network ? std::chrono::seconds (1) : std::chrono::seconds (5 * 60);
 	static std::chrono::seconds constexpr peer_interval = search_pending_interval;
-	static std::chrono::hours constexpr unchecked_cleaning_interval = std::chrono::hours (2);
+	static std::chrono::hours constexpr unchecked_cleanup_interval = std::chrono::hours (1);
-	std::chrono::seconds unchecked_cutoff = std::chrono::seconds (7 * 24 * 60 * 60); // Week
 	static std::chrono::milliseconds constexpr process_confirmed_interval = nano::is_test_network ? std::chrono::milliseconds (50) : std::chrono::milliseconds (500);
 };
 

nano/node/nodeconfig.cpp

@@ -36,7 +36,8 @@ bootstrap_connections_max (64),
 callback_port (0),
 lmdb_max_dbs (128),
 allow_local_peers (false),
-block_processor_batch_max_time (std::chrono::milliseconds (5000))
+block_processor_batch_max_time (std::chrono::milliseconds (5000)),
+unchecked_cutoff_time (std::chrono::seconds (4 * 60 * 60)) // 4 hours
 {
 	const char * epoch_message ("epoch v1 block");
 	strncpy ((char *)epoch_block_link.bytes.data (), epoch_message, epoch_block_link.bytes.size ());
@@ -120,6 +121,7 @@ nano::error nano::node_config::serialize_json (nano::jsonconfig & json) const
 	json.put ("block_processor_batch_max_time", block_processor_batch_max_time.count ());
 	json.put ("allow_local_peers", allow_local_peers);
 	json.put ("vote_minimum", vote_minimum.to_string_dec ());
+	json.put ("unchecked_cutoff_time", unchecked_cutoff_time.count ());
 
 	nano::jsonconfig ipc_l;
 	ipc_config.serialize_json (ipc_l);
@@ -239,6 +241,7 @@ bool nano::node_config::upgrade_json (unsigned version_a, nano::jsonconfig & jso
 			json.put_child ("ipc", ipc_l);
 
 			json.put (signature_checker_threads_key, signature_checker_threads);
+			json.put ("unchecked_cutoff_time", unchecked_cutoff_time.count ());
 
 			upgraded = true;
 		}
@@ -338,6 +341,9 @@ nano::error nano::node_config::deserialize_json (bool & upgraded_a, nano::jsonco
 
 		auto block_processor_batch_max_time_l (json.get<unsigned long> ("block_processor_batch_max_time"));
 		block_processor_batch_max_time = std::chrono::milliseconds (block_processor_batch_max_time_l);
+		unsigned long unchecked_cutoff_time_l (unchecked_cutoff_time.count ());
+		json.get ("unchecked_cutoff_time", unchecked_cutoff_time_l);
+		unchecked_cutoff_time = std::chrono::seconds (unchecked_cutoff_time_l);
 
 		auto ipc_config_l (json.get_optional_child ("ipc"));
 		if (ipc_config_l)
@@ -398,7 +404,7 @@ disable_lazy_bootstrap (false),
 disable_legacy_bootstrap (false),
 disable_wallet_bootstrap (false),
 disable_bootstrap_listener (false),
-disable_unchecked_cleaning (false),
+disable_unchecked_cleanup (false),
 disable_unchecked_drop (true),
 fast_bootstrap (false),
 sideband_batch_size (512)

nano/node/nodeconfig.hpp

@@ -51,6 +51,7 @@ public:
 	nano::uint256_union epoch_block_link;
 	nano::account epoch_block_signer;
 	std::chrono::milliseconds block_processor_batch_max_time;
+	std::chrono::seconds unchecked_cutoff_time;
 	static std::chrono::seconds constexpr keepalive_period = std::chrono::seconds (60);
 	static std::chrono::seconds constexpr keepalive_cutoff = keepalive_period * 5;
 	static std::chrono::minutes constexpr wallet_backup_interval = std::chrono::minutes (5);
@@ -69,7 +70,7 @@ public:
 	bool disable_legacy_bootstrap;
 	bool disable_wallet_bootstrap;
 	bool disable_bootstrap_listener;
-	bool disable_unchecked_cleaning;
+	bool disable_unchecked_cleanup;
 	bool disable_unchecked_drop;
 	bool fast_bootstrap;
 	size_t sideband_batch_size;