diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index c8b91eb93..515dd0dca 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -2249,7 +2249,7 @@ TEST (node, peers) node->stop (); } -TEST (node, unchecked_cleaning) +TEST (node, unchecked_cleanup) { nano::system system (24000, 1); nano::keypair key; @@ -2257,21 +2257,21 @@ TEST (node, unchecked_cleaning) auto open (std::make_shared (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, system.work.generate (key.pub))); node.process_active (open); node.block_processor.flush (); - node.unchecked_cutoff = std::chrono::seconds (2); + node.config.unchecked_cutoff_time = std::chrono::seconds (2); { auto transaction (node.store.tx_begin ()); auto unchecked_count (node.store.unchecked_count (transaction)); ASSERT_EQ (unchecked_count, 1); } std::this_thread::sleep_for (std::chrono::seconds (1)); - node.unchecked_cleaning (); + node.unchecked_cleanup (); { auto transaction (node.store.tx_begin ()); auto unchecked_count (node.store.unchecked_count (transaction)); ASSERT_EQ (unchecked_count, 1); } std::this_thread::sleep_for (std::chrono::seconds (2)); - node.unchecked_cleaning (); + node.unchecked_cleanup (); { auto transaction (node.store.tx_begin ()); auto unchecked_count (node.store.unchecked_count (transaction)); diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index 1867553ac..41061e47e 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -28,7 +28,7 @@ int main (int argc, char * const * argv) ("disable_legacy_bootstrap", "Disables legacy bootstrap") ("disable_wallet_bootstrap", "Disables wallet lazy bootstrap") ("disable_bootstrap_listener", "Disables bootstrap listener (incoming connections)") - ("disable_unchecked_cleaning", "Disables periodic cleaning of old records from unchecked table") + ("disable_unchecked_cleanup", "Disables periodic cleanup of old records from unchecked table") ("disable_unchecked_drop", "Disables drop of unchecked table at startup") ("fast_bootstrap", "Increase bootstrap speed for high end nodes with higher limits") ("batch_size",boost::program_options::value (), "Increase sideband batch size, default 512") @@ -98,7 +98,7 @@ int main (int argc, char * const * argv) flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0); flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0); flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0); - flags.disable_unchecked_cleaning = (vm.count ("disable_unchecked_cleaning") > 0); + flags.disable_unchecked_cleanup = (vm.count ("disable_unchecked_cleanup") > 0); flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0); flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0); daemon.run (data_path, flags); diff --git a/nano/nano_wallet/entry.cpp b/nano/nano_wallet/entry.cpp index 4c36e167b..43f51d502 100644 --- a/nano/nano_wallet/entry.cpp +++ b/nano/nano_wallet/entry.cpp @@ -369,7 +369,7 @@ int main (int argc, char * const * argv) flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0); flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0); flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0); - flags.disable_unchecked_cleaning = (vm.count ("disable_unchecked_cleaning") > 0); + flags.disable_unchecked_cleanup = (vm.count ("disable_unchecked_cleanup") > 0); flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0); flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0); result = run_wallet (application, argc, argv, data_path, flags); diff --git a/nano/node/bootstrap.cpp b/nano/node/bootstrap.cpp index 2370b6383..9052fbc43 100644 --- a/nano/node/bootstrap.cpp +++ b/nano/node/bootstrap.cpp @@ -1110,6 +1110,10 @@ void nano::bootstrap_attempt::run () lazy_run (); lock.lock (); } + if (!node->flags.disable_unchecked_cleanup) + { + node->unchecked_cleanup (); + } } stopped = true; condition.notify_all (); diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 7fbc36fee..a859bf520 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -22,7 +22,7 @@ std::chrono::seconds constexpr nano::node::syn_cookie_cutoff; std::chrono::minutes constexpr nano::node::backup_interval; std::chrono::seconds constexpr nano::node::search_pending_interval; std::chrono::seconds constexpr nano::node::peer_interval; -std::chrono::hours constexpr nano::node::unchecked_cleaning_interval; +std::chrono::hours constexpr nano::node::unchecked_cleanup_interval; std::chrono::milliseconds constexpr nano::node::process_confirmed_interval; int constexpr nano::port_mapping::mapping_timeout; @@ -1895,6 +1895,10 @@ void nano::node::start () { ongoing_bootstrap (); } + else if (!flags.disable_unchecked_cleanup) + { + ongoing_unchecked_cleanup (); + } ongoing_store_flush (); ongoing_rep_crawl (); ongoing_rep_calculation (); @@ -1918,10 +1922,6 @@ void nano::node::start () }); } port_mapping.start (); - if (!flags.disable_unchecked_cleaning) - { - unchecked_cleaning (); - } } void nano::node::stop () @@ -2165,41 +2165,47 @@ void nano::node::bootstrap_wallet () bootstrap_initiator.bootstrap_wallet (accounts); } -void nano::node::unchecked_cleaning () +void nano::node::unchecked_cleanup () { - std::deque cleaning; + std::deque cleaning_list; // Collect old unchecked keys - if (!bootstrap_initiator.in_progress ()) { auto now (nano::seconds_since_epoch ()); auto transaction (store.tx_begin_read ()); - // Max 32k records to clean, max 60 seconds reading to prevent slow i/o systems start issues - for (auto i (store.unchecked_begin (transaction)), n (store.unchecked_end ()); i != n && cleaning.size () < 64 * 1024 && nano::seconds_since_epoch () - now < 60; ++i) + // Max 128k records to clean, max 2 minutes reading to prevent slow i/o systems start issues + for (auto i (store.unchecked_begin (transaction)), n (store.unchecked_end ()); i != n && cleaning_list.size () < 128 * 1024 && nano::seconds_since_epoch () - now < 120; ++i) { nano::unchecked_key key (i->first); nano::unchecked_info info (i->second); - if ((now - info.modified) > unchecked_cutoff.count ()) + if ((now - info.modified) > config.unchecked_cutoff_time.count ()) { - cleaning.push_back (key); + cleaning_list.push_back (key); } } } // Delete old unchecked keys in batches - while (!cleaning.empty () && !bootstrap_initiator.in_progress ()) + while (!cleaning_list.empty ()) { size_t deleted_count (0); auto transaction (store.tx_begin_write ()); - while (deleted_count++ < 2 * 1024 && !cleaning.empty ()) + while (deleted_count++ < 2 * 1024 && !cleaning_list.empty ()) { - auto key (cleaning.front ()); - cleaning.pop_front (); + auto key (cleaning_list.front ()); + cleaning_list.pop_front (); store.unchecked_del (transaction, key); } } - cleaning.clear (); +} + +void nano::node::ongoing_unchecked_cleanup () +{ + if (!bootstrap_initiator.in_progress ()) + { + unchecked_cleanup (); + } auto this_l (shared ()); - alarm.add (std::chrono::steady_clock::now () + unchecked_cleaning_interval, [this_l]() { - this_l->unchecked_cleaning (); + alarm.add (std::chrono::steady_clock::now () + unchecked_cleanup_interval, [this_l]() { + this_l->ongoing_unchecked_cleanup (); }); } diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 96dcf993f..bda997e0d 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -462,10 +462,11 @@ public: void ongoing_bootstrap (); void ongoing_store_flush (); void ongoing_peer_store (); + void ongoing_unchecked_cleanup (); void backup_wallet (); void search_pending (); void bootstrap_wallet (); - void unchecked_cleaning (); + void unchecked_cleanup (); int price (nano::uint128_t const &, int); void work_generate_blocking (nano::block &, uint64_t = nano::work_pool::publish_threshold); uint64_t work_generate_blocking (nano::uint256_union const &, uint64_t = nano::work_pool::publish_threshold); @@ -521,8 +522,7 @@ public: static std::chrono::minutes constexpr backup_interval = std::chrono::minutes (5); static std::chrono::seconds constexpr search_pending_interval = nano::is_test_network ? std::chrono::seconds (1) : std::chrono::seconds (5 * 60); static std::chrono::seconds constexpr peer_interval = search_pending_interval; - static std::chrono::hours constexpr unchecked_cleaning_interval = std::chrono::hours (2); - std::chrono::seconds unchecked_cutoff = std::chrono::seconds (7 * 24 * 60 * 60); // Week + static std::chrono::hours constexpr unchecked_cleanup_interval = std::chrono::hours (1); static std::chrono::milliseconds constexpr process_confirmed_interval = nano::is_test_network ? std::chrono::milliseconds (50) : std::chrono::milliseconds (500); }; diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index 963fb13fe..29f2310da 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -36,7 +36,8 @@ bootstrap_connections_max (64), callback_port (0), lmdb_max_dbs (128), allow_local_peers (false), -block_processor_batch_max_time (std::chrono::milliseconds (5000)) +block_processor_batch_max_time (std::chrono::milliseconds (5000)), +unchecked_cutoff_time (std::chrono::seconds (4 * 60 * 60)) // 4 hours { const char * epoch_message ("epoch v1 block"); strncpy ((char *)epoch_block_link.bytes.data (), epoch_message, epoch_block_link.bytes.size ()); @@ -120,6 +121,7 @@ nano::error nano::node_config::serialize_json (nano::jsonconfig & json) const json.put ("block_processor_batch_max_time", block_processor_batch_max_time.count ()); json.put ("allow_local_peers", allow_local_peers); json.put ("vote_minimum", vote_minimum.to_string_dec ()); + json.put ("unchecked_cutoff_time", unchecked_cutoff_time.count ()); nano::jsonconfig ipc_l; ipc_config.serialize_json (ipc_l); @@ -239,6 +241,7 @@ bool nano::node_config::upgrade_json (unsigned version_a, nano::jsonconfig & jso json.put_child ("ipc", ipc_l); json.put (signature_checker_threads_key, signature_checker_threads); + json.put ("unchecked_cutoff_time", unchecked_cutoff_time.count ()); upgraded = true; } @@ -338,6 +341,9 @@ nano::error nano::node_config::deserialize_json (bool & upgraded_a, nano::jsonco auto block_processor_batch_max_time_l (json.get ("block_processor_batch_max_time")); block_processor_batch_max_time = std::chrono::milliseconds (block_processor_batch_max_time_l); + unsigned long unchecked_cutoff_time_l (unchecked_cutoff_time.count ()); + json.get ("unchecked_cutoff_time", unchecked_cutoff_time_l); + unchecked_cutoff_time = std::chrono::seconds (unchecked_cutoff_time_l); auto ipc_config_l (json.get_optional_child ("ipc")); if (ipc_config_l) @@ -398,7 +404,7 @@ disable_lazy_bootstrap (false), disable_legacy_bootstrap (false), disable_wallet_bootstrap (false), disable_bootstrap_listener (false), -disable_unchecked_cleaning (false), +disable_unchecked_cleanup (false), disable_unchecked_drop (true), fast_bootstrap (false), sideband_batch_size (512) diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index dc17eb768..cbe34b603 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -51,6 +51,7 @@ public: nano::uint256_union epoch_block_link; nano::account epoch_block_signer; std::chrono::milliseconds block_processor_batch_max_time; + std::chrono::seconds unchecked_cutoff_time; static std::chrono::seconds constexpr keepalive_period = std::chrono::seconds (60); static std::chrono::seconds constexpr keepalive_cutoff = keepalive_period * 5; static std::chrono::minutes constexpr wallet_backup_interval = std::chrono::minutes (5); @@ -69,7 +70,7 @@ public: bool disable_legacy_bootstrap; bool disable_wallet_bootstrap; bool disable_bootstrap_listener; - bool disable_unchecked_cleaning; + bool disable_unchecked_cleanup; bool disable_unchecked_drop; bool fast_bootstrap; size_t sideband_batch_size;