dn724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Cleanup unchecked table after each bootstrap attempt (#1724)

Browse source 
* Cleanup unchecked table after each bootstrap attempt

* Add default unchecked_cutoff_time to deserialize_json

* Correct 4 hours
This commit is contained in:
Sergey Kroshnin 2019-02-15 20:47:10 +03:00 • committed by GitHub
commit 50d50498dd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 49 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

8
nano/core_test/node.cpp
View file

@ -2249,7 +2249,7 @@ TEST (node, peers)
node->stop ();
}
TEST (node, unchecked_cleaning)
TEST (node, unchecked_cleanup)
{
nano::system system (24000, 1);
nano::keypair key;
@ -2257,21 +2257,21 @@ TEST (node, unchecked_cleaning)
auto open (std::make_shared<nano::state_block> (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, system.work.generate (key.pub)));
node.process_active (open);
node.block_processor.flush ();
node.unchecked_cutoff = std::chrono::seconds (2);
node.config.unchecked_cutoff_time = std::chrono::seconds (2);
{
auto transaction (node.store.tx_begin ());
auto unchecked_count (node.store.unchecked_count (transaction));
ASSERT_EQ (unchecked_count, 1);
}
std::this_thread::sleep_for (std::chrono::seconds (1));
node.unchecked_cleaning ();
node.unchecked_cleanup ();
{
auto transaction (node.store.tx_begin ());
auto unchecked_count (node.store.unchecked_count (transaction));
ASSERT_EQ (unchecked_count, 1);
}
std::this_thread::sleep_for (std::chrono::seconds (2));
node.unchecked_cleaning ();
node.unchecked_cleanup ();
{
auto transaction (node.store.tx_begin ());
auto unchecked_count (node.store.unchecked_count (transaction));

4
nano/nano_node/entry.cpp
View file

@ -28,7 +28,7 @@ int main (int argc, char * const * argv)
("disable_legacy_bootstrap", "Disables legacy bootstrap")
("disable_wallet_bootstrap", "Disables wallet lazy bootstrap")
("disable_bootstrap_listener", "Disables bootstrap listener (incoming connections)")
("disable_unchecked_cleaning", "Disables periodic cleaning of old records from unchecked table")
("disable_unchecked_cleanup", "Disables periodic cleanup of old records from unchecked table")
("disable_unchecked_drop", "Disables drop of unchecked table at startup")
("fast_bootstrap", "Increase bootstrap speed for high end nodes with higher limits")
("batch_size",boost::program_options::value<std::size_t> (), "Increase sideband batch size, default 512")
@ -98,7 +98,7 @@ int main (int argc, char * const * argv)
flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0);
flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0);
flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0);
flags.disable_unchecked_cleaning = (vm.count ("disable_unchecked_cleaning") > 0);
flags.disable_unchecked_cleanup = (vm.count ("disable_unchecked_cleanup") > 0);
flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
daemon.run (data_path, flags);

2
nano/nano_wallet/entry.cpp
View file

@ -369,7 +369,7 @@ int main (int argc, char * const * argv)
flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0);
flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0);
flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0);
flags.disable_unchecked_cleaning = (vm.count ("disable_unchecked_cleaning") > 0);
flags.disable_unchecked_cleanup = (vm.count ("disable_unchecked_cleanup") > 0);
flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
result = run_wallet (application, argc, argv, data_path, flags);

4
nano/node/bootstrap.cpp
View file

@ -1110,6 +1110,10 @@ void nano::bootstrap_attempt::run ()
lazy_run ();
lock.lock ();
}
if (!node->flags.disable_unchecked_cleanup)
{
node->unchecked_cleanup ();
}
}
stopped = true;
condition.notify_all ();

44
nano/node/node.cpp
View file

@ -22,7 +22,7 @@ std::chrono::seconds constexpr nano::node::syn_cookie_cutoff;
std::chrono::minutes constexpr nano::node::backup_interval;
std::chrono::seconds constexpr nano::node::search_pending_interval;
std::chrono::seconds constexpr nano::node::peer_interval;
std::chrono::hours constexpr nano::node::unchecked_cleaning_interval;
std::chrono::hours constexpr nano::node::unchecked_cleanup_interval;
std::chrono::milliseconds constexpr nano::node::process_confirmed_interval;
int constexpr nano::port_mapping::mapping_timeout;
@ -1895,6 +1895,10 @@ void nano::node::start ()
{
ongoing_bootstrap ();
}
else if (!flags.disable_unchecked_cleanup)
{
ongoing_unchecked_cleanup ();
}
ongoing_store_flush ();
ongoing_rep_crawl ();
ongoing_rep_calculation ();
@ -1918,10 +1922,6 @@ void nano::node::start ()
});
}
port_mapping.start ();
if (!flags.disable_unchecked_cleaning)
{
unchecked_cleaning ();
}
}
void nano::node::stop ()
@ -2165,41 +2165,47 @@ void nano::node::bootstrap_wallet ()
bootstrap_initiator.bootstrap_wallet (accounts);
}
void nano::node::unchecked_cleaning ()
void nano::node::unchecked_cleanup ()
{
std::deque<nano::unchecked_key> cleaning;
std::deque<nano::unchecked_key> cleaning_list;
// Collect old unchecked keys
if (!bootstrap_initiator.in_progress ())
{
auto now (nano::seconds_since_epoch ());
auto transaction (store.tx_begin_read ());
// Max 32k records to clean, max 60 seconds reading to prevent slow i/o systems start issues
for (auto i (store.unchecked_begin (transaction)), n (store.unchecked_end ()); i != n && cleaning.size () < 64 * 1024 && nano::seconds_since_epoch () - now < 60; ++i)
// Max 128k records to clean, max 2 minutes reading to prevent slow i/o systems start issues
for (auto i (store.unchecked_begin (transaction)), n (store.unchecked_end ()); i != n && cleaning_list.size () < 128 * 1024 && nano::seconds_since_epoch () - now < 120; ++i)
{
nano::unchecked_key key (i->first);
nano::unchecked_info info (i->second);
if ((now - info.modified) > unchecked_cutoff.count ())
if ((now - info.modified) > config.unchecked_cutoff_time.count ())
{
cleaning.push_back (key);
cleaning_list.push_back (key);
}
}
}
// Delete old unchecked keys in batches
while (!cleaning.empty () && !bootstrap_initiator.in_progress ())
while (!cleaning_list.empty ())
{
size_t deleted_count (0);
auto transaction (store.tx_begin_write ());
while (deleted_count++ < 2 * 1024 && !cleaning.empty ())
while (deleted_count++ < 2 * 1024 && !cleaning_list.empty ())
{
auto key (cleaning.front ());
cleaning.pop_front ();
auto key (cleaning_list.front ());
cleaning_list.pop_front ();
store.unchecked_del (transaction, key);
}
}
cleaning.clear ();
}
void nano::node::ongoing_unchecked_cleanup ()
{
if (!bootstrap_initiator.in_progress ())
{
unchecked_cleanup ();
}
auto this_l (shared ());
alarm.add (std::chrono::steady_clock::now () + unchecked_cleaning_interval, [this_l]() {
this_l->unchecked_cleaning ();
alarm.add (std::chrono::steady_clock::now () + unchecked_cleanup_interval, [this_l]() {
this_l->ongoing_unchecked_cleanup ();
});
}

6
nano/node/node.hpp
View file

@ -462,10 +462,11 @@ public:
void ongoing_bootstrap ();
void ongoing_store_flush ();
void ongoing_peer_store ();
void ongoing_unchecked_cleanup ();
void backup_wallet ();
void search_pending ();
void bootstrap_wallet ();
void unchecked_cleaning ();
void unchecked_cleanup ();
int price (nano::uint128_t const &, int);
void work_generate_blocking (nano::block &, uint64_t = nano::work_pool::publish_threshold);
uint64_t work_generate_blocking (nano::uint256_union const &, uint64_t = nano::work_pool::publish_threshold);
@ -521,8 +522,7 @@ public:
static std::chrono::minutes constexpr backup_interval = std::chrono::minutes (5);
static std::chrono::seconds constexpr search_pending_interval = nano::is_test_network ? std::chrono::seconds (1) : std::chrono::seconds (5 * 60);
static std::chrono::seconds constexpr peer_interval = search_pending_interval;
static std::chrono::hours constexpr unchecked_cleaning_interval = std::chrono::hours (2);
std::chrono::seconds unchecked_cutoff = std::chrono::seconds (7 * 24 * 60 * 60); // Week
static std::chrono::hours constexpr unchecked_cleanup_interval = std::chrono::hours (1);
static std::chrono::milliseconds constexpr process_confirmed_interval = nano::is_test_network ? std::chrono::milliseconds (50) : std::chrono::milliseconds (500);
};

10
nano/node/nodeconfig.cpp
View file

@ -36,7 +36,8 @@ bootstrap_connections_max (64),
callback_port (0),
lmdb_max_dbs (128),
allow_local_peers (false),
block_processor_batch_max_time (std::chrono::milliseconds (5000))
block_processor_batch_max_time (std::chrono::milliseconds (5000)),
unchecked_cutoff_time (std::chrono::seconds (4 * 60 * 60)) // 4 hours
{
const char * epoch_message ("epoch v1 block");
strncpy ((char *)epoch_block_link.bytes.data (), epoch_message, epoch_block_link.bytes.size ());
@ -120,6 +121,7 @@ nano::error nano::node_config::serialize_json (nano::jsonconfig & json) const
json.put ("block_processor_batch_max_time", block_processor_batch_max_time.count ());
json.put ("allow_local_peers", allow_local_peers);
json.put ("vote_minimum", vote_minimum.to_string_dec ());
json.put ("unchecked_cutoff_time", unchecked_cutoff_time.count ());
nano::jsonconfig ipc_l;
ipc_config.serialize_json (ipc_l);
@ -239,6 +241,7 @@ bool nano::node_config::upgrade_json (unsigned version_a, nano::jsonconfig & jso
json.put_child ("ipc", ipc_l);
json.put (signature_checker_threads_key, signature_checker_threads);
json.put ("unchecked_cutoff_time", unchecked_cutoff_time.count ());
upgraded = true;
}
@ -338,6 +341,9 @@ nano::error nano::node_config::deserialize_json (bool & upgraded_a, nano::jsonco
auto block_processor_batch_max_time_l (json.get<unsigned long> ("block_processor_batch_max_time"));
block_processor_batch_max_time = std::chrono::milliseconds (block_processor_batch_max_time_l);
unsigned long unchecked_cutoff_time_l (unchecked_cutoff_time.count ());
json.get ("unchecked_cutoff_time", unchecked_cutoff_time_l);
unchecked_cutoff_time = std::chrono::seconds (unchecked_cutoff_time_l);
auto ipc_config_l (json.get_optional_child ("ipc"));
if (ipc_config_l)
@ -398,7 +404,7 @@ disable_lazy_bootstrap (false),
disable_legacy_bootstrap (false),
disable_wallet_bootstrap (false),
disable_bootstrap_listener (false),
disable_unchecked_cleaning (false),
disable_unchecked_cleanup (false),
disable_unchecked_drop (true),
fast_bootstrap (false),
sideband_batch_size (512)

3
nano/node/nodeconfig.hpp
View file

@ -51,6 +51,7 @@ public:
nano::uint256_union epoch_block_link;
nano::account epoch_block_signer;
std::chrono::milliseconds block_processor_batch_max_time;
std::chrono::seconds unchecked_cutoff_time;
static std::chrono::seconds constexpr keepalive_period = std::chrono::seconds (60);
static std::chrono::seconds constexpr keepalive_cutoff = keepalive_period * 5;
static std::chrono::minutes constexpr wallet_backup_interval = std::chrono::minutes (5);
@ -69,7 +70,7 @@ public:
bool disable_legacy_bootstrap;
bool disable_wallet_bootstrap;
bool disable_bootstrap_listener;
bool disable_unchecked_cleaning;
bool disable_unchecked_cleanup;
bool disable_unchecked_drop;
bool fast_bootstrap;
size_t sideband_batch_size;