From 5c19c49ff800b5b7839d9071bf3b273b4acaead1 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Thu, 29 Aug 2019 23:24:23 +0300 Subject: [PATCH] Prevent dropping of unchecked on restart if node is not synchronized (#2257) * Prevent dropping of unchecked on restart if node is not synchronized. Based on built-in bootstrap_weight_max_blocks * Disable droping of unchecked & using bootstrap weights for inactive_node * Log dropping unchecked blocks --- nano/core_test/block_store.cpp | 2 +- nano/node/lmdb/lmdb.cpp | 8 +------- nano/node/lmdb/lmdb.hpp | 2 +- nano/node/node.cpp | 27 +++++++++++++++++++-------- nano/node/nodeconfig.hpp | 1 + nano/node/rocksdb/rocksdb.cpp | 10 ++-------- nano/node/rocksdb/rocksdb.hpp | 2 +- nano/secure/blockstore.hpp | 2 +- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index f5146ee5..e0682d0c 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -1692,7 +1692,7 @@ TEST (mdb_block_store, upgrade_backup) // Now do the upgrade and confirm that backup is saved nano::logger_mt logger; - nano::mdb_store store (logger, path, nano::txn_tracking_config{}, std::chrono::seconds (5), 128, false, 512, true); + nano::mdb_store store (logger, path, nano::txn_tracking_config{}, std::chrono::seconds (5), 128, 512, true); ASSERT_FALSE (store.init_error ()); auto transaction (store.tx_begin_read ()); ASSERT_LT (14, store.version_get (transaction)); diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 4bbab271..7a15147d 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -39,7 +39,7 @@ void mdb_val::convert_buffer_to_value () } } -nano::mdb_store::mdb_store (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, bool drop_unchecked, size_t const batch_size, bool backup_before_upgrade) : +nano::mdb_store::mdb_store (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, size_t const batch_size, bool backup_before_upgrade) : logger (logger_a), env (error, path_a, lmdb_max_dbs, true), mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a), @@ -79,12 +79,6 @@ txn_tracking_enabled (txn_tracking_config_a.enable) auto transaction (tx_begin_read ()); open_databases (error, transaction, 0); } - - if (!error && drop_unchecked) - { - auto transaction (tx_begin_write ({ nano::tables::cached_counts, tables::unchecked })); - unchecked_clear (transaction); - } } } diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index a86d93a8..08f66afd 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -32,7 +32,7 @@ public: using block_store_partial::block_exists; using block_store_partial::unchecked_put; - mdb_store (nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512, bool backup_before_upgrade = false); + mdb_store (nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, size_t batch_size = 512, bool backup_before_upgrade = false); nano::write_transaction tx_begin_write (std::vector const & tables_requiring_lock = {}, std::vector const & tables_no_lock = {}) override; nano::read_transaction tx_begin_read () override; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 412bf06c..2997511b 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -123,7 +123,7 @@ alarm (alarm_a), work (work_a), distributed_work (*this), logger (config_a.logging.min_time_between_log_output), -store_impl (nano::make_store (logger, application_path_a, flags.read_only, true, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_max_dbs, !flags.disable_unchecked_drop, flags.sideband_batch_size, config_a.backup_before_upgrade)), +store_impl (nano::make_store (logger, application_path_a, flags.read_only, true, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_max_dbs, flags.sideband_batch_size, config_a.backup_before_upgrade)), store (*store_impl), wallets_store_impl (std::make_unique (application_path_a / "wallets.ldb", config_a.lmdb_max_dbs)), wallets_store (*wallets_store_impl), @@ -419,17 +419,20 @@ startup_time (std::chrono::steady_clock::now ()) node_id = nano::keypair (); logger.always_log ("Node ID: ", node_id.pub.to_node_id ()); - const uint8_t * weight_buffer = network_params.network.is_live_network () ? nano_bootstrap_weights_live : nano_bootstrap_weights_beta; - size_t weight_size = network_params.network.is_live_network () ? nano_bootstrap_weights_live_size : nano_bootstrap_weights_beta_size; - if (network_params.network.is_live_network () || network_params.network.is_beta_network ()) + if ((network_params.network.is_live_network () || network_params.network.is_beta_network ()) && !flags.inactive_node) { + // Use bootstrap weights if initial bootstrap is not completed + bool use_bootstrap_weight (false); + const uint8_t * weight_buffer = network_params.network.is_live_network () ? nano_bootstrap_weights_live : nano_bootstrap_weights_beta; + size_t weight_size = network_params.network.is_live_network () ? nano_bootstrap_weights_live_size : nano_bootstrap_weights_beta_size; nano::bufferstream weight_stream ((const uint8_t *)weight_buffer, weight_size); nano::uint128_union block_height; if (!nano::try_read (weight_stream, block_height)) { auto max_blocks = (uint64_t)block_height.number (); auto transaction (store.tx_begin_read ()); - if (ledger.store.block_count (transaction).sum () < max_blocks) + use_bootstrap_weight = ledger.store.block_count (transaction).sum () < max_blocks; + if (use_bootstrap_weight) { ledger.bootstrap_weight_max_blocks = max_blocks; while (true) @@ -449,6 +452,13 @@ startup_time (std::chrono::steady_clock::now ()) } } } + // Drop unchecked blocks if initial bootstrap is completed + if (!flags.disable_unchecked_drop && !use_bootstrap_weight && !flags.read_only) + { + auto transaction (store.tx_begin_write ()); + store.unchecked_clear (transaction); + logger.always_log ("Dropping unchecked blocks"); + } } } node_initialized_latch.count_down (); @@ -1333,17 +1343,18 @@ nano::inactive_node::~inactive_node () nano::node_flags const & nano::inactive_node_flag_defaults () { static nano::node_flags node_flags; + node_flags.inactive_node = true; node_flags.read_only = true; node_flags.cache_representative_weights_from_frontiers = false; node_flags.cache_cemented_count_from_frontiers = false; return node_flags; } -std::unique_ptr nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, bool drop_unchecked, size_t batch_size, bool backup_before_upgrade) +std::unique_ptr nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, size_t batch_size, bool backup_before_upgrade) { #if NANO_ROCKSDB - return std::make_unique (logger, add_db_postfix ? path / "rocksdb" : path, drop_unchecked, read_only); + return std::make_unique (logger, add_db_postfix ? path / "rocksdb" : path, read_only); #else - return std::make_unique (logger, add_db_postfix ? path / "data.ldb" : path, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_max_dbs, drop_unchecked, batch_size, backup_before_upgrade); + return std::make_unique (logger, add_db_postfix ? path / "data.ldb" : path, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_max_dbs, batch_size, backup_before_upgrade); #endif } diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index f2a7ed3c..6a82337e 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -121,6 +121,7 @@ public: bool cache_representative_weights_from_frontiers{ true }; /** Whether to read all frontiers and construct the total cemented count */ bool cache_cemented_count_from_frontiers{ true }; + bool inactive_node{ false }; size_t sideband_batch_size{ 512 }; size_t block_processor_batch_size{ 0 }; size_t block_processor_full_size{ 65536 }; diff --git a/nano/node/rocksdb/rocksdb.cpp b/nano/node/rocksdb/rocksdb.cpp index f2f0840e..235ad015 100644 --- a/nano/node/rocksdb/rocksdb.cpp +++ b/nano/node/rocksdb/rocksdb.cpp @@ -40,7 +40,7 @@ void rocksdb_val::convert_buffer_to_value () } } -nano::rocksdb_store::rocksdb_store (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, bool drop_unchecked_a, bool open_read_only_a) : +nano::rocksdb_store::rocksdb_store (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, bool open_read_only_a) : logger (logger_a) { boost::system::error_code error_mkdir, error_chmod; @@ -58,12 +58,6 @@ logger (logger_a) } open (error, path_a, open_read_only_a); } - - if (!error && !open_read_only_a && drop_unchecked_a) - { - auto transaction (tx_begin_write ({ nano::tables::cached_counts, tables::unchecked })); - unchecked_clear (transaction); - } } nano::rocksdb_store::~rocksdb_store () @@ -608,7 +602,7 @@ bool nano::rocksdb_store::copy_db (boost::filesystem::path const & destination_p // Open it so that it flushes all WAL files if (status.ok ()) { - nano::rocksdb_store rocksdb_store (logger, destination_path.string (), false, false); + nano::rocksdb_store rocksdb_store (logger, destination_path.string (), false); return !rocksdb_store.init_error (); } return false; diff --git a/nano/node/rocksdb/rocksdb.hpp b/nano/node/rocksdb/rocksdb.hpp index 09511230..9086bbf2 100644 --- a/nano/node/rocksdb/rocksdb.hpp +++ b/nano/node/rocksdb/rocksdb.hpp @@ -24,7 +24,7 @@ class logging_mt; class rocksdb_store : public block_store_partial { public: - rocksdb_store (nano::logger_mt &, boost::filesystem::path const &, bool drop_unchecked = false, bool open_read_only = false); + rocksdb_store (nano::logger_mt &, boost::filesystem::path const &, bool open_read_only = false); ~rocksdb_store (); nano::write_transaction tx_begin_write (std::vector const & tables_requiring_lock = {}, std::vector const & tables_no_lock = {}) override; nano::read_transaction tx_begin_read () override; diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index 4cb8573a..f6d269ae 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -695,7 +695,7 @@ public: virtual nano::read_transaction tx_begin_read () = 0; }; -std::unique_ptr make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool open_read_only = false, bool add_db_postfix = false, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512, bool backup_before_upgrade = false); +std::unique_ptr make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool open_read_only = false, bool add_db_postfix = false, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, size_t batch_size = 512, bool backup_before_upgrade = false); } namespace std