diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 2f3ca5ca..9fc0f906 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -17,10 +17,10 @@ #include +#include #include #include - -#include +#include using namespace std::chrono_literals; @@ -2177,6 +2177,89 @@ namespace lmdb auto transaction (store.tx_begin_read ()); ASSERT_LT (19, store.version.get (transaction)); } + + TEST (mdb_block_store, upgrade_v21_v22) + { + if (nano::rocksdb_config::using_rocksdb_in_tests ()) + { + // Don't test this in rocksdb mode + GTEST_SKIP (); + } + + auto path (nano::unique_path ()); + nano::logger_mt logger; + nano::stats stats; + auto const check_correct_state = [&] () { + nano::lmdb::store store (logger, path, nano::dev::constants); + auto transaction (store.tx_begin_write ()); + ASSERT_EQ (store.version.get (transaction), store.version_current); + MDB_dbi unchecked_handle{ 0 }; + ASSERT_EQ (MDB_NOTFOUND, mdb_dbi_open (store.env.tx (transaction), "unchecked", 0, &unchecked_handle)); + }; + + // Testing current version doesn't contain the unchecked table + check_correct_state (); + + // Setting the database to its 21st version state + { + nano::lmdb::store store (logger, path, nano::dev::constants); + auto transaction (store.tx_begin_write ()); + store.version.put (transaction, 21); + MDB_dbi unchecked_handle{ 0 }; + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE, &unchecked_handle)); + ASSERT_EQ (store.version.get (transaction), 21); + } + + // Testing the upgrade code worked + check_correct_state (); + } +} + +namespace rocksdb +{ + TEST (rocksdb_block_store, upgrade_v21_v22) + { + if (!nano::rocksdb_config::using_rocksdb_in_tests ()) + { + // Don't test this in LMDB mode + GTEST_SKIP (); + } + + auto const path = nano::unique_path (); + nano::logger_mt logger; + nano::stats stats; + auto const check_correct_state = [&] () { + nano::rocksdb::store store (logger, path, nano::dev::constants); + auto transaction (store.tx_begin_write ()); + ASSERT_EQ (store.version.get (transaction), store.version_current); + ASSERT_FALSE (store.column_family_exists ("unchecked")); + }; + + // Testing current version doesn't contain the unchecked table + check_correct_state (); + + // Setting the database to its 21st version state + { + nano::rocksdb::store store (logger, path, nano::dev::constants); + + // Create a column family for "unchecked" + ::rocksdb::ColumnFamilyOptions new_cf_options; + ::rocksdb::ColumnFamilyHandle * new_cf_handle; + ::rocksdb::Status status = store.db->CreateColumnFamily (new_cf_options, "unchecked", &new_cf_handle); + store.handles.emplace_back (new_cf_handle); + + // The new column family was created successfully, and 'new_cf_handle' now points to it. + ASSERT_TRUE (status.ok ()); + + // Rollback the database version number. + auto transaction (store.tx_begin_write ()); + store.version.put (transaction, 21); + ASSERT_EQ (store.version.get (transaction), 21); + } + + // Testing the upgrade code worked + check_correct_state (); + } } } diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index 3fc61244..4092de1c 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -5541,7 +5541,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb) boost::asio::ip::address_v6 address (boost::asio::ip::make_address_v6 ("::ffff:127.0.0.1")); uint16_t port = 100; nano::lmdb::store store{ logger, path / "data.ldb", nano::dev::constants }; - nano::unchecked_map unchecked{ system.stats, false }; nano::ledger ledger{ store, system.stats, nano::dev::constants }; nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; @@ -5556,7 +5555,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb) .build_shared (); nano::endpoint_key endpoint_key (address.to_bytes (), port); - auto version = 99; + auto version = nano::store::version_current; { auto transaction = store.tx_begin_write (); @@ -5582,7 +5581,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb) ASSERT_FALSE (error); nano::rocksdb::store rocksdb_store{ logger, path / "rocksdb", nano::dev::constants }; - nano::unchecked_map rocksdb_unchecked{ system.stats, false }; auto rocksdb_transaction (rocksdb_store.tx_begin_read ()); nano::pending_info pending_info{}; diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index 6016ac96..e1b7ce44 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -136,7 +136,6 @@ add_library( lmdb/pruned_store.cpp lmdb/version_store.hpp lmdb/version_store.cpp - lmdb/unchecked_store.hpp lmdb/lmdb.hpp lmdb/lmdb.cpp lmdb/lmdb_env.hpp diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index a9d324fa..931821bc 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -62,7 +62,6 @@ nano::lmdb::store::store (nano::logger_mt & logger_a, boost::filesystem::path co peer_store{ *this }, confirmation_height_store{ *this }, final_vote_store{ *this }, - unchecked_store{}, version_store{ *this }, logger (logger_a), env (error, path_a, nano::mdb_env::options::make ().set_config (lmdb_config_a).set_use_no_mem_init (true)), @@ -208,7 +207,6 @@ nano::mdb_txn_callbacks nano::lmdb::store::create_txn_callbacks () const void nano::lmdb::store::open_databases (bool & error_a, nano::transaction const & transaction_a, unsigned flags) { error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontier_store.frontiers_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "unchecked", flags, &unchecked_store.unchecked_handle) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight_store.online_weight_handle) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &block_store.meta_handle) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peer_store.peers_handle) != 0; @@ -303,6 +301,9 @@ bool nano::lmdb::store::do_upgrades (nano::write_transaction & transaction_a, na upgrade_v20_to_v21 (transaction_a); [[fallthrough]]; case 21: + upgrade_v21_to_v22 (transaction_a); + [[fallthrough]]; + case 22: break; default: logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); @@ -776,6 +777,16 @@ void nano::lmdb::store::upgrade_v20_to_v21 (nano::write_transaction const & tran logger.always_log ("Finished creating new final_vote table"); } +void nano::lmdb::store::upgrade_v21_to_v22 (nano::write_transaction const & transaction_a) +{ + logger.always_log ("Preparing v21 to v22 database upgrade..."); + MDB_dbi unchecked_handle{ 0 }; + release_assert (!mdb_dbi_open (env.tx (transaction_a), "unchecked", MDB_CREATE, &unchecked_handle)); + release_assert (!mdb_drop (env.tx (transaction_a), unchecked_handle, 1)); // del = 1, to delete it from the environment and close the DB handle. + version.put (transaction_a, 22); + logger.always_log ("Finished removing unchecked table"); +} + /** Takes a filepath, appends '_backup_' to the end (but before any extension) and saves that file in the same directory */ void nano::lmdb::store::create_backup_file (nano::mdb_env & env_a, boost::filesystem::path const & filepath_a, nano::logger_mt & logger_a) { @@ -865,8 +876,6 @@ MDB_dbi nano::lmdb::store::table_to_dbi (tables table_a) const return block_store.blocks_handle; case tables::pending: return pending_store.pending_handle; - case tables::unchecked: - return unchecked_store.unchecked_handle; case tables::online_weight: return online_weight_store.online_weight_handle; case tables::meta: diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index 670f3777..84c4b432 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -57,7 +56,6 @@ namespace lmdb nano::lmdb::peer_store peer_store; nano::lmdb::pending_store pending_store; nano::lmdb::pruned_store pruned_store; - nano::lmdb::unchecked_store unchecked_store; nano::lmdb::version_store version_store; friend class nano::lmdb::account_store; @@ -69,7 +67,6 @@ namespace lmdb friend class nano::lmdb::peer_store; friend class nano::lmdb::pending_store; friend class nano::lmdb::pruned_store; - friend class nano::lmdb::unchecked_store; friend class nano::lmdb::version_store; public: @@ -136,6 +133,7 @@ namespace lmdb void upgrade_v18_to_v19 (nano::write_transaction const &); void upgrade_v19_to_v20 (nano::write_transaction const &); void upgrade_v20_to_v21 (nano::write_transaction const &); + void upgrade_v21_to_v22 (nano::write_transaction const &); std::shared_ptr block_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; nano::mdb_val block_raw_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const; @@ -188,6 +186,7 @@ namespace lmdb friend class mdb_block_store_upgrade_v18_v19_Test; friend class mdb_block_store_upgrade_v19_v20_Test; friend class mdb_block_store_upgrade_v20_v21_Test; + friend class mdb_block_store_upgrade_v21_v22_Test; friend class block_store_DISABLED_change_dupsort_Test; friend void write_sideband_v14 (nano::lmdb::store &, nano::transaction &, nano::block const &, MDB_dbi); friend void write_sideband_v15 (nano::lmdb::store &, nano::transaction &, nano::block const &); diff --git a/nano/node/lmdb/unchecked_store.hpp b/nano/node/lmdb/unchecked_store.hpp deleted file mode 100644 index 7a491bef..00000000 --- a/nano/node/lmdb/unchecked_store.hpp +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include - -namespace nano::lmdb -{ -class unchecked_store -{ -public: - /** - * Unchecked bootstrap blocks info. - * nano::block_hash -> nano::unchecked_info - */ - MDB_dbi unchecked_handle{ 0 }; -}; -} diff --git a/nano/node/node.cpp b/nano/node/node.cpp index fab425ac..6a981110 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -441,7 +441,6 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co // Drop unchecked blocks if initial bootstrap is completed if (!flags.disable_unchecked_drop && !use_bootstrap_weight && !flags.read_only) { - auto const transaction (store.tx_begin_write ({ tables::unchecked })); unchecked.clear (); logger.always_log ("Dropping unchecked blocks"); } @@ -996,7 +995,6 @@ void nano::node::unchecked_cleanup () while (!cleaning_list.empty ()) { std::size_t deleted_count (0); - auto const transaction (store.tx_begin_write ({ tables::unchecked })); while (deleted_count++ < 2 * 1024 && !cleaning_list.empty ()) { auto key (cleaning_list.front ()); diff --git a/nano/node/rocksdb/rocksdb.cpp b/nano/node/rocksdb/rocksdb.cpp index c7797522..95ad527f 100644 --- a/nano/node/rocksdb/rocksdb.cpp +++ b/nano/node/rocksdb/rocksdb.cpp @@ -96,15 +96,86 @@ nano::rocksdb::store::store (nano::logger_mt & logger_a, boost::filesystem::path nano::set_secure_perm_directory (path_a, error_chmod); error = static_cast (error_mkdir); + if (error) + { + return; + } + + generate_tombstone_map (); + small_table_factory.reset (::rocksdb::NewBlockBasedTableFactory (get_small_table_options ())); + + // TODO: get_db_options () registers a listener for resetting tombstones, needs to check if it is a problem calling it more than once. + auto options = get_db_options (); + + // The only certain column family is "meta" which contains the DB version info. + // RocksDB requires this operation to be in read-only mode. + auto is_fresh_db = false; + open (is_fresh_db, path_a, true, options, get_single_column_family ("meta")); + + auto is_fully_upgraded = false; + if (!is_fresh_db) + { + auto transaction = tx_begin_read (); + auto version_l = version.get (transaction); + if (version_l > version_current) + { + error = true; + logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); + return; + } + else if (version_l < version_minimum) + { + error = true; + logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is lower than the minimum (%2%) which is supported for upgrades. Either upgrade to a v19, v20 or v21 node first or delete the ledger.") % version_l % version_minimum)); + return; + } + is_fully_upgraded = (version_l == version_current); + } + + if (db) + { + // Needs to clear the store references before reopening the DB. + handles.clear (); + db.reset (nullptr); + } + + if (!open_read_only_a) + { + construct_column_family_mutexes (); + } + + if (is_fully_upgraded) + { + open (error, path_a, open_read_only_a, options, create_column_families ()); + return; + } + + if (open_read_only_a) + { + // Either following cases cannot run in read-only mode: + // a) there is no database yet, the access needs to be in write mode for it to be created; + // b) it will upgrade, and it is not possible to do it in read-only mode. + error = true; + return; + } + + if (is_fresh_db) + { + open (error, path_a, open_read_only_a, options, create_column_families ()); + if (!error) + { + version.put (tx_begin_write (), version_current); // It is fresh, someone needs to tell it its version. + } + return; + } + + // The database is not upgraded, and it may not be compatible with the current column family set. + open (error, path_a, open_read_only_a, options, get_current_column_families (path_a.string (), options)); if (!error) { - generate_tombstone_map (); - small_table_factory.reset (::rocksdb::NewBlockBasedTableFactory (get_small_table_options ())); - if (!open_read_only_a) - { - construct_column_family_mutexes (); - } - open (error, path_a, open_read_only_a); + logger.always_log ("Upgrade in progress..."); + auto transaction = tx_begin_write (); + error |= do_upgrades (transaction); } } @@ -115,7 +186,6 @@ std::unordered_map nano::rocksdb::store::create_cf_n { "accounts", tables::accounts }, { "blocks", tables::blocks }, { "pending", tables::pending }, - { "unchecked", tables::unchecked }, { "vote", tables::vote }, { "online_weight", tables::online_weight }, { "meta", tables::meta }, @@ -128,22 +198,21 @@ std::unordered_map nano::rocksdb::store::create_cf_n return map; } -void nano::rocksdb::store::open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a) +void nano::rocksdb::store::open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a, ::rocksdb::Options const & options_a, std::vector<::rocksdb::ColumnFamilyDescriptor> column_families) { - auto column_families = create_column_families (); - auto options = get_db_options (); + // auto options = get_db_options (); ::rocksdb::Status s; std::vector<::rocksdb::ColumnFamilyHandle *> handles_l; if (open_read_only_a) { ::rocksdb::DB * db_l; - s = ::rocksdb::DB::OpenForReadOnly (options, path_a.string (), column_families, &handles_l, &db_l); + s = ::rocksdb::DB::OpenForReadOnly (options_a, path_a.string (), column_families, &handles_l, &db_l); db.reset (db_l); } else { - s = ::rocksdb::OptimisticTransactionDB::Open (options, path_a.string (), column_families, &handles_l, &optimistic_db); + s = ::rocksdb::OptimisticTransactionDB::Open (options_a, path_a.string (), column_families, &handles_l, &optimistic_db); if (optimistic_db) { db.reset (optimistic_db); @@ -158,22 +227,74 @@ void nano::rocksdb::store::open (bool & error_a, boost::filesystem::path const & // Assign handles to supplied error_a |= !s.ok (); +} - if (!error_a) +bool nano::rocksdb::store::do_upgrades (nano::write_transaction const & transaction_a) +{ + bool error_l{ false }; + auto version_l = version.get (transaction_a); + switch (version_l) { - auto transaction = tx_begin_read (); - auto version_l = version.get (transaction); - if (version_l > version_current) - { - error_a = true; + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + case 10: + case 11: + case 12: + case 13: + release_assert (false && "do_upgrades () for RocksDB requires the version_minimum already checked."); + error_l = true; + break; + case 14: + case 15: + case 16: + case 17: + case 18: + case 19: + case 20: + case 21: + upgrade_v21_to_v22 (transaction_a); + [[fallthrough]]; + case 22: + break; + default: logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); - } + error_l = true; + break; } + return error_l; +} + +void nano::rocksdb::store::upgrade_v21_to_v22 (nano::write_transaction const & transaction_a) +{ + logger.always_log ("Preparing v21 to v22 database upgrade..."); + if (column_family_exists ("unchecked")) + { + auto const unchecked_handle = get_column_family ("unchecked"); + db->DropColumnFamily (unchecked_handle); + db->DestroyColumnFamilyHandle (unchecked_handle); + std::erase_if (handles, [unchecked_handle] (auto & handle) { + if (handle.get () == unchecked_handle) + { + // The handle resource is deleted by RocksDB. + [[maybe_unused]] auto ptr = handle.release (); + return true; + } + return false; + }); + } + version.put (transaction_a, 22); + logger.always_log ("Finished removing unchecked table"); } void nano::rocksdb::store::generate_tombstone_map () { - tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::unchecked), std::forward_as_tuple (0, 50000)); tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::blocks), std::forward_as_tuple (0, 25000)); tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::accounts), std::forward_as_tuple (0, 25000)); tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::pending), std::forward_as_tuple (0, 25000)); @@ -216,21 +337,7 @@ rocksdb::ColumnFamilyOptions nano::rocksdb::store::get_cf_options (std::string c ::rocksdb::ColumnFamilyOptions cf_options; auto const memtable_size_bytes = base_memtable_size_bytes (); auto const block_cache_size_bytes = 1024ULL * 1024 * rocksdb_config.memory_multiplier * base_block_cache_size; - if (cf_name_a == "unchecked") - { - std::shared_ptr<::rocksdb::TableFactory> table_factory (::rocksdb::NewBlockBasedTableFactory (get_active_table_options (block_cache_size_bytes * 4))); - cf_options = get_active_cf_options (table_factory, memtable_size_bytes); - - // Create prefix bloom for memtable with the size of write_buffer_size * memtable_prefix_bloom_size_ratio - cf_options.memtable_prefix_bloom_size_ratio = 0.25; - - // Number of files in level 0 which triggers compaction. Size of L0 and L1 should be kept similar as this is the only compaction which is single threaded - cf_options.level0_file_num_compaction_trigger = 2; - - // L1 size, compaction is triggered for L0 at this size (2 SST files in L1) - cf_options.max_bytes_for_level_base = memtable_size_bytes * 2; - } - else if (cf_name_a == "blocks") + if (cf_name_a == "blocks") { std::shared_ptr<::rocksdb::TableFactory> table_factory (::rocksdb::NewBlockBasedTableFactory (get_active_table_options (block_cache_size_bytes * 4))); cf_options = get_active_cf_options (table_factory, blocks_memtable_size_bytes ()); @@ -346,46 +453,81 @@ std::string nano::rocksdb::store::vendor_get () const return boost::str (boost::format ("RocksDB %1%.%2%.%3%") % ROCKSDB_MAJOR % ROCKSDB_MINOR % ROCKSDB_PATCH); } -rocksdb::ColumnFamilyHandle * nano::rocksdb::store::table_to_column_family (tables table_a) const +std::vector<::rocksdb::ColumnFamilyDescriptor> nano::rocksdb::store::get_single_column_family (std::string cf_name) const +{ + std::vector<::rocksdb::ColumnFamilyDescriptor> minimum_cf_set{ + { ::rocksdb::kDefaultColumnFamilyName, ::rocksdb::ColumnFamilyOptions{} }, + { cf_name, get_cf_options (cf_name) } + }; + return minimum_cf_set; +} + +std::vector<::rocksdb::ColumnFamilyDescriptor> nano::rocksdb::store::get_current_column_families (std::string const & path_a, ::rocksdb::Options const & options_a) const +{ + std::vector<::rocksdb::ColumnFamilyDescriptor> column_families; + + // Retrieve the column families available in the database. + std::vector current_cf_names; + auto s = ::rocksdb::DB::ListColumnFamilies (options_a, path_a, ¤t_cf_names); + debug_assert (s.ok ()); + + column_families.reserve (current_cf_names.size ()); + for (const auto & cf : current_cf_names) + { + column_families.emplace_back (cf, ::rocksdb::ColumnFamilyOptions ()); + } + + return column_families; +} + +rocksdb::ColumnFamilyHandle * nano::rocksdb::store::get_column_family (char const * name) const { auto & handles_l = handles; - auto get_handle = [&handles_l] (char const * name) { - auto iter = std::find_if (handles_l.begin (), handles_l.end (), [name] (auto & handle) { - return (handle->GetName () == name); - }); - debug_assert (iter != handles_l.end ()); - return (*iter).get (); - }; + auto iter = std::find_if (handles_l.begin (), handles_l.end (), [name] (auto & handle) { + return (handle->GetName () == name); + }); + debug_assert (iter != handles_l.end ()); + return (*iter).get (); +} +bool nano::rocksdb::store::column_family_exists (char const * name) const +{ + auto & handles_l = handles; + auto iter = std::find_if (handles_l.begin (), handles_l.end (), [name] (auto & handle) { + return (handle->GetName () == name); + }); + return (iter != handles_l.end ()); +} + +rocksdb::ColumnFamilyHandle * nano::rocksdb::store::table_to_column_family (tables table_a) const +{ switch (table_a) { case tables::frontiers: - return get_handle ("frontiers"); + return get_column_family ("frontiers"); case tables::accounts: - return get_handle ("accounts"); + return get_column_family ("accounts"); case tables::blocks: - return get_handle ("blocks"); + return get_column_family ("blocks"); case tables::pending: - return get_handle ("pending"); - case tables::unchecked: - return get_handle ("unchecked"); + return get_column_family ("pending"); case tables::vote: - return get_handle ("vote"); + return get_column_family ("vote"); case tables::online_weight: - return get_handle ("online_weight"); + return get_column_family ("online_weight"); case tables::meta: - return get_handle ("meta"); + return get_column_family ("meta"); case tables::peers: - return get_handle ("peers"); + return get_column_family ("peers"); case tables::pruned: - return get_handle ("pruned"); + return get_column_family ("pruned"); case tables::confirmation_height: - return get_handle ("confirmation_height"); + return get_column_family ("confirmation_height"); case tables::final_votes: - return get_handle ("final_votes"); + return get_column_family ("final_votes"); default: release_assert (false); - return get_handle (""); + return get_column_family (""); } } @@ -506,11 +648,6 @@ uint64_t nano::rocksdb::store::count (nano::transaction const & transaction_a, t ++sum; } } - // This is only an estimation - else if (table_a == tables::unchecked) - { - db->GetIntProperty (table_to_column_family (table_a), "rocksdb.estimate-num-keys", &sum); - } // This should be correct at node start, later only cache should be used else if (table_a == tables::pruned) { @@ -613,6 +750,7 @@ rocksdb::Options nano::rocksdb::store::get_db_options () db_options.create_if_missing = true; db_options.create_missing_column_families = true; + // TODO: review if this should be changed due to the unchecked table removal. // Enable whole key bloom filter in memtables for ones with memtable_prefix_bloom_size_ratio set (unchecked table currently). // It can potentially reduce CPU usage for point-look-ups. db_options.memtable_whole_key_filtering = true; @@ -640,7 +778,9 @@ rocksdb::Options nano::rocksdb::store::get_db_options () // Not compressing any SST files for compatibility reasons. db_options.compression = ::rocksdb::kNoCompression; - auto event_listener_l = new event_listener ([this] (::rocksdb::FlushJobInfo const & flush_job_info_a) { this->on_flush (flush_job_info_a); }); + auto event_listener_l = new event_listener ([this] (::rocksdb::FlushJobInfo const & flush_job_info_a) { + this->on_flush (flush_job_info_a); + }); db_options.listeners.emplace_back (event_listener_l); return db_options; @@ -726,7 +866,7 @@ void nano::rocksdb::store::on_flush (::rocksdb::FlushJobInfo const & flush_job_i std::vector nano::rocksdb::store::all_tables () const { - return std::vector{ tables::accounts, tables::blocks, tables::confirmation_height, tables::final_votes, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::pruned, tables::unchecked, tables::vote }; + return std::vector{ tables::accounts, tables::blocks, tables::confirmation_height, tables::final_votes, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::pruned, tables::vote }; } bool nano::rocksdb::store::copy_db (boost::filesystem::path const & destination_path) diff --git a/nano/node/rocksdb/rocksdb.hpp b/nano/node/rocksdb/rocksdb.hpp index 7edafc5d..47f3e718 100644 --- a/nano/node/rocksdb/rocksdb.hpp +++ b/nano/node/rocksdb/rocksdb.hpp @@ -31,6 +31,8 @@ class rocksdb_block_store_tombstone_count_Test; namespace rocksdb { + class rocksdb_block_store_upgrade_v21_v22_Test; + /** * rocksdb implementation of the block store */ @@ -136,10 +138,17 @@ namespace rocksdb int status_code_not_found () const override; int drop (nano::write_transaction const &, tables) override; + std::vector<::rocksdb::ColumnFamilyDescriptor> get_single_column_family (std::string cf_name) const; + std::vector<::rocksdb::ColumnFamilyDescriptor> get_current_column_families (std::string const & path_a, ::rocksdb::Options const & options_a) const; + ::rocksdb::ColumnFamilyHandle * get_column_family (char const * name) const; + bool column_family_exists (char const * name) const; ::rocksdb::ColumnFamilyHandle * table_to_column_family (tables table_a) const; int clear (::rocksdb::ColumnFamilyHandle * column_family); - void open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a); + void open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a, ::rocksdb::Options const & options_a, std::vector<::rocksdb::ColumnFamilyDescriptor> column_families); + + bool do_upgrades (nano::write_transaction const &); + void upgrade_v21_to_v22 (nano::write_transaction const &); void construct_column_family_mutexes (); ::rocksdb::Options get_db_options (); @@ -164,6 +173,7 @@ namespace rocksdb constexpr static int base_block_cache_size = 8; friend class nano::rocksdb_block_store_tombstone_count_Test; + friend class nano::rocksdb::rocksdb_block_store_upgrade_v21_v22_Test; }; } // namespace rocksdb } // namespace nano diff --git a/nano/secure/store.hpp b/nano/secure/store.hpp index 8d16eb81..50c6a3a9 100644 --- a/nano/secure/store.hpp +++ b/nano/secure/store.hpp @@ -515,7 +515,6 @@ enum class tables peers, pending, pruned, - unchecked, vote }; @@ -772,6 +771,7 @@ public: class store { friend class rocksdb_block_store_tombstone_count_Test; + friend class mdb_block_store_upgrade_v21_v22_Test; public: // clang-format off @@ -802,7 +802,7 @@ public: account_store & account; pending_store & pending; static int constexpr version_minimum{ 14 }; - static int constexpr version_current{ 21 }; + static int constexpr version_current{ 22 }; public: online_weight_store & online_weight;