Merge pull request #4131 from simpago/clear-unchecked-table
Add LMDB and RocksDB upgrades to v22: Remove unchecked table
This commit is contained in:
clemahieu
GitHub
commit
0f19fab219
10 changed files with 319 additions and 99 deletions
|
|
@ -17,10 +17,10 @@
|
|||
|
||||
#include <boost/filesystem.hpp>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <fstream>
|
||||
#include <unordered_set>
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <vector>
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
|
|
@ -2177,6 +2177,89 @@ namespace lmdb
|
|||
auto transaction (store.tx_begin_read ());
|
||||
ASSERT_LT (19, store.version.get (transaction));
|
||||
}
|
||||
|
||||
TEST (mdb_block_store, upgrade_v21_v22)
|
||||
{
|
||||
if (nano::rocksdb_config::using_rocksdb_in_tests ())
|
||||
{
|
||||
// Don't test this in rocksdb mode
|
||||
GTEST_SKIP ();
|
||||
}
|
||||
|
||||
auto path (nano::unique_path ());
|
||||
nano::logger_mt logger;
|
||||
nano::stats stats;
|
||||
auto const check_correct_state = [&] () {
|
||||
nano::lmdb::store store (logger, path, nano::dev::constants);
|
||||
auto transaction (store.tx_begin_write ());
|
||||
ASSERT_EQ (store.version.get (transaction), store.version_current);
|
||||
MDB_dbi unchecked_handle{ 0 };
|
||||
ASSERT_EQ (MDB_NOTFOUND, mdb_dbi_open (store.env.tx (transaction), "unchecked", 0, &unchecked_handle));
|
||||
};
|
||||
|
||||
// Testing current version doesn't contain the unchecked table
|
||||
check_correct_state ();
|
||||
|
||||
// Setting the database to its 21st version state
|
||||
{
|
||||
nano::lmdb::store store (logger, path, nano::dev::constants);
|
||||
auto transaction (store.tx_begin_write ());
|
||||
store.version.put (transaction, 21);
|
||||
MDB_dbi unchecked_handle{ 0 };
|
||||
ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE, &unchecked_handle));
|
||||
ASSERT_EQ (store.version.get (transaction), 21);
|
||||
}
|
||||
|
||||
// Testing the upgrade code worked
|
||||
check_correct_state ();
|
||||
}
|
||||
}
|
||||
|
||||
namespace rocksdb
|
||||
{
|
||||
TEST (rocksdb_block_store, upgrade_v21_v22)
|
||||
{
|
||||
if (!nano::rocksdb_config::using_rocksdb_in_tests ())
|
||||
{
|
||||
// Don't test this in LMDB mode
|
||||
GTEST_SKIP ();
|
||||
}
|
||||
|
||||
auto const path = nano::unique_path ();
|
||||
nano::logger_mt logger;
|
||||
nano::stats stats;
|
||||
auto const check_correct_state = [&] () {
|
||||
nano::rocksdb::store store (logger, path, nano::dev::constants);
|
||||
auto transaction (store.tx_begin_write ());
|
||||
ASSERT_EQ (store.version.get (transaction), store.version_current);
|
||||
ASSERT_FALSE (store.column_family_exists ("unchecked"));
|
||||
};
|
||||
|
||||
// Testing current version doesn't contain the unchecked table
|
||||
check_correct_state ();
|
||||
|
||||
// Setting the database to its 21st version state
|
||||
{
|
||||
nano::rocksdb::store store (logger, path, nano::dev::constants);
|
||||
|
||||
// Create a column family for "unchecked"
|
||||
::rocksdb::ColumnFamilyOptions new_cf_options;
|
||||
::rocksdb::ColumnFamilyHandle * new_cf_handle;
|
||||
::rocksdb::Status status = store.db->CreateColumnFamily (new_cf_options, "unchecked", &new_cf_handle);
|
||||
store.handles.emplace_back (new_cf_handle);
|
||||
|
||||
// The new column family was created successfully, and 'new_cf_handle' now points to it.
|
||||
ASSERT_TRUE (status.ok ());
|
||||
|
||||
// Rollback the database version number.
|
||||
auto transaction (store.tx_begin_write ());
|
||||
store.version.put (transaction, 21);
|
||||
ASSERT_EQ (store.version.get (transaction), 21);
|
||||
}
|
||||
|
||||
// Testing the upgrade code worked
|
||||
check_correct_state ();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5541,7 +5541,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
|
|||
boost::asio::ip::address_v6 address (boost::asio::ip::make_address_v6 ("::ffff:127.0.0.1"));
|
||||
uint16_t port = 100;
|
||||
nano::lmdb::store store{ logger, path / "data.ldb", nano::dev::constants };
|
||||
nano::unchecked_map unchecked{ system.stats, false };
|
||||
nano::ledger ledger{ store, system.stats, nano::dev::constants };
|
||||
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
|
||||
|
||||
|
|
@ -5556,7 +5555,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
|
|||
.build_shared ();
|
||||
|
||||
nano::endpoint_key endpoint_key (address.to_bytes (), port);
|
||||
auto version = 99;
|
||||
auto version = nano::store::version_current;
|
||||
|
||||
{
|
||||
auto transaction = store.tx_begin_write ();
|
||||
|
|
@ -5582,7 +5581,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
|
|||
ASSERT_FALSE (error);
|
||||
|
||||
nano::rocksdb::store rocksdb_store{ logger, path / "rocksdb", nano::dev::constants };
|
||||
nano::unchecked_map rocksdb_unchecked{ system.stats, false };
|
||||
auto rocksdb_transaction (rocksdb_store.tx_begin_read ());
|
||||
|
||||
nano::pending_info pending_info{};
|
||||
|
|
|
|||
|
|
@ -136,7 +136,6 @@ add_library(
|
|||
lmdb/pruned_store.cpp
|
||||
lmdb/version_store.hpp
|
||||
lmdb/version_store.cpp
|
||||
lmdb/unchecked_store.hpp
|
||||
lmdb/lmdb.hpp
|
||||
lmdb/lmdb.cpp
|
||||
lmdb/lmdb_env.hpp
|
||||
|
|
|
|||
|
|
@ -62,7 +62,6 @@ nano::lmdb::store::store (nano::logger_mt & logger_a, boost::filesystem::path co
|
|||
peer_store{ *this },
|
||||
confirmation_height_store{ *this },
|
||||
final_vote_store{ *this },
|
||||
unchecked_store{},
|
||||
version_store{ *this },
|
||||
logger (logger_a),
|
||||
env (error, path_a, nano::mdb_env::options::make ().set_config (lmdb_config_a).set_use_no_mem_init (true)),
|
||||
|
|
@ -208,7 +207,6 @@ nano::mdb_txn_callbacks nano::lmdb::store::create_txn_callbacks () const
|
|||
void nano::lmdb::store::open_databases (bool & error_a, nano::transaction const & transaction_a, unsigned flags)
|
||||
{
|
||||
error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontier_store.frontiers_handle) != 0;
|
||||
error_a |= mdb_dbi_open (env.tx (transaction_a), "unchecked", flags, &unchecked_store.unchecked_handle) != 0;
|
||||
error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight_store.online_weight_handle) != 0;
|
||||
error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &block_store.meta_handle) != 0;
|
||||
error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peer_store.peers_handle) != 0;
|
||||
|
|
@ -303,6 +301,9 @@ bool nano::lmdb::store::do_upgrades (nano::write_transaction & transaction_a, na
|
|||
upgrade_v20_to_v21 (transaction_a);
|
||||
[[fallthrough]];
|
||||
case 21:
|
||||
upgrade_v21_to_v22 (transaction_a);
|
||||
[[fallthrough]];
|
||||
case 22:
|
||||
break;
|
||||
default:
|
||||
logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l));
|
||||
|
|
@ -776,6 +777,16 @@ void nano::lmdb::store::upgrade_v20_to_v21 (nano::write_transaction const & tran
|
|||
logger.always_log ("Finished creating new final_vote table");
|
||||
}
|
||||
|
||||
void nano::lmdb::store::upgrade_v21_to_v22 (nano::write_transaction const & transaction_a)
|
||||
{
|
||||
logger.always_log ("Preparing v21 to v22 database upgrade...");
|
||||
MDB_dbi unchecked_handle{ 0 };
|
||||
release_assert (!mdb_dbi_open (env.tx (transaction_a), "unchecked", MDB_CREATE, &unchecked_handle));
|
||||
release_assert (!mdb_drop (env.tx (transaction_a), unchecked_handle, 1)); // del = 1, to delete it from the environment and close the DB handle.
|
||||
version.put (transaction_a, 22);
|
||||
logger.always_log ("Finished removing unchecked table");
|
||||
}
|
||||
|
||||
/** Takes a filepath, appends '_backup_<timestamp>' to the end (but before any extension) and saves that file in the same directory */
|
||||
void nano::lmdb::store::create_backup_file (nano::mdb_env & env_a, boost::filesystem::path const & filepath_a, nano::logger_mt & logger_a)
|
||||
{
|
||||
|
|
@ -865,8 +876,6 @@ MDB_dbi nano::lmdb::store::table_to_dbi (tables table_a) const
|
|||
return block_store.blocks_handle;
|
||||
case tables::pending:
|
||||
return pending_store.pending_handle;
|
||||
case tables::unchecked:
|
||||
return unchecked_store.unchecked_handle;
|
||||
case tables::online_weight:
|
||||
return online_weight_store.online_weight_handle;
|
||||
case tables::meta:
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@
|
|||
#include <nano/node/lmdb/peer_store.hpp>
|
||||
#include <nano/node/lmdb/pending_store.hpp>
|
||||
#include <nano/node/lmdb/pruned_store.hpp>
|
||||
#include <nano/node/lmdb/unchecked_store.hpp>
|
||||
#include <nano/node/lmdb/version_store.hpp>
|
||||
#include <nano/secure/common.hpp>
|
||||
#include <nano/secure/versioning.hpp>
|
||||
|
|
@ -57,7 +56,6 @@ namespace lmdb
|
|||
nano::lmdb::peer_store peer_store;
|
||||
nano::lmdb::pending_store pending_store;
|
||||
nano::lmdb::pruned_store pruned_store;
|
||||
nano::lmdb::unchecked_store unchecked_store;
|
||||
nano::lmdb::version_store version_store;
|
||||
|
||||
friend class nano::lmdb::account_store;
|
||||
|
|
@ -69,7 +67,6 @@ namespace lmdb
|
|||
friend class nano::lmdb::peer_store;
|
||||
friend class nano::lmdb::pending_store;
|
||||
friend class nano::lmdb::pruned_store;
|
||||
friend class nano::lmdb::unchecked_store;
|
||||
friend class nano::lmdb::version_store;
|
||||
|
||||
public:
|
||||
|
|
@ -136,6 +133,7 @@ namespace lmdb
|
|||
void upgrade_v18_to_v19 (nano::write_transaction const &);
|
||||
void upgrade_v19_to_v20 (nano::write_transaction const &);
|
||||
void upgrade_v20_to_v21 (nano::write_transaction const &);
|
||||
void upgrade_v21_to_v22 (nano::write_transaction const &);
|
||||
|
||||
std::shared_ptr<nano::block> block_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const;
|
||||
nano::mdb_val block_raw_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const;
|
||||
|
|
@ -188,6 +186,7 @@ namespace lmdb
|
|||
friend class mdb_block_store_upgrade_v18_v19_Test;
|
||||
friend class mdb_block_store_upgrade_v19_v20_Test;
|
||||
friend class mdb_block_store_upgrade_v20_v21_Test;
|
||||
friend class mdb_block_store_upgrade_v21_v22_Test;
|
||||
friend class block_store_DISABLED_change_dupsort_Test;
|
||||
friend void write_sideband_v14 (nano::lmdb::store &, nano::transaction &, nano::block const &, MDB_dbi);
|
||||
friend void write_sideband_v15 (nano::lmdb::store &, nano::transaction &, nano::block const &);
|
||||
|
|
|
|||
|
|
@ -1,16 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <lmdb/libraries/liblmdb/lmdb.h>
|
||||
|
||||
namespace nano::lmdb
|
||||
{
|
||||
class unchecked_store
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Unchecked bootstrap blocks info.
|
||||
* nano::block_hash -> nano::unchecked_info
|
||||
*/
|
||||
MDB_dbi unchecked_handle{ 0 };
|
||||
};
|
||||
}
|
||||
|
|
@ -441,7 +441,6 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co
|
|||
// Drop unchecked blocks if initial bootstrap is completed
|
||||
if (!flags.disable_unchecked_drop && !use_bootstrap_weight && !flags.read_only)
|
||||
{
|
||||
auto const transaction (store.tx_begin_write ({ tables::unchecked }));
|
||||
unchecked.clear ();
|
||||
logger.always_log ("Dropping unchecked blocks");
|
||||
}
|
||||
|
|
@ -996,7 +995,6 @@ void nano::node::unchecked_cleanup ()
|
|||
while (!cleaning_list.empty ())
|
||||
{
|
||||
std::size_t deleted_count (0);
|
||||
auto const transaction (store.tx_begin_write ({ tables::unchecked }));
|
||||
while (deleted_count++ < 2 * 1024 && !cleaning_list.empty ())
|
||||
{
|
||||
auto key (cleaning_list.front ());
|
||||
|
|
|
|||
|
|
@ -96,15 +96,86 @@ nano::rocksdb::store::store (nano::logger_mt & logger_a, boost::filesystem::path
|
|||
nano::set_secure_perm_directory (path_a, error_chmod);
|
||||
error = static_cast<bool> (error_mkdir);
|
||||
|
||||
if (error)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
generate_tombstone_map ();
|
||||
small_table_factory.reset (::rocksdb::NewBlockBasedTableFactory (get_small_table_options ()));
|
||||
|
||||
// TODO: get_db_options () registers a listener for resetting tombstones, needs to check if it is a problem calling it more than once.
|
||||
auto options = get_db_options ();
|
||||
|
||||
// The only certain column family is "meta" which contains the DB version info.
|
||||
// RocksDB requires this operation to be in read-only mode.
|
||||
auto is_fresh_db = false;
|
||||
open (is_fresh_db, path_a, true, options, get_single_column_family ("meta"));
|
||||
|
||||
auto is_fully_upgraded = false;
|
||||
if (!is_fresh_db)
|
||||
{
|
||||
auto transaction = tx_begin_read ();
|
||||
auto version_l = version.get (transaction);
|
||||
if (version_l > version_current)
|
||||
{
|
||||
error = true;
|
||||
logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l));
|
||||
return;
|
||||
}
|
||||
else if (version_l < version_minimum)
|
||||
{
|
||||
error = true;
|
||||
logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is lower than the minimum (%2%) which is supported for upgrades. Either upgrade to a v19, v20 or v21 node first or delete the ledger.") % version_l % version_minimum));
|
||||
return;
|
||||
}
|
||||
is_fully_upgraded = (version_l == version_current);
|
||||
}
|
||||
|
||||
if (db)
|
||||
{
|
||||
// Needs to clear the store references before reopening the DB.
|
||||
handles.clear ();
|
||||
db.reset (nullptr);
|
||||
}
|
||||
|
||||
if (!open_read_only_a)
|
||||
{
|
||||
construct_column_family_mutexes ();
|
||||
}
|
||||
|
||||
if (is_fully_upgraded)
|
||||
{
|
||||
open (error, path_a, open_read_only_a, options, create_column_families ());
|
||||
return;
|
||||
}
|
||||
|
||||
if (open_read_only_a)
|
||||
{
|
||||
// Either following cases cannot run in read-only mode:
|
||||
// a) there is no database yet, the access needs to be in write mode for it to be created;
|
||||
// b) it will upgrade, and it is not possible to do it in read-only mode.
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (is_fresh_db)
|
||||
{
|
||||
open (error, path_a, open_read_only_a, options, create_column_families ());
|
||||
if (!error)
|
||||
{
|
||||
version.put (tx_begin_write (), version_current); // It is fresh, someone needs to tell it its version.
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// The database is not upgraded, and it may not be compatible with the current column family set.
|
||||
open (error, path_a, open_read_only_a, options, get_current_column_families (path_a.string (), options));
|
||||
if (!error)
|
||||
{
|
||||
generate_tombstone_map ();
|
||||
small_table_factory.reset (::rocksdb::NewBlockBasedTableFactory (get_small_table_options ()));
|
||||
if (!open_read_only_a)
|
||||
{
|
||||
construct_column_family_mutexes ();
|
||||
}
|
||||
open (error, path_a, open_read_only_a);
|
||||
logger.always_log ("Upgrade in progress...");
|
||||
auto transaction = tx_begin_write ();
|
||||
error |= do_upgrades (transaction);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -115,7 +186,6 @@ std::unordered_map<char const *, nano::tables> nano::rocksdb::store::create_cf_n
|
|||
{ "accounts", tables::accounts },
|
||||
{ "blocks", tables::blocks },
|
||||
{ "pending", tables::pending },
|
||||
{ "unchecked", tables::unchecked },
|
||||
{ "vote", tables::vote },
|
||||
{ "online_weight", tables::online_weight },
|
||||
{ "meta", tables::meta },
|
||||
|
|
@ -128,22 +198,21 @@ std::unordered_map<char const *, nano::tables> nano::rocksdb::store::create_cf_n
|
|||
return map;
|
||||
}
|
||||
|
||||
void nano::rocksdb::store::open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a)
|
||||
void nano::rocksdb::store::open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a, ::rocksdb::Options const & options_a, std::vector<::rocksdb::ColumnFamilyDescriptor> column_families)
|
||||
{
|
||||
auto column_families = create_column_families ();
|
||||
auto options = get_db_options ();
|
||||
// auto options = get_db_options ();
|
||||
::rocksdb::Status s;
|
||||
|
||||
std::vector<::rocksdb::ColumnFamilyHandle *> handles_l;
|
||||
if (open_read_only_a)
|
||||
{
|
||||
::rocksdb::DB * db_l;
|
||||
s = ::rocksdb::DB::OpenForReadOnly (options, path_a.string (), column_families, &handles_l, &db_l);
|
||||
s = ::rocksdb::DB::OpenForReadOnly (options_a, path_a.string (), column_families, &handles_l, &db_l);
|
||||
db.reset (db_l);
|
||||
}
|
||||
else
|
||||
{
|
||||
s = ::rocksdb::OptimisticTransactionDB::Open (options, path_a.string (), column_families, &handles_l, &optimistic_db);
|
||||
s = ::rocksdb::OptimisticTransactionDB::Open (options_a, path_a.string (), column_families, &handles_l, &optimistic_db);
|
||||
if (optimistic_db)
|
||||
{
|
||||
db.reset (optimistic_db);
|
||||
|
|
@ -158,22 +227,74 @@ void nano::rocksdb::store::open (bool & error_a, boost::filesystem::path const &
|
|||
|
||||
// Assign handles to supplied
|
||||
error_a |= !s.ok ();
|
||||
}
|
||||
|
||||
if (!error_a)
|
||||
bool nano::rocksdb::store::do_upgrades (nano::write_transaction const & transaction_a)
|
||||
{
|
||||
bool error_l{ false };
|
||||
auto version_l = version.get (transaction_a);
|
||||
switch (version_l)
|
||||
{
|
||||
auto transaction = tx_begin_read ();
|
||||
auto version_l = version.get (transaction);
|
||||
if (version_l > version_current)
|
||||
{
|
||||
error_a = true;
|
||||
case 1:
|
||||
case 2:
|
||||
case 3:
|
||||
case 4:
|
||||
case 5:
|
||||
case 6:
|
||||
case 7:
|
||||
case 8:
|
||||
case 9:
|
||||
case 10:
|
||||
case 11:
|
||||
case 12:
|
||||
case 13:
|
||||
release_assert (false && "do_upgrades () for RocksDB requires the version_minimum already checked.");
|
||||
error_l = true;
|
||||
break;
|
||||
case 14:
|
||||
case 15:
|
||||
case 16:
|
||||
case 17:
|
||||
case 18:
|
||||
case 19:
|
||||
case 20:
|
||||
case 21:
|
||||
upgrade_v21_to_v22 (transaction_a);
|
||||
[[fallthrough]];
|
||||
case 22:
|
||||
break;
|
||||
default:
|
||||
logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l));
|
||||
}
|
||||
error_l = true;
|
||||
break;
|
||||
}
|
||||
return error_l;
|
||||
}
|
||||
|
||||
void nano::rocksdb::store::upgrade_v21_to_v22 (nano::write_transaction const & transaction_a)
|
||||
{
|
||||
logger.always_log ("Preparing v21 to v22 database upgrade...");
|
||||
if (column_family_exists ("unchecked"))
|
||||
{
|
||||
auto const unchecked_handle = get_column_family ("unchecked");
|
||||
db->DropColumnFamily (unchecked_handle);
|
||||
db->DestroyColumnFamilyHandle (unchecked_handle);
|
||||
std::erase_if (handles, [unchecked_handle] (auto & handle) {
|
||||
if (handle.get () == unchecked_handle)
|
||||
{
|
||||
// The handle resource is deleted by RocksDB.
|
||||
[[maybe_unused]] auto ptr = handle.release ();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
}
|
||||
version.put (transaction_a, 22);
|
||||
logger.always_log ("Finished removing unchecked table");
|
||||
}
|
||||
|
||||
void nano::rocksdb::store::generate_tombstone_map ()
|
||||
{
|
||||
tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::unchecked), std::forward_as_tuple (0, 50000));
|
||||
tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::blocks), std::forward_as_tuple (0, 25000));
|
||||
tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::accounts), std::forward_as_tuple (0, 25000));
|
||||
tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::pending), std::forward_as_tuple (0, 25000));
|
||||
|
|
@ -216,21 +337,7 @@ rocksdb::ColumnFamilyOptions nano::rocksdb::store::get_cf_options (std::string c
|
|||
::rocksdb::ColumnFamilyOptions cf_options;
|
||||
auto const memtable_size_bytes = base_memtable_size_bytes ();
|
||||
auto const block_cache_size_bytes = 1024ULL * 1024 * rocksdb_config.memory_multiplier * base_block_cache_size;
|
||||
if (cf_name_a == "unchecked")
|
||||
{
|
||||
std::shared_ptr<::rocksdb::TableFactory> table_factory (::rocksdb::NewBlockBasedTableFactory (get_active_table_options (block_cache_size_bytes * 4)));
|
||||
cf_options = get_active_cf_options (table_factory, memtable_size_bytes);
|
||||
|
||||
// Create prefix bloom for memtable with the size of write_buffer_size * memtable_prefix_bloom_size_ratio
|
||||
cf_options.memtable_prefix_bloom_size_ratio = 0.25;
|
||||
|
||||
// Number of files in level 0 which triggers compaction. Size of L0 and L1 should be kept similar as this is the only compaction which is single threaded
|
||||
cf_options.level0_file_num_compaction_trigger = 2;
|
||||
|
||||
// L1 size, compaction is triggered for L0 at this size (2 SST files in L1)
|
||||
cf_options.max_bytes_for_level_base = memtable_size_bytes * 2;
|
||||
}
|
||||
else if (cf_name_a == "blocks")
|
||||
if (cf_name_a == "blocks")
|
||||
{
|
||||
std::shared_ptr<::rocksdb::TableFactory> table_factory (::rocksdb::NewBlockBasedTableFactory (get_active_table_options (block_cache_size_bytes * 4)));
|
||||
cf_options = get_active_cf_options (table_factory, blocks_memtable_size_bytes ());
|
||||
|
|
@ -346,46 +453,81 @@ std::string nano::rocksdb::store::vendor_get () const
|
|||
return boost::str (boost::format ("RocksDB %1%.%2%.%3%") % ROCKSDB_MAJOR % ROCKSDB_MINOR % ROCKSDB_PATCH);
|
||||
}
|
||||
|
||||
rocksdb::ColumnFamilyHandle * nano::rocksdb::store::table_to_column_family (tables table_a) const
|
||||
std::vector<::rocksdb::ColumnFamilyDescriptor> nano::rocksdb::store::get_single_column_family (std::string cf_name) const
|
||||
{
|
||||
std::vector<::rocksdb::ColumnFamilyDescriptor> minimum_cf_set{
|
||||
{ ::rocksdb::kDefaultColumnFamilyName, ::rocksdb::ColumnFamilyOptions{} },
|
||||
{ cf_name, get_cf_options (cf_name) }
|
||||
};
|
||||
return minimum_cf_set;
|
||||
}
|
||||
|
||||
std::vector<::rocksdb::ColumnFamilyDescriptor> nano::rocksdb::store::get_current_column_families (std::string const & path_a, ::rocksdb::Options const & options_a) const
|
||||
{
|
||||
std::vector<::rocksdb::ColumnFamilyDescriptor> column_families;
|
||||
|
||||
// Retrieve the column families available in the database.
|
||||
std::vector<std::string> current_cf_names;
|
||||
auto s = ::rocksdb::DB::ListColumnFamilies (options_a, path_a, ¤t_cf_names);
|
||||
debug_assert (s.ok ());
|
||||
|
||||
column_families.reserve (current_cf_names.size ());
|
||||
for (const auto & cf : current_cf_names)
|
||||
{
|
||||
column_families.emplace_back (cf, ::rocksdb::ColumnFamilyOptions ());
|
||||
}
|
||||
|
||||
return column_families;
|
||||
}
|
||||
|
||||
rocksdb::ColumnFamilyHandle * nano::rocksdb::store::get_column_family (char const * name) const
|
||||
{
|
||||
auto & handles_l = handles;
|
||||
auto get_handle = [&handles_l] (char const * name) {
|
||||
auto iter = std::find_if (handles_l.begin (), handles_l.end (), [name] (auto & handle) {
|
||||
return (handle->GetName () == name);
|
||||
});
|
||||
debug_assert (iter != handles_l.end ());
|
||||
return (*iter).get ();
|
||||
};
|
||||
auto iter = std::find_if (handles_l.begin (), handles_l.end (), [name] (auto & handle) {
|
||||
return (handle->GetName () == name);
|
||||
});
|
||||
debug_assert (iter != handles_l.end ());
|
||||
return (*iter).get ();
|
||||
}
|
||||
|
||||
bool nano::rocksdb::store::column_family_exists (char const * name) const
|
||||
{
|
||||
auto & handles_l = handles;
|
||||
auto iter = std::find_if (handles_l.begin (), handles_l.end (), [name] (auto & handle) {
|
||||
return (handle->GetName () == name);
|
||||
});
|
||||
return (iter != handles_l.end ());
|
||||
}
|
||||
|
||||
rocksdb::ColumnFamilyHandle * nano::rocksdb::store::table_to_column_family (tables table_a) const
|
||||
{
|
||||
switch (table_a)
|
||||
{
|
||||
case tables::frontiers:
|
||||
return get_handle ("frontiers");
|
||||
return get_column_family ("frontiers");
|
||||
case tables::accounts:
|
||||
return get_handle ("accounts");
|
||||
return get_column_family ("accounts");
|
||||
case tables::blocks:
|
||||
return get_handle ("blocks");
|
||||
return get_column_family ("blocks");
|
||||
case tables::pending:
|
||||
return get_handle ("pending");
|
||||
case tables::unchecked:
|
||||
return get_handle ("unchecked");
|
||||
return get_column_family ("pending");
|
||||
case tables::vote:
|
||||
return get_handle ("vote");
|
||||
return get_column_family ("vote");
|
||||
case tables::online_weight:
|
||||
return get_handle ("online_weight");
|
||||
return get_column_family ("online_weight");
|
||||
case tables::meta:
|
||||
return get_handle ("meta");
|
||||
return get_column_family ("meta");
|
||||
case tables::peers:
|
||||
return get_handle ("peers");
|
||||
return get_column_family ("peers");
|
||||
case tables::pruned:
|
||||
return get_handle ("pruned");
|
||||
return get_column_family ("pruned");
|
||||
case tables::confirmation_height:
|
||||
return get_handle ("confirmation_height");
|
||||
return get_column_family ("confirmation_height");
|
||||
case tables::final_votes:
|
||||
return get_handle ("final_votes");
|
||||
return get_column_family ("final_votes");
|
||||
default:
|
||||
release_assert (false);
|
||||
return get_handle ("");
|
||||
return get_column_family ("");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -506,11 +648,6 @@ uint64_t nano::rocksdb::store::count (nano::transaction const & transaction_a, t
|
|||
++sum;
|
||||
}
|
||||
}
|
||||
// This is only an estimation
|
||||
else if (table_a == tables::unchecked)
|
||||
{
|
||||
db->GetIntProperty (table_to_column_family (table_a), "rocksdb.estimate-num-keys", &sum);
|
||||
}
|
||||
// This should be correct at node start, later only cache should be used
|
||||
else if (table_a == tables::pruned)
|
||||
{
|
||||
|
|
@ -613,6 +750,7 @@ rocksdb::Options nano::rocksdb::store::get_db_options ()
|
|||
db_options.create_if_missing = true;
|
||||
db_options.create_missing_column_families = true;
|
||||
|
||||
// TODO: review if this should be changed due to the unchecked table removal.
|
||||
// Enable whole key bloom filter in memtables for ones with memtable_prefix_bloom_size_ratio set (unchecked table currently).
|
||||
// It can potentially reduce CPU usage for point-look-ups.
|
||||
db_options.memtable_whole_key_filtering = true;
|
||||
|
|
@ -640,7 +778,9 @@ rocksdb::Options nano::rocksdb::store::get_db_options ()
|
|||
// Not compressing any SST files for compatibility reasons.
|
||||
db_options.compression = ::rocksdb::kNoCompression;
|
||||
|
||||
auto event_listener_l = new event_listener ([this] (::rocksdb::FlushJobInfo const & flush_job_info_a) { this->on_flush (flush_job_info_a); });
|
||||
auto event_listener_l = new event_listener ([this] (::rocksdb::FlushJobInfo const & flush_job_info_a) {
|
||||
this->on_flush (flush_job_info_a);
|
||||
});
|
||||
db_options.listeners.emplace_back (event_listener_l);
|
||||
|
||||
return db_options;
|
||||
|
|
@ -726,7 +866,7 @@ void nano::rocksdb::store::on_flush (::rocksdb::FlushJobInfo const & flush_job_i
|
|||
|
||||
std::vector<nano::tables> nano::rocksdb::store::all_tables () const
|
||||
{
|
||||
return std::vector<nano::tables>{ tables::accounts, tables::blocks, tables::confirmation_height, tables::final_votes, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::pruned, tables::unchecked, tables::vote };
|
||||
return std::vector<nano::tables>{ tables::accounts, tables::blocks, tables::confirmation_height, tables::final_votes, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::pruned, tables::vote };
|
||||
}
|
||||
|
||||
bool nano::rocksdb::store::copy_db (boost::filesystem::path const & destination_path)
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ class rocksdb_block_store_tombstone_count_Test;
|
|||
|
||||
namespace rocksdb
|
||||
{
|
||||
class rocksdb_block_store_upgrade_v21_v22_Test;
|
||||
|
||||
/**
|
||||
* rocksdb implementation of the block store
|
||||
*/
|
||||
|
|
@ -136,10 +138,17 @@ namespace rocksdb
|
|||
int status_code_not_found () const override;
|
||||
int drop (nano::write_transaction const &, tables) override;
|
||||
|
||||
std::vector<::rocksdb::ColumnFamilyDescriptor> get_single_column_family (std::string cf_name) const;
|
||||
std::vector<::rocksdb::ColumnFamilyDescriptor> get_current_column_families (std::string const & path_a, ::rocksdb::Options const & options_a) const;
|
||||
::rocksdb::ColumnFamilyHandle * get_column_family (char const * name) const;
|
||||
bool column_family_exists (char const * name) const;
|
||||
::rocksdb::ColumnFamilyHandle * table_to_column_family (tables table_a) const;
|
||||
int clear (::rocksdb::ColumnFamilyHandle * column_family);
|
||||
|
||||
void open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a);
|
||||
void open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a, ::rocksdb::Options const & options_a, std::vector<::rocksdb::ColumnFamilyDescriptor> column_families);
|
||||
|
||||
bool do_upgrades (nano::write_transaction const &);
|
||||
void upgrade_v21_to_v22 (nano::write_transaction const &);
|
||||
|
||||
void construct_column_family_mutexes ();
|
||||
::rocksdb::Options get_db_options ();
|
||||
|
|
@ -164,6 +173,7 @@ namespace rocksdb
|
|||
constexpr static int base_block_cache_size = 8;
|
||||
|
||||
friend class nano::rocksdb_block_store_tombstone_count_Test;
|
||||
friend class nano::rocksdb::rocksdb_block_store_upgrade_v21_v22_Test;
|
||||
};
|
||||
} // namespace rocksdb
|
||||
} // namespace nano
|
||||
|
|
|
|||
|
|
@ -515,7 +515,6 @@ enum class tables
|
|||
peers,
|
||||
pending,
|
||||
pruned,
|
||||
unchecked,
|
||||
vote
|
||||
};
|
||||
|
||||
|
|
@ -772,6 +771,7 @@ public:
|
|||
class store
|
||||
{
|
||||
friend class rocksdb_block_store_tombstone_count_Test;
|
||||
friend class mdb_block_store_upgrade_v21_v22_Test;
|
||||
|
||||
public:
|
||||
// clang-format off
|
||||
|
|
@ -802,7 +802,7 @@ public:
|
|||
account_store & account;
|
||||
pending_store & pending;
|
||||
static int constexpr version_minimum{ 14 };
|
||||
static int constexpr version_current{ 21 };
|
||||
static int constexpr version_current{ 22 };
|
||||
|
||||
public:
|
||||
online_weight_store & online_weight;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue