Minecon724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Add CLI command for better LMDB compaction (#2435)

Browse source 
* Add CLI command for better LMDB compaction
* Special path for pending table
* Fix RocksDB function
This commit is contained in:
Sergey Kroshnin 2019-12-17 11:53:22 +03:00 • committed by GitHub
commit 8e6cde718b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 65 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

8
nano/node/cli.cpp
View file

@ -51,6 +51,7 @@ void nano::add_node_options (boost::program_options::options_description & descr
("peer_clear", "Clear online peers database dump")
("unchecked_clear", "Clear unchecked blocks")
("confirmation_height_clear", "Clear confirmation height")
("rebuild_database", "Rebuild LMDB database with vacuum for best compaction")
("diagnostics", "Run internal diagnostics")
("generate_config", boost::program_options::value<std::string> (), "Write configuration to stdout, populated with defaults suitable for this system. Pass the configuration type node or rpc. See also use_defaults.")
("key_create", "Generates a adhoc random keypair and prints it to stdout")
@ -155,7 +156,7 @@ void database_write_lock_error (std::error_code & ec)
bool copy_database (boost::filesystem::path const & data_path, boost::program_options::variables_map & vm, boost::filesystem::path const & output_path, std::error_code & ec)
{
bool success = false;
bool needs_to_write = vm.count ("unchecked_clear") || vm.count ("clear_send_ids") || vm.count ("online_weight_clear") || vm.count ("peer_clear") || vm.count ("confirmation_height_clear");
bool needs_to_write = vm.count ("unchecked_clear") || vm.count ("clear_send_ids") || vm.count ("online_weight_clear") || vm.count ("peer_clear") || vm.count ("confirmation_height_clear") || vm.count ("rebuild_database");
auto node_flags = nano::inactive_node_flag_defaults ();
node_flags.read_only = !needs_to_write;
@ -186,6 +187,11 @@ bool copy_database (boost::filesystem::path const & data_path, boost::program_op
{
reset_confirmation_heights (node.node->store);
}
if (vm.count ("rebuild_database"))
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.rebuild_db (transaction);
}
success = node.node->copy_with_compaction (output_path);
}

50
nano/node/lmdb/lmdb.cpp
View file

@ -868,6 +868,56 @@ bool nano::mdb_store::copy_db (boost::filesystem::path const & destination_file)
return !mdb_env_copy2 (env.environment, destination_file.string ().c_str (), MDB_CP_COMPACT);
}
void nano::mdb_store::rebuild_db (nano::write_transaction const & transaction_a)
{
// Tables with uint256_union key
std::vector<MDB_dbi> tables = { accounts, send_blocks, receive_blocks, open_blocks, change_blocks, state_blocks, vote, confirmation_height };
for (auto const & table : tables)
{
MDB_dbi temp;
mdb_dbi_open (env.tx (transaction_a), "temp_table", MDB_CREATE, &temp);
// Copy all values to temporary table
for (auto i (nano::store_iterator<nano::uint256_union, nano::mdb_val> (std::make_unique<nano::mdb_iterator<nano::uint256_union, nano::mdb_val>> (transaction_a, table))), n (nano::store_iterator<nano::uint256_union, nano::mdb_val> (nullptr)); i != n; ++i)
{
auto s = mdb_put (env.tx (transaction_a), temp, nano::mdb_val (i->first), i->second, MDB_APPEND);
release_assert (success (s));
}
release_assert (count (transaction_a, table) == count (transaction_a, temp));
// Clear existing table
mdb_drop (env.tx (transaction_a), table, 0);
// Put values from copy
for (auto i (nano::store_iterator<nano::uint256_union, nano::mdb_val> (std::make_unique<nano::mdb_iterator<nano::uint256_union, nano::mdb_val>> (transaction_a, temp))), n (nano::store_iterator<nano::uint256_union, nano::mdb_val> (nullptr)); i != n; ++i)
{
auto s = mdb_put (env.tx (transaction_a), table, nano::mdb_val (i->first), i->second, MDB_APPEND);
release_assert (success (s));
}
release_assert (count (transaction_a, table) == count (transaction_a, temp));
// Remove temporary table
mdb_drop (env.tx (transaction_a), temp, 1);
}
// Pending table
{
MDB_dbi temp;
mdb_dbi_open (env.tx (transaction_a), "temp_table", MDB_CREATE, &temp);
// Copy all values to temporary table
for (auto i (nano::store_iterator<nano::pending_key, nano::pending_info> (std::make_unique<nano::mdb_iterator<nano::pending_key, nano::pending_info>> (transaction_a, pending))), n (nano::store_iterator<nano::pending_key, nano::pending_info> (nullptr)); i != n; ++i)
{
auto s = mdb_put (env.tx (transaction_a), temp, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND);
release_assert (success (s));
}
release_assert (count (transaction_a, pending) == count (transaction_a, temp));
mdb_drop (env.tx (transaction_a), pending, 0);
// Put values from copy
for (auto i (nano::store_iterator<nano::pending_key, nano::pending_info> (std::make_unique<nano::mdb_iterator<nano::pending_key, nano::pending_info>> (transaction_a, temp))), n (nano::store_iterator<nano::pending_key, nano::pending_info> (nullptr)); i != n; ++i)
{
auto s = mdb_put (env.tx (transaction_a), pending, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND);
release_assert (success (s));
}
release_assert (count (transaction_a, pending) == count (transaction_a, temp));
mdb_drop (env.tx (transaction_a), temp, 1);
}
}
bool nano::mdb_store::init_error () const
{
return error;

1
nano/node/lmdb/lmdb.hpp
View file

@ -196,6 +196,7 @@ public:
int del (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const;
bool copy_db (boost::filesystem::path const & destination_file) override;
void rebuild_db (nano::write_transaction const & transaction_a) override;
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a) const

5
nano/node/rocksdb/rocksdb.cpp
View file

@ -603,6 +603,11 @@ bool nano::rocksdb_store::copy_db (boost::filesystem::path const & destination_p
return false;
}
void nano::rocksdb_store::rebuild_db (nano::write_transaction const & transaction_a)
{
release_assert (false && "Not available for RocksDB");
}
bool nano::rocksdb_store::init_error () const
{
return error;

1
nano/node/rocksdb/rocksdb.hpp
View file

@ -55,6 +55,7 @@ public:
}
bool copy_db (boost::filesystem::path const & destination) override;
void rebuild_db (nano::write_transaction const & transaction_a) override;
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a) const

1
nano/secure/blockstore.hpp
View file

@ -732,6 +732,7 @@ public:
virtual std::mutex & get_cache_mutex () = 0;
virtual bool copy_db (boost::filesystem::path const & destination) = 0;
virtual void rebuild_db (nano::write_transaction const & transaction_a) = 0;
/** Not applicable to all sub-classes */
virtual void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) = 0;