LMDB sync options and new config settings (#2588)
* LMDB sync options and new config settings * Force sync always for wallet store
This commit is contained in:
cryptocode
GitHub
parent
3ba48bbd35
commit
695dc5f5dc
17 changed files with 269 additions and 38 deletions
|
|
@ -1,5 +1,6 @@
|
|||
#include <nano/core_test/testutil.hpp>
|
||||
#include <nano/crypto_lib/random_pool.hpp>
|
||||
#include <nano/lib/lmdbconfig.hpp>
|
||||
#include <nano/lib/stats.hpp>
|
||||
#include <nano/lib/utility.hpp>
|
||||
#include <nano/lib/work.hpp>
|
||||
|
|
@ -2063,7 +2064,7 @@ TEST (mdb_block_store, upgrade_backup)
|
|||
|
||||
// Now do the upgrade and confirm that backup is saved
|
||||
nano::logger_mt logger;
|
||||
nano::mdb_store store (logger, path, nano::txn_tracking_config{}, std::chrono::seconds (5), 128, 512, true);
|
||||
nano::mdb_store store (logger, path, nano::txn_tracking_config{}, std::chrono::seconds (5), nano::lmdb_config{}, 512, true);
|
||||
ASSERT_FALSE (store.init_error ());
|
||||
auto transaction (store.tx_begin_read ());
|
||||
ASSERT_LT (14, store.version_get (transaction));
|
||||
|
|
|
|||
|
|
@ -555,7 +555,7 @@ TEST (node_config, serialization)
|
|||
config1.callback_address = "test";
|
||||
config1.callback_port = 10;
|
||||
config1.callback_target = "test";
|
||||
config1.lmdb_max_dbs = 256;
|
||||
config1.deprecated_lmdb_max_dbs = 256;
|
||||
nano::jsonconfig tree;
|
||||
config1.serialize_json (tree);
|
||||
nano::logging logging2;
|
||||
|
|
@ -572,7 +572,7 @@ TEST (node_config, serialization)
|
|||
ASSERT_NE (config2.callback_address, config1.callback_address);
|
||||
ASSERT_NE (config2.callback_port, config1.callback_port);
|
||||
ASSERT_NE (config2.callback_target, config1.callback_target);
|
||||
ASSERT_NE (config2.lmdb_max_dbs, config1.lmdb_max_dbs);
|
||||
ASSERT_NE (config2.deprecated_lmdb_max_dbs, config1.deprecated_lmdb_max_dbs);
|
||||
|
||||
ASSERT_FALSE (tree.get_optional<std::string> ("epoch_block_link"));
|
||||
ASSERT_FALSE (tree.get_optional<std::string> ("epoch_block_signer"));
|
||||
|
|
@ -590,7 +590,7 @@ TEST (node_config, serialization)
|
|||
ASSERT_EQ (config2.callback_address, config1.callback_address);
|
||||
ASSERT_EQ (config2.callback_port, config1.callback_port);
|
||||
ASSERT_EQ (config2.callback_target, config1.callback_target);
|
||||
ASSERT_EQ (config2.lmdb_max_dbs, config1.lmdb_max_dbs);
|
||||
ASSERT_EQ (config2.deprecated_lmdb_max_dbs, config1.deprecated_lmdb_max_dbs);
|
||||
}
|
||||
|
||||
TEST (node_config, v1_v2_upgrade)
|
||||
|
|
|
|||
|
|
@ -120,6 +120,7 @@ TEST (toml, daemon_config_deserialize_defaults)
|
|||
[node.statistics.log]
|
||||
[node.statistics.sampling]
|
||||
[node.websocket]
|
||||
[node.lmdb]
|
||||
[node.rocksdb]
|
||||
[opencl]
|
||||
[rpc]
|
||||
|
|
@ -158,7 +159,7 @@ TEST (toml, daemon_config_deserialize_defaults)
|
|||
ASSERT_EQ (conf.node.external_address, defaults.node.external_address);
|
||||
ASSERT_EQ (conf.node.external_port, defaults.node.external_port);
|
||||
ASSERT_EQ (conf.node.io_threads, defaults.node.io_threads);
|
||||
ASSERT_EQ (conf.node.lmdb_max_dbs, defaults.node.lmdb_max_dbs);
|
||||
ASSERT_EQ (conf.node.deprecated_lmdb_max_dbs, defaults.node.deprecated_lmdb_max_dbs);
|
||||
ASSERT_EQ (conf.node.max_work_generate_multiplier, defaults.node.max_work_generate_multiplier);
|
||||
ASSERT_EQ (conf.node.network_threads, defaults.node.network_threads);
|
||||
ASSERT_EQ (conf.node.secondary_work_peers, defaults.node.secondary_work_peers);
|
||||
|
|
@ -245,6 +246,10 @@ TEST (toml, daemon_config_deserialize_defaults)
|
|||
ASSERT_EQ (conf.node.stat_config.log_counters_filename, defaults.node.stat_config.log_counters_filename);
|
||||
ASSERT_EQ (conf.node.stat_config.log_samples_filename, defaults.node.stat_config.log_samples_filename);
|
||||
|
||||
ASSERT_EQ (conf.node.lmdb_config.sync, defaults.node.lmdb_config.sync);
|
||||
ASSERT_EQ (conf.node.lmdb_config.max_databases, defaults.node.lmdb_config.max_databases);
|
||||
ASSERT_EQ (conf.node.lmdb_config.map_size, defaults.node.lmdb_config.map_size);
|
||||
|
||||
ASSERT_EQ (conf.node.rocksdb_config.enable, defaults.node.rocksdb_config.enable);
|
||||
ASSERT_EQ (conf.node.rocksdb_config.bloom_filter_bits, defaults.node.rocksdb_config.bloom_filter_bits);
|
||||
ASSERT_EQ (conf.node.rocksdb_config.block_cache, defaults.node.rocksdb_config.block_cache);
|
||||
|
|
@ -495,6 +500,11 @@ TEST (toml, daemon_config_deserialize_no_defaults)
|
|||
enable = true
|
||||
port = 999
|
||||
|
||||
[node.lmdb]
|
||||
sync = "nosync_safe"
|
||||
max_databases = 999
|
||||
map_size = 999
|
||||
|
||||
[node.rocksdb]
|
||||
enable = true
|
||||
bloom_filter_bits = 10
|
||||
|
|
@ -557,7 +567,7 @@ TEST (toml, daemon_config_deserialize_no_defaults)
|
|||
ASSERT_NE (conf.node.external_address, defaults.node.external_address);
|
||||
ASSERT_NE (conf.node.external_port, defaults.node.external_port);
|
||||
ASSERT_NE (conf.node.io_threads, defaults.node.io_threads);
|
||||
ASSERT_NE (conf.node.lmdb_max_dbs, defaults.node.lmdb_max_dbs);
|
||||
ASSERT_NE (conf.node.deprecated_lmdb_max_dbs, defaults.node.deprecated_lmdb_max_dbs);
|
||||
ASSERT_NE (conf.node.max_work_generate_multiplier, defaults.node.max_work_generate_multiplier);
|
||||
ASSERT_NE (conf.node.frontiers_confirmation, defaults.node.frontiers_confirmation);
|
||||
ASSERT_NE (conf.node.network_threads, defaults.node.network_threads);
|
||||
|
|
@ -645,6 +655,10 @@ TEST (toml, daemon_config_deserialize_no_defaults)
|
|||
ASSERT_NE (conf.node.stat_config.log_counters_filename, defaults.node.stat_config.log_counters_filename);
|
||||
ASSERT_NE (conf.node.stat_config.log_samples_filename, defaults.node.stat_config.log_samples_filename);
|
||||
|
||||
ASSERT_NE (conf.node.lmdb_config.sync, defaults.node.lmdb_config.sync);
|
||||
ASSERT_NE (conf.node.lmdb_config.max_databases, defaults.node.lmdb_config.max_databases);
|
||||
ASSERT_NE (conf.node.lmdb_config.map_size, defaults.node.lmdb_config.map_size);
|
||||
|
||||
ASSERT_NE (conf.node.rocksdb_config.enable, defaults.node.rocksdb_config.enable);
|
||||
ASSERT_NE (conf.node.rocksdb_config.bloom_filter_bits, defaults.node.rocksdb_config.bloom_filter_bits);
|
||||
ASSERT_NE (conf.node.rocksdb_config.block_cache, defaults.node.rocksdb_config.block_cache);
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ TEST (wallets, DISABLED_wallet_create_max)
|
|||
bool error (false);
|
||||
nano::wallets wallets (error, *system.nodes[0]);
|
||||
const int nonWalletDbs = 19;
|
||||
for (int i = 0; i < system.nodes[0]->config.lmdb_max_dbs - nonWalletDbs; i++)
|
||||
for (int i = 0; i < system.nodes[0]->config.deprecated_lmdb_max_dbs - nonWalletDbs; i++)
|
||||
{
|
||||
auto wallet_id = nano::random_wallet_id ();
|
||||
auto wallet = wallets.create (wallet_id);
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ add_library (nano_lib
|
|||
json_error_response.hpp
|
||||
jsonconfig.hpp
|
||||
jsonconfig.cpp
|
||||
lmdbconfig.hpp
|
||||
lmdbconfig.cpp
|
||||
locks.hpp
|
||||
locks.cpp
|
||||
logger_mt.hpp
|
||||
|
|
|
|||
72
nano/lib/lmdbconfig.cpp
Normal file
72
nano/lib/lmdbconfig.cpp
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
#include <nano/lib/lmdbconfig.hpp>
|
||||
#include <nano/lib/tomlconfig.hpp>
|
||||
#include <nano/secure/common.hpp>
|
||||
|
||||
#include <iostream>
|
||||
|
||||
nano::error nano::lmdb_config::serialize_toml (nano::tomlconfig & toml) const
|
||||
{
|
||||
std::string sync_string;
|
||||
switch (sync)
|
||||
{
|
||||
case nano::lmdb_config::sync_strategy::always:
|
||||
sync_string = "always";
|
||||
break;
|
||||
case nano::lmdb_config::sync_strategy::nosync_safe:
|
||||
sync_string = "nosync_safe";
|
||||
break;
|
||||
case nano::lmdb_config::sync_strategy::nosync_unsafe:
|
||||
sync_string = "nosync_unsafe";
|
||||
break;
|
||||
case nano::lmdb_config::sync_strategy::nosync_unsafe_large_memory:
|
||||
sync_string = "nosync_unsafe_large_memory";
|
||||
break;
|
||||
}
|
||||
|
||||
toml.put ("sync", sync_string, "Sync strategy for flushing commits to the ledger database. This does not affect the wallet database.\ntype:string,{always, nosync_safe, nosync_unsafe, nosync_unsafe_large_memory}");
|
||||
toml.put ("max_databases", max_databases, "Maximum open lmdb databases. Increase default if more than 100 wallets is required.\nNote: external management is recommended when a large amounts of wallets are required (see https://docs.nano.org/integration-guides/key-management/).\ntype:uin32");
|
||||
toml.put ("map_size", map_size, "Maximum ledger database map size in bytes.\ntype:uint64");
|
||||
return toml.get_error ();
|
||||
}
|
||||
|
||||
nano::error nano::lmdb_config::deserialize_toml (nano::tomlconfig & toml, bool is_deprecated_lmdb_dbs_used)
|
||||
{
|
||||
static nano::network_params params;
|
||||
auto default_max_databases = max_databases;
|
||||
toml.get_optional<uint32_t> ("max_databases", max_databases);
|
||||
toml.get_optional<size_t> ("map_size", map_size);
|
||||
|
||||
// For now we accept either setting, but not both
|
||||
if (!params.network.is_test_network () && is_deprecated_lmdb_dbs_used && default_max_databases != max_databases)
|
||||
{
|
||||
toml.get_error ().set ("Both the deprecated node.lmdb_max_dbs and the new node.lmdb.max_databases setting are used. Please use max_databases only.");
|
||||
}
|
||||
|
||||
if (!toml.get_error ())
|
||||
{
|
||||
std::string sync_string = "always";
|
||||
toml.get_optional<std::string> ("sync", sync_string);
|
||||
if (sync_string == "always")
|
||||
{
|
||||
sync = nano::lmdb_config::sync_strategy::always;
|
||||
}
|
||||
else if (sync_string == "nosync_safe")
|
||||
{
|
||||
sync = nano::lmdb_config::sync_strategy::nosync_safe;
|
||||
}
|
||||
else if (sync_string == "nosync_unsafe")
|
||||
{
|
||||
sync = nano::lmdb_config::sync_strategy::nosync_unsafe;
|
||||
}
|
||||
else if (sync_string == "nosync_unsafe_large_memory")
|
||||
{
|
||||
sync = nano::lmdb_config::sync_strategy::nosync_unsafe_large_memory;
|
||||
}
|
||||
else
|
||||
{
|
||||
toml.get_error ().set (sync_string + " is not a valid sync option");
|
||||
}
|
||||
}
|
||||
|
||||
return toml.get_error ();
|
||||
}
|
||||
50
nano/lib/lmdbconfig.hpp
Normal file
50
nano/lib/lmdbconfig.hpp
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
#pragma once
|
||||
|
||||
#include <nano/lib/errors.hpp>
|
||||
|
||||
#include <thread>
|
||||
|
||||
namespace nano
|
||||
{
|
||||
class tomlconfig;
|
||||
|
||||
/** Configuration options for LMDB */
|
||||
class lmdb_config final
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Dictates how lmdb flushes to disk on commit.
|
||||
* These options only apply to the ledger database; the wallet database
|
||||
* always flush.
|
||||
*/
|
||||
enum sync_strategy
|
||||
{
|
||||
/** Always flush to disk on commit. This is default. */
|
||||
always,
|
||||
|
||||
/** Do not flush meta data eagerly. This may cause loss of transactions, but maintains integrity. */
|
||||
nosync_safe,
|
||||
|
||||
/**
|
||||
* Let the OS decide when to flush to disk. On filesystems with write ordering, this has the same
|
||||
* guarantees as nosync_safe, otherwise corruption may occur on system crash.
|
||||
*/
|
||||
nosync_unsafe,
|
||||
/**
|
||||
* Use a writeable memory map. Let the OS decide when to flush to disk, and make the request asynchronous.
|
||||
* This may give better performance on systems where the database fits entirely in memory, otherwise is
|
||||
* may be slower.
|
||||
* @warning Do not use this option if external processes uses the database concurrently.
|
||||
*/
|
||||
nosync_unsafe_large_memory
|
||||
};
|
||||
|
||||
nano::error serialize_toml (nano::tomlconfig & toml_a) const;
|
||||
nano::error deserialize_toml (nano::tomlconfig & toml_a, bool is_deprecated_lmdb_dbs_used);
|
||||
|
||||
/** Sync strategy for the ledger database */
|
||||
sync_strategy sync{ always };
|
||||
uint32_t max_databases{ 128 };
|
||||
size_t map_size{ 128ULL * 1024 * 1024 * 1024 };
|
||||
};
|
||||
}
|
||||
|
|
@ -40,9 +40,9 @@ void mdb_val::convert_buffer_to_value ()
|
|||
}
|
||||
}
|
||||
|
||||
nano::mdb_store::mdb_store (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, size_t const batch_size, bool backup_before_upgrade) :
|
||||
nano::mdb_store::mdb_store (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, size_t const batch_size, bool backup_before_upgrade) :
|
||||
logger (logger_a),
|
||||
env (error, path_a, lmdb_max_dbs, true),
|
||||
env (error, path_a, nano::mdb_env::options::make ().set_config (lmdb_config_a).set_use_no_mem_init (true)),
|
||||
mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a),
|
||||
txn_tracking_enabled (txn_tracking_config_a.enable)
|
||||
{
|
||||
|
|
@ -91,7 +91,7 @@ txn_tracking_enabled (txn_tracking_config_a.enable)
|
|||
if (needs_vacuuming && !network_constants.is_test_network ())
|
||||
{
|
||||
logger.always_log ("Preparing vacuum...");
|
||||
auto vacuum_success = vacuum_after_upgrade (path_a, lmdb_max_dbs);
|
||||
auto vacuum_success = vacuum_after_upgrade (path_a, lmdb_config_a);
|
||||
logger.always_log (vacuum_success ? "Vacuum succeeded." : "Failed to vacuum. (Optional) Ensure enough disk space is available for a copy of the database and try to vacuum after shutting down the node");
|
||||
}
|
||||
}
|
||||
|
|
@ -103,7 +103,7 @@ txn_tracking_enabled (txn_tracking_config_a.enable)
|
|||
}
|
||||
}
|
||||
|
||||
bool nano::mdb_store::vacuum_after_upgrade (boost::filesystem::path const & path_a, int lmdb_max_dbs)
|
||||
bool nano::mdb_store::vacuum_after_upgrade (boost::filesystem::path const & path_a, nano::lmdb_config const & lmdb_config_a)
|
||||
{
|
||||
// Vacuum the database. This is not a required step and may actually fail if there isn't enough storage space.
|
||||
auto vacuum_path = path_a.parent_path () / "vacuumed.ldb";
|
||||
|
|
@ -112,6 +112,7 @@ bool nano::mdb_store::vacuum_after_upgrade (boost::filesystem::path const & path
|
|||
if (vacuum_success)
|
||||
{
|
||||
// Need to close the database to release the file handle
|
||||
mdb_env_sync (env.environment, true);
|
||||
mdb_env_close (env.environment);
|
||||
env.environment = nullptr;
|
||||
|
||||
|
|
@ -119,7 +120,10 @@ bool nano::mdb_store::vacuum_after_upgrade (boost::filesystem::path const & path
|
|||
boost::filesystem::rename (vacuum_path, path_a);
|
||||
|
||||
// Set up the environment again
|
||||
env.init (error, path_a, lmdb_max_dbs, true);
|
||||
auto options = nano::mdb_env::options::make ()
|
||||
.set_config (lmdb_config_a)
|
||||
.set_use_no_mem_init (true);
|
||||
env.init (error, path_a, options);
|
||||
if (!error)
|
||||
{
|
||||
auto transaction (tx_begin_read ());
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#pragma once
|
||||
|
||||
#include <nano/lib/diagnosticsconfig.hpp>
|
||||
#include <nano/lib/lmdbconfig.hpp>
|
||||
#include <nano/lib/logger_mt.hpp>
|
||||
#include <nano/lib/numbers.hpp>
|
||||
#include <nano/node/lmdb/lmdb_env.hpp>
|
||||
|
|
@ -36,7 +37,7 @@ public:
|
|||
using block_store_partial::block_exists;
|
||||
using block_store_partial::unchecked_put;
|
||||
|
||||
mdb_store (nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, size_t batch_size = 512, bool backup_before_upgrade = false);
|
||||
mdb_store (nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, size_t batch_size = 512, bool backup_before_upgrade = false);
|
||||
nano::write_transaction tx_begin_write (std::vector<nano::tables> const & tables_requiring_lock = {}, std::vector<nano::tables> const & tables_no_lock = {}) override;
|
||||
nano::read_transaction tx_begin_read () override;
|
||||
|
||||
|
|
@ -261,7 +262,7 @@ private:
|
|||
|
||||
size_t count (nano::transaction const & transaction_a, tables table_a) const override;
|
||||
|
||||
bool vacuum_after_upgrade (boost::filesystem::path const & path_a, int lmdb_max_dbs);
|
||||
bool vacuum_after_upgrade (boost::filesystem::path const & path_a, nano::lmdb_config const & lmdb_config_a);
|
||||
|
||||
class upgrade_counters
|
||||
{
|
||||
|
|
|
|||
|
|
@ -2,12 +2,12 @@
|
|||
|
||||
#include <boost/filesystem/operations.hpp>
|
||||
|
||||
nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a, int max_dbs_a, bool use_no_mem_init_a, size_t map_size_a)
|
||||
nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a, nano::mdb_env::options options_a)
|
||||
{
|
||||
init (error_a, path_a, max_dbs_a, use_no_mem_init_a, map_size_a);
|
||||
init (error_a, path_a, options_a);
|
||||
}
|
||||
|
||||
void nano::mdb_env::init (bool & error_a, boost::filesystem::path const & path_a, int max_dbs_a, bool use_no_mem_init_a, size_t map_size_a)
|
||||
void nano::mdb_env::init (bool & error_a, boost::filesystem::path const & path_a, nano::mdb_env::options options_a)
|
||||
{
|
||||
boost::system::error_code error_mkdir, error_chmod;
|
||||
if (path_a.has_parent_path ())
|
||||
|
|
@ -18,11 +18,11 @@ void nano::mdb_env::init (bool & error_a, boost::filesystem::path const & path_a
|
|||
{
|
||||
auto status1 (mdb_env_create (&environment));
|
||||
release_assert (status1 == 0);
|
||||
auto status2 (mdb_env_set_maxdbs (environment, max_dbs_a));
|
||||
auto status2 (mdb_env_set_maxdbs (environment, options_a.config.max_databases));
|
||||
release_assert (status2 == 0);
|
||||
auto map_size = map_size_a;
|
||||
auto map_size = options_a.config.map_size;
|
||||
auto max_valgrind_map_size = 16 * 1024 * 1024;
|
||||
if (running_within_valgrind () && map_size_a > max_valgrind_map_size)
|
||||
if (running_within_valgrind () && map_size > max_valgrind_map_size)
|
||||
{
|
||||
// In order to run LMDB under Valgrind, the maximum map size must be smaller than half your available RAM
|
||||
map_size = max_valgrind_map_size;
|
||||
|
|
@ -34,7 +34,20 @@ void nano::mdb_env::init (bool & error_a, boost::filesystem::path const & path_a
|
|||
// MDB_NORDAHEAD will allow platforms that support it to load the DB in memory as needed.
|
||||
// MDB_NOMEMINIT prevents zeroing malloc'ed pages. Can provide improvement for non-sensitive data but may make memory checkers noisy (e.g valgrind).
|
||||
auto environment_flags = MDB_NOSUBDIR | MDB_NOTLS | MDB_NORDAHEAD;
|
||||
if (!running_within_valgrind () && use_no_mem_init_a)
|
||||
if (options_a.config.sync == nano::lmdb_config::sync_strategy::nosync_safe)
|
||||
{
|
||||
environment_flags |= MDB_NOMETASYNC;
|
||||
}
|
||||
else if (options_a.config.sync == nano::lmdb_config::sync_strategy::nosync_unsafe)
|
||||
{
|
||||
environment_flags |= MDB_NOSYNC;
|
||||
}
|
||||
else if (options_a.config.sync == nano::lmdb_config::sync_strategy::nosync_unsafe_large_memory)
|
||||
{
|
||||
environment_flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC;
|
||||
}
|
||||
|
||||
if (!running_within_valgrind () && options_a.use_no_mem_init)
|
||||
{
|
||||
environment_flags |= MDB_NOMEMINIT;
|
||||
}
|
||||
|
|
@ -69,6 +82,8 @@ nano::mdb_env::~mdb_env ()
|
|||
{
|
||||
if (environment != nullptr)
|
||||
{
|
||||
// Make sure the commits are flushed. This is a no-op unless MDB_NOSYNC is used.
|
||||
mdb_env_sync (environment, true);
|
||||
mdb_env_close (environment);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#pragma once
|
||||
|
||||
#include <nano/lib/lmdbconfig.hpp>
|
||||
#include <nano/node/lmdb/lmdb_txn.hpp>
|
||||
#include <nano/secure/blockstore.hpp>
|
||||
|
||||
|
|
@ -11,8 +12,50 @@ namespace nano
|
|||
class mdb_env final
|
||||
{
|
||||
public:
|
||||
mdb_env (bool &, boost::filesystem::path const &, int max_dbs = 128, bool use_no_mem_init = false, size_t map_size = 128ULL * 1024 * 1024 * 1024);
|
||||
void init (bool &, boost::filesystem::path const &, int max_dbs, bool use_no_mem_init, size_t map_size = 128ULL * 1024 * 1024 * 1024);
|
||||
/** Environment options, most of which originates from the config file. */
|
||||
class options final
|
||||
{
|
||||
friend class mdb_env;
|
||||
|
||||
public:
|
||||
static options make ()
|
||||
{
|
||||
return options ();
|
||||
}
|
||||
|
||||
options & set_config (nano::lmdb_config config_a)
|
||||
{
|
||||
config = config_a;
|
||||
return *this;
|
||||
}
|
||||
|
||||
options & set_use_no_mem_init (int use_no_mem_init_a)
|
||||
{
|
||||
use_no_mem_init = use_no_mem_init_a;
|
||||
return *this;
|
||||
}
|
||||
|
||||
/** Used by the wallet to override the config map size */
|
||||
options & override_config_map_size (size_t map_size_a)
|
||||
{
|
||||
config.map_size = map_size_a;
|
||||
return *this;
|
||||
}
|
||||
|
||||
/** Used by the wallet to override the sync strategy */
|
||||
options & override_config_sync (nano::lmdb_config::sync_strategy sync_a)
|
||||
{
|
||||
config.sync = sync_a;
|
||||
return *this;
|
||||
}
|
||||
|
||||
private:
|
||||
bool use_no_mem_init{ false };
|
||||
nano::lmdb_config config;
|
||||
};
|
||||
|
||||
mdb_env (bool &, boost::filesystem::path const &, nano::mdb_env::options options_a = nano::mdb_env::options::make ());
|
||||
void init (bool &, boost::filesystem::path const &, nano::mdb_env::options options_a = nano::mdb_env::options::make ());
|
||||
~mdb_env ();
|
||||
operator MDB_env * () const;
|
||||
nano::read_transaction tx_begin_read (mdb_txn_callbacks txn_callbacks = mdb_txn_callbacks{}) const;
|
||||
|
|
|
|||
|
|
@ -119,9 +119,9 @@ alarm (alarm_a),
|
|||
work (work_a),
|
||||
distributed_work (*this),
|
||||
logger (config_a.logging.min_time_between_log_output),
|
||||
store_impl (nano::make_store (logger, application_path_a, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_max_dbs, flags.sideband_batch_size, config_a.backup_before_upgrade, config_a.rocksdb_config.enable)),
|
||||
store_impl (nano::make_store (logger, application_path_a, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_config, flags.sideband_batch_size, config_a.backup_before_upgrade, config_a.rocksdb_config.enable)),
|
||||
store (*store_impl),
|
||||
wallets_store_impl (std::make_unique<nano::mdb_wallets_store> (application_path_a / "wallets.ldb", config_a.lmdb_max_dbs)),
|
||||
wallets_store_impl (std::make_unique<nano::mdb_wallets_store> (application_path_a / "wallets.ldb", config_a.lmdb_config)),
|
||||
wallets_store (*wallets_store_impl),
|
||||
gap_cache (*this),
|
||||
ledger (store, stats, flags_a.generate_cache),
|
||||
|
|
@ -1395,7 +1395,7 @@ nano::node_flags const & nano::inactive_node_flag_defaults ()
|
|||
return node_flags;
|
||||
}
|
||||
|
||||
std::unique_ptr<nano::block_store> nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, size_t batch_size, bool backup_before_upgrade, bool use_rocksdb_backend)
|
||||
std::unique_ptr<nano::block_store> nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, size_t batch_size, bool backup_before_upgrade, bool use_rocksdb_backend)
|
||||
{
|
||||
#if NANO_ROCKSDB
|
||||
auto make_rocksdb = [&logger, add_db_postfix, &path, &rocksdb_config, read_only]() {
|
||||
|
|
@ -1426,5 +1426,5 @@ std::unique_ptr<nano::block_store> nano::make_store (nano::logger_mt & logger, b
|
|||
#endif
|
||||
}
|
||||
|
||||
return std::make_unique<nano::mdb_store> (logger, add_db_postfix ? path / "data.ldb" : path, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_max_dbs, batch_size, backup_before_upgrade);
|
||||
return std::make_unique<nano::mdb_store> (logger, add_db_postfix ? path / "data.ldb" : path, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_config_a, batch_size, backup_before_upgrade);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
#include <nano/crypto_lib/random_pool.hpp>
|
||||
#include <nano/lib/config.hpp>
|
||||
#include <nano/lib/jsonconfig.hpp>
|
||||
#include <nano/lib/rocksdbconfig.hpp>
|
||||
#include <nano/lib/rpcconfig.hpp>
|
||||
#include <nano/lib/tomlconfig.hpp>
|
||||
#include <nano/node/nodeconfig.hpp>
|
||||
|
|
@ -81,7 +80,7 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
|
|||
toml.put ("bootstrap_connections", bootstrap_connections, "Number of outbound bootstrap connections. Must be a power of 2. Defaults to 4.\nWarning: a larger amount of connections may use substantially more system memory.\ntype:uint64");
|
||||
toml.put ("bootstrap_connections_max", bootstrap_connections_max, "Maximum number of inbound bootstrap connections. Defaults to 64.\nWarning: a larger amount of connections may use additional system memory.\ntype:uint64");
|
||||
toml.put ("bootstrap_initiator_threads", bootstrap_initiator_threads, "Number of threads dedicated to concurrent bootstrap attempts. Defaults to 2 (if the number of CPU threads is more than 1), otherwise 1.\nWarning: a larger amount of attempts may use additional system memory and disk IO.\ntype:uint64");
|
||||
toml.put ("lmdb_max_dbs", lmdb_max_dbs, "Maximum open lmdb databases. Increase default if more than 100 wallets is required.\nNote: external management is recommended when a large amounts of wallets are required (see https://docs.nano.org/integration-guides/key-management/).\ntype:uint64");
|
||||
toml.put ("lmdb_max_dbs", deprecated_lmdb_max_dbs, "DEPRECATED: use node.lmdb.max_databases instead.\nMaximum open lmdb databases. Increase default if more than 100 wallets is required.\nNote: external management is recommended when a large amounts of wallets are required (see https://docs.nano.org/integration-guides/key-management/).\ntype:uint64");
|
||||
toml.put ("block_processor_batch_max_time", block_processor_batch_max_time.count (), "The maximum time the block processor can continously process blocks for.\ntype:milliseconds");
|
||||
toml.put ("allow_local_peers", allow_local_peers, "Enable or disable local host peering.\ntype:bool");
|
||||
toml.put ("vote_minimum", vote_minimum.to_string_dec (), "Local representatives do not vote if the delegated weight is under this threshold. Saves on system resources.\ntype:string,amount,raw");
|
||||
|
|
@ -161,6 +160,10 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
|
|||
rocksdb_config.serialize_toml (rocksdb_l);
|
||||
toml.put_child ("rocksdb", rocksdb_l);
|
||||
|
||||
nano::tomlconfig lmdb_l;
|
||||
lmdb_config.serialize_toml (lmdb_l);
|
||||
toml.put_child ("lmdb", lmdb_l);
|
||||
|
||||
return toml.get_error ();
|
||||
}
|
||||
|
||||
|
|
@ -304,11 +307,33 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
|
|||
toml.get<unsigned> ("bootstrap_connections", bootstrap_connections);
|
||||
toml.get<unsigned> ("bootstrap_connections_max", bootstrap_connections_max);
|
||||
toml.get<unsigned> ("bootstrap_initiator_threads", bootstrap_initiator_threads);
|
||||
toml.get<int> ("lmdb_max_dbs", lmdb_max_dbs);
|
||||
toml.get<bool> ("enable_voting", enable_voting);
|
||||
toml.get<bool> ("allow_local_peers", allow_local_peers);
|
||||
toml.get<unsigned> (signature_checker_threads_key, signature_checker_threads);
|
||||
|
||||
auto lmdb_max_dbs_default = deprecated_lmdb_max_dbs;
|
||||
toml.get<int> ("lmdb_max_dbs", deprecated_lmdb_max_dbs);
|
||||
bool is_deprecated_lmdb_dbs_used = lmdb_max_dbs_default != deprecated_lmdb_max_dbs;
|
||||
|
||||
// Note: using the deprecated setting will result in a fail-fast config error in the future
|
||||
if (!network_params.network.is_test_network () && is_deprecated_lmdb_dbs_used)
|
||||
{
|
||||
std::cerr << "WARNING: The node.lmdb_max_dbs setting is deprecated and will be removed in a future version." << std::endl;
|
||||
std::cerr << "Please use the node.lmdb.max_databases setting instead." << std::endl;
|
||||
}
|
||||
|
||||
if (toml.has_key ("lmdb"))
|
||||
{
|
||||
auto lmdb_config_l (toml.get_required_child ("lmdb"));
|
||||
lmdb_config.deserialize_toml (lmdb_config_l, is_deprecated_lmdb_dbs_used);
|
||||
|
||||
// Note that the lmdb config fails is both the deprecated and new setting are changed.
|
||||
if (is_deprecated_lmdb_dbs_used)
|
||||
{
|
||||
lmdb_config.max_databases = deprecated_lmdb_max_dbs;
|
||||
}
|
||||
}
|
||||
|
||||
boost::asio::ip::address_v6 external_address_l;
|
||||
toml.get<boost::asio::ip::address_v6> ("external_address", external_address_l);
|
||||
external_address = external_address_l.to_string ();
|
||||
|
|
@ -446,7 +471,7 @@ nano::error nano::node_config::serialize_json (nano::jsonconfig & json) const
|
|||
json.put ("callback_address", callback_address);
|
||||
json.put ("callback_port", callback_port);
|
||||
json.put ("callback_target", callback_target);
|
||||
json.put ("lmdb_max_dbs", lmdb_max_dbs);
|
||||
json.put ("lmdb_max_dbs", deprecated_lmdb_max_dbs);
|
||||
json.put ("block_processor_batch_max_time", block_processor_batch_max_time.count ());
|
||||
json.put ("allow_local_peers", allow_local_peers);
|
||||
json.put ("vote_minimum", vote_minimum.to_string_dec ());
|
||||
|
|
@ -740,7 +765,7 @@ nano::error nano::node_config::deserialize_json (bool & upgraded_a, nano::jsonco
|
|||
json.get<std::string> ("callback_address", callback_address);
|
||||
json.get<uint16_t> ("callback_port", callback_port);
|
||||
json.get<std::string> ("callback_target", callback_target);
|
||||
json.get<int> ("lmdb_max_dbs", lmdb_max_dbs);
|
||||
json.get<int> ("lmdb_max_dbs", deprecated_lmdb_max_dbs);
|
||||
json.get<bool> ("enable_voting", enable_voting);
|
||||
json.get<bool> ("allow_local_peers", allow_local_peers);
|
||||
json.get<unsigned> (signature_checker_threads_key, signature_checker_threads);
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
#include <nano/lib/diagnosticsconfig.hpp>
|
||||
#include <nano/lib/errors.hpp>
|
||||
#include <nano/lib/jsonconfig.hpp>
|
||||
#include <nano/lib/lmdbconfig.hpp>
|
||||
#include <nano/lib/numbers.hpp>
|
||||
#include <nano/lib/rocksdbconfig.hpp>
|
||||
#include <nano/lib/stats.hpp>
|
||||
|
|
@ -70,7 +71,7 @@ public:
|
|||
std::string callback_address;
|
||||
uint16_t callback_port{ 0 };
|
||||
std::string callback_target;
|
||||
int lmdb_max_dbs{ 128 };
|
||||
int deprecated_lmdb_max_dbs{ 128 };
|
||||
bool allow_local_peers{ !network_params.network.is_live_network () }; // disable by default for live network
|
||||
nano::stat_config stat_config;
|
||||
nano::ipc::ipc_config ipc_config;
|
||||
|
|
@ -96,6 +97,7 @@ public:
|
|||
uint64_t max_work_generate_difficulty{ nano::network_constants::publish_full_threshold };
|
||||
uint32_t max_queued_requests{ 512 };
|
||||
nano::rocksdb_config rocksdb_config;
|
||||
nano::lmdb_config lmdb_config;
|
||||
nano::frontiers_confirmation_mode frontiers_confirmation{ nano::frontiers_confirmation_mode::automatic };
|
||||
std::string serialize_frontiers_confirmation (nano::frontiers_confirmation_mode) const;
|
||||
nano::frontiers_confirmation_mode deserialize_frontiers_confirmation (std::string const &);
|
||||
|
|
|
|||
|
|
@ -1989,8 +1989,8 @@ nano::store_iterator<nano::account, nano::wallet_value> nano::wallet_store::end
|
|||
{
|
||||
return nano::store_iterator<nano::account, nano::wallet_value> (nullptr);
|
||||
}
|
||||
nano::mdb_wallets_store::mdb_wallets_store (boost::filesystem::path const & path_a, int lmdb_max_dbs) :
|
||||
environment (error, path_a, lmdb_max_dbs, false, 1ULL * 1024 * 1024 * 1024)
|
||||
nano::mdb_wallets_store::mdb_wallets_store (boost::filesystem::path const & path_a, nano::lmdb_config const & lmdb_config_a) :
|
||||
environment (error, path_a, nano::mdb_env::options::make ().set_config (lmdb_config_a).override_config_sync (nano::lmdb_config::sync_strategy::always).override_config_map_size (1ULL * 1024 * 1024 * 1024))
|
||||
{
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#pragma once
|
||||
|
||||
#include <nano/lib/lmdbconfig.hpp>
|
||||
#include <nano/lib/work.hpp>
|
||||
#include <nano/node/lmdb/lmdb.hpp>
|
||||
#include <nano/node/lmdb/wallet_value.hpp>
|
||||
|
|
@ -257,7 +258,7 @@ public:
|
|||
class mdb_wallets_store final : public wallets_store
|
||||
{
|
||||
public:
|
||||
mdb_wallets_store (boost::filesystem::path const &, int lmdb_max_dbs = 128);
|
||||
mdb_wallets_store (boost::filesystem::path const &, nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{});
|
||||
nano::mdb_env environment;
|
||||
bool init_error () const override;
|
||||
bool error{ false };
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
#include <nano/crypto_lib/random_pool.hpp>
|
||||
#include <nano/lib/diagnosticsconfig.hpp>
|
||||
#include <nano/lib/lmdbconfig.hpp>
|
||||
#include <nano/lib/logger_mt.hpp>
|
||||
#include <nano/lib/memory.hpp>
|
||||
#include <nano/lib/rocksdbconfig.hpp>
|
||||
|
|
@ -776,7 +777,7 @@ public:
|
|||
virtual std::string vendor_get () const = 0;
|
||||
};
|
||||
|
||||
std::unique_ptr<nano::block_store> make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool open_read_only = false, bool add_db_postfix = false, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, size_t batch_size = 512, bool backup_before_upgrade = false, bool rocksdb_backend = false);
|
||||
std::unique_ptr<nano::block_store> make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool open_read_only = false, bool add_db_postfix = false, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, size_t batch_size = 512, bool backup_before_upgrade = false, bool rocksdb_backend = false);
|
||||
}
|
||||
|
||||
namespace std
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue