dn724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 1

Use the MDB_NOMEMINIT flag with the ledger db environment (#2077)

Browse source 
* Use MDB_NOMEMINIT with data.ldb

* Modify map size if running under Valgrind

* Lower maximum map size under valgrind to account for multiple nodes in tests
This commit is contained in:
Wesley Shillingford 2019-06-25 12:11:21 +01:00 • committed by GitHub
commit e9da1da61a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 4816 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

26
nano/node/lmdb.cpp
View file

@ -9,7 +9,9 @@
#include <queue>
nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a, int max_dbs, size_t map_size_a)
#include <valgrind/valgrind.h>
nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a, int max_dbs_a, bool use_no_mem_init_a, size_t map_size_a)
{
boost::system::error_code error_mkdir, error_chmod;
if (path_a.has_parent_path ())
@ -20,14 +22,28 @@ nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a,
{
auto status1 (mdb_env_create (&environment));
release_assert (status1 == 0);
auto status2 (mdb_env_set_maxdbs (environment, max_dbs));
auto status2 (mdb_env_set_maxdbs (environment, max_dbs_a));
release_assert (status2 == 0);
auto status3 (mdb_env_set_mapsize (environment, map_size_a));
auto running_within_valgrind = (RUNNING_ON_VALGRIND > 0);
auto map_size = map_size_a;
auto max_valgrind_map_size = 16 * 1024 * 1024;
if (running_within_valgrind && map_size_a > max_valgrind_map_size)
{
// In order to run LMDB under Valgrind, the maximum map size must be smaller than half your available RAM
map_size = max_valgrind_map_size;
}
auto status3 (mdb_env_set_mapsize (environment, map_size));
release_assert (status3 == 0);
// It seems if there's ever more threads than mdb_env_set_maxreaders has read slots available, we get failures on transaction creation unless MDB_NOTLS is specified
// This can happen if something like 256 io_threads are specified in the node config
// MDB_NORDAHEAD will allow platforms that support it to load the DB in memory as needed.
auto status4 (mdb_env_open (environment, path_a.string ().c_str (), MDB_NOSUBDIR | MDB_NOTLS | MDB_NORDAHEAD, 00600));
// MDB_NOMEMINIT prevents zeroing malloc'ed pages. Can provide improvement for non-sensitive data but may make memory checkers noisy (e.g valgrind).
auto environment_flags = MDB_NOSUBDIR | MDB_NOTLS | MDB_NORDAHEAD;
if (!running_within_valgrind && use_no_mem_init_a)
{
environment_flags |= MDB_NOMEMINIT;
}
auto status4 (mdb_env_open (environment, path_a.string ().c_str (), environment_flags, 00600));
if (status4 != 0)
{
std::cerr << "Could not open lmdb environment: " << status4;
@ -824,7 +840,7 @@ nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> nano::mdb_store
nano::mdb_store::mdb_store (bool & error_a, nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, bool drop_unchecked, size_t const batch_size) :
logger (logger_a),
env (error_a, path_a, lmdb_max_dbs),
env (error_a, path_a, lmdb_max_dbs, true),
mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a),
txn_tracking_enabled (txn_tracking_config_a.enable)
{

4
nano/node/lmdb.hpp
View file

@ -56,10 +56,10 @@ public:
/**
* RAII wrapper for MDB_env
*/
class mdb_env
class mdb_env final
{
public:
mdb_env (bool &, boost::filesystem::path const &, int max_dbs = 128, size_t map_size = 128ULL * 1024 * 1024 * 1024);
mdb_env (bool &, boost::filesystem::path const &, int max_dbs = 128, bool use_no_mem_init = false, size_t map_size = 128ULL * 1024 * 1024 * 1024);
~mdb_env ();
operator MDB_env * () const;
// clang-format off

2
nano/node/wallet.cpp
View file

@ -1962,7 +1962,7 @@ nano::store_iterator<nano::uint256_union, nano::wallet_value> nano::wallet_store
return nano::store_iterator<nano::uint256_union, nano::wallet_value> (nullptr);
}
nano::mdb_wallets_store::mdb_wallets_store (bool & error_a, boost::filesystem::path const & path_a, int lmdb_max_dbs) :
environment (error_a, path_a, lmdb_max_dbs, 1ULL * 1024 * 1024 * 1024)
environment (error_a, path_a, lmdb_max_dbs, false, 1ULL * 1024 * 1024 * 1024)
{
}
MDB_txn * nano::wallet_store::tx (nano::transaction const & transaction_a) const

4792
valgrind/valgrind.h Normal file
View file

File diff suppressed because it is too large Load diff