From d3c4e9e2ff5445db4edd9f3d9121955008859f1d Mon Sep 17 00:00:00 2001 From: clemahieu Date: Sat, 8 Apr 2017 15:20:13 -0500 Subject: [PATCH] Moving resizing outside of creating a transaction. --- rai/config.hpp | 4 ++-- rai/node/node.cpp | 8 +++++++ rai/node/testing.cpp | 24 +++++++++++++-------- rai/utility.cpp | 50 +++++++++++++++++++++++++------------------- rai/utility.hpp | 6 ++++-- 5 files changed, 58 insertions(+), 34 deletions(-) diff --git a/rai/config.hpp b/rai/config.hpp index f9a08de5..f64d4d28 100644 --- a/rai/config.hpp +++ b/rai/config.hpp @@ -15,6 +15,6 @@ enum class rai_networks rai_live_network }; rai::rai_networks const rai_network = rai_networks::ACTIVE_NETWORK; -int const database_check_interval = rai_network == rai::rai_networks::rai_test_network ? 4 : 1024; +int const database_check_interval = rai_network == rai::rai_networks::rai_test_network ? 64 : 4096; size_t const database_size_increment = rai_network == rai::rai_networks::rai_test_network ? 2 * 1024 * 1024 : 256 * 1024 * 1024; -} \ No newline at end of file +} diff --git a/rai/node/node.cpp b/rai/node/node.cpp index fb852701..6fe6b238 100755 --- a/rai/node/node.cpp +++ b/rai/node/node.cpp @@ -1043,6 +1043,14 @@ port_mapping (*this), vote_processor (*this), warmed_up (0) { + store.environment.sizing_action = [this] () + { + auto this_l (shared_from_this ()); + background ([this_l] () + { + this_l->store.environment.handle_environment_sizing (); + }); + }; wallets.observer = [this] (rai::account const & account_a, bool active) { observers.wallet (account_a, active); diff --git a/rai/node/testing.cpp b/rai/node/testing.cpp index 623681a4..fe051734 100644 --- a/rai/node/testing.cpp +++ b/rai/node/testing.cpp @@ -161,16 +161,22 @@ void rai::system::generate_rollback (rai::node & node_a, std::vector send_block; + { + rai::transaction transaction (node_a.store.environment, nullptr, false); + rai::uint256_union random_block; + random_pool.GenerateBlock (random_block.bytes.data (), sizeof (random_block.bytes)); + auto i (node_a.store.pending_begin (transaction, rai::pending_key (random_block, 0))); + if (i != node_a.store.pending_end ()) + { + rai::pending_key send_hash (i->first); + rai::pending_info info (i->second); + send_block = node_a.store.block_get (transaction, send_hash.hash); + assert (send_block != nullptr); + } + } + if (send_block != nullptr) { - rai::pending_key send_hash (i->first); - rai::pending_info info (i->second); - auto send_block (node_a.store.block_get (transaction, send_hash.hash)); - assert (send_block != nullptr); auto receive_error (wallet (0)->receive_sync (static_cast (*send_block), rai::genesis_account, std::numeric_limits::max ())); (void) receive_error; } diff --git a/rai/utility.cpp b/rai/utility.cpp index 212bddbd..6d15cbbe 100644 --- a/rai/utility.cpp +++ b/rai/utility.cpp @@ -45,7 +45,8 @@ bool rai::from_string_hex (std::string const & value_a, uint64_t & target_a) rai::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a) : open_transactions (0), transaction_iteration (0), -resizing (false) +resizing (false), +sizing_action ([this] () { handle_environment_sizing (); }) { boost::system::error_code error; if (path_a.has_parent_path ()) @@ -57,7 +58,7 @@ resizing (false) assert (status1 == 0); auto status2 (mdb_env_set_maxdbs (environment, 128)); assert (status2 == 0); - auto status3 (mdb_env_set_mapsize (environment, 2 * database_size_increment)); + auto status3 (mdb_env_set_mapsize (environment, 0)); assert (status3 == 0); auto status4 (mdb_env_open (environment, path_a.string ().c_str (), MDB_NOSUBDIR, 00600)); error_a = status4 != 0; @@ -90,38 +91,45 @@ rai::mdb_env::operator MDB_env * () const void rai::mdb_env::add_transaction () { + // Minimize I/O from checking if we need to resive + // Too high and the enviroment might overflow + // Too low and we're making lots of trips to the disk + if ((transaction_iteration++ % rai::database_check_interval) == 0) + { + sizing_action (); + } std::unique_lock lock_l (lock); + // Wait for any resizing operations to complete while (resizing) { resize_notify.wait (lock_l); } - if ((transaction_iteration % rai::database_check_interval) == 0) + ++open_transactions; +} + +void rai::mdb_env::handle_environment_sizing () +{ + MDB_stat stats; + mdb_env_stat (environment, &stats); + MDB_envinfo info; + mdb_env_info (environment, &info); + size_t load (info.me_last_pgno * stats.ms_psize); + auto slack (info.me_mapsize - load); + if (slack < (rai::database_size_increment / 4)) { - MDB_stat stats; - mdb_env_stat (environment, &stats); - MDB_envinfo info; - mdb_env_info (environment, &info); - size_t load (info.me_last_pgno * stats.ms_psize); - auto slack (info.me_mapsize - load); - if (slack < (rai::database_size_increment / 4)) + if (!resizing.exchange (true)) { - resizing = true; - auto done (std::chrono::system_clock::now () + std::chrono::milliseconds (500)); - while (std::chrono::system_clock::now () < done && open_transactions > 0) + std::unique_lock lock_l (lock); + while (open_transactions > 0) { - open_notify.wait_for (lock_l, std::chrono::milliseconds (50)); - } - if (open_transactions == 0) - { - auto next_size (((info.me_mapsize / database_size_increment) + 1) * database_size_increment); - mdb_env_set_mapsize (environment, next_size); + open_notify.wait (lock_l); } + auto next_size (((info.me_mapsize / database_size_increment) + 1) * database_size_increment); + mdb_env_set_mapsize (environment, next_size); resizing = false; resize_notify.notify_all (); } } - ++transaction_iteration; - ++open_transactions; } void rai::mdb_env::remove_transaction () diff --git a/rai/utility.hpp b/rai/utility.hpp index a6c5c749..8a9932d3 100644 --- a/rai/utility.hpp +++ b/rai/utility.hpp @@ -138,13 +138,15 @@ public: operator MDB_env * () const; void add_transaction (); void remove_transaction (); + void handle_environment_sizing (); MDB_env * environment; std::mutex lock; std::condition_variable open_notify; unsigned open_transactions; - unsigned transaction_iteration; + std::atomic_uint transaction_iteration; std::condition_variable resize_notify; - bool resizing; + std::atomic_bool resizing; + std::function sizing_action; }; class mdb_val {