Minecon724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

add CLI switch for adjusting Sideband upgrade batch sizes (#1684)

Browse source 
* resize batching for sideband upgrade to improve UI and RPC responsiveness

* Command line switch for configuring sideband batch_size, default is 512

* Simplification and updating max to use batch_size

* formatting :/

* cleanup

* Apply flags at node creation time

* Properly display options for unchecked_drop
This commit is contained in:
Russel Waters 2019-02-01 14:18:02 -05:00 • committed by GitHub
commit 7899d82cef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 36 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

3
nano/nano_node/daemon.cpp
View file

@ -133,10 +133,9 @@ void nano_daemon::daemon::run (boost::filesystem::path const & data_path, nano::
nano::node_init init;
try
{
auto node (std::make_shared<nano::node> (init, io_ctx, data_path, alarm, config.node, opencl_work, true));
auto node (std::make_shared<nano::node> (init, io_ctx, data_path, alarm, config.node, opencl_work, flags));
if (!init.error ())
{
node->flags = flags;
node->start ();
std::unique_ptr<nano::rpc> rpc = get_rpc (io_ctx, *node, config.rpc);
if (rpc)

9
nano/nano_node/entry.cpp
View file

@ -29,7 +29,9 @@ int main (int argc, char * const * argv)
("disable_wallet_bootstrap", "Disables wallet lazy bootstrap")
("disable_bootstrap_listener", "Disables bootstrap listener (incoming connections)")
("disable_unchecked_cleaning", "Disables periodic cleaning of old records from unchecked table")
("disable_unchecked_drop", "Disables drop of unchecked table at startup")
("fast_bootstrap", "Increase bootstrap speed for high end nodes with higher limits")
("batch_size",boost::program_options::value<std::size_t> (), "Increase sideband batch size, default 512")
("debug_block_count", "Display the number of block")
("debug_bootstrap_generate", "Generate bootstrap sequence of blocks")
("debug_dump_representatives", "List representatives and weights")
@ -80,19 +82,24 @@ int main (int argc, char * const * argv)
boost::filesystem::path data_path ((data_path_it != vm.end ()) ? data_path_it->second.as<std::string> () : nano::working_path ());
auto ec = nano::handle_node_options (vm);
if (ec == nano::error_cli::unknown_command)
{
if (vm.count ("daemon") > 0)
{
nano_daemon::daemon daemon;
nano::node_flags flags;
auto batch_size_it = vm.find ("batch_size");
if (batch_size_it != vm.end ())
{
flags.sideband_batch_size = batch_size_it->second.as<size_t> ();
}
flags.disable_backup = (vm.count ("disable_backup") > 0);
flags.disable_lazy_bootstrap = (vm.count ("disable_lazy_bootstrap") > 0);
flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0);
flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0);
flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0);
flags.disable_unchecked_cleaning = (vm.count ("disable_unchecked_cleaning") > 0);
flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
daemon.run (data_path, flags);
}

9
nano/nano_wallet/entry.cpp
View file

@ -243,7 +243,7 @@ int run_wallet (QApplication & application, int argc, char * const * argv, boost
: std::function<boost::optional<uint64_t> (nano::uint256_union const &)> (nullptr));
nano::alarm alarm (io_ctx);
nano::node_init init;
node = std::make_shared<nano::node> (init, io_ctx, data_path, alarm, config.node, work, true);
node = std::make_shared<nano::node> (init, io_ctx, data_path, alarm, config.node, work, flags);
if (!init.error ())
{
auto wallet (node->wallets.open (config.wallet));
@ -359,11 +359,18 @@ int main (int argc, char * const * argv)
data_path = nano::working_path ();
}
nano::node_flags flags;
auto batch_size_it = vm.find ("batch_size");
if (batch_size_it != vm.end ())
{
flags.sideband_batch_size = batch_size_it->second.as<size_t> ();
}
flags.disable_backup = (vm.count ("disable_backup") > 0);
flags.disable_lazy_bootstrap = (vm.count ("disable_lazy_bootstrap") > 0);
flags.disable_legacy_bootstrap = (vm.count ("disable_legacy_bootstrap") > 0);
flags.disable_wallet_bootstrap = (vm.count ("disable_wallet_bootstrap") > 0);
flags.disable_bootstrap_listener = (vm.count ("disable_bootstrap_listener") > 0);
flags.disable_unchecked_cleaning = (vm.count ("disable_unchecked_cleaning") > 0);
flags.disable_unchecked_drop = (vm.count ("disable_unchecked_drop") > 0);
flags.fast_bootstrap = (vm.count ("fast_bootstrap") > 0);
result = run_wallet (application, argc, argv, data_path, flags);
}

15
nano/node/lmdb.cpp
View file

@ -743,7 +743,7 @@ nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> nano::mdb_store
return nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> (nullptr);
}
nano::mdb_store::mdb_store (bool & error_a, nano::logging & logging_a, boost::filesystem::path const & path_a, int lmdb_max_dbs, bool drop_unchecked) :
nano::mdb_store::mdb_store (bool & error_a, nano::logging & logging_a, boost::filesystem::path const & path_a, int lmdb_max_dbs, bool drop_unchecked, size_t const batch_size) :
logging (logging_a),
env (error_a, path_a, lmdb_max_dbs)
{
@ -783,9 +783,9 @@ env (error_a, path_a, lmdb_max_dbs)
}
if (slow_upgrade)
{
upgrades = std::thread ([this]() {
upgrades = std::thread ([this, batch_size]() {
nano::thread_role::set (nano::thread_role::name::slow_db_upgrade);
do_slow_upgrades ();
do_slow_upgrades (batch_size);
});
}
}
@ -1162,7 +1162,7 @@ void nano::mdb_store::upgrade_v10_to_v11 (nano::transaction const & transaction_
mdb_drop (env.tx (transaction_a), unsynced, 1);
}
void nano::mdb_store::do_slow_upgrades ()
void nano::mdb_store::do_slow_upgrades (size_t const batch_size)
{
int version;
{
@ -1185,7 +1185,7 @@ void nano::mdb_store::do_slow_upgrades ()
case 11:
break;
case 12:
upgrade_v12_to_v13 ();
upgrade_v12_to_v13 (batch_size);
break;
case 13:
break;
@ -1205,10 +1205,9 @@ void nano::mdb_store::upgrade_v11_to_v12 (nano::transaction const & transaction_
mdb_drop (env.tx (transaction_a), checksum, 1);
}
void nano::mdb_store::upgrade_v12_to_v13 ()
void nano::mdb_store::upgrade_v12_to_v13 (size_t const batch_size)
{
size_t cost (0);
size_t const max (16384);
nano::account account (0);
auto transaction (tx_begin_write ());
while (!stopped && account != nano::not_an_account)
@ -1230,7 +1229,7 @@ void nano::mdb_store::upgrade_v12_to_v13 ()
nano::block_sideband sideband;
while (!stopped && !hash.is_zero ())
{
if (cost >= max)
if (cost >= batch_size)
{
BOOST_LOG (logging.log) << boost::str (boost::format ("Upgrading sideband information for account %1%... height %2%") % first.to_account ().substr (0, 24) % std::to_string (height));
auto tx (boost::polymorphic_downcast<nano::mdb_txn *> (transaction.impl.get ()));

6
nano/node/lmdb.hpp
View file

@ -150,7 +150,7 @@ class mdb_store : public block_store
friend class nano::block_predecessor_set;
public:
mdb_store (bool &, nano::logging &, boost::filesystem::path const &, int lmdb_max_dbs = 128, bool drop_unchecked = false);
mdb_store (bool &, nano::logging &, boost::filesystem::path const &, int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512);
~mdb_store ();
nano::transaction tx_begin_write () override;
@ -263,8 +263,8 @@ public:
void upgrade_v9_to_v10 (nano::transaction const &);
void upgrade_v10_to_v11 (nano::transaction const &);
void upgrade_v11_to_v12 (nano::transaction const &);
void do_slow_upgrades ();
void upgrade_v12_to_v13 ();
void do_slow_upgrades (size_t const);
void upgrade_v12_to_v13 (size_t const);
bool full_sideband (nano::transaction const &);
// Requires a write transaction

5
nano/node/node.cpp
View file

@ -2055,12 +2055,13 @@ node (init_a, io_ctx_a, application_path_a, alarm_a, nano::node_config (peering_
{
}
nano::node::node (nano::node_init & init_a, boost::asio::io_context & io_ctx_a, boost::filesystem::path const & application_path_a, nano::alarm & alarm_a, nano::node_config const & config_a, nano::work_pool & work_a, bool drop_unchecked) :
nano::node::node (nano::node_init & init_a, boost::asio::io_context & io_ctx_a, boost::filesystem::path const & application_path_a, nano::alarm & alarm_a, nano::node_config const & config_a, nano::work_pool & work_a, nano::node_flags flags_a) :
io_ctx (io_ctx_a),
config (config_a),
alarm (alarm_a),
work (work_a),
store_impl (std::make_unique<nano::mdb_store> (init_a.block_store_init, config.logging, application_path_a / "data.ldb", config_a.lmdb_max_dbs, drop_unchecked)),
flags (flags_a),
store_impl (std::make_unique<nano::mdb_store> (init_a.block_store_init, config.logging, application_path_a / "data.ldb", config_a.lmdb_max_dbs, !flags.disable_unchecked_drop, flags.sideband_batch_size)),
store (*store_impl),
wallets_store_impl (std::make_unique<nano::mdb_wallets_store> (init_a.wallets_store_init, application_path_a / "wallets.ldb", config_a.lmdb_max_dbs)),
wallets_store (*wallets_store_impl),

2
nano/node/node.hpp
View file

@ -531,7 +531,7 @@ class node : public std::enable_shared_from_this<nano::node>
{
public:
node (nano::node_init &, boost::asio::io_context &, uint16_t, boost::filesystem::path const &, nano::alarm &, nano::logging const &, nano::work_pool &);
node (nano::node_init &, boost::asio::io_context &, boost::filesystem::path const &, nano::alarm &, nano::node_config const &, nano::work_pool &, bool drop_unchecked = false);
node (nano::node_init &, boost::asio::io_context &, boost::filesystem::path const &, nano::alarm &, nano::node_config const &, nano::work_pool &, nano::node_flags = nano::node_flags ());
~node ();
template <typename T>
void background (T action_a)

4
nano/node/nodeconfig.cpp
View file

@ -399,6 +399,8 @@ disable_legacy_bootstrap (false),
disable_wallet_bootstrap (false),
disable_bootstrap_listener (false),
disable_unchecked_cleaning (false),
fast_bootstrap (false)
disable_unchecked_drop (true),
fast_bootstrap (false),
sideband_batch_size (512)
{
}

2
nano/node/nodeconfig.hpp
View file

@ -70,6 +70,8 @@ public:
bool disable_wallet_bootstrap;
bool disable_bootstrap_listener;
bool disable_unchecked_cleaning;
bool disable_unchecked_drop;
bool fast_bootstrap;
size_t sideband_batch_size;
};
}