From d5fd87c1f73d8e5edac33e7969c1b3d5a04d90f4 Mon Sep 17 00:00:00 2001 From: Wesley Shillingford Date: Thu, 3 Oct 2019 20:32:31 +0100 Subject: [PATCH] CLI command to output the total number of (un)opened account versions (#2323) * CLI command to output the number of (un)opened account versions * Fix build on older gcc versions * Fix failing upgrade tests * Better CLI description --- nano/nano_node/entry.cpp | 77 ++++++++++++++++++++++++++++++++ nano/node/lmdb/lmdb.cpp | 38 ++++++++++++---- nano/node/lmdb/lmdb.hpp | 12 +++++ nano/node/lmdb/lmdb_iterator.hpp | 1 + nano/secure/epoch.cpp | 9 ++++ nano/secure/epoch.hpp | 6 ++- 6 files changed, 134 insertions(+), 9 deletions(-) diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index 3b095fe38..fdc8aae81 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -134,6 +134,7 @@ int main (int argc, char * const * argv) ("debug_peers", "Display peer IPv6:port connections") ("debug_cemented_block_count", "Displays the number of cemented (confirmed) blocks") ("debug_stacktrace", "Display an example stacktrace") + ("debug_account_versions", "Display the total counts of each version for all accounts (including unpocketed)") ("platform", boost::program_options::value (), "Defines the for OpenCL commands") ("device", boost::program_options::value (), "Defines for OpenCL command") ("threads", boost::program_options::value (), "Defines count for OpenCL command") @@ -1096,6 +1097,82 @@ int main (int argc, char * const * argv) nano::inactive_node node (data_path); node.node->logger.always_log (nano::severity_level::error, "Testing system logger"); } + else if (vm.count ("debug_account_versions")) + { + nano::inactive_node node (data_path); + + auto transaction (node.node->store.tx_begin_read ()); + std::vector> opened_account_versions (nano::normalized_epoch (nano::epoch::max)); + + // Cache the accounts in a collection to make searching quicker against unchecked keys. Group by epoch + for (auto i (node.node->store.latest_begin (transaction)), n (node.node->store.latest_end ()); i != n; ++i) + { + auto const & account (i->first); + auto const & account_info (i->second); + + // Epoch 0 will be index 0 for instance + auto epoch_idx = nano::normalized_epoch (account_info.epoch ()); + opened_account_versions[epoch_idx].emplace (account); + } + + // Iterate all pending blocks and collect the highest version for each unopened account + std::unordered_map> unopened_highest_pending; + for (auto i (node.node->store.pending_begin (transaction)), n (node.node->store.pending_end ()); i != n; ++i) + { + nano::pending_key const & key (i->first); + nano::pending_info const & info (i->second); + // clang-format off + auto & account = key.account; + auto exists = std::any_of (opened_account_versions.begin (), opened_account_versions.end (), [&account](auto const & account_version) { + return account_version.find (account) != account_version.end (); + }); + // clang-format on + if (!exists) + { + // This is an unopened account, store the highest pending version + auto it = unopened_highest_pending.find (key.account); + auto epoch = nano::normalized_epoch (info.epoch); + if (it != unopened_highest_pending.cend ()) + { + // Found it, compare against existing value + if (epoch > it->second) + { + it->second = epoch; + } + } + else + { + // New unopened account + unopened_highest_pending.emplace (key.account, epoch); + } + } + } + + auto output_account_version_number = [](auto version, auto num_accounts) { + std::cout << "Account version " << version << " num accounts: " << num_accounts << "\n"; + }; + + // Output total version counts for the opened accounts + std::cout << "Opened accounts:\n"; + for (auto i = 0u; i < opened_account_versions.size (); ++i) + { + output_account_version_number (i, opened_account_versions[i].size ()); + } + + // Accumulate the version numbers for the highest pending epoch for each unopened account. + std::vector unopened_account_version_totals (nano::normalized_epoch (nano::epoch::max)); + for (auto & pair : unopened_highest_pending) + { + ++unopened_account_version_totals[pair.second]; + } + + // Output total version counts for the unopened accounts + std::cout << "\nUnopened accounts:\n"; + for (auto i = 0u; i < unopened_account_version_totals.size (); ++i) + { + output_account_version_number (i, unopened_account_version_totals[i]); + } + } else if (vm.count ("version")) { std::cout << "Version " << NANO_VERSION_STRING << "\n" diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 6b7c2019a..af42a5b0e 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -503,22 +503,25 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa { // Move confirmation height from account_info database to its own table std::vector> account_infos; - account_infos.reserve (count (transaction_a, accounts_v0) + count (transaction_a, accounts_v1)); + upgrade_counters account_counters (count (transaction_a, accounts_v0), count (transaction_a, accounts_v1)); + account_infos.reserve (account_counters.before_v0 + account_counters.before_v1); - nano::mdb_merge_iterator i (transaction_a, accounts_v0, accounts_v1); - nano::mdb_merge_iterator n{}; - for (; i != n; ++i) + nano::mdb_merge_iterator i_account (transaction_a, accounts_v0, accounts_v1); + nano::mdb_merge_iterator n_account{}; + for (; i_account != n_account; ++i_account) { - nano::account account (i->first); - nano::account_info_v14 account_info_v14 (i->second); + nano::account account (i_account->first); + nano::account_info_v14 account_info_v14 (i_account->second); // Upgrade rep block to representative account auto rep_block = block_get_v14 (transaction_a, account_info_v14.rep_block); release_assert (rep_block != nullptr); - account_infos.emplace_back (account, nano::account_info{ account_info_v14.head, rep_block->representative (), account_info_v14.open_block, account_info_v14.balance, account_info_v14.modified, account_info_v14.block_count, i.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 }); + account_infos.emplace_back (account, nano::account_info{ account_info_v14.head, rep_block->representative (), account_info_v14.open_block, account_info_v14.balance, account_info_v14.modified, account_info_v14.block_count, i_account.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 }); confirmation_height_put (transaction_a, account, account_info_v14.confirmation_height); + i_account.from_first_database ? ++account_counters.after_v0 : ++account_counters.after_v1; } + assert (account_counters.are_equal ()); // No longer need accounts_v1, keep v0 but clear it mdb_drop (env.tx (transaction_a), accounts_v1, 1); mdb_drop (env.tx (transaction_a), accounts_v0, 0); @@ -537,6 +540,8 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa MDB_dbi state_blocks_new; mdb_dbi_open (env.tx (transaction_a), "state_blocks", MDB_CREATE, &state_blocks_new); + upgrade_counters state_counters (count (transaction_a, state_blocks_v0), count (transaction_a, state_blocks_v1)); + nano::mdb_merge_iterator i_state (transaction_a, state_blocks_v0, state_blocks_v1); nano::mdb_merge_iterator n_state{}; auto num = 0u; @@ -566,8 +571,10 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa { logger.always_log (boost::str (boost::format ("Database epoch merge upgrade %1% million state blocks upgraded") % (num / output_cutoff))); } + i_state.from_first_database ? ++state_counters.after_v0 : ++state_counters.after_v1; } + assert (state_counters.are_equal ()); logger.always_log ("Epoch merge upgrade. Finished state blocks, now doing pending blocks"); state_blocks = state_blocks_new; @@ -578,8 +585,9 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa state_blocks_v0 = state_blocks; + upgrade_counters pending_counters (count (transaction_a, pending_v0), count (transaction_a, pending_v1)); std::vector> pending_infos; - pending_infos.reserve (count (transaction_a, pending_v0) + count (transaction_a, pending_v1)); + pending_infos.reserve (pending_counters.before_v0 + pending_counters.before_v1); nano::mdb_merge_iterator i_pending (transaction_a, pending_v0, pending_v1); nano::mdb_merge_iterator n_pending{}; @@ -587,8 +595,11 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa { nano::pending_info_v14 info (i_pending->second); pending_infos.emplace_back (nano::pending_key (i_pending->first), nano::pending_info{ info.source, info.amount, i_pending.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 }); + i_pending.from_first_database ? ++pending_counters.after_v0 : ++pending_counters.after_v1; } + assert (pending_counters.are_equal ()); + // No longer need the pending v1 table mdb_drop (env.tx (transaction_a), pending_v1, 1); mdb_drop (env.tx (transaction_a), pending_v0, 0); @@ -1004,3 +1015,14 @@ std::shared_ptr nano::mdb_store::block_get_v14 (nano::transaction c } return result; } + +nano::mdb_store::upgrade_counters::upgrade_counters (uint64_t count_before_v0, uint64_t count_before_v1) : +before_v0 (count_before_v0), +before_v1 (count_before_v1) +{ +} + +bool nano::mdb_store::upgrade_counters::are_equal () const +{ + return (before_v0 == after_v0) && (before_v1 == after_v1); +} diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index 7221f53d7..08b7d4633 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -249,6 +249,18 @@ private: bool txn_tracking_enabled; size_t count (nano::transaction const & transaction_a, tables table_a) const override; + + class upgrade_counters + { + public: + upgrade_counters (uint64_t count_before_v0, uint64_t count_before_v1); + bool are_equal () const; + + uint64_t before_v0; + uint64_t before_v1; + uint64_t after_v0{ 0 }; + uint64_t after_v1{ 0 }; + }; }; template <> diff --git a/nano/node/lmdb/lmdb_iterator.hpp b/nano/node/lmdb/lmdb_iterator.hpp index 88e3bf19b..6f1e7e130 100644 --- a/nano/node/lmdb/lmdb_iterator.hpp +++ b/nano/node/lmdb/lmdb_iterator.hpp @@ -264,6 +264,7 @@ private: else if (key_cmp > 0) { result = impl2.get (); + from_first_database = false; } else { diff --git a/nano/secure/epoch.cpp b/nano/secure/epoch.cpp index 9947912a5..29938d09a 100644 --- a/nano/secure/epoch.cpp +++ b/nano/secure/epoch.cpp @@ -34,3 +34,12 @@ bool nano::epochs::is_sequential (nano::epoch epoch_a, nano::epoch new_epoch_a) bool is_valid_epoch (head_epoch >= std::underlying_type_t (nano::epoch::epoch_0)); return is_valid_epoch && (std::underlying_type_t (new_epoch_a) == (head_epoch + 1)); } + +std::underlying_type_t nano::normalized_epoch (nano::epoch epoch_a) +{ + // Currently assumes that the epoch versions in the enum are sequential. + auto start = std::underlying_type_t (nano::epoch::epoch_0); + auto end = std::underlying_type_t (epoch_a); + assert (end >= start); + return end - start; +} diff --git a/nano/secure/epoch.hpp b/nano/secure/epoch.hpp index f3132b923..63ed9f6c4 100644 --- a/nano/secure/epoch.hpp +++ b/nano/secure/epoch.hpp @@ -17,8 +17,12 @@ enum class epoch : uint8_t epoch_begin = 2, epoch_0 = 2, epoch_1 = 3, - epoch_2 = 4 + epoch_2 = 4, + max = epoch_2 }; + +/* This turns epoch_0 into 0 for instance */ +std::underlying_type_t normalized_epoch (nano::epoch epoch_a); } namespace std {