CLI command to output the total number of (un)opened account versions (#2323)

* CLI command to output the number of (un)opened account versions

* Fix build on older gcc versions

* Fix failing upgrade tests

* Better CLI description

nano/nano_node/entry.cpp

@@ -134,6 +134,7 @@ int main (int argc, char * const * argv)
 		("debug_peers", "Display peer IPv6:port connections")
 		("debug_cemented_block_count", "Displays the number of cemented (confirmed) blocks")
 		("debug_stacktrace", "Display an example stacktrace")
+		("debug_account_versions", "Display the total counts of each version for all accounts (including unpocketed)")
 		("platform", boost::program_options::value<std::string> (), "Defines the <platform> for OpenCL commands")
 		("device", boost::program_options::value<std::string> (), "Defines <device> for OpenCL command")
 		("threads", boost::program_options::value<std::string> (), "Defines <threads> count for OpenCL command")
@@ -1096,6 +1097,82 @@ int main (int argc, char * const * argv)
 			nano::inactive_node node (data_path);
 			node.node->logger.always_log (nano::severity_level::error, "Testing system logger");
 		}
+		else if (vm.count ("debug_account_versions"))
+		{
+			nano::inactive_node node (data_path);
+
+			auto transaction (node.node->store.tx_begin_read ());
+			std::vector<std::unordered_set<nano::account>> opened_account_versions (nano::normalized_epoch (nano::epoch::max));
+
+			// Cache the accounts in a collection to make searching quicker against unchecked keys. Group by epoch
+			for (auto i (node.node->store.latest_begin (transaction)), n (node.node->store.latest_end ()); i != n; ++i)
+			{
+				auto const & account (i->first);
+				auto const & account_info (i->second);
+
+				// Epoch 0 will be index 0 for instance
+				auto epoch_idx = nano::normalized_epoch (account_info.epoch ());
+				opened_account_versions[epoch_idx].emplace (account);
+			}
+
+			// Iterate all pending blocks and collect the highest version for each unopened account
+			std::unordered_map<nano::account, std::underlying_type_t<nano::epoch>> unopened_highest_pending;
+			for (auto i (node.node->store.pending_begin (transaction)), n (node.node->store.pending_end ()); i != n; ++i)
+			{
+				nano::pending_key const & key (i->first);
+				nano::pending_info const & info (i->second);
+				// clang-format off
+				auto & account = key.account;
+				auto exists = std::any_of (opened_account_versions.begin (), opened_account_versions.end (), [&account](auto const & account_version) {
+					return account_version.find (account) != account_version.end ();
+				});
+				// clang-format on
+				if (!exists)
+				{
+					// This is an unopened account, store the highest pending version
+					auto it = unopened_highest_pending.find (key.account);
+					auto epoch = nano::normalized_epoch (info.epoch);
+					if (it != unopened_highest_pending.cend ())
+					{
+						// Found it, compare against existing value
+						if (epoch > it->second)
+						{
+							it->second = epoch;
+						}
+					}
+					else
+					{
+						// New unopened account
+						unopened_highest_pending.emplace (key.account, epoch);
+					}
+				}
+			}
+
+			auto output_account_version_number = [](auto version, auto num_accounts) {
+				std::cout << "Account version " << version << " num accounts: " << num_accounts << "\n";
+			};
+
+			// Output total version counts for the opened accounts
+			std::cout << "Opened accounts:\n";
+			for (auto i = 0u; i < opened_account_versions.size (); ++i)
+			{
+				output_account_version_number (i, opened_account_versions[i].size ());
+			}
+
+			// Accumulate the version numbers for the highest pending epoch for each unopened account.
+			std::vector<size_t> unopened_account_version_totals (nano::normalized_epoch (nano::epoch::max));
+			for (auto & pair : unopened_highest_pending)
+			{
+				++unopened_account_version_totals[pair.second];
+			}
+
+			// Output total version counts for the unopened accounts
+			std::cout << "\nUnopened accounts:\n";
+			for (auto i = 0u; i < unopened_account_version_totals.size (); ++i)
+			{
+				output_account_version_number (i, unopened_account_version_totals[i]);
+			}
+		}
 		else if (vm.count ("version"))
 		{
 			std::cout << "Version " << NANO_VERSION_STRING << "\n"

nano/node/lmdb/lmdb.cpp

@@ -503,22 +503,25 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa
 {
 	// Move confirmation height from account_info database to its own table
 	std::vector<std::pair<nano::account, nano::account_info>> account_infos;
-	account_infos.reserve (count (transaction_a, accounts_v0) + count (transaction_a, accounts_v1));
+	upgrade_counters account_counters (count (transaction_a, accounts_v0), count (transaction_a, accounts_v1));
+	account_infos.reserve (account_counters.before_v0 + account_counters.before_v1);
 
-	nano::mdb_merge_iterator<nano::account, nano::account_info_v14> i (transaction_a, accounts_v0, accounts_v1);
+	nano::mdb_merge_iterator<nano::account, nano::account_info_v14> i_account (transaction_a, accounts_v0, accounts_v1);
-	nano::mdb_merge_iterator<nano::account, nano::account_info_v14> n{};
+	nano::mdb_merge_iterator<nano::account, nano::account_info_v14> n_account{};
-	for (; i != n; ++i)
+	for (; i_account != n_account; ++i_account)
 	{
-		nano::account account (i->first);
+		nano::account account (i_account->first);
-		nano::account_info_v14 account_info_v14 (i->second);
+		nano::account_info_v14 account_info_v14 (i_account->second);
 
 		// Upgrade rep block to representative account
 		auto rep_block = block_get_v14 (transaction_a, account_info_v14.rep_block);
 		release_assert (rep_block != nullptr);
-		account_infos.emplace_back (account, nano::account_info{ account_info_v14.head, rep_block->representative (), account_info_v14.open_block, account_info_v14.balance, account_info_v14.modified, account_info_v14.block_count, i.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 });
+		account_infos.emplace_back (account, nano::account_info{ account_info_v14.head, rep_block->representative (), account_info_v14.open_block, account_info_v14.balance, account_info_v14.modified, account_info_v14.block_count, i_account.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 });
 		confirmation_height_put (transaction_a, account, account_info_v14.confirmation_height);
+		i_account.from_first_database ? ++account_counters.after_v0 : ++account_counters.after_v1;
 	}
 
+	assert (account_counters.are_equal ());
 	// No longer need accounts_v1, keep v0 but clear it
 	mdb_drop (env.tx (transaction_a), accounts_v1, 1);
 	mdb_drop (env.tx (transaction_a), accounts_v0, 0);
@@ -537,6 +540,8 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa
 	MDB_dbi state_blocks_new;
 	mdb_dbi_open (env.tx (transaction_a), "state_blocks", MDB_CREATE, &state_blocks_new);
 
+	upgrade_counters state_counters (count (transaction_a, state_blocks_v0), count (transaction_a, state_blocks_v1));
+
 	nano::mdb_merge_iterator<nano::block_hash, nano::state_block_w_sideband_v14> i_state (transaction_a, state_blocks_v0, state_blocks_v1);
 	nano::mdb_merge_iterator<nano::block_hash, nano::state_block_w_sideband_v14> n_state{};
 	auto num = 0u;
@@ -566,8 +571,10 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa
 		{
 			logger.always_log (boost::str (boost::format ("Database epoch merge upgrade %1% million state blocks upgraded") % (num / output_cutoff)));
 		}
+		i_state.from_first_database ? ++state_counters.after_v0 : ++state_counters.after_v1;
 	}
 
+	assert (state_counters.are_equal ());
 	logger.always_log ("Epoch merge upgrade. Finished state blocks, now doing pending blocks");
 
 	state_blocks = state_blocks_new;
@@ -578,8 +585,9 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa
 
 	state_blocks_v0 = state_blocks;
 
+	upgrade_counters pending_counters (count (transaction_a, pending_v0), count (transaction_a, pending_v1));
 	std::vector<std::pair<nano::pending_key, nano::pending_info>> pending_infos;
-	pending_infos.reserve (count (transaction_a, pending_v0) + count (transaction_a, pending_v1));
+	pending_infos.reserve (pending_counters.before_v0 + pending_counters.before_v1);
 
 	nano::mdb_merge_iterator<nano::pending_key, nano::pending_info_v14> i_pending (transaction_a, pending_v0, pending_v1);
 	nano::mdb_merge_iterator<nano::pending_key, nano::pending_info_v14> n_pending{};
@@ -587,8 +595,11 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transa
 	{
 		nano::pending_info_v14 info (i_pending->second);
 		pending_infos.emplace_back (nano::pending_key (i_pending->first), nano::pending_info{ info.source, info.amount, i_pending.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 });
+		i_pending.from_first_database ? ++pending_counters.after_v0 : ++pending_counters.after_v1;
 	}
 
+	assert (pending_counters.are_equal ());
+
 	// No longer need the pending v1 table
 	mdb_drop (env.tx (transaction_a), pending_v1, 1);
 	mdb_drop (env.tx (transaction_a), pending_v0, 0);
@@ -1004,3 +1015,14 @@ std::shared_ptr<nano::block> nano::mdb_store::block_get_v14 (nano::transaction c
 	}
 	return result;
 }
+
+nano::mdb_store::upgrade_counters::upgrade_counters (uint64_t count_before_v0, uint64_t count_before_v1) :
+before_v0 (count_before_v0),
+before_v1 (count_before_v1)
+{
+}
+
+bool nano::mdb_store::upgrade_counters::are_equal () const
+{
+	return (before_v0 == after_v0) && (before_v1 == after_v1);
+}

nano/node/lmdb/lmdb.hpp

@@ -249,6 +249,18 @@ private:
 	bool txn_tracking_enabled;
 
 	size_t count (nano::transaction const & transaction_a, tables table_a) const override;
+
+	class upgrade_counters
+	{
+	public:
+		upgrade_counters (uint64_t count_before_v0, uint64_t count_before_v1);
+		bool are_equal () const;
+
+		uint64_t before_v0;
+		uint64_t before_v1;
+		uint64_t after_v0{ 0 };
+		uint64_t after_v1{ 0 };
+	};
 };
 
 template <>

nano/node/lmdb/lmdb_iterator.hpp

@@ -264,6 +264,7 @@ private:
 			else if (key_cmp > 0)
 			{
 				result = impl2.get ();
+				from_first_database = false;
 			}
 			else
 			{

nano/secure/epoch.cpp

@@ -34,3 +34,12 @@ bool nano::epochs::is_sequential (nano::epoch epoch_a, nano::epoch new_epoch_a)
 	bool is_valid_epoch (head_epoch >= std::underlying_type_t<nano::epoch> (nano::epoch::epoch_0));
 	return is_valid_epoch && (std::underlying_type_t<nano::epoch> (new_epoch_a) == (head_epoch + 1));
 }
+
+std::underlying_type_t<nano::epoch> nano::normalized_epoch (nano::epoch epoch_a)
+{
+	// Currently assumes that the epoch versions in the enum are sequential.
+	auto start = std::underlying_type_t<nano::epoch> (nano::epoch::epoch_0);
+	auto end = std::underlying_type_t<nano::epoch> (epoch_a);
+	assert (end >= start);
+	return end - start;
+}

nano/secure/epoch.hpp

@@ -17,8 +17,12 @@ enum class epoch : uint8_t
 	epoch_begin = 2,
 	epoch_0 = 2,
 	epoch_1 = 3,
-	epoch_2 = 4
+	epoch_2 = 4,
+	max = epoch_2
 };
+
+/* This turns epoch_0 into 0 for instance */
+std::underlying_type_t<nano::epoch> normalized_epoch (nano::epoch epoch_a);
 }
 namespace std
 {