Merge pull request #4916 from pwojcikdev/ledger-verify-consistency

Verify ledger balance consistency on startup

nano/core_test/ledger.cpp

@@ -4623,7 +4623,7 @@ TEST (ledger, confirmation_height_not_updated)
 	ASSERT_EQ (nano::block_hash (0), confirmation_height_info.frontier);
 }
 
-TEST (ledger, zero_rep)
+TEST (ledger, zero_rep_weight)
 {
 	nano::test::system system (1);
 	auto & node1 (*system.nodes[0]);
@@ -4640,7 +4640,7 @@ TEST (ledger, zero_rep)
 	auto transaction = node1.ledger.tx_begin_write ();
 	ASSERT_EQ (nano::block_status::progress, node1.ledger.process (transaction, block1));
 	ASSERT_EQ (0, node1.ledger.rep_weights.get (nano::dev::genesis_key.pub));
-	ASSERT_EQ (nano::dev::constants.genesis_amount, node1.ledger.rep_weights.get (0));
+	ASSERT_EQ (0, node1.ledger.rep_weights.get (0));
 	auto block2 = builder.state ()
 				  .account (nano::dev::genesis_key.pub)
 				  .previous (block1->hash ())

nano/nano_node/entry.cpp

@@ -1373,7 +1373,7 @@ int main (int argc, char * const * argv)
 
 			auto node_flags = nano::inactive_node_flag_defaults ();
 			nano::update_flags (node_flags, vm);
-			node_flags.generate_cache.enable_all ();
+			node_flags.generate_cache = nano::generate_cache_flags::all_enabled ();
 			nano::inactive_node inactive_node_l (data_path, node_flags);
 
 			nano::node_rpc_config config;

nano/node/block_processor.cpp

@@ -369,7 +369,7 @@ void nano::block_processor::process_batch (nano::unique_lock<nano::mutex> & lock
 	}
 
 	// We had rocksdb issues in the past, ensure that rep weights are always consistent
-	ledger.rep_weights.verify_consistency ();
+	ledger.verify_consistency (transaction);
 
 	if (number_of_blocks_processed != 0 && timer.stop () > std::chrono::milliseconds (100))
 	{

nano/node/bounded_backlog.cpp

@@ -345,7 +345,7 @@ std::deque<nano::block_hash> nano::bounded_backlog::perform_rollbacks (std::dequ
 	}
 
 	// We had rocksdb issues in the past, ensure that rep weights are always consistent
-	ledger.rep_weights.verify_consistency ();
+	ledger.verify_consistency (transaction);
 
 	return processed;
 }

nano/node/inactive_node.cpp

@@ -19,10 +19,7 @@ nano::node_flags const & nano::inactive_node_flag_defaults ()
 	static nano::node_flags node_flags;
 	node_flags.inactive_node = true;
 	node_flags.read_only = true;
-	node_flags.generate_cache.reps = false;
-	node_flags.generate_cache.cemented_count = false;
-	node_flags.generate_cache.unchecked_count = false;
-	node_flags.generate_cache.account_count = false;
+	node_flags.generate_cache = nano::generate_cache_flags::all_disabled ();
 	node_flags.disable_bootstrap_listener = true;
 	node_flags.disable_tcp_realtime = true;
 	return node_flags;

nano/secure/CMakeLists.txt

@@ -22,7 +22,6 @@ add_library(
   common.cpp
   fwd.hpp
   generate_cache_flags.hpp
-  generate_cache_flags.cpp
   ledger.hpp
   ledger.cpp
   ledger_set_any.hpp

nano/secure/generate_cache_flags.cpp

@@ -1,9 +0,0 @@
-#include <nano/secure/generate_cache_flags.hpp>
-
-void nano::generate_cache_flags::enable_all ()
-{
-	reps = true;
-	cemented_count = true;
-	unchecked_count = true;
-	account_count = true;
-}

nano/secure/generate_cache_flags.hpp

@@ -7,24 +7,29 @@ namespace nano
 class generate_cache_flags
 {
 public:
-	bool reps = true;
-	bool cemented_count = true;
-	bool unchecked_count = true;
-	bool account_count = true;
-	bool block_count = true;
-
-	void enable_all ();
+	bool reps{ true };
+	bool cemented_count{ true };
+	bool unchecked_count{ true };
+	bool account_count{ true };
+	bool block_count{ true };
+	bool consistency_check{ true };
 
 public:
+	static generate_cache_flags all_enabled ()
+	{
+		return {};
+	}
+
 	static generate_cache_flags all_disabled ()
 	{
-		generate_cache_flags flags;
-		flags.reps = false;
-		flags.cemented_count = false;
-		flags.unchecked_count = false;
-		flags.account_count = false;
-		flags.block_count = false;
-		return flags;
+		return {
+			.reps = false,
+			.cemented_count = false,
+			.unchecked_count = false,
+			.account_count = false,
+			.block_count = false,
+			.consistency_check = false,
+		};
 	}
 };
 }

nano/secure/ledger.cpp

@@ -790,6 +790,8 @@ void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache
 
 	if (generate_cache_flags.account_count || generate_cache_flags.block_count)
 	{
+		logger.debug (nano::log::type::ledger, "Generating block count cache...");
+
 		store.account.for_each_par (
 		[this] (store::read_transaction const &, auto i, auto n) {
 			uint64_t block_count_l{ 0 };
@@ -803,10 +805,40 @@ void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache
 			this->cache.block_count += block_count_l;
 			this->cache.account_count += account_count_l;
 		});
+
+		logger.debug (nano::log::type::ledger, "Block count cache generated");
+	}
+
+	if (generate_cache_flags.cemented_count)
+	{
+		logger.debug (nano::log::type::ledger, "Generating cemented count cache...");
+
+		store.confirmation_height.for_each_par (
+		[this] (store::read_transaction const &, auto i, auto n) {
+			uint64_t cemented_count_l (0);
+			for (; i != n; ++i)
+			{
+				cemented_count_l += i->second.height;
+			}
+			this->cache.cemented_count += cemented_count_l;
+		});
+
+		logger.debug (nano::log::type::ledger, "Cemented count cache generated");
+	}
+
+	{
+		logger.debug (nano::log::type::ledger, "Generating pruned count cache...");
+
+		auto transaction = store.tx_begin_read ();
+		cache.pruned_count = store.pruned.count (transaction);
+
+		logger.debug (nano::log::type::ledger, "Pruned count cache generated");
 	}
 
 	if (generate_cache_flags.reps)
 	{
+		logger.debug (nano::log::type::ledger, "Generating representative weights cache...");
+
 		store.rep_weight.for_each_par (
 		[this] (store::read_transaction const &, auto i, auto n) {
 			nano::rep_weights rep_weights_l{ this->store.rep_weight };
@@ -827,25 +859,80 @@ void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache
 			this->rep_weights.append_from (rep_weights_l);
 		});
 
-		rep_weights.verify_consistency ();
+		logger.debug (nano::log::type::ledger, "Representative weights cache generated");
 	}
 
-	if (generate_cache_flags.cemented_count)
+	// Use larger precision types to detect potential overflow issues
+	nano::uint256_t active_balance, pending_balance, burned_balance;
+
+	if (generate_cache_flags.consistency_check)
 	{
-		store.confirmation_height.for_each_par (
-		[this] (store::read_transaction const &, auto i, auto n) {
-			uint64_t cemented_count_l (0);
+		logger.debug (nano::log::type::ledger, "Verifying ledger balance consistency...");
+
+		// Verify sum of all account and pending balances
+		nano::locked<nano::uint256_t> active_balance_s{ 0 };
+		nano::locked<nano::uint256_t> pending_balance_s{ 0 };
+		nano::locked<nano::uint256_t> burned_balance_s{ 0 };
+
+		store.account.for_each_par (
+		[&] (store::read_transaction const &, auto i, auto n) {
+			nano::uint256_t balance_l{ 0 };
+			nano::uint256_t burned_l{ 0 };
 			for (; i != n; ++i)
 			{
-				cemented_count_l += i->second.height;
+				nano::account_info const & info = i->second;
+				if (i->first == constants.burn_account)
+				{
+					burned_l += info.balance.number ();
+				}
+				else
+				{
+					balance_l += info.balance.number ();
+				}
 			}
-			this->cache.cemented_count += cemented_count_l;
+			(*active_balance_s.lock ()) += balance_l;
+			release_assert (burned_l == 0); // The burn account should not have any active balance
 		});
+
+		store.pending.for_each_par (
+		[&] (store::read_transaction const &, auto i, auto n) {
+			nano::uint256_t balance_l{ 0 };
+			nano::uint256_t burned_l{ 0 };
+			for (; i != n; ++i)
+			{
+				nano::pending_key const & key = i->first;
+				nano::pending_info const & info = i->second;
+				if (key.account == constants.burn_account)
+				{
+					burned_l += info.amount.number ();
+				}
+				else
+				{
+					balance_l += info.amount.number ();
+				}
+			}
+			(*pending_balance_s.lock ()) += balance_l;
+			(*burned_balance_s.lock ()) += burned_l;
+		});
+
+		active_balance = *active_balance_s.lock ();
+		pending_balance = *pending_balance_s.lock ();
+		burned_balance = *burned_balance_s.lock ();
+
+		release_assert (active_balance <= std::numeric_limits<nano::uint128_t>::max ());
+		release_assert (pending_balance <= std::numeric_limits<nano::uint128_t>::max ());
+		release_assert (burned_balance <= std::numeric_limits<nano::uint128_t>::max ());
+
+		release_assert (active_balance + pending_balance + burned_balance == constants.genesis_amount, "ledger corruption detected: account and pending balances do not match genesis amount", to_string (active_balance) + " + " + to_string (pending_balance) + " + " + to_string (burned_balance) + " != " + to_string (constants.genesis_amount));
+		release_assert (active_balance == rep_weights.get_weight_committed (), "ledger corruption detected: active balance does not match committed representative weights", to_string (active_balance) + " != " + to_string (rep_weights.get_weight_committed ()));
+		release_assert (pending_balance + burned_balance == rep_weights.get_weight_unused (), "ledger corruption detected: pending balance does not match unused representative weights", to_string (pending_balance) + " != " + to_string (rep_weights.get_weight_unused ()));
+
+		logger.debug (nano::log::type::ledger, "Ledger balance consistency verified");
 	}
 
+	if (generate_cache_flags.reps && generate_cache_flags.consistency_check)
 	{
-		auto transaction (store.tx_begin_read ());
-		cache.pruned_count = store.pruned.count (transaction);
+		rep_weights.verify_consistency (static_cast<nano::uint128_t> (burned_balance));
 	}
 
 	logger.info (nano::log::type::ledger, "Block count:    {:>11}", cache.block_count.load ());
@@ -853,17 +940,26 @@ void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache
 	logger.info (nano::log::type::ledger, "Account count:  {:>11}", cache.account_count.load ());
 	logger.info (nano::log::type::ledger, "Pruned count:   {:>11}", cache.pruned_count.load ());
 	logger.info (nano::log::type::ledger, "Representative count: {:>5}", rep_weights.size ());
-	logger.info (nano::log::type::ledger, "Weight commited: {} | unused: {}",
+	logger.info (nano::log::type::ledger, "Active balance: {} | pending: {} | burned: {}",
+	nano::uint128_union{ static_cast<nano::uint128_t> (active_balance) }.format_balance (nano::nano_ratio, 0, true),
+	nano::uint128_union{ static_cast<nano::uint128_t> (pending_balance) }.format_balance (nano::nano_ratio, 0, true),
+	nano::uint128_union{ static_cast<nano::uint128_t> (burned_balance) }.format_balance (nano::nano_ratio, 0, true));
+	logger.info (nano::log::type::ledger, "Weight committed: {} | unused: {}",
 	nano::uint128_union{ rep_weights.get_weight_committed () }.format_balance (nano::nano_ratio, 0, true),
 	nano::uint128_union{ rep_weights.get_weight_unused () }.format_balance (nano::nano_ratio, 0, true));
 }
 
-bool nano::ledger::unconfirmed_exists (secure::transaction const & transaction, nano::block_hash const & hash)
+void nano::ledger::verify_consistency (secure::transaction const & transaction) const
+{
+	rep_weights.verify_consistency (0); // It's impractical to recompute burned weight, so we skip it here
+}
+
+bool nano::ledger::unconfirmed_exists (secure::transaction const & transaction, nano::block_hash const & hash) const
 {
 	return any.block_exists (transaction, hash) && !confirmed.block_exists (transaction, hash);
 }
 
-nano::uint128_t nano::ledger::account_receivable (secure::transaction const & transaction_a, nano::account const & account_a, bool only_confirmed_a)
+nano::uint128_t nano::ledger::account_receivable (secure::transaction const & transaction_a, nano::account const & account_a, bool only_confirmed_a) const
 {
 	nano::uint128_t result (0);
 	nano::account end (account_a.number () + 1);
@@ -1267,7 +1363,6 @@ std::optional<nano::account> nano::ledger::linked_account (secure::transaction c
 	{
 		return any.block_account (transaction, block.source ());
 	}
-
 	return std::nullopt;
 }
 
@@ -1602,7 +1697,6 @@ nano::epoch nano::ledger::version (nano::block const & block)
 	{
 		return block.sideband ().details.epoch;
 	}
-
 	return nano::epoch::epoch_0;
 }
 

nano/secure/ledger.hpp

@@ -45,8 +45,8 @@ public:
 	/** Start read-only transaction */
 	secure::read_transaction tx_begin_read () const;
 
-	bool unconfirmed_exists (secure::transaction const &, nano::block_hash const &);
-	nano::uint128_t account_receivable (secure::transaction const &, nano::account const &, bool = false);
+	bool unconfirmed_exists (secure::transaction const &, nano::block_hash const &) const;
+	nano::uint128_t account_receivable (secure::transaction const &, nano::account const &, bool = false) const;
 	/**
 	 * Returns the cached vote weight for the given representative.
 	 * If the weight is below the cache limit it returns 0.
@@ -95,6 +95,8 @@ public:
 	using block_priority_result = std::pair<nano::amount, nano::priority_timestamp>;
 	block_priority_result block_priority (secure::transaction const &, nano::block const &) const;
 
+	void verify_consistency (secure::transaction const &) const;
+
 	nano::container_info container_info () const;
 
 public:

nano/secure/rep_weights.cpp

@@ -123,12 +123,17 @@ void nano::rep_weights::append_from (nano::rep_weights const & other)
 	weight_unused += other.weight_unused;
 }
 
-void nano::rep_weights::verify_consistency () const
+void nano::rep_weights::verify_consistency (nano::uint128_t const burn_balance) const
 {
 	std::shared_lock guard{ mutex };
-	auto total_weight = weight_committed + weight_unused;
+
+	auto const total_weight = weight_committed + weight_unused;
 	release_assert (total_weight == std::numeric_limits<nano::uint128_t>::max (), "total weight exceeds maximum value", to_string (weight_committed) + " + " + to_string (weight_unused));
-	auto cached_weight = std::accumulate (rep_amounts.begin (), rep_amounts.end (), nano::uint256_t{ 0 }, [] (nano::uint256_t sum, const auto & entry) {
+
+	auto const expected_total = std::numeric_limits<nano::uint128_t>::max () - burn_balance;
+	release_assert (weight_committed <= expected_total, "total weight does not match expected value accounting for burn", to_string (weight_committed) + " + " + to_string (weight_unused) + " != " + to_string (expected_total) + " (burn: " + to_string (burn_balance) + ")");
+
+	auto const cached_weight = std::accumulate (rep_amounts.begin (), rep_amounts.end (), nano::uint256_t{ 0 }, [] (nano::uint256_t sum, const auto & entry) {
 		return sum + entry.second;
 	});
 	release_assert (cached_weight <= weight_committed, "total cached weight must match the sum of all committed weights", to_string (cached_weight) + " <= " + to_string (weight_committed));
@@ -178,6 +183,11 @@ void nano::rep_weights::put_store (store::write_transaction const & txn, nano::a
 
 nano::uint128_t nano::rep_weights::get_impl (nano::account const & rep) const
 {
+	if (rep.is_zero ())
+	{
+		return 0; // Zero account always has zero weight
+	}
+
 	auto it = rep_amounts.find (rep);
 	if (it != rep_amounts.end ())
 	{

nano/secure/rep_weights.hpp

@@ -41,7 +41,7 @@ public:
 	nano::uint128_t get_weight_committed () const;
 	nano::uint128_t get_weight_unused () const;
 
-	void verify_consistency () const;
+	void verify_consistency (nano::uint128_t burn_balance) const;
 
 private:
 	nano::store::rep_weight & rep_weight_store;