Add gap pending epoch open blocks to unchecked (#3108)

* Add gap pending epoch open blocks to unchecked
to address found by @Srayman full bootstrap slowdown

* Expanding test to show that these epochs are picked up when a pending entry comes in.
* Using block_processor.flush () instead of ASSERT_TIMELY.
* More checks in ledger.epoch_open_pending test
* Do not check last epoch blocks
* Explain settings

Co-authored-by: clemahieu <clemahieu@gmail.com>

nano/core_test/ledger.cpp

@@ -2631,13 +2631,44 @@ TEST (ledger, successor_epoch)
 
 TEST (ledger, epoch_open_pending)
 {
+	nano::block_builder builder;
 	nano::system system (1);
 	auto & node1 (*system.nodes[0]);
 	nano::work_pool pool (std::numeric_limits<unsigned>::max ());
 	nano::keypair key1;
-	nano::state_block epoch_open (key1.pub, 0, 0, 0, node1.ledger.epoch_link (nano::epoch::epoch_1), nano::dev_genesis_key.prv, nano::dev_genesis_key.pub, *pool.generate (key1.pub));
-	auto transaction (node1.store.tx_begin_write ());
-	ASSERT_EQ (nano::process_result::block_position, node1.ledger.process (transaction, epoch_open).code);
+	auto epoch_open = builder.state ()
+	                  .account (key1.pub)
+	                  .previous (0)
+	                  .representative (0)
+	                  .balance (0)
+	                  .link (node1.ledger.epoch_link (nano::epoch::epoch_1))
+	                  .sign (nano::dev_genesis_key.prv, nano::dev_genesis_key.pub)
+	                  .work (*pool.generate (key1.pub))
+	                  .build_shared ();
+	auto process_result (node1.ledger.process (node1.store.tx_begin_write (), *epoch_open));
+	ASSERT_EQ (nano::process_result::gap_epoch_open_pending, process_result.code);
+	ASSERT_EQ (nano::signature_verification::valid_epoch, process_result.verified);
+	node1.block_processor.add (epoch_open);
+	node1.block_processor.flush ();
+	ASSERT_FALSE (node1.ledger.block_exists (epoch_open->hash ()));
+	// Open block should be inserted into unchecked
+	auto blocks (node1.store.unchecked_get (node1.store.tx_begin_read (), nano::hash_or_account (epoch_open->account ()).hash));
+	ASSERT_EQ (blocks.size (), 1);
+	ASSERT_EQ (blocks[0].block->full_hash (), epoch_open->full_hash ());
+	ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid_epoch);
+	// New block to process epoch open
+	auto send1 = builder.state ()
+	             .account (nano::genesis_account)
+	             .previous (nano::genesis_hash)
+	             .representative (nano::genesis_account)
+	             .balance (nano::genesis_amount - 100)
+	             .link (key1.pub)
+	             .sign (nano::dev_genesis_key.prv, nano::dev_genesis_key.pub)
+	             .work (*pool.generate (nano::genesis_hash))
+	             .build_shared ();
+	node1.block_processor.add (send1);
+	node1.block_processor.flush ();
+	ASSERT_TRUE (node1.ledger.block_exists (epoch_open->hash ()));
 }
 
 TEST (ledger, block_hash_account_conflict)

nano/lib/errors.cpp

@@ -248,6 +248,8 @@ std::string nano::error_process_messages::message (int ev) const
 			return "Gap previous block";
 		case nano::error_process::gap_source:
 			return "Gap source block";
+		case nano::error_process::gap_epoch_open_pending:
+			return "Gap pending for open epoch block";
 		case nano::error_process::opened_burn_account:
 			return "Burning account";
 		case nano::error_process::balance_mismatch:

nano/lib/errors.hpp

@@ -137,6 +137,7 @@ enum class error_process
 	unreceivable, // Source block doesn't exist or has already been received
 	gap_previous, // Block marked as previous is unknown
 	gap_source, // Block marked as source is unknown
+	gap_epoch_open_pending, // Block marked as pending blocks required for epoch open block are unknown
 	opened_burn_account, // The impossible happened, someone found the private key associated with the public key '0'.
 	balance_mismatch, // Balance and amount delta don't match
 	block_position, // This block cannot follow the previous block

nano/node/blockprocessor.cpp

@@ -389,6 +389,15 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction
 				events_a.events.emplace_back ([this, hash, block = info_a.block, result, watch_work_a, origin_a](nano::transaction const & post_event_transaction_a) { process_live (post_event_transaction_a, hash, block, result, watch_work_a, origin_a); });
 			}
 			queue_unchecked (transaction_a, hash);
+			/* For send blocks check epoch open unchecked (gap pending).
+			For state blocks check only send subtype and only if block epoch is not last epoch.
+			If epoch is last, then pending entry shouldn't trigger same epoch open block for destination account. */
+			if (block->type () == nano::block_type::send || (block->type () == nano::block_type::state && block->sideband ().details.is_send && std::underlying_type_t<nano::epoch> (block->sideband ().details.epoch) < std::underlying_type_t<nano::epoch> (nano::epoch::max)))
+			{
+				/* block->destination () for legacy send blocks
+				block->link () for state blocks (send subtype) */
+				queue_unchecked (transaction_a, block->destination ().is_zero () ? block->link () : block->destination ());
+			}
 			break;
 		}
 		case nano::process_result::gap_previous:
@@ -431,6 +440,24 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction
 			node.stats.inc (nano::stat::type::ledger, nano::stat::detail::gap_source);
 			break;
 		}
+		case nano::process_result::gap_epoch_open_pending:
+		{
+			if (node.config.logging.ledger_logging ())
+			{
+				node.logger.try_log (boost::str (boost::format ("Gap pending entries for epoch open: %1%") % hash.to_string ()));
+			}
+			info_a.verified = result.verified;
+			if (info_a.modified == 0)
+			{
+				info_a.modified = nano::seconds_since_epoch ();
+			}
+
+			nano::unchecked_key unchecked_key (block->account (), hash); // Specific unchecked key starting with epoch open block account public key
+			node.store.unchecked_put (transaction_a, unchecked_key, info_a);
+
+			node.stats.inc (nano::stat::type::ledger, nano::stat::detail::gap_source);
+			break;
+		}
 		case nano::process_result::old:
 		{
 			if (node.config.logging.ledger_duplicate_logging ())
@@ -537,18 +564,18 @@ void nano::block_processor::process_old (nano::transaction const & transaction_a
 	}
 }
 
-void nano::block_processor::queue_unchecked (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a)
+void nano::block_processor::queue_unchecked (nano::write_transaction const & transaction_a, nano::hash_or_account const & hash_or_account_a)
 {
-	auto unchecked_blocks (node.store.unchecked_get (transaction_a, hash_a));
+	auto unchecked_blocks (node.store.unchecked_get (transaction_a, hash_or_account_a.hash));
 	for (auto & info : unchecked_blocks)
 	{
 		if (!node.flags.disable_block_processor_unchecked_deletion)
 		{
-			node.store.unchecked_del (transaction_a, nano::unchecked_key (hash_a, info.block->hash ()));
+			node.store.unchecked_del (transaction_a, nano::unchecked_key (hash_or_account_a, info.block->hash ()));
 		}
 		add (info, true);
 	}
-	node.gap_cache.erase (hash_a);
+	node.gap_cache.erase (hash_or_account_a.hash);
 }
 
 void nano::block_processor::requeue_invalid (nano::block_hash const & hash_a, nano::unchecked_info const & info_a)

nano/node/blockprocessor.hpp

@@ -70,7 +70,7 @@ public:
 	static std::chrono::milliseconds constexpr confirmation_request_delay{ 1500 };
 
 private:
-	void queue_unchecked (nano::write_transaction const &, nano::block_hash const &);
+	void queue_unchecked (nano::write_transaction const &, nano::hash_or_account const &);
 	void process_batch (nano::unique_lock<nano::mutex> &);
 	void process_live (nano::transaction const &, nano::block_hash const &, std::shared_ptr<nano::block> const &, nano::process_return const &, const bool = false, nano::block_origin const = nano::block_origin::remote);
 	void process_old (nano::transaction const &, std::shared_ptr<nano::block> const &, nano::block_origin const);

nano/node/json_handler.cpp

@@ -3107,6 +3107,11 @@ void nano::json_handler::process ()
 							rpc_l->ec = nano::error_process::block_position;
 							break;
 						}
+						case nano::process_result::gap_epoch_open_pending:
+						{
+							rpc_l->ec = nano::error_process::gap_epoch_open_pending;
+							break;
+						}
 						case nano::process_result::fork:
 						{
 							const bool force = rpc_l->request.get<bool> ("force", false);

nano/secure/blockstore_partial.hpp

@@ -806,10 +806,8 @@ public:
 	{
 		parallel_traversal<nano::uint512_t> (
 		[&action_a, this](nano::uint512_t const & start, nano::uint512_t const & end, bool const is_last) {
-			nano::uint512_union union_start (start);
-			nano::uint512_union union_end (end);
-			nano::unchecked_key key_start (union_start.uint256s[0].number (), union_start.uint256s[1].number ());
-			nano::unchecked_key key_end (union_end.uint256s[0].number (), union_end.uint256s[1].number ());
+			nano::unchecked_key key_start (start);
+			nano::unchecked_key key_end (end);
 			auto transaction (this->tx_begin_read ());
 			action_a (transaction, this->unchecked_begin (transaction, key_start), !is_last ? this->unchecked_begin (transaction, key_end) : this->unchecked_end ());
 		});

nano/secure/common.cpp

@@ -822,12 +822,18 @@ nano::wallet_id nano::random_wallet_id ()
 	return wallet_id;
 }
 
-nano::unchecked_key::unchecked_key (nano::block_hash const & previous_a, nano::block_hash const & hash_a) :
-previous (previous_a),
+nano::unchecked_key::unchecked_key (nano::hash_or_account const & previous_a, nano::block_hash const & hash_a) :
+previous (previous_a.hash),
 hash (hash_a)
 {
 }
 
+nano::unchecked_key::unchecked_key (nano::uint512_union const & union_a) :
+previous (union_a.uint256s[0].number ()),
+hash (union_a.uint256s[1].number ())
+{
+}
+
 bool nano::unchecked_key::deserialize (nano::stream & stream_a)
 {
 	auto error (false);

nano/secure/common.hpp

@@ -171,7 +171,8 @@ class unchecked_key final
 {
 public:
 	unchecked_key () = default;
-	unchecked_key (nano::block_hash const &, nano::block_hash const &);
+	unchecked_key (nano::hash_or_account const &, nano::block_hash const &);
+	unchecked_key (nano::uint512_union const &);
 	bool deserialize (nano::stream &);
 	bool operator== (nano::unchecked_key const &) const;
 	nano::block_hash const & key () const;
@@ -312,6 +313,7 @@ enum class process_result
 	unreceivable, // Source block doesn't exist, has already been received, or requires an account upgrade (epoch blocks)
 	gap_previous, // Block marked as previous is unknown
 	gap_source, // Block marked as source is unknown
+	gap_epoch_open_pending, // Block marked as pending blocks required for epoch open block are unknown
 	opened_burn_account, // The impossible happened, someone found the private key associated with the public key '0'.
 	balance_mismatch, // Balance and amount delta don't match
 	representative_mismatch, // Representative is changed when it is not allowed

nano/secure/ledger.cpp

@@ -431,7 +431,7 @@ void ledger_processor::epoch_block_impl (nano::state_block & block_a)
 					if (result.code == nano::process_result::progress)
 					{
 						bool pending_exists = ledger.store.pending_any (transaction, block_a.hashables.account);
-						result.code = pending_exists ? nano::process_result::progress : nano::process_result::block_position;
+						result.code = pending_exists ? nano::process_result::progress : nano::process_result::gap_epoch_open_pending;
 					}
 				}
 				if (result.code == nano::process_result::progress)