[Pruning] Support pruned blocks in confirmation height processor & active transactions (#2978)

* [Pruning] Confirmation height processor support
Splitting https://github.com/nanocurrency/nano-node/pull/2881
* Support pruned blocks in active transactions
* Remove old commentary
* Use node.ledger.pruning instead of node.flags.enable_pruning
* Guilherme review
active transactions previous block should be confirmed & not pruned

nano/core_test/confirmation_height.cpp

@@ -1416,3 +1416,48 @@ TEST (confirmation_height, unbounded_block_cache_iteration)
 	ASSERT_EQ (2, stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_unbounded, nano::stat::dir::in));
 	ASSERT_EQ (3, ledger.cache.cemented_count);
 }
+
+TEST (confirmation_height, pruned_source)
+{
+	nano::logger_mt logger;
+	nano::logging logging;
+	auto path (nano::unique_path ());
+	auto store = nano::make_store (logger, path);
+	ASSERT_TRUE (!store->init_error ());
+	nano::genesis genesis;
+	nano::stat stats;
+	nano::ledger ledger (*store, stats);
+	ledger.pruning = true;
+	nano::write_database_queue write_database_queue (false);
+	nano::work_pool pool (std::numeric_limits<unsigned>::max ());
+	nano::keypair key1, key2;
+	auto send1 (std::make_shared<nano::state_block> (nano::dev_genesis_key.pub, genesis.hash (), nano::dev_genesis_key.pub, nano::genesis_amount - 100, key1.pub, nano::dev_genesis_key.prv, nano::dev_genesis_key.pub, *pool.generate (nano::genesis_hash)));
+	auto open1 (std::make_shared<nano::state_block> (key1.pub, 0, key1.pub, 100, send1->hash (), key1.prv, key1.pub, *pool.generate (key1.pub)));
+	auto send2 (std::make_shared<nano::state_block> (key1.pub, open1->hash (), key1.pub, 50, key2.pub, key1.prv, key1.pub, *pool.generate (open1->hash ())));
+	auto send3 (std::make_shared<nano::state_block> (key1.pub, send2->hash (), key1.pub, 25, key2.pub, key1.prv, key1.pub, *pool.generate (send2->hash ())));
+	auto open2 (std::make_shared<nano::state_block> (key2.pub, 0, key1.pub, 50, send2->hash (), key2.prv, key2.pub, *pool.generate (key2.pub)));
+	{
+		auto transaction (store->tx_begin_write ());
+		store->initialize (transaction, genesis, ledger.cache);
+		ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *send1).code);
+		ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *open1).code);
+		ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *send2).code);
+		ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *send3).code);
+		ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *open2).code);
+	}
+	auto block_hash_being_processed (open2->hash ());
+	uint64_t batch_write_size = 2;
+	std::atomic<bool> stopped{ false };
+	bool first_time{ true };
+	nano::confirmation_height_bounded bounded_processor (
+	ledger, write_database_queue, 10ms, logging, logger, stopped, block_hash_being_processed, batch_write_size, [&](auto const & cemented_blocks_a) {
+		if (first_time)
+		{
+			// Prune the send
+			auto transaction (store->tx_begin_write ());
+			ASSERT_EQ (2, ledger.pruning_action (transaction, send2->hash (), 2));
+		}
+		first_time = false; },
+	[](auto const &) {}, []() { return 0; });
+	bounded_processor.process ();
+}

nano/node/active_transactions.cpp

@@ -520,15 +520,12 @@ void nano::active_transactions::confirm_expired_frontiers_pessimistically (nano:
 
 bool nano::active_transactions::should_do_frontiers_confirmation () const
 {
-	/*
- 	 * Confirm frontiers when there aren't many confirmations already pending and node finished initial bootstrap
- 	 */
 	auto pending_confirmation_height_size (confirmation_height_processor.awaiting_processing_size ());
 	auto bootstrap_weight_reached (node.ledger.cache.block_count >= node.ledger.bootstrap_weight_max_blocks);
 	auto disabled_confirmation_mode = (node.config.frontiers_confirmation == nano::frontiers_confirmation_mode::disabled);
 	auto conf_height_capacity_reached = pending_confirmation_height_size > confirmed_frontiers_max_pending_size;
 	auto all_cemented = node.ledger.cache.block_count == node.ledger.cache.cemented_count;
-	return (!disabled_confirmation_mode && bootstrap_weight_reached && !conf_height_capacity_reached && !all_cemented);
+	return (!disabled_confirmation_mode && (bootstrap_weight_reached || node.ledger.pruning) && !conf_height_capacity_reached && !all_cemented);
 }
 
 void nano::active_transactions::request_loop ()
@@ -1398,7 +1395,7 @@ nano::inactive_cache_status nano::active_transactions::inactive_votes_bootstrap_
 				insert_impl (block);
 			}
 		}
-		else if (!block && status.bootstrap_started && !previously_a.bootstrap_started)
+		else if (!block && status.bootstrap_started && !previously_a.bootstrap_started && (!node.ledger.pruning || !node.store.pruned_exists (transaction, hash_a)))
 		{
 			node.gap_cache.bootstrap_start (hash_a);
 		}

nano/node/confirmation_height_bounded.cpp

@@ -80,11 +80,22 @@ void nano::confirmation_height_bounded::process ()
 		auto block = ledger.store.block_get (transaction, current);
 		if (!block)
 		{
-			auto error_str = (boost::format ("Ledger mismatch trying to set confirmation height for block %1% (bounded processor)") % current.to_string ()).str ();
-			logger.always_log (error_str);
-			std::cerr << error_str << std::endl;
+			if (ledger.pruning && ledger.store.pruned_exists (transaction, current))
+			{
+				if (!receive_source_pairs.empty ())
+				{
+					receive_source_pairs.pop_back ();
+				}
+				continue;
+			}
+			else
+			{
+				auto error_str = (boost::format ("Ledger mismatch trying to set confirmation height for block %1% (bounded processor)") % current.to_string ()).str ();
+				logger.always_log (error_str);
+				std::cerr << error_str << std::endl;
+				release_assert (block);
+			}
 		}
-		release_assert (block);
 		nano::account account (block->account ());
 		if (account.is_zero ())
 		{

nano/node/confirmation_height_unbounded.cpp

@@ -360,16 +360,25 @@ void nano::confirmation_height_unbounded::cement_blocks (nano::write_guard & sco
 			if (!error && pending.height > confirmation_height)
 			{
 				auto block = ledger.store.block_get (transaction, pending.hash);
-				debug_assert (network_params.network.is_dev_network () || block != nullptr);
-				debug_assert (network_params.network.is_dev_network () || block->sideband ().height == pending.height);
+				debug_assert (network_params.network.is_dev_network () || ledger.pruning || block != nullptr);
+				debug_assert (network_params.network.is_dev_network () || ledger.pruning || block->sideband ().height == pending.height);
 
 				if (!block)
 				{
-					auto error_str = (boost::format ("Failed to write confirmation height for block %1% (unbounded processor)") % pending.hash.to_string ()).str ();
-					logger.always_log (error_str);
-					std::cerr << error_str << std::endl;
-					error = true;
-					break;
+					if (ledger.pruning && ledger.store.pruned_exists (transaction, pending.hash))
+					{
+						pending_writes.erase (pending_writes.begin ());
+						--pending_writes_size;
+						continue;
+					}
+					else
+					{
+						auto error_str = (boost::format ("Failed to write confirmation height for block %1% (unbounded processor)") % pending.hash.to_string ()).str ();
+						logger.always_log (error_str);
+						std::cerr << error_str << std::endl;
+						error = true;
+						break;
+					}
 				}
 				ledger.stats.add (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in, pending.height - confirmation_height);
 				ledger.stats.add (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_unbounded, nano::stat::dir::in, pending.height - confirmation_height);