Local block broadcaster (#4454)

* Add rolled back event

* Local block broadcaster

* Rate limit block broadcasting

* Rename to `local_block_broadcaster`

* Node initialization order

* Fix tests

* Bump local block queue size to 8k elements

nano/lib/stats_enums.hpp

@@ -49,6 +49,7 @@ enum class type : uint8_t
 	election_scheduler,
 	optimistic_scheduler,
 	handshake,
+	local_block_broadcaster,
 
 	bootstrap_ascending,
 	bootstrap_ascending_accounts,
@@ -102,6 +103,7 @@ enum class detail : uint8_t
 	old,
 	gap_previous,
 	gap_source,
+	rollback,
 	rollback_failed,
 	progress,
 	bad_signature,
@@ -328,6 +330,12 @@ enum class detail : uint8_t
 	deprioritize,
 	deprioritize_failed,
 
+	// block broadcaster
+	broadcast_normal,
+	broadcast_aggressive,
+	erase_old,
+	erase_confirmed,
+
 	_last // Must be the last enum
 };
 

nano/lib/thread_roles.cpp

@@ -100,6 +100,9 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
 		case nano::thread_role::name::scheduler_priority:
 			thread_role_name_string = "Sched Priority";
 			break;
+		case nano::thread_role::name::local_block_broadcasting:
+			thread_role_name_string = "Local broadcast";
+			break;
 		default:
 			debug_assert (false && "nano::thread_role::get_string unhandled thread role");
 	}

nano/lib/thread_roles.hpp

@@ -42,6 +42,7 @@ enum class name
 	scheduler_manual,
 	scheduler_optimistic,
 	scheduler_priority,
+	local_block_broadcasting,
 };
 
 /*

nano/node/CMakeLists.txt

@@ -20,8 +20,6 @@ add_library(
   backlog_population.cpp
   bandwidth_limiter.hpp
   bandwidth_limiter.cpp
-  block_broadcast.cpp
-  block_broadcast.hpp
   blockprocessor.hpp
   blockprocessor.cpp
   bootstrap/block_deserializer.hpp
@@ -100,6 +98,8 @@ add_library(
   ipc/ipc_server.cpp
   json_handler.hpp
   json_handler.cpp
+  local_block_broadcaster.cpp
+  local_block_broadcaster.hpp
   make_store.hpp
   make_store.cpp
   network.hpp

nano/node/block_broadcast.cpp

@@ -1,51 +0,0 @@
-#include <nano/node/block_broadcast.hpp>
-#include <nano/node/blockprocessor.hpp>
-#include <nano/node/network.hpp>
-
-nano::block_broadcast::block_broadcast (nano::network & network, bool enabled) :
-	network{ network },
-	enabled{ enabled }
-{
-}
-
-void nano::block_broadcast::connect (nano::block_processor & block_processor)
-{
-	if (!enabled)
-	{
-		return;
-	}
-	block_processor.block_processed.add ([this] (auto const & result, auto const & context) {
-		switch (result)
-		{
-			case nano::block_status::progress:
-				observe (context);
-				break;
-			default:
-				break;
-		}
-	});
-}
-
-void nano::block_broadcast::observe (nano::block_processor::context const & context)
-{
-	auto const & block = context.block;
-	if (context.source == nano::block_source::local)
-	{
-		// Block created on this node
-		// Perform more agressive initial flooding
-		network.flood_block_initial (block);
-	}
-	else
-	{
-		if (context.source != nano::block_source::bootstrap && context.source != nano::block_source::bootstrap_legacy)
-		{
-			// Block arrived from realtime traffic, do normal gossip.
-			network.flood_block (block, nano::transport::buffer_drop_policy::limiter);
-		}
-		else
-		{
-			// Block arrived from bootstrap
-			// Don't broadcast blocks we're bootstrapping
-		}
-	}
-}

nano/node/block_broadcast.hpp

@@ -1,28 +0,0 @@
-#pragma once
-
-#include <nano/lib/blocks.hpp>
-#include <nano/node/blockprocessor.hpp>
-
-#include <memory>
-#include <unordered_set>
-
-namespace nano
-{
-class network;
-
-// This class tracks blocks that originated from this node.
-class block_broadcast
-{
-public:
-	block_broadcast (nano::network & network, bool enabled = false);
-	// Add batch_processed observer to block_processor if enabled
-	void connect (nano::block_processor & block_processor);
-
-private:
-	// Block_processor observer
-	void observe (nano::block_processor::context const &);
-
-	nano::network & network;
-	bool enabled;
-};
-}

nano/node/blockprocessor.cpp

@@ -152,11 +152,15 @@ void nano::block_processor::rollback_competitor (store::write_transaction const
 		}
 		else
 		{
+			node.stats.inc (nano::stat::type::ledger, nano::stat::detail::rollback);
 			node.logger.debug (nano::log::type::blockprocessor, "Blocks rolled back: {}", rollback_list.size ());
 		}
+
 		// Deleting from votes cache, stop active transaction
 		for (auto & i : rollback_list)
 		{
+			rolled_back.notify (i);
+
 			node.history.erase (i->root ());
 			// Stop all rolled back active transactions except initial
 			if (i->hash () != successor->hash ())

nano/node/blockprocessor.hpp

@@ -86,6 +86,7 @@ public: // Events
 	// The batch observer feeds the processed observer
 	nano::observer_set<nano::block_status const &, context const &> block_processed;
 	nano::observer_set<processed_batch_t const &> batch_processed;
+	nano::observer_set<std::shared_ptr<nano::block> const &> rolled_back;
 
 private:
 	// Roll back block in the ledger that conflicts with 'block'

nano/node/local_block_broadcaster.cpp

@@ -0,0 +1,170 @@
+#include <nano/lib/threading.hpp>
+#include <nano/lib/utility.hpp>
+#include <nano/node/blockprocessor.hpp>
+#include <nano/node/local_block_broadcaster.hpp>
+#include <nano/node/network.hpp>
+#include <nano/node/node.hpp>
+
+nano::local_block_broadcaster::local_block_broadcaster (nano::node & node_a, nano::block_processor & block_processor_a, nano::network & network_a, nano::stats & stats_a, bool enabled_a) :
+	node{ node_a },
+	block_processor{ block_processor_a },
+	network{ network_a },
+	stats{ stats_a },
+	enabled{ enabled_a }
+{
+	if (!enabled)
+	{
+		return;
+	}
+
+	block_processor.batch_processed.add ([this] (auto const & batch) {
+		bool should_notify = false;
+		for (auto const & [result, context] : batch)
+		{
+			// Only rebroadcast local blocks that were successfully processed (no forks or gaps)
+			if (result == nano::block_status::progress && context.source == nano::block_source::local)
+			{
+				nano::lock_guard<nano::mutex> guard{ mutex };
+				local_blocks.emplace_back (local_entry{ context.block, std::chrono::steady_clock::now () });
+				stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::insert);
+				should_notify = true;
+			}
+		}
+		if (should_notify)
+		{
+			condition.notify_all ();
+		}
+	});
+
+	block_processor.rolled_back.add ([this] (auto const & block) {
+		nano::lock_guard<nano::mutex> guard{ mutex };
+		auto erased = local_blocks.get<tag_hash> ().erase (block->hash ());
+		stats.add (nano::stat::type::local_block_broadcaster, nano::stat::detail::rollback, nano::stat::dir::in, erased);
+	});
+}
+
+nano::local_block_broadcaster::~local_block_broadcaster ()
+{
+	// Thread must be stopped before destruction
+	debug_assert (!thread.joinable ());
+}
+
+void nano::local_block_broadcaster::start ()
+{
+	if (!enabled)
+	{
+		return;
+	}
+
+	debug_assert (!thread.joinable ());
+
+	thread = std::thread{ [this] () {
+		nano::thread_role::set (nano::thread_role::name::local_block_broadcasting);
+		run ();
+	} };
+}
+
+void nano::local_block_broadcaster::stop ()
+{
+	{
+		nano::lock_guard<nano::mutex> lock{ mutex };
+		stopped = true;
+	}
+	condition.notify_all ();
+	nano::join_or_pass (thread);
+}
+
+void nano::local_block_broadcaster::run ()
+{
+	nano::unique_lock<nano::mutex> lock{ mutex };
+	while (!stopped)
+	{
+		stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::loop);
+
+		condition.wait_for (lock, check_interval);
+		debug_assert ((std::this_thread::yield (), true)); // Introduce some random delay in debug builds
+
+		if (!stopped)
+		{
+			cleanup ();
+			run_broadcasts (lock);
+			debug_assert (lock.owns_lock ());
+		}
+	}
+}
+
+void nano::local_block_broadcaster::run_broadcasts (nano::unique_lock<nano::mutex> & lock)
+{
+	debug_assert (lock.owns_lock ());
+
+	std::vector<std::shared_ptr<nano::block>> to_broadcast;
+
+	auto const now = std::chrono::steady_clock::now ();
+	for (auto & entry : local_blocks)
+	{
+		if (elapsed (entry.last_broadcast, broadcast_interval, now))
+		{
+			entry.last_broadcast = now;
+			to_broadcast.push_back (entry.block);
+		}
+	}
+
+	lock.unlock ();
+
+	for (auto const & block : to_broadcast)
+	{
+		while (!limiter.should_pass (1))
+		{
+			std::this_thread::sleep_for (std::chrono::milliseconds{ 100 });
+			if (stopped)
+			{
+				return;
+			}
+		}
+
+		stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::broadcast, nano::stat::dir::out);
+
+		network.flood_block_initial (block);
+	}
+
+	lock.lock ();
+}
+
+void nano::local_block_broadcaster::cleanup ()
+{
+	debug_assert (!mutex.try_lock ());
+
+	// Erase oldest blocks if the queue gets too big
+	while (local_blocks.size () > max_size)
+	{
+		stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::erase_oldest);
+		local_blocks.pop_front ();
+	}
+
+	// TODO: Mutex is held during IO, but it should be fine since it's not performance critical
+	auto transaction = node.store.tx_begin_read ();
+	erase_if (local_blocks, [this, &transaction] (auto const & entry) {
+		transaction.refresh_if_needed ();
+
+		if (entry.last_broadcast == std::chrono::steady_clock::time_point{})
+		{
+			// This block has never been broadcasted, keep it so it's broadcasted at least once
+			return false;
+		}
+		if (node.block_confirmed_or_being_confirmed (transaction, entry.block->hash ()))
+		{
+			stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::erase_confirmed);
+			return true;
+		}
+		return false;
+	});
+}
+
+std::unique_ptr<nano::container_info_component> nano::local_block_broadcaster::collect_container_info (const std::string & name) const
+{
+	nano::lock_guard<nano::mutex> guard{ mutex };
+
+	auto composite = std::make_unique<container_info_composite> (name);
+	composite->add_component (std::make_unique<container_info_leaf> (container_info{ "local", local_blocks.size (), sizeof (decltype (local_blocks)::value_type) }));
+	return composite;
+}

nano/node/local_block_broadcaster.hpp

@@ -0,0 +1,106 @@
+#pragma once
+
+#include <nano/lib/blocks.hpp>
+#include <nano/lib/locks.hpp>
+#include <nano/lib/processing_queue.hpp>
+#include <nano/node/bandwidth_limiter.hpp>
+#include <nano/node/blockprocessor.hpp>
+#include <nano/secure/common.hpp>
+
+#include <boost/multi_index/hashed_index.hpp>
+#include <boost/multi_index/mem_fun.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/sequenced_index.hpp>
+#include <boost/multi_index_container.hpp>
+
+#include <memory>
+#include <thread>
+#include <unordered_set>
+
+namespace mi = boost::multi_index;
+
+namespace nano
+{
+class node;
+class network;
+}
+
+namespace nano
+{
+/**
+ * Broadcasts blocks to the network
+ * Tracks local blocks for more aggressive propagation
+ */
+class local_block_broadcaster
+{
+	enum class broadcast_strategy
+	{
+		normal,
+		aggressive,
+	};
+
+public:
+	local_block_broadcaster (nano::node &, nano::block_processor &, nano::network &, nano::stats &, bool enabled = false);
+	~local_block_broadcaster ();
+
+	void start ();
+	void stop ();
+
+	std::unique_ptr<container_info_component> collect_container_info (std::string const & name) const;
+
+private:
+	void run ();
+	void run_broadcasts (nano::unique_lock<nano::mutex> &);
+	void cleanup ();
+
+private: // Dependencies
+	nano::node & node;
+	nano::block_processor & block_processor;
+	nano::network & network;
+	nano::stats & stats;
+
+private:
+	struct local_entry
+	{
+		std::shared_ptr<nano::block> const block;
+		std::chrono::steady_clock::time_point const arrival;
+		mutable std::chrono::steady_clock::time_point last_broadcast{}; // Not part of any index
+
+		nano::block_hash hash () const
+		{
+			return block->hash ();
+		}
+	};
+
+	// clang-format off
+	class tag_sequenced	{};
+	class tag_hash {};
+
+	using ordered_locals = boost::multi_index_container<local_entry,
+	mi::indexed_by<
+		mi::sequenced<mi::tag<tag_sequenced>>,
+		mi::hashed_unique<mi::tag<tag_hash>,
+			mi::const_mem_fun<local_entry, nano::block_hash, &local_entry::hash>>
+	>>;
+	// clang-format on
+
+	ordered_locals local_blocks;
+
+private:
+	bool enabled{ false };
+
+	nano::bandwidth_limiter limiter{ broadcast_rate_limit, broadcast_rate_burst_ratio };
+
+	std::atomic<bool> stopped{ false };
+	nano::condition_variable condition;
+	mutable nano::mutex mutex;
+	std::thread thread;
+
+	// TODO: Make these configurable
+	static std::size_t constexpr max_size{ 1024 * 8 };
+	static std::chrono::seconds constexpr check_interval{ 30 };
+	static std::chrono::seconds constexpr broadcast_interval{ 60 };
+	static std::size_t constexpr broadcast_rate_limit{ 32 };
+	static double constexpr broadcast_rate_burst_ratio{ 3 };
+};
+}

nano/node/node.cpp

@@ -195,14 +195,13 @@ nano::node::node (boost::asio::io_context & io_ctx_a, std::filesystem::path cons
 	ascendboot{ config, block_processor, ledger, network, stats },
 	websocket{ config.websocket_config, observers, wallets, ledger, io_ctx, logger },
 	epoch_upgrader{ *this, ledger, store, network_params, logger },
+	local_block_broadcaster{ *this, block_processor, network, stats, !flags.disable_block_processor_republishing },
+	process_live_dispatcher{ ledger, scheduler.priority, vote_cache, websocket },
 	startup_time (std::chrono::steady_clock::now ()),
-	node_seq (seq),
+	node_seq (seq)
-	block_broadcast{ network, !flags.disable_block_processor_republishing },
-	process_live_dispatcher{ ledger, scheduler.priority, vote_cache, websocket }
 {
 	logger.debug (nano::log::type::node, "Constructing node...");
 
-	block_broadcast.connect (block_processor);
 	process_live_dispatcher.connect (block_processor);
 
 	unchecked.satisfied.add ([this] (nano::unchecked_info const & info) {
@@ -551,6 +550,7 @@ std::unique_ptr<nano::container_info_component> nano::collect_container_info (no
 	composite->add_component (collect_container_info (node.final_generator, "vote_generator_final"));
 	composite->add_component (node.ascendboot.collect_container_info ("bootstrap_ascending"));
 	composite->add_component (node.unchecked.collect_container_info ("unchecked"));
+	composite->add_component (node.local_block_broadcaster.collect_container_info ("local_block_broadcaster"));
 	return composite;
 }
 
@@ -659,6 +659,7 @@ void nano::node::start ()
 	}
 	websocket.start ();
 	telemetry.start ();
+	local_block_broadcaster.start ();
 }
 
 void nano::node::stop ()
@@ -699,6 +700,7 @@ void nano::node::stop ()
 	stats.stop ();
 	epoch_upgrader.stop ();
 	workers.stop ();
+	local_block_broadcaster.stop ();
 	// work pool is not stopped on purpose due to testing setup
 }
 

nano/node/node.hpp

@@ -8,7 +8,6 @@
 #include <nano/node/active_transactions.hpp>
 #include <nano/node/backlog_population.hpp>
 #include <nano/node/bandwidth_limiter.hpp>
-#include <nano/node/block_broadcast.hpp>
 #include <nano/node/blockprocessor.hpp>
 #include <nano/node/bootstrap/bootstrap.hpp>
 #include <nano/node/bootstrap/bootstrap_attempt.hpp>
@@ -18,6 +17,7 @@
 #include <nano/node/distributed_work_factory.hpp>
 #include <nano/node/election.hpp>
 #include <nano/node/epoch_upgrader.hpp>
+#include <nano/node/local_block_broadcaster.hpp>
 #include <nano/node/network.hpp>
 #include <nano/node/node_observers.hpp>
 #include <nano/node/nodeconfig.hpp>
@@ -186,7 +186,7 @@ public:
 	nano::bootstrap_ascending::service ascendboot;
 	nano::websocket_server websocket;
 	nano::epoch_upgrader epoch_upgrader;
-	nano::block_broadcast block_broadcast;
+	nano::local_block_broadcaster local_block_broadcaster;
 	nano::process_live_dispatcher process_live_dispatcher;
 
 	std::chrono::steady_clock::time_point const startup_time;

nano/rpc_test/rpc.cpp

@@ -2228,7 +2228,7 @@ TEST (rpc, block_count_pruning)
 				 .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
 				 .work (*node1->work_generate_blocking (latest))
 				 .build ();
-	node1->process_active (send1);
+	node1->process_local (send1);
 	auto receive1 = builder
 					.receive ()
 					.previous (send1->hash ())
@@ -2236,7 +2236,7 @@ TEST (rpc, block_count_pruning)
 					.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
 					.work (*node1->work_generate_blocking (send1->hash ()))
 					.build ();
-	node1->process_active (receive1);
+	node1->process_local (receive1);
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 	ASSERT_TIMELY (5s, node1->block_confirmed (receive1->hash ()));
 	// Pruning action