dn724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Local block broadcaster (#4454)

Browse source 
* Add rolled back event

* Local block broadcaster

* Rate limit block broadcasting

* Rename to `local_block_broadcaster`

* Node initialization order

* Fix tests

* Bump local block queue size to 8k elements
This commit is contained in:
Piotr Wójcik 2024-03-05 18:42:20 +01:00 • committed by GitHub
commit 5f28f1a8b8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 305 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

8
nano/lib/stats_enums.hpp
View file

@ -49,6 +49,7 @@ enum class type : uint8_t
election_scheduler,
optimistic_scheduler,
handshake,
local_block_broadcaster,
bootstrap_ascending,
bootstrap_ascending_accounts,
@ -102,6 +103,7 @@ enum class detail : uint8_t
old,
gap_previous,
gap_source,
rollback,
rollback_failed,
progress,
bad_signature,
@ -328,6 +330,12 @@ enum class detail : uint8_t
deprioritize,
deprioritize_failed,
// block broadcaster
broadcast_normal,
broadcast_aggressive,
erase_old,
erase_confirmed,
_last // Must be the last enum
};

3
nano/lib/thread_roles.cpp
View file

@ -100,6 +100,9 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
case nano::thread_role::name::scheduler_priority:
thread_role_name_string = "Sched Priority";
break;
case nano::thread_role::name::local_block_broadcasting:
thread_role_name_string = "Local broadcast";
break;
default:
debug_assert (false && "nano::thread_role::get_string unhandled thread role");
}

1
nano/lib/thread_roles.hpp
View file

@ -42,6 +42,7 @@ enum class name
scheduler_manual,
scheduler_optimistic,
scheduler_priority,
local_block_broadcasting,
};
/*

4
nano/node/CMakeLists.txt
View file

@ -20,8 +20,6 @@ add_library(
backlog_population.cpp
bandwidth_limiter.hpp
bandwidth_limiter.cpp
block_broadcast.cpp
block_broadcast.hpp
blockprocessor.hpp
blockprocessor.cpp
bootstrap/block_deserializer.hpp
@ -100,6 +98,8 @@ add_library(
ipc/ipc_server.cpp
json_handler.hpp
json_handler.cpp
local_block_broadcaster.cpp
local_block_broadcaster.hpp
make_store.hpp
make_store.cpp
network.hpp

51
nano/node/block_broadcast.cpp
View file

@ -1,51 +0,0 @@
#include <nano/node/block_broadcast.hpp>
#include <nano/node/blockprocessor.hpp>
#include <nano/node/network.hpp>
nano::block_broadcast::block_broadcast (nano::network & network, bool enabled) :
network{ network },
enabled{ enabled }
{
}
void nano::block_broadcast::connect (nano::block_processor & block_processor)
{
if (!enabled)
{
return;
}
block_processor.block_processed.add ([this] (auto const & result, auto const & context) {
switch (result)
{
case nano::block_status::progress:
observe (context);
break;
default:
break;
}
});
}
void nano::block_broadcast::observe (nano::block_processor::context const & context)
{
auto const & block = context.block;
if (context.source == nano::block_source::local)
{
// Block created on this node
// Perform more agressive initial flooding
network.flood_block_initial (block);
}
else
{
if (context.source != nano::block_source::bootstrap && context.source != nano::block_source::bootstrap_legacy)
{
// Block arrived from realtime traffic, do normal gossip.
network.flood_block (block, nano::transport::buffer_drop_policy::limiter);
}
else
{
// Block arrived from bootstrap
// Don't broadcast blocks we're bootstrapping
}
}
}

28
nano/node/block_broadcast.hpp
View file

@ -1,28 +0,0 @@
#pragma once
#include <nano/lib/blocks.hpp>
#include <nano/node/blockprocessor.hpp>
#include <memory>
#include <unordered_set>
namespace nano
{
class network;
// This class tracks blocks that originated from this node.
class block_broadcast
{
public:
block_broadcast (nano::network & network, bool enabled = false);
// Add batch_processed observer to block_processor if enabled
void connect (nano::block_processor & block_processor);
private:
// Block_processor observer
void observe (nano::block_processor::context const &);
nano::network & network;
bool enabled;
};
}

4
nano/node/blockprocessor.cpp
View file

@ -152,11 +152,15 @@ void nano::block_processor::rollback_competitor (store::write_transaction const
}
else
{
node.stats.inc (nano::stat::type::ledger, nano::stat::detail::rollback);
node.logger.debug (nano::log::type::blockprocessor, "Blocks rolled back: {}", rollback_list.size ());
}
// Deleting from votes cache, stop active transaction
for (auto & i : rollback_list)
{
rolled_back.notify (i);
node.history.erase (i->root ());
// Stop all rolled back active transactions except initial
if (i->hash () != successor->hash ())

1
nano/node/blockprocessor.hpp
View file

@ -86,6 +86,7 @@ public: // Events
// The batch observer feeds the processed observer
nano::observer_set<nano::block_status const &, context const &> block_processed;
nano::observer_set<processed_batch_t const &> batch_processed;
nano::observer_set<std::shared_ptr<nano::block> const &> rolled_back;
private:
// Roll back block in the ledger that conflicts with 'block'

170
nano/node/local_block_broadcaster.cpp Normal file
View file

@ -0,0 +1,170 @@
#include <nano/lib/threading.hpp>
#include <nano/lib/utility.hpp>
#include <nano/node/blockprocessor.hpp>
#include <nano/node/local_block_broadcaster.hpp>
#include <nano/node/network.hpp>
#include <nano/node/node.hpp>
nano::local_block_broadcaster::local_block_broadcaster (nano::node & node_a, nano::block_processor & block_processor_a, nano::network & network_a, nano::stats & stats_a, bool enabled_a) :
node{ node_a },
block_processor{ block_processor_a },
network{ network_a },
stats{ stats_a },
enabled{ enabled_a }
{
if (!enabled)
{
return;
}
block_processor.batch_processed.add ([this] (auto const & batch) {
bool should_notify = false;
for (auto const & [result, context] : batch)
{
// Only rebroadcast local blocks that were successfully processed (no forks or gaps)
if (result == nano::block_status::progress && context.source == nano::block_source::local)
{
nano::lock_guard<nano::mutex> guard{ mutex };
local_blocks.emplace_back (local_entry{ context.block, std::chrono::steady_clock::now () });
stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::insert);
should_notify = true;
}
}
if (should_notify)
{
condition.notify_all ();
}
});
block_processor.rolled_back.add ([this] (auto const & block) {
nano::lock_guard<nano::mutex> guard{ mutex };
auto erased = local_blocks.get<tag_hash> ().erase (block->hash ());
stats.add (nano::stat::type::local_block_broadcaster, nano::stat::detail::rollback, nano::stat::dir::in, erased);
});
}
nano::local_block_broadcaster::~local_block_broadcaster ()
{
// Thread must be stopped before destruction
debug_assert (!thread.joinable ());
}
void nano::local_block_broadcaster::start ()
{
if (!enabled)
{
return;
}
debug_assert (!thread.joinable ());
thread = std::thread{ [this] () {
nano::thread_role::set (nano::thread_role::name::local_block_broadcasting);
run ();
} };
}
void nano::local_block_broadcaster::stop ()
{
{
nano::lock_guard<nano::mutex> lock{ mutex };
stopped = true;
}
condition.notify_all ();
nano::join_or_pass (thread);
}
void nano::local_block_broadcaster::run ()
{
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::loop);
condition.wait_for (lock, check_interval);
debug_assert ((std::this_thread::yield (), true)); // Introduce some random delay in debug builds
if (!stopped)
{
cleanup ();
run_broadcasts (lock);
debug_assert (lock.owns_lock ());
}
}
}
void nano::local_block_broadcaster::run_broadcasts (nano::unique_lock<nano::mutex> & lock)
{
debug_assert (lock.owns_lock ());
std::vector<std::shared_ptr<nano::block>> to_broadcast;
auto const now = std::chrono::steady_clock::now ();
for (auto & entry : local_blocks)
{
if (elapsed (entry.last_broadcast, broadcast_interval, now))
{
entry.last_broadcast = now;
to_broadcast.push_back (entry.block);
}
}
lock.unlock ();
for (auto const & block : to_broadcast)
{
while (!limiter.should_pass (1))
{
std::this_thread::sleep_for (std::chrono::milliseconds{ 100 });
if (stopped)
{
return;
}
}
stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::broadcast, nano::stat::dir::out);
network.flood_block_initial (block);
}
lock.lock ();
}
void nano::local_block_broadcaster::cleanup ()
{
debug_assert (!mutex.try_lock ());
// Erase oldest blocks if the queue gets too big
while (local_blocks.size () > max_size)
{
stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::erase_oldest);
local_blocks.pop_front ();
}
// TODO: Mutex is held during IO, but it should be fine since it's not performance critical
auto transaction = node.store.tx_begin_read ();
erase_if (local_blocks, [this, &transaction] (auto const & entry) {
transaction.refresh_if_needed ();
if (entry.last_broadcast == std::chrono::steady_clock::time_point{})
{
// This block has never been broadcasted, keep it so it's broadcasted at least once
return false;
}
if (node.block_confirmed_or_being_confirmed (transaction, entry.block->hash ()))
{
stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::erase_confirmed);
return true;
}
return false;
});
}
std::unique_ptr<nano::container_info_component> nano::local_block_broadcaster::collect_container_info (const std::string & name) const
{
nano::lock_guard<nano::mutex> guard{ mutex };
auto composite = std::make_unique<container_info_composite> (name);
composite->add_component (std::make_unique<container_info_leaf> (container_info{ "local", local_blocks.size (), sizeof (decltype (local_blocks)::value_type) }));
return composite;
}

106
nano/node/local_block_broadcaster.hpp Normal file
View file

@ -0,0 +1,106 @@
#pragma once
#include <nano/lib/blocks.hpp>
#include <nano/lib/locks.hpp>
#include <nano/lib/processing_queue.hpp>
#include <nano/node/bandwidth_limiter.hpp>
#include <nano/node/blockprocessor.hpp>
#include <nano/secure/common.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/mem_fun.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/sequenced_index.hpp>
#include <boost/multi_index_container.hpp>
#include <memory>
#include <thread>
#include <unordered_set>
namespace mi = boost::multi_index;
namespace nano
{
class node;
class network;
}
namespace nano
{
/**
* Broadcasts blocks to the network
* Tracks local blocks for more aggressive propagation
*/
class local_block_broadcaster
{
enum class broadcast_strategy
{
normal,
aggressive,
};
public:
local_block_broadcaster (nano::node &, nano::block_processor &, nano::network &, nano::stats &, bool enabled = false);
~local_block_broadcaster ();
void start ();
void stop ();
std::unique_ptr<container_info_component> collect_container_info (std::string const & name) const;
private:
void run ();
void run_broadcasts (nano::unique_lock<nano::mutex> &);
void cleanup ();
private: // Dependencies
nano::node & node;
nano::block_processor & block_processor;
nano::network & network;
nano::stats & stats;
private:
struct local_entry
{
std::shared_ptr<nano::block> const block;
std::chrono::steady_clock::time_point const arrival;
mutable std::chrono::steady_clock::time_point last_broadcast{}; // Not part of any index
nano::block_hash hash () const
{
return block->hash ();
}
};
// clang-format off
class tag_sequenced {};
class tag_hash {};
using ordered_locals = boost::multi_index_container<local_entry,
mi::indexed_by<
mi::sequenced<mi::tag<tag_sequenced>>,
mi::hashed_unique<mi::tag<tag_hash>,
mi::const_mem_fun<local_entry, nano::block_hash, &local_entry::hash>>
>>;
// clang-format on
ordered_locals local_blocks;
private:
bool enabled{ false };
nano::bandwidth_limiter limiter{ broadcast_rate_limit, broadcast_rate_burst_ratio };
std::atomic<bool> stopped{ false };
nano::condition_variable condition;
mutable nano::mutex mutex;
std::thread thread;
// TODO: Make these configurable
static std::size_t constexpr max_size{ 1024 * 8 };
static std::chrono::seconds constexpr check_interval{ 30 };
static std::chrono::seconds constexpr broadcast_interval{ 60 };
static std::size_t constexpr broadcast_rate_limit{ 32 };
static double constexpr broadcast_rate_burst_ratio{ 3 };
};
}

10
nano/node/node.cpp
View file

@ -195,14 +195,13 @@ nano::node::node (boost::asio::io_context & io_ctx_a, std::filesystem::path cons
ascendboot{ config, block_processor, ledger, network, stats },
websocket{ config.websocket_config, observers, wallets, ledger, io_ctx, logger },
epoch_upgrader{ *this, ledger, store, network_params, logger },
local_block_broadcaster{ *this, block_processor, network, stats, !flags.disable_block_processor_republishing },
process_live_dispatcher{ ledger, scheduler.priority, vote_cache, websocket },
startup_time (std::chrono::steady_clock::now ()),
node_seq (seq),
block_broadcast{ network, !flags.disable_block_processor_republishing },
process_live_dispatcher{ ledger, scheduler.priority, vote_cache, websocket }
node_seq (seq)
{
logger.debug (nano::log::type::node, "Constructing node...");
block_broadcast.connect (block_processor);
process_live_dispatcher.connect (block_processor);
unchecked.satisfied.add ([this] (nano::unchecked_info const & info) {
@ -551,6 +550,7 @@ std::unique_ptr<nano::container_info_component> nano::collect_container_info (no
composite->add_component (collect_container_info (node.final_generator, "vote_generator_final"));
composite->add_component (node.ascendboot.collect_container_info ("bootstrap_ascending"));
composite->add_component (node.unchecked.collect_container_info ("unchecked"));
composite->add_component (node.local_block_broadcaster.collect_container_info ("local_block_broadcaster"));
return composite;
}
@ -659,6 +659,7 @@ void nano::node::start ()
}
websocket.start ();
telemetry.start ();
local_block_broadcaster.start ();
}
void nano::node::stop ()
@ -699,6 +700,7 @@ void nano::node::stop ()
stats.stop ();
epoch_upgrader.stop ();
workers.stop ();
local_block_broadcaster.stop ();
// work pool is not stopped on purpose due to testing setup
}

4
nano/node/node.hpp
View file

@ -8,7 +8,6 @@
#include <nano/node/active_transactions.hpp>
#include <nano/node/backlog_population.hpp>
#include <nano/node/bandwidth_limiter.hpp>
#include <nano/node/block_broadcast.hpp>
#include <nano/node/blockprocessor.hpp>
#include <nano/node/bootstrap/bootstrap.hpp>
#include <nano/node/bootstrap/bootstrap_attempt.hpp>
@ -18,6 +17,7 @@
#include <nano/node/distributed_work_factory.hpp>
#include <nano/node/election.hpp>
#include <nano/node/epoch_upgrader.hpp>
#include <nano/node/local_block_broadcaster.hpp>
#include <nano/node/network.hpp>
#include <nano/node/node_observers.hpp>
#include <nano/node/nodeconfig.hpp>
@ -186,7 +186,7 @@ public:
nano::bootstrap_ascending::service ascendboot;
nano::websocket_server websocket;
nano::epoch_upgrader epoch_upgrader;
nano::block_broadcast block_broadcast;
nano::local_block_broadcaster local_block_broadcaster;
nano::process_live_dispatcher process_live_dispatcher;
std::chrono::steady_clock::time_point const startup_time;

4
nano/rpc_test/rpc.cpp
View file

@ -2228,7 +2228,7 @@ TEST (rpc, block_count_pruning)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*node1->work_generate_blocking (latest))
.build ();
node1->process_active (send1);
node1->process_local (send1);
auto receive1 = builder
.receive ()
.previous (send1->hash ())
@ -2236,7 +2236,7 @@ TEST (rpc, block_count_pruning)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*node1->work_generate_blocking (send1->hash ()))
.build ();
node1->process_active (receive1);
node1->process_local (receive1);
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
ASSERT_TIMELY (5s, node1->block_confirmed (receive1->hash ()));
// Pruning action