diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index 2a1216f59..bef1a262e 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -156,6 +156,7 @@ TEST (toml, daemon_config_deserialize_defaults) ASSERT_EQ (conf.node.bootstrap_connections, defaults.node.bootstrap_connections); ASSERT_EQ (conf.node.bootstrap_connections_max, defaults.node.bootstrap_connections_max); ASSERT_EQ (conf.node.bootstrap_initiator_threads, defaults.node.bootstrap_initiator_threads); + ASSERT_EQ (conf.node.bootstrap_serving_threads, defaults.node.bootstrap_serving_threads); ASSERT_EQ (conf.node.bootstrap_frontier_request_count, defaults.node.bootstrap_frontier_request_count); ASSERT_EQ (conf.node.bootstrap_fraction_numerator, defaults.node.bootstrap_fraction_numerator); ASSERT_EQ (conf.node.conf_height_processor_batch_min_time, defaults.node.conf_height_processor_batch_min_time); @@ -395,6 +396,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) bootstrap_connections = 999 bootstrap_connections_max = 999 bootstrap_initiator_threads = 999 + bootstrap_serving_threads = 999 bootstrap_frontier_request_count = 9999 bootstrap_fraction_numerator = 999 conf_height_processor_batch_min_time = 999 @@ -559,6 +561,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) ASSERT_NE (conf.node.bootstrap_connections, defaults.node.bootstrap_connections); ASSERT_NE (conf.node.bootstrap_connections_max, defaults.node.bootstrap_connections_max); ASSERT_NE (conf.node.bootstrap_initiator_threads, defaults.node.bootstrap_initiator_threads); + ASSERT_NE (conf.node.bootstrap_serving_threads, defaults.node.bootstrap_serving_threads); ASSERT_NE (conf.node.bootstrap_frontier_request_count, defaults.node.bootstrap_frontier_request_count); ASSERT_NE (conf.node.bootstrap_fraction_numerator, defaults.node.bootstrap_fraction_numerator); ASSERT_NE (conf.node.conf_height_processor_batch_min_time, defaults.node.conf_height_processor_batch_min_time); diff --git a/nano/lib/threading.cpp b/nano/lib/threading.cpp index b6ddc64ac..e705ea2fe 100644 --- a/nano/lib/threading.cpp +++ b/nano/lib/threading.cpp @@ -72,6 +72,9 @@ std::string nano::thread_role::get_string (nano::thread_role::name role) case nano::thread_role::name::worker: thread_role_name_string = "Worker"; break; + case nano::thread_role::name::bootstrap_worker: + thread_role_name_string = "Bootstrap work"; + break; case nano::thread_role::name::request_aggregator: thread_role_name_string = "Req aggregator"; break; diff --git a/nano/lib/threading.hpp b/nano/lib/threading.hpp index a06e2eb9f..0c21e17e0 100644 --- a/nano/lib/threading.hpp +++ b/nano/lib/threading.hpp @@ -36,6 +36,7 @@ namespace thread_role rpc_process_container, confirmation_height_processing, worker, + bootstrap_worker, request_aggregator, state_block_signature_verification, epoch_upgrader, diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.cpp b/nano/node/bootstrap/bootstrap_bulk_pull.cpp index 1856fef60..75ac5d106 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.cpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.cpp @@ -491,7 +491,9 @@ void nano::bulk_pull_server::sent_action (boost::system::error_code const & ec, { if (!ec) { - send_next (); + connection->node->bootstrap_workers.push_task ([this_l = shared_from_this ()] () { + this_l->send_next (); + }); } else { @@ -774,7 +776,9 @@ void nano::bulk_pull_account_server::sent_action (boost::system::error_code cons { if (!ec) { - send_next_block (); + connection->node->bootstrap_workers.push_task ([this_l = shared_from_this ()] () { + this_l->send_next_block (); + }); } else { diff --git a/nano/node/bootstrap/bootstrap_frontier.cpp b/nano/node/bootstrap/bootstrap_frontier.cpp index ebce00ff4..bac266941 100644 --- a/nano/node/bootstrap/bootstrap_frontier.cpp +++ b/nano/node/bootstrap/bootstrap_frontier.cpp @@ -316,7 +316,10 @@ void nano::frontier_req_server::sent_action (boost::system::error_code const & e if (!ec) { count++; - send_next (); + + connection->node->bootstrap_workers.push_task ([this_l = shared_from_this ()] () { + this_l->send_next (); + }); } else { diff --git a/nano/node/bootstrap/bootstrap_server.cpp b/nano/node/bootstrap/bootstrap_server.cpp index 7148d2ca1..2959e672c 100644 --- a/nano/node/bootstrap/bootstrap_server.cpp +++ b/nano/node/bootstrap/bootstrap_server.cpp @@ -663,13 +663,17 @@ public: } void bulk_pull (nano::bulk_pull const &) override { - auto response (std::make_shared (connection, std::unique_ptr (static_cast (connection->requests.front ().release ())))); - response->send_next (); + connection->node->bootstrap_workers.push_task ([connection = connection] () { + auto response (std::make_shared (connection, std::unique_ptr (static_cast (connection->requests.front ().release ())))); + response->send_next (); + }); } void bulk_pull_account (nano::bulk_pull_account const &) override { - auto response (std::make_shared (connection, std::unique_ptr (static_cast (connection->requests.front ().release ())))); - response->send_frontier (); + connection->node->bootstrap_workers.push_task ([connection = connection] () { + auto response (std::make_shared (connection, std::unique_ptr (static_cast (connection->requests.front ().release ())))); + response->send_frontier (); + }); } void bulk_push (nano::bulk_push const &) override { @@ -678,8 +682,10 @@ public: } void frontier_req (nano::frontier_req const &) override { - auto response (std::make_shared (connection, std::unique_ptr (static_cast (connection->requests.front ().release ())))); - response->send_next (); + connection->node->bootstrap_workers.push_task ([connection = connection] () { + auto response (std::make_shared (connection, std::unique_ptr (static_cast (connection->requests.front ().release ())))); + response->send_next (); + }); } void telemetry_req (nano::telemetry_req const & message_a) override { diff --git a/nano/node/node.cpp b/nano/node/node.cpp index bc0f1c8b9..8df5e833a 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -123,6 +123,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co network_params{ config.network_params }, stats (config.stat_config), workers (std::max (3u, config.io_threads / 4), nano::thread_role::name::worker), + bootstrap_workers{ config.bootstrap_serving_threads, nano::thread_role::name::bootstrap_worker }, flags (flags_a), work (work_a), distributed_work (*this), diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 7a062b1b9..4452685a2 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -161,6 +161,7 @@ public: nano::network_params & network_params; nano::stat stats; nano::thread_pool workers; + nano::thread_pool bootstrap_workers; std::shared_ptr websocket_server; nano::node_flags flags; nano::work_pool & work; diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index 9488c0504..167c4d32b 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -99,6 +99,7 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const toml.put ("bootstrap_connections", bootstrap_connections, "Number of outbound bootstrap connections. Must be a power of 2. Defaults to 4.\nWarning: a larger amount of connections may use substantially more system memory.\ntype:uint64"); toml.put ("bootstrap_connections_max", bootstrap_connections_max, "Maximum number of inbound bootstrap connections. Defaults to 64.\nWarning: a larger amount of connections may use additional system memory.\ntype:uint64"); toml.put ("bootstrap_initiator_threads", bootstrap_initiator_threads, "Number of threads dedicated to concurrent bootstrap attempts. Defaults to 1.\nWarning: a larger amount of attempts may use additional system memory and disk IO.\ntype:uint64"); + toml.put ("bootstrap_serving_threads", bootstrap_serving_threads, "Number of threads dedicated to serving bootstrap data to other peers. Defaults to half the number of CPU threads, and at least 2.\ntype:uint64"); toml.put ("bootstrap_frontier_request_count", bootstrap_frontier_request_count, "Number frontiers per bootstrap frontier request. Defaults to 1048576.\ntype:uint32,[1024..4294967295]"); toml.put ("block_processor_batch_max_time", block_processor_batch_max_time.count (), "The maximum time the block processor can continuously process blocks for.\ntype:milliseconds"); toml.put ("allow_local_peers", allow_local_peers, "Enable or disable local host peering.\ntype:bool"); @@ -335,6 +336,7 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml) toml.get ("bootstrap_connections", bootstrap_connections); toml.get ("bootstrap_connections_max", bootstrap_connections_max); toml.get ("bootstrap_initiator_threads", bootstrap_initiator_threads); + toml.get ("bootstrap_serving_threads", bootstrap_serving_threads); toml.get ("bootstrap_frontier_request_count", bootstrap_frontier_request_count); toml.get ("enable_voting", enable_voting); toml.get ("allow_local_peers", allow_local_peers); diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index ea7bb9b5e..71a109aa3 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -65,6 +65,7 @@ public: unsigned bootstrap_connections{ 4 }; unsigned bootstrap_connections_max{ 64 }; unsigned bootstrap_initiator_threads{ 1 }; + unsigned bootstrap_serving_threads{ std::max (2, std::thread::hardware_concurrency () / 2) }; uint32_t bootstrap_frontier_request_count{ 1024 * 1024 }; nano::websocket::config websocket_config; nano::diagnostics_config diagnostics_config;