From 684fb3a8bb241bdf10cc5254f5ad9cecf235354c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Wed, 21 Sep 2022 13:04:43 +0200 Subject: [PATCH] Allow overriding of `std::thread::hardware concurrency` (#3948) * Add `get_env_int_or_default` * Add `nano::hardware_concurrency` * Use `nano::hardware_concurrency` when checking available cores * Print info about logical cores at startup * Lexical casts may throw exceptions if the string is invalid. Since this is explicitly overriden by the user, and called in several non-exception safe contexts, log to cerr the reason why the exception was thrown. * Comment Co-authored-by: clemahieu --- nano/lib/config.cpp | 44 +++++++++++++++++++++++++----- nano/lib/config.hpp | 16 +++++++++++ nano/lib/rocksdbconfig.hpp | 3 +- nano/lib/rpcconfig.hpp | 4 ++- nano/lib/threading.cpp | 12 ++++++++ nano/lib/threading.hpp | 5 ++++ nano/lib/work.cpp | 2 +- nano/nano_node/daemon.cpp | 3 ++ nano/node/nodeconfig.hpp | 10 +++---- nano/node/rocksdb/rocksdb.cpp | 2 +- nano/secure/parallel_traversal.hpp | 2 +- nano/test_common/system.hpp | 2 +- 12 files changed, 87 insertions(+), 18 deletions(-) diff --git a/nano/lib/config.cpp b/nano/lib/config.cpp index 7700dad3..fb3a01af 100644 --- a/nano/lib/config.cpp +++ b/nano/lib/config.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -229,12 +230,6 @@ uint8_t get_pre_release_node_version () return boost::numeric_cast (boost::lexical_cast (NANO_PRE_RELEASE_VERSION_STRING)); } -std::string get_env_or_default (char const * variable_name, std::string default_value) -{ - auto value = getenv (variable_name); - return value ? value : default_value; -} - uint64_t get_env_threshold_or_default (char const * variable_name, uint64_t const default_value) { auto * value = getenv (variable_name); @@ -321,8 +316,43 @@ std::string get_tls_toml_config_path (boost::filesystem::path const & data_path) } } // namespace nano +std::optional nano::get_env (const char * variable_name) +{ + auto value = std::getenv (variable_name); + if (value) + { + return value; + } + return {}; +} + +std::string nano::get_env_or_default (char const * variable_name, std::string default_value) +{ + auto value = nano::get_env (variable_name); + return value ? *value : default_value; +} + +int nano::get_env_int_or_default (const char * variable_name, const int default_value) +{ + auto value = nano::get_env (variable_name); + if (value) + { + try + { + return boost::lexical_cast (*value); + } + catch (...) + { + // It is unexpected that this exception will be caught, log to cerr the reason. + std::cerr << boost::str (boost::format ("Error parsing environment variable: %1% value: %2%") % variable_name % *value); + throw; + } + } + return default_value; +} + uint32_t nano::test_scan_wallet_reps_delay () { auto test_env = nano::get_env_or_default ("NANO_TEST_WALLET_SCAN_REPS_DELAY", "900000"); // 15 minutes by default return boost::lexical_cast (test_env); -} \ No newline at end of file +} diff --git a/nano/lib/config.hpp b/nano/lib/config.hpp index 0caeabae..a8f42305 100644 --- a/nano/lib/config.hpp +++ b/nano/lib/config.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include namespace boost @@ -87,7 +88,22 @@ uint8_t get_minor_node_version (); uint8_t get_patch_node_version (); uint8_t get_pre_release_node_version (); +/* + * Environment variables + */ + +/* + * Get environment variable as string or none if variable is not present + */ +std::optional get_env (char const * variable_name); +/* + * Get environment variable as string or `default_value` if variable is not present + */ std::string get_env_or_default (char const * variable_name, std::string const default_value); +/* + * Get environment variable as int or `default_value` if variable is not present + */ +int get_env_int_or_default (char const * variable_name, int const default_value); uint64_t get_env_threshold_or_default (char const * variable_name, uint64_t const default_value); uint16_t test_node_port (); diff --git a/nano/lib/rocksdbconfig.hpp b/nano/lib/rocksdbconfig.hpp index ff185ada..20c0f876 100644 --- a/nano/lib/rocksdbconfig.hpp +++ b/nano/lib/rocksdbconfig.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -24,6 +25,6 @@ public: bool enable{ false }; uint8_t memory_multiplier{ 2 }; - unsigned io_threads{ std::thread::hardware_concurrency () }; + unsigned io_threads{ nano::hardware_concurrency () }; }; } diff --git a/nano/lib/rpcconfig.hpp b/nano/lib/rpcconfig.hpp index ad5efa1b..f45e2c7e 100644 --- a/nano/lib/rpcconfig.hpp +++ b/nano/lib/rpcconfig.hpp @@ -2,7 +2,9 @@ #include #include +#include +#include #include #include #include @@ -53,7 +55,7 @@ class rpc_process_config final public: rpc_process_config (nano::network_constants & network_constants); nano::network_constants & network_constants; - unsigned io_threads{ (4 < std::thread::hardware_concurrency ()) ? std::thread::hardware_concurrency () : 4 }; + unsigned io_threads{ std::max (nano::hardware_concurrency (), 4u) }; std::string ipc_address; uint16_t ipc_port{ network_constants.default_ipc_port }; unsigned num_ipc_connections{ (network_constants.is_live_network () || network_constants.is_test_network ()) ? 8u : network_constants.is_beta_network () ? 4u diff --git a/nano/lib/threading.cpp b/nano/lib/threading.cpp index c809b1b3..7f37d0a8 100644 --- a/nano/lib/threading.cpp +++ b/nano/lib/threading.cpp @@ -308,3 +308,15 @@ std::unique_ptr nano::collect_container_info (th composite->add_component (std::make_unique (container_info{ "count", thread_pool.num_queued_tasks (), sizeof (std::function) })); return composite; } + +unsigned int nano::hardware_concurrency () +{ + // Try to read overridden value from environment variable + static int value = nano::get_env_int_or_default ("NANO_HARDWARE_CONCURRENCY", 0); + if (value <= 0) + { + // Not present or invalid, use default + return std::thread::hardware_concurrency (); + } + return value; +} \ No newline at end of file diff --git a/nano/lib/threading.hpp b/nano/lib/threading.hpp index 66ff2fc3..d0e4f409 100644 --- a/nano/lib/threading.hpp +++ b/nano/lib/threading.hpp @@ -201,4 +201,9 @@ private: }; std::unique_ptr collect_container_info (thread_pool & thread_pool, std::string const & name); + +/* + * Number of available logical processor cores. Might be overridden by setting `NANO_HARDWARE_CONCURRENCY` environment variable + */ +unsigned int hardware_concurrency (); } diff --git a/nano/lib/work.cpp b/nano/lib/work.cpp index 1bbcd133..ca41e03c 100644 --- a/nano/lib/work.cpp +++ b/nano/lib/work.cpp @@ -32,7 +32,7 @@ nano::work_pool::work_pool (nano::network_constants & network_constants, unsigne static_assert (ATOMIC_INT_LOCK_FREE == 2, "Atomic int needed"); boost::thread::attributes attrs; nano::thread_attributes::set (attrs); - auto count (network_constants.is_dev_network () ? std::min (max_threads_a, 1u) : std::min (max_threads_a, std::max (1u, boost::thread::hardware_concurrency ()))); + auto count (network_constants.is_dev_network () ? std::min (max_threads_a, 1u) : std::min (max_threads_a, std::max (1u, nano::hardware_concurrency ()))); if (opencl) { // One thread to handle OpenCL diff --git a/nano/nano_node/daemon.cpp b/nano/nano_node/daemon.cpp index e795e349..700b5a77 100644 --- a/nano/nano_node/daemon.cpp +++ b/nano/nano_node/daemon.cpp @@ -117,6 +117,9 @@ void nano_daemon::daemon::run (boost::filesystem::path const & data_path, nano:: std::cout << initialization_text << std::endl; logger.always_log (initialization_text); + // Print info about number of logical cores detected, those are used to decide how many IO, worker and signature checker threads to spawn + logger.always_log (boost::format ("Hardware concurrency: %1% ( configured: %2% )") % std::thread::hardware_concurrency () % nano::hardware_concurrency ()); + nano::set_file_descriptor_limit (OPEN_FILE_DESCRIPTORS_LIMIT); auto const file_descriptor_limit = nano::get_file_descriptor_limit (); if (file_descriptor_limit < OPEN_FILE_DESCRIPTORS_LIMIT) diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index 2e2e86ce..10f04e9f 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -56,16 +56,16 @@ public: nano::amount online_weight_minimum{ 60000 * nano::Gxrb_ratio }; unsigned election_hint_weight_percent{ 50 }; unsigned password_fanout{ 1024 }; - unsigned io_threads{ std::max (4, std::thread::hardware_concurrency ()) }; - unsigned network_threads{ std::max (4, std::thread::hardware_concurrency ()) }; - unsigned work_threads{ std::max (4, std::thread::hardware_concurrency ()) }; + unsigned io_threads{ std::max (4u, nano::hardware_concurrency ()) }; + unsigned network_threads{ std::max (4u, nano::hardware_concurrency ()) }; + unsigned work_threads{ std::max (4u, nano::hardware_concurrency ()) }; /* Use half available threads on the system for signature checking. The calling thread does checks as well, so these are extra worker threads */ - unsigned signature_checker_threads{ std::thread::hardware_concurrency () / 2 }; + unsigned signature_checker_threads{ std::max (2u, nano::hardware_concurrency () / 2) }; bool enable_voting{ false }; unsigned bootstrap_connections{ 4 }; unsigned bootstrap_connections_max{ 64 }; unsigned bootstrap_initiator_threads{ 1 }; - unsigned bootstrap_serving_threads{ std::max (2, std::thread::hardware_concurrency () / 2) }; + unsigned bootstrap_serving_threads{ std::max (2u, nano::hardware_concurrency () / 2) }; uint32_t bootstrap_frontier_request_count{ 1024 * 1024 }; nano::websocket::config websocket_config; nano::diagnostics_config diagnostics_config; diff --git a/nano/node/rocksdb/rocksdb.cpp b/nano/node/rocksdb/rocksdb.cpp index 2a18a072..82a3f920 100644 --- a/nano/node/rocksdb/rocksdb.cpp +++ b/nano/node/rocksdb/rocksdb.cpp @@ -744,7 +744,7 @@ bool nano::rocksdb::store::copy_db (boost::filesystem::path const & destination_ backup_options.share_table_files = true; // Increase number of threads used for copying - backup_options.max_background_operations = std::thread::hardware_concurrency (); + backup_options.max_background_operations = nano::hardware_concurrency (); auto status = ::rocksdb::BackupEngine::Open (::rocksdb::Env::Default (), backup_options, &backup_engine_raw); backup_engine.reset (backup_engine_raw); if (!status.ok ()) diff --git a/nano/secure/parallel_traversal.hpp b/nano/secure/parallel_traversal.hpp index 80187b21..3bc261f8 100644 --- a/nano/secure/parallel_traversal.hpp +++ b/nano/secure/parallel_traversal.hpp @@ -9,7 +9,7 @@ template void parallel_traversal (std::function const & action) { // Between 10 and 40 threads, scales well even in low power systems as long as actions are I/O bound - unsigned const thread_count = std::max (10u, std::min (40u, 10 * std::thread::hardware_concurrency ())); + unsigned const thread_count = std::max (10u, std::min (40u, 10 * nano::hardware_concurrency ())); T const value_max{ std::numeric_limits::max () }; T const split = value_max / thread_count; std::vector threads; diff --git a/nano/test_common/system.hpp b/nano/test_common/system.hpp index f5677ae4..4fe7305c 100644 --- a/nano/test_common/system.hpp +++ b/nano/test_common/system.hpp @@ -63,7 +63,7 @@ namespace test boost::asio::io_context io_ctx; std::vector> nodes; nano::logging logging; - nano::work_pool work{ nano::dev::network_params.network, std::max (std::thread::hardware_concurrency (), 1u) }; + nano::work_pool work{ nano::dev::network_params.network, std::max (nano::hardware_concurrency (), 1u) }; std::chrono::time_point> deadline{ std::chrono::steady_clock::time_point::max () }; double deadline_scaling_factor{ 1.0 }; unsigned node_sequence{ 0 };