Merge pull request #4287 from clemahieu/nano_store_cleanup

This is a sequence of cleanup commits preparing for the extraction of the nano::store library.
This commit is contained in:
clemahieu 2023-09-20 18:31:04 +01:00 committed by GitHub
commit 3e0ff0471f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
67 changed files with 446 additions and 478 deletions

View file

@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.4)
cmake_minimum_required(VERSION 3.14)
if(CMAKE_VERSION VERSION_GREATER 3.12 OR CMAKE_VERSION VERSION_EQUAL 3.12)
# find_package uses <PACKAGENAME>_ROOT variables
@ -547,7 +547,8 @@ endif()
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
add_definitions(-DCRYPTOPP_DISABLE_MIXED_ASM -DCRYPTOPP_DISABLE_ASM)
message(
"CryptoPP with disabled ASM for ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}"
STATUS
"CryptoPP with disabled ASM for ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}"
)
endif()
add_definitions(-DCRYPTOPP_DISABLE_SHANI)
@ -774,7 +775,7 @@ if(NANO_GUI OR RAIBLOCKS_GUI)
nano_wallet ${PLATFORM_GUI_TARGET_PROPERTIES} ${PLATFORM_WALLET_SOURCE}
nano/nano_wallet/entry.cpp nano/nano_wallet/icon.hpp ${RES})
target_link_libraries(nano_wallet rpc node qt)
target_link_libraries(nano_wallet Boost::process rpc node qt)
if(WIN32)
target_link_libraries(nano_wallet Qt5::WinExtras)

View file

@ -6,6 +6,7 @@
#include <nano/lib/work.hpp>
#include <nano/node/common.hpp>
#include <nano/node/lmdb/lmdb.hpp>
#include <nano/node/make_store.hpp>
#include <nano/node/rocksdb/rocksdb.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/utility.hpp>

View file

@ -1,5 +1,6 @@
#include <nano/lib/stats.hpp>
#include <nano/node/bootstrap_ascending/service.hpp>
#include <nano/node/make_store.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>

View file

@ -1,4 +1,5 @@
#include <nano/node/election.hpp>
#include <nano/node/make_store.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>

View file

@ -1,6 +1,7 @@
#include <nano/lib/stats.hpp>
#include <nano/lib/threading.hpp>
#include <nano/node/election.hpp>
#include <nano/node/make_store.hpp>
#include <nano/node/rocksdb/rocksdb.hpp>
#include <nano/node/scheduler/component.hpp>
#include <nano/node/scheduler/priority.hpp>

View file

@ -5,6 +5,8 @@
#include <gtest/gtest.h>
#include <boost/filesystem.hpp>
#include <chrono>
#include <regex>
#include <thread>

View file

@ -1,5 +1,6 @@
#include <nano/lib/config.hpp>
#include <nano/node/election.hpp>
#include <nano/node/make_store.hpp>
#include <nano/node/scheduler/component.hpp>
#include <nano/node/scheduler/manual.hpp>
#include <nano/node/scheduler/priority.hpp>

View file

@ -1,5 +1,6 @@
#include <nano/lib/stats.hpp>
#include <nano/lib/work.hpp>
#include <nano/node/make_store.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/store.hpp>
#include <nano/secure/utility.hpp>

View file

@ -1,5 +1,6 @@
#include <nano/boost/asio/ip/address_v6.hpp>
#include <nano/boost/asio/ip/network_v6.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/threading.hpp>
#include <nano/node/transport/socket.hpp>
#include <nano/test_common/system.hpp>

View file

@ -1,3 +1,4 @@
#include <nano/lib/thread_runner.hpp>
#include <nano/node/transport/inproc.hpp>
#include <nano/test_common/network.hpp>
#include <nano/test_common/system.hpp>

View file

@ -8,6 +8,8 @@
#include <gtest/gtest.h>
#include <boost/filesystem.hpp>
#include <numeric>
#include <sstream>
#include <string>

View file

@ -1,6 +1,6 @@
#include <nano/lib/optional_ptr.hpp>
#include <nano/lib/rate_limiting.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/thread_pool.hpp>
#include <nano/lib/timer.hpp>
#include <nano/lib/utility.hpp>
#include <nano/secure/utility.hpp>

View file

@ -1,4 +1,5 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/threading.hpp>
#include <nano/node/lmdb/wallet_value.hpp>
#include <nano/test_common/system.hpp>

View file

@ -46,7 +46,8 @@ file(GLOB files
foreach(file ${files})
get_filename_component(flatbuffers_filename ${file} NAME_WE)
message(
"Generating flatbuffers code for: ${flatbuffers_filename} into ${CMAKE_CURRENT_SOURCE_DIR}/generated/flatbuffers"
STATUS
"Generating flatbuffers code for: ${flatbuffers_filename} into ${CMAKE_CURRENT_SOURCE_DIR}/generated/flatbuffers"
)
configure_file(

View file

@ -74,8 +74,12 @@ add_library(
stats_enums.hpp
stats_enums.cpp
stream.hpp
thread_pool.hpp
thread_pool.cpp
thread_roles.hpp
thread_roles.cpp
thread_runner.hpp
thread_runner.cpp
threading.hpp
threading.cpp
timer.hpp

View file

@ -282,48 +282,11 @@ std::string nano::error_config_messages::message (int ev) const
return "Invalid error code";
}
#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE)
char const * nano::error_conversion::detail::generic_category::name () const noexcept
{
return boost::system::generic_category ().name ();
}
std::string nano::error_conversion::detail::generic_category::message (int value) const
{
return boost::system::generic_category ().message (value);
}
std::error_category const & nano::error_conversion::generic_category ()
{
static detail::generic_category instance;
return instance;
}
std::error_code nano::error_conversion::convert (boost::system::error_code const & error)
{
if (error.category () == boost::system::generic_category ())
{
return std::error_code (error.value (),
nano::error_conversion::generic_category ());
}
debug_assert (false);
return nano::error_common::invalid_type_conversion;
}
#endif
nano::error::error (std::error_code code_a)
{
code = code_a;
}
#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE)
nano::error::error (boost::system::error_code const & code_a)
{
code = std::make_error_code (static_cast<std::errc> (code_a.value ()));
}
#endif
nano::error::error (std::string message_a)
{
code = nano::error_common::generic;
@ -358,24 +321,6 @@ nano::error & nano::error::operator= (std::error_code const code_a)
return *this;
}
#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE)
/** Assign boost error code (as converted to std::error_code) */
nano::error & nano::error::operator= (boost::system::error_code const & code_a)
{
code = nano::error_conversion::convert (code_a);
message.clear ();
return *this;
}
/** Assign boost error code (as converted to std::error_code) */
nano::error & nano::error::operator= (boost::system::errc::errc_t const & code_a)
{
code = nano::error_conversion::convert (boost::system::errc::make_error_code (code_a));
message.clear ();
return *this;
}
#endif
/** Set the error to nano::error_common::generic and the error message to \p message_a */
nano::error & nano::error::operator= (std::string message_a)
{
@ -398,14 +343,6 @@ bool nano::error::operator== (std::error_code const code_a) const
return code == code_a;
}
#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE)
/** Return true if this#error_code equals the parameter */
bool nano::error::operator== (boost::system::error_code const code_a) const
{
return code.value () == code_a.value ();
}
#endif
/** Call the function iff the current error is zero */
nano::error & nano::error::then (std::function<nano::error &()> next)
{
@ -495,14 +432,3 @@ nano::error & nano::error::clear ()
message.clear ();
return *this;
}
#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE)
// TODO: theoretically, nothing besides template (partial) specializations should ever be added inside std...
namespace std
{
std::error_code make_error_code (boost::system::errc::errc_t const & e)
{
return std::error_code (static_cast<int> (e), ::nano::error_conversion::generic_category ());
}
}
#endif

View file

@ -1,8 +1,5 @@
#pragma once
#include <boost/filesystem/operations.hpp>
#include <boost/system/error_code.hpp>
#include <algorithm>
#include <functional>
#include <memory>
@ -197,51 +194,6 @@ REGISTER_ERROR_CODES (nano, error_rpc);
REGISTER_ERROR_CODES (nano, error_process);
REGISTER_ERROR_CODES (nano, error_config);
#if BOOST_VERSION >= 107800
/* no need for error_code bridge */
#else
#define NANO_USE_BOOST_TO_STD_ERROR_BRIDGE
#endif
#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE)
/* boost->std error_code bridge */
namespace nano
{
namespace error_conversion
{
std::error_category const & generic_category ();
}
}
namespace std
{
template <>
struct is_error_code_enum<boost::system::errc::errc_t>
: public std::true_type
{
};
std::error_code make_error_code (boost::system::errc::errc_t const & e);
}
namespace nano
{
namespace error_conversion
{
namespace detail
{
class generic_category : public std::error_category
{
public:
char const * name () const noexcept override;
std::string message (int value) const override;
};
}
std::error_category const & generic_category ();
std::error_code convert (boost::system::error_code const & error);
}
}
#endif
namespace nano
{
/** Adapter for std/boost::error_code, std::exception and bool flags to facilitate unified error handling */
@ -253,24 +205,14 @@ public:
error (nano::error && error_a) = default;
error (std::error_code code_a);
#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE)
error (boost::system::error_code const & code_a);
#endif
error (std::string message_a);
error (std::exception const & exception_a);
error & operator= (nano::error const & err_a);
error & operator= (nano::error && err_a);
error & operator= (std::error_code code_a);
#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE)
error & operator= (boost::system::error_code const & code_a);
error & operator= (boost::system::errc::errc_t const & code_a);
#endif
error & operator= (std::string message_a);
error & operator= (std::exception const & exception_a);
bool operator== (std::error_code code_a) const;
#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE)
bool operator== (boost::system::error_code code_a) const;
#endif
error & then (std::function<nano::error &()> next);
template <typename... ErrorCode>
error & accept (ErrorCode... err)

View file

@ -1,4 +1,4 @@
#include <nano/lib/threading.hpp>
#include <nano/lib/thread_roles.hpp>
#include <pthread.h>

View file

@ -1,4 +1,4 @@
#include <nano/lib/threading.hpp>
#include <nano/lib/thread_roles.hpp>
#include <pthread.h>

View file

@ -1,4 +1,4 @@
#include <nano/lib/threading.hpp>
#include <nano/lib/thread_roles.hpp>
#include <processthreadsapi.h>

View file

@ -3,6 +3,7 @@
#include <nano/lib/locks.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/stats.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/utility.hpp>
@ -189,4 +190,4 @@ private:
nano::condition_variable condition;
std::vector<std::thread> threads;
};
}
}

97
nano/lib/thread_pool.cpp Normal file
View file

@ -0,0 +1,97 @@
#include <nano/lib/thread_pool.hpp>
#include <boost/asio/post.hpp>
#include <boost/asio/steady_timer.hpp>
#include <boost/asio/thread_pool.hpp>
/*
* thread_pool
*/
nano::thread_pool::thread_pool (unsigned num_threads, nano::thread_role::name thread_name) :
num_threads (num_threads),
thread_pool_m (std::make_unique<boost::asio::thread_pool> (num_threads)),
thread_names_latch{ num_threads }
{
set_thread_names (thread_name);
}
nano::thread_pool::~thread_pool ()
{
stop ();
}
void nano::thread_pool::stop ()
{
nano::unique_lock<nano::mutex> lk (mutex);
if (!stopped)
{
stopped = true;
#if defined(BOOST_ASIO_HAS_IOCP)
// A hack needed for Windows to prevent deadlock during destruction, described here: https://github.com/chriskohlhoff/asio/issues/431
boost::asio::use_service<boost::asio::detail::win_iocp_io_context> (*thread_pool_m).stop ();
#endif
lk.unlock ();
thread_pool_m->stop ();
thread_pool_m->join ();
lk.lock ();
thread_pool_m = nullptr;
}
}
void nano::thread_pool::push_task (std::function<void ()> task)
{
++num_tasks;
nano::lock_guard<nano::mutex> guard (mutex);
if (!stopped)
{
boost::asio::post (*thread_pool_m, [this, task] () {
task ();
--num_tasks;
});
}
}
void nano::thread_pool::add_timed_task (std::chrono::steady_clock::time_point const & expiry_time, std::function<void ()> task)
{
nano::lock_guard<nano::mutex> guard (mutex);
if (!stopped && thread_pool_m)
{
auto timer = std::make_shared<boost::asio::steady_timer> (thread_pool_m->get_executor (), expiry_time);
timer->async_wait ([this, task, timer] (boost::system::error_code const & ec) {
if (!ec)
{
push_task (task);
}
});
}
}
unsigned nano::thread_pool::get_num_threads () const
{
return num_threads;
}
uint64_t nano::thread_pool::num_queued_tasks () const
{
return num_tasks;
}
void nano::thread_pool::set_thread_names (nano::thread_role::name thread_name)
{
for (auto i = 0u; i < num_threads; ++i)
{
boost::asio::post (*thread_pool_m, [this, thread_name] () {
nano::thread_role::set (thread_name);
thread_names_latch.arrive_and_wait ();
});
}
thread_names_latch.wait ();
}
std::unique_ptr<nano::container_info_component> nano::collect_container_info (thread_pool & thread_pool, std::string const & name)
{
auto composite = std::make_unique<container_info_composite> (name);
composite->add_component (std::make_unique<container_info_leaf> (container_info{ "count", thread_pool.num_queued_tasks (), sizeof (std::function<void ()>) }));
return composite;
}

53
nano/lib/thread_pool.hpp Normal file
View file

@ -0,0 +1,53 @@
#pragma once
#include <nano/lib/relaxed_atomic.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/threading.hpp>
#include <atomic>
#include <chrono>
#include <functional>
#include <latch>
namespace boost::asio
{
class thread_pool;
}
namespace nano
{
class thread_pool final
{
public:
explicit thread_pool (unsigned, nano::thread_role::name);
~thread_pool ();
/** This will run when there is an available thread for execution */
void push_task (std::function<void ()>);
/** Run a task at a certain point in time */
void add_timed_task (std::chrono::steady_clock::time_point const & expiry_time, std::function<void ()> task);
/** Stops any further pushed tasks from executing */
void stop ();
/** Number of threads in the thread pool */
unsigned get_num_threads () const;
/** Returns the number of tasks which are awaiting execution by the thread pool **/
uint64_t num_queued_tasks () const;
private:
nano::mutex mutex;
std::atomic<bool> stopped{ false };
unsigned num_threads;
std::unique_ptr<boost::asio::thread_pool> thread_pool_m;
nano::relaxed_atomic_integral<uint64_t> num_tasks{ 0 };
/** Set the names of all the threads in the thread pool for easier identification */
std::latch thread_names_latch;
void set_thread_names (nano::thread_role::name thread_name);
};
std::unique_ptr<nano::container_info_component> collect_container_info (thread_pool & thread_pool, std::string const & name);
} // namespace nano

View file

@ -0,0 +1,82 @@
#include <nano/lib/thread_runner.hpp>
#include <iostream>
/*
* thread_runner
*/
nano::thread_runner::thread_runner (boost::asio::io_context & io_ctx_a, unsigned num_threads, const nano::thread_role::name thread_role_a) :
io_guard{ boost::asio::make_work_guard (io_ctx_a) },
role{ thread_role_a }
{
for (auto i (0u); i < num_threads; ++i)
{
threads.emplace_back (nano::thread_attributes::get_default (), [this, &io_ctx_a] () {
nano::thread_role::set (role);
try
{
run (io_ctx_a);
}
catch (std::exception const & ex)
{
std::cerr << ex.what () << std::endl;
#ifndef NDEBUG
throw; // Re-throw to debugger in debug mode
#endif
}
catch (...)
{
#ifndef NDEBUG
throw; // Re-throw to debugger in debug mode
#endif
}
});
}
}
nano::thread_runner::~thread_runner ()
{
join ();
}
void nano::thread_runner::run (boost::asio::io_context & io_ctx_a)
{
#if NANO_ASIO_HANDLER_TRACKING == 0
io_ctx_a.run ();
#else
nano::timer<> timer;
timer.start ();
while (true)
{
timer.restart ();
// Run at most 1 completion handler and record the time it took to complete (non-blocking)
auto count = io_ctx_a.poll_one ();
if (count == 1 && timer.since_start ().count () >= NANO_ASIO_HANDLER_TRACKING)
{
auto timestamp = std::chrono::duration_cast<std::chrono::microseconds> (std::chrono::system_clock::now ().time_since_epoch ()).count ();
std::cout << (boost::format ("[%1%] io_thread held for %2%ms") % timestamp % timer.since_start ().count ()).str () << std::endl;
}
// Sleep for a bit to give more time slices to other threads
std::this_thread::sleep_for (std::chrono::milliseconds (5));
std::this_thread::yield ();
}
#endif
}
void nano::thread_runner::join ()
{
io_guard.reset ();
for (auto & i : threads)
{
if (i.joinable ())
{
i.join ();
}
}
}
void nano::thread_runner::stop_event_processing ()
{
io_guard.get_executor ().context ().stop ();
}

View file

@ -0,0 +1,31 @@
#pragma once
#include <nano/boost/asio/deadline_timer.hpp>
#include <nano/boost/asio/executor_work_guard.hpp>
#include <nano/boost/asio/io_context.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/threading.hpp>
#include <boost/thread.hpp>
namespace nano
{
class thread_runner final
{
public:
thread_runner (boost::asio::io_context &, unsigned num_threads, nano::thread_role::name thread_role = nano::thread_role::name::io);
~thread_runner ();
/** Tells the IO context to stop processing events.*/
void stop_event_processing ();
/** Wait for IO threads to complete */
void join ();
private:
nano::thread_role::name const role;
std::vector<boost::thread> threads;
boost::asio::executor_work_guard<boost::asio::io_context::executor_type> io_guard;
private:
void run (boost::asio::io_context &);
};
} // namespace nano

View file

@ -1,11 +1,7 @@
#include <nano/boost/asio/post.hpp>
#include <nano/lib/config.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/threading.hpp>
#include <boost/format.hpp>
#include <future>
#include <iostream>
#include <thread>
/*
@ -19,179 +15,6 @@ boost::thread::attributes nano::thread_attributes::get_default ()
return attrs;
}
/*
* thread_runner
*/
nano::thread_runner::thread_runner (boost::asio::io_context & io_ctx_a, unsigned num_threads, const nano::thread_role::name thread_role_a) :
io_guard{ boost::asio::make_work_guard (io_ctx_a) },
role{ thread_role_a }
{
for (auto i (0u); i < num_threads; ++i)
{
threads.emplace_back (nano::thread_attributes::get_default (), [this, &io_ctx_a] () {
nano::thread_role::set (role);
// In a release build, catch and swallow any exceptions,
// In debug mode let if fall through
#ifndef NDEBUG
run (io_ctx_a);
#else
try
{
run (io_ctx_a);
}
catch (std::exception const & ex)
{
std::cerr << ex.what () << std::endl;
}
catch (...)
{
}
#endif
});
}
}
nano::thread_runner::~thread_runner ()
{
join ();
}
void nano::thread_runner::run (boost::asio::io_context & io_ctx_a)
{
#if NANO_ASIO_HANDLER_TRACKING == 0
io_ctx_a.run ();
#else
nano::timer<> timer;
timer.start ();
while (true)
{
timer.restart ();
// Run at most 1 completion handler and record the time it took to complete (non-blocking)
auto count = io_ctx_a.poll_one ();
if (count == 1 && timer.since_start ().count () >= NANO_ASIO_HANDLER_TRACKING)
{
auto timestamp = std::chrono::duration_cast<std::chrono::microseconds> (std::chrono::system_clock::now ().time_since_epoch ()).count ();
std::cout << (boost::format ("[%1%] io_thread held for %2%ms") % timestamp % timer.since_start ().count ()).str () << std::endl;
}
// Sleep for a bit to give more time slices to other threads
std::this_thread::sleep_for (std::chrono::milliseconds (5));
std::this_thread::yield ();
}
#endif
}
void nano::thread_runner::join ()
{
io_guard.reset ();
for (auto & i : threads)
{
if (i.joinable ())
{
i.join ();
}
}
}
void nano::thread_runner::stop_event_processing ()
{
io_guard.get_executor ().context ().stop ();
}
/*
* thread_pool
*/
nano::thread_pool::thread_pool (unsigned num_threads, nano::thread_role::name thread_name) :
num_threads (num_threads),
thread_pool_m (std::make_unique<boost::asio::thread_pool> (num_threads)),
thread_names_latch{ num_threads }
{
set_thread_names (thread_name);
}
nano::thread_pool::~thread_pool ()
{
stop ();
}
void nano::thread_pool::stop ()
{
nano::unique_lock<nano::mutex> lk (mutex);
if (!stopped)
{
stopped = true;
#if defined(BOOST_ASIO_HAS_IOCP)
// A hack needed for Windows to prevent deadlock during destruction, described here: https://github.com/chriskohlhoff/asio/issues/431
boost::asio::use_service<boost::asio::detail::win_iocp_io_context> (*thread_pool_m).stop ();
#endif
lk.unlock ();
thread_pool_m->stop ();
thread_pool_m->join ();
lk.lock ();
thread_pool_m = nullptr;
}
}
void nano::thread_pool::push_task (std::function<void ()> task)
{
++num_tasks;
nano::lock_guard<nano::mutex> guard (mutex);
if (!stopped)
{
boost::asio::post (*thread_pool_m, [this, task] () {
task ();
--num_tasks;
});
}
}
void nano::thread_pool::add_timed_task (std::chrono::steady_clock::time_point const & expiry_time, std::function<void ()> task)
{
nano::lock_guard<nano::mutex> guard (mutex);
if (!stopped && thread_pool_m)
{
auto timer = std::make_shared<boost::asio::steady_timer> (thread_pool_m->get_executor (), expiry_time);
timer->async_wait ([this, task, timer] (boost::system::error_code const & ec) {
if (!ec)
{
push_task (task);
}
});
}
}
unsigned nano::thread_pool::get_num_threads () const
{
return num_threads;
}
uint64_t nano::thread_pool::num_queued_tasks () const
{
return num_tasks;
}
void nano::thread_pool::set_thread_names (nano::thread_role::name thread_name)
{
for (auto i = 0u; i < num_threads; ++i)
{
boost::asio::post (*thread_pool_m, [this, thread_name] () {
nano::thread_role::set (thread_name);
thread_names_latch.arrive_and_wait ();
});
}
thread_names_latch.wait ();
}
std::unique_ptr<nano::container_info_component> nano::collect_container_info (thread_pool & thread_pool, std::string const & name)
{
auto composite = std::make_unique<container_info_composite> (name);
composite->add_component (std::make_unique<container_info_leaf> (container_info{ "count", thread_pool.num_queued_tasks (), sizeof (std::function<void ()>) }));
return composite;
}
unsigned int nano::hardware_concurrency ()
{
// Try to read overridden value from environment variable

View file

@ -1,80 +1,15 @@
#pragma once
#include <nano/boost/asio/deadline_timer.hpp>
#include <nano/boost/asio/executor_work_guard.hpp>
#include <nano/boost/asio/io_context.hpp>
#include <nano/boost/asio/steady_timer.hpp>
#include <nano/boost/asio/thread_pool.hpp>
#include <nano/lib/relaxed_atomic.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/utility.hpp>
#include <boost/thread/thread.hpp>
#include <latch>
#include <thread>
namespace nano
{
namespace thread_attributes
{
boost::thread::attributes get_default ();
}
class thread_runner final
{
public:
thread_runner (boost::asio::io_context &, unsigned num_threads, nano::thread_role::name thread_role = nano::thread_role::name::io);
~thread_runner ();
/** Tells the IO context to stop processing events.*/
void stop_event_processing ();
/** Wait for IO threads to complete */
void join ();
private:
nano::thread_role::name const role;
std::vector<boost::thread> threads;
boost::asio::executor_work_guard<boost::asio::io_context::executor_type> io_guard;
private:
void run (boost::asio::io_context &);
};
class thread_pool final
{
public:
explicit thread_pool (unsigned, nano::thread_role::name);
~thread_pool ();
/** This will run when there is an available thread for execution */
void push_task (std::function<void ()>);
/** Run a task at a certain point in time */
void add_timed_task (std::chrono::steady_clock::time_point const & expiry_time, std::function<void ()> task);
/** Stops any further pushed tasks from executing */
void stop ();
/** Number of threads in the thread pool */
unsigned get_num_threads () const;
/** Returns the number of tasks which are awaiting execution by the thread pool **/
uint64_t num_queued_tasks () const;
private:
nano::mutex mutex;
std::atomic<bool> stopped{ false };
unsigned num_threads;
std::unique_ptr<boost::asio::thread_pool> thread_pool_m;
nano::relaxed_atomic_integral<uint64_t> num_tasks{ 0 };
/** Set the names of all the threads in the thread pool for easier identification */
std::latch thread_names_latch;
void set_thread_names (nano::thread_role::name thread_name);
};
std::unique_ptr<nano::container_info_component> collect_container_info (thread_pool & thread_pool, std::string const & name);
} // namespace thread_attributes
/**
* Number of available logical processor cores. Might be overridden by setting `NANO_HARDWARE_CONCURRENCY` environment variable
@ -83,6 +18,7 @@ unsigned int hardware_concurrency ();
/**
* If thread is joinable joins it, otherwise does nothing
* Returns thread.joinable()
*/
bool join_or_pass (std::thread &);
}
} // namespace nano

View file

@ -3,6 +3,7 @@
#include <nano/lib/tlsconfig.hpp>
#include <nano/lib/tomlconfig.hpp>
#include <boost/filesystem.hpp>
#include <boost/format.hpp>
#include <iostream>

View file

@ -1,6 +1,7 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/epoch.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/work.hpp>
#include <nano/node/xorshift.hpp>

View file

@ -4,6 +4,7 @@
#include <nano/boost/beast/core/flat_buffer.hpp>
#include <nano/boost/beast/http.hpp>
#include <nano/boost/process/child.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/tomlconfig.hpp>
#include <nano/node/daemonconfig.hpp>

View file

@ -1,6 +1,7 @@
#include <nano/boost/process/child.hpp>
#include <nano/lib/signal_manager.hpp>
#include <nano/lib/stacktrace.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/tlsconfig.hpp>
#include <nano/lib/utility.hpp>

View file

@ -1,5 +1,6 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/cli.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/utility.hpp>
#include <nano/nano_node/daemon.hpp>
#include <nano/node/cli.hpp>

View file

@ -1,6 +1,7 @@
#include <nano/lib/cli.hpp>
#include <nano/lib/errors.hpp>
#include <nano/lib/signal_manager.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/tlsconfig.hpp>
#include <nano/lib/utility.hpp>

View file

@ -3,7 +3,7 @@
#include <nano/lib/cli.hpp>
#include <nano/lib/errors.hpp>
#include <nano/lib/rpcconfig.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/tlsconfig.hpp>
#include <nano/lib/tomlconfig.hpp>
#include <nano/lib/utility.hpp>

View file

@ -143,6 +143,8 @@ add_library(
lmdb/wallet_value.cpp
logging.hpp
logging.cpp
make_store.hpp
make_store.cpp
network.hpp
network.cpp
nodeconfig.hpp

View file

@ -6,6 +6,7 @@
#include <nano/node/daemonconfig.hpp>
#include <nano/node/node.hpp>
#include <boost/filesystem.hpp>
#include <boost/format.hpp>
namespace

View file

@ -1,6 +1,7 @@
#pragma once
#include <nano/lib/numbers.hpp>
#include <nano/lib/relaxed_atomic.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/timer.hpp>
#include <nano/secure/store.hpp>

View file

@ -1,6 +1,6 @@
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/utility.hpp>
#include <nano/node/confirmation_height_processor.hpp>
#include <nano/node/write_database_queue.hpp>

View file

@ -1,3 +1,4 @@
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/stats.hpp>
#include <nano/node/confirmation_height_unbounded.hpp>
#include <nano/node/logging.hpp>

View file

@ -1,6 +1,7 @@
#pragma once
#include <nano/lib/numbers.hpp>
#include <nano/lib/relaxed_atomic.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/timer.hpp>
#include <nano/secure/store.hpp>

View file

@ -3,6 +3,8 @@
#include <nano/lib/tomlconfig.hpp>
#include <nano/node/daemonconfig.hpp>
#include <boost/filesystem.hpp>
#include <sstream>
#include <vector>

View file

@ -2,6 +2,7 @@
#include <nano/node/ipc/ipc_access_config.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/filesystem.hpp>
namespace
{

View file

@ -5,6 +5,7 @@
#include <nano/lib/config.hpp>
#include <nano/lib/ipc.hpp>
#include <nano/lib/locks.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/timer.hpp>
#include <nano/node/ipc/action_handler.hpp>

View file

@ -9,6 +9,7 @@
#include <boost/filesystem.hpp>
#include <boost/format.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <queue>

View file

@ -1,6 +1,6 @@
#include <nano/lib/jsonconfig.hpp>
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/utility.hpp>
#include <nano/node/lmdb/lmdb_env.hpp>
#include <nano/node/lmdb/lmdb_txn.hpp>

View file

@ -5,6 +5,7 @@
#include <nano/node/logging.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/filesystem.hpp>
#include <boost/log/expressions.hpp>
#include <boost/log/utility/exception_handler.hpp>
#include <boost/log/utility/setup/common_attributes.hpp>

15
nano/node/make_store.cpp Normal file
View file

@ -0,0 +1,15 @@
#include <nano/node/lmdb/lmdb.hpp>
#include <nano/node/make_store.hpp>
#include <nano/node/rocksdb/rocksdb.hpp>
#include <boost/filesystem/path.hpp>
std::unique_ptr<nano::store> nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, nano::ledger_constants & constants, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade)
{
if (rocksdb_config.enable)
{
return std::make_unique<nano::rocksdb::store> (logger, add_db_postfix ? path / "rocksdb" : path, constants, rocksdb_config, read_only);
}
return std::make_unique<nano::lmdb::store> (logger, add_db_postfix ? path / "data.ldb" : path, constants, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_config_a, backup_before_upgrade);
}

27
nano/node/make_store.hpp Normal file
View file

@ -0,0 +1,27 @@
#pragma once
#include <nano/lib/diagnosticsconfig.hpp>
#include <nano/lib/lmdbconfig.hpp>
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/rocksdbconfig.hpp>
#include <chrono>
namespace boost::filesystem
{
class path;
}
namespace nano
{
class ledger_constants;
class lmdb_config;
class rocksdb_config;
class store;
class txn_tracking_config;
}
namespace nano
{
std::unique_ptr<nano::store> make_store (nano::logger_mt & logger, boost::filesystem::path const & path, nano::ledger_constants & constants, bool open_read_only = false, bool add_db_postfix = true, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false);
}

View file

@ -3,6 +3,7 @@
#include <nano/lib/utility.hpp>
#include <nano/node/common.hpp>
#include <nano/node/daemonconfig.hpp>
#include <nano/node/make_store.hpp>
#include <nano/node/node.hpp>
#include <nano/node/rocksdb/rocksdb.hpp>
#include <nano/node/scheduler/component.hpp>
@ -1570,13 +1571,3 @@ nano::node_flags const & nano::inactive_node_flag_defaults ()
node_flags.disable_tcp_realtime = true;
return node_flags;
}
std::unique_ptr<nano::store> nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, nano::ledger_constants & constants, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade)
{
if (rocksdb_config.enable)
{
return std::make_unique<nano::rocksdb::store> (logger, add_db_postfix ? path / "rocksdb" : path, constants, rocksdb_config, read_only);
}
return std::make_unique<nano::lmdb::store> (logger, add_db_postfix ? path / "data.ldb" : path, constants, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_config_a, backup_before_upgrade);
}

View file

@ -2,6 +2,7 @@
#include <nano/lib/config.hpp>
#include <nano/lib/stats.hpp>
#include <nano/lib/thread_pool.hpp>
#include <nano/lib/work.hpp>
#include <nano/node/active_transactions.hpp>
#include <nano/node/backlog_population.hpp>

View file

@ -3,6 +3,7 @@
#include <nano/node/rocksdb/rocksdb_iterator.hpp>
#include <nano/node/rocksdb/rocksdb_txn.hpp>
#include <boost/filesystem.hpp>
#include <boost/format.hpp>
#include <boost/polymorphic_cast.hpp>
#include <boost/property_tree/ptree.hpp>

View file

@ -3,6 +3,7 @@
#include <nano/lib/config.hpp>
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/rocksdbconfig.hpp>
#include <nano/node/rocksdb/account_store.hpp>
#include <nano/node/rocksdb/block_store.hpp>
#include <nano/node/rocksdb/confirmation_height_store.hpp>

View file

@ -1,6 +1,6 @@
#pragma once
#include <nano/lib/threading.hpp>
#include <nano/lib/thread_pool.hpp>
#include <nano/lib/utility.hpp>
#include <atomic>

View file

@ -1,7 +1,7 @@
#include <nano/lib/locks.hpp>
#include <nano/lib/stats.hpp>
#include <nano/lib/stats_enums.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/timer.hpp>
#include <nano/node/unchecked_map.hpp>

View file

@ -1,7 +1,6 @@
#include <nano/lib/config.hpp>
#include <nano/qt/qt.hpp>
#include <boost/foreach.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/property_tree/ptree.hpp>
@ -845,7 +844,7 @@ void nano_qt::stats_viewer::refresh_stats ()
if (json)
{
// Format the stat data to make totals and values easier to read
BOOST_FOREACH (boost::property_tree::ptree::value_type const & child, json->get_child ("entries"))
for (boost::property_tree::ptree::value_type const & child : json->get_child ("entries"))
{
auto time = child.second.get<std::string> ("time");
auto type = child.second.get<std::string> ("type");

View file

@ -1,3 +1,4 @@
#include <nano/node/make_store.hpp>
#include <nano/qt/qt.hpp>
#include <nano/test_common/network.hpp>
#include <nano/test_common/system.hpp>

View file

@ -1,6 +1,6 @@
#include <nano/lib/asio.hpp>
#include <nano/lib/json_error_response.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/rpc/rpc_request_processor.hpp>
#include <boost/endian/conversion.hpp>

View file

@ -1,6 +1,7 @@
#include <nano/boost/beast/core/flat_buffer.hpp>
#include <nano/boost/beast/http.hpp>
#include <nano/lib/rpcconfig.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/threading.hpp>
#include <nano/node/ipc/ipc_server.hpp>
#include <nano/node/json_handler.hpp>

View file

@ -2,10 +2,13 @@
#include <nano/lib/stats.hpp>
#include <nano/lib/utility.hpp>
#include <nano/lib/work.hpp>
#include <nano/node/make_store.hpp>
#include <nano/secure/common.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/store.hpp>
#include <boost/filesystem.hpp>
#include <cryptopp/words.h>
namespace
@ -676,6 +679,69 @@ ledger_processor::ledger_processor (nano::ledger & ledger_a, nano::write_transac
transaction (transaction_a)
{
}
/**
* Determine the representative for this block
*/
class representative_visitor final : public nano::block_visitor
{
public:
representative_visitor (nano::transaction const & transaction_a, nano::store & store_a);
~representative_visitor () = default;
void compute (nano::block_hash const & hash_a);
void send_block (nano::send_block const & block_a) override;
void receive_block (nano::receive_block const & block_a) override;
void open_block (nano::open_block const & block_a) override;
void change_block (nano::change_block const & block_a) override;
void state_block (nano::state_block const & block_a) override;
nano::transaction const & transaction;
nano::store & store;
nano::block_hash current;
nano::block_hash result;
};
representative_visitor::representative_visitor (nano::transaction const & transaction_a, nano::store & store_a) :
transaction (transaction_a),
store (store_a),
result (0)
{
}
void representative_visitor::compute (nano::block_hash const & hash_a)
{
current = hash_a;
while (result.is_zero ())
{
auto block (store.block.get (transaction, current));
debug_assert (block != nullptr);
block->visit (*this);
}
}
void representative_visitor::send_block (nano::send_block const & block_a)
{
current = block_a.previous ();
}
void representative_visitor::receive_block (nano::receive_block const & block_a)
{
current = block_a.previous ();
}
void representative_visitor::open_block (nano::open_block const & block_a)
{
result = block_a.hash ();
}
void representative_visitor::change_block (nano::change_block const & block_a)
{
result = block_a.hash ();
}
void representative_visitor::state_block (nano::state_block const & block_a)
{
result = block_a.hash ();
}
} // namespace
nano::ledger::ledger (nano::store & store_a, nano::stats & stat_a, nano::ledger_constants & constants, nano::generate_cache const & generate_cache_a) :

View file

@ -1,5 +1,6 @@
#pragma once
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/threading.hpp>
#include <thread>

View file

@ -1,5 +1,7 @@
#include <nano/secure/working.hpp>
#include <boost/filesystem.hpp>
#include <Foundation/Foundation.h>
namespace nano
@ -12,4 +14,4 @@ boost::filesystem::path app_path ()
[dir_string release];
return result;
}
}
}

View file

@ -1,6 +1,8 @@
#include <nano/lib/utility.hpp>
#include <nano/secure/working.hpp>
#include <boost/filesystem.hpp>
#include <pwd.h>
#include <sys/types.h>

View file

@ -1,50 +1,7 @@
#include <nano/lib/threading.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/timer.hpp>
#include <nano/secure/store.hpp>
nano::representative_visitor::representative_visitor (nano::transaction const & transaction_a, nano::store & store_a) :
transaction (transaction_a),
store (store_a),
result (0)
{
}
void nano::representative_visitor::compute (nano::block_hash const & hash_a)
{
current = hash_a;
while (result.is_zero ())
{
auto block (store.block.get (transaction, current));
debug_assert (block != nullptr);
block->visit (*this);
}
}
void nano::representative_visitor::send_block (nano::send_block const & block_a)
{
current = block_a.previous ();
}
void nano::representative_visitor::receive_block (nano::receive_block const & block_a)
{
current = block_a.previous ();
}
void nano::representative_visitor::open_block (nano::open_block const & block_a)
{
result = block_a.hash ();
}
void nano::representative_visitor::change_block (nano::change_block const & block_a)
{
result = block_a.hash ();
}
void nano::representative_visitor::state_block (nano::state_block const & block_a)
{
result = block_a.hash ();
}
nano::read_transaction::read_transaction (std::unique_ptr<nano::read_transaction_impl> read_transaction_impl) :
impl (std::move (read_transaction_impl))
{

View file

@ -1,11 +1,7 @@
#pragma once
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/diagnosticsconfig.hpp>
#include <nano/lib/lmdbconfig.hpp>
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/memory.hpp>
#include <nano/lib/rocksdbconfig.hpp>
#include <nano/secure/buffer.hpp>
#include <nano/secure/common.hpp>
#include <nano/secure/versioning.hpp>
@ -401,29 +397,6 @@ private:
return result;
}
};
class transaction;
class store;
/**
* Determine the representative for this block
*/
class representative_visitor final : public nano::block_visitor
{
public:
representative_visitor (nano::transaction const & transaction_a, nano::store & store_a);
~representative_visitor () = default;
void compute (nano::block_hash const & hash_a);
void send_block (nano::send_block const & block_a) override;
void receive_block (nano::receive_block const & block_a) override;
void open_block (nano::open_block const & block_a) override;
void change_block (nano::change_block const & block_a) override;
void state_block (nano::state_block const & block_a) override;
nano::transaction const & transaction;
nano::store & store;
nano::block_hash current;
nano::block_hash result;
};
template <typename T, typename U>
class store_iterator_impl
{
@ -824,8 +797,6 @@ public:
virtual std::string vendor_get () const = 0;
};
std::unique_ptr<nano::store> make_store (nano::logger_mt & logger, boost::filesystem::path const & path, nano::ledger_constants & constants, bool open_read_only = false, bool add_db_postfix = true, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false);
}
namespace std

View file

@ -1,4 +1,5 @@
#include <nano/lib/rpcconfig.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/node/bootstrap/bootstrap_server.hpp>
#include <nano/node/bootstrap_ascending/service.hpp>
#include <nano/node/ipc/ipc_server.hpp>

View file

@ -1,6 +1,8 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/threading.hpp>
#include <nano/node/election.hpp>
#include <nano/node/make_store.hpp>
#include <nano/node/scheduler/component.hpp>
#include <nano/node/scheduler/manual.hpp>
#include <nano/node/scheduler/priority.hpp>

View file

@ -1,3 +1,4 @@
#include <nano/node/make_store.hpp>
#include <nano/node/node.hpp>
#include <nano/test_common/ledger.hpp>