Minecon724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 1

Merge remote-tracking branch 'nano/develop' into remove-process-confirmed

Browse source 
# Conflicts:
#	nano/lib/stats_enums.hpp
This commit is contained in:
Piotr Wójcik 2024-11-23 17:54:41 +01:00
commit 8221c05b09
74 changed files with 877 additions and 651 deletions
Download patch file Download diff file Expand all files Collapse all files

2
nano/core_test/CMakeLists.txt
View file

@ -4,6 +4,7 @@ add_executable(
fakes/websocket_client.hpp
fakes/work_peer.hpp
active_elections.cpp
assert.cpp
async.cpp
backlog.cpp
block.cpp
@ -42,6 +43,7 @@ add_executable(
processor_service.cpp
random.cpp
random_pool.cpp
rate_limiting.cpp
rep_crawler.cpp
receivable.cpp
peer_history.cpp

15
nano/core_test/assert.cpp Normal file
View file

@ -0,0 +1,15 @@
#include <nano/lib/assert.hpp>
#include <gtest/gtest.h>
TEST (assert_DeathTest, debug_assert)
{
debug_assert (true);
ASSERT_DEATH (debug_assert (false), ".*Assertion \\(false\\) failed.*");
}
TEST (assert_DeathTest, release_assert)
{
release_assert (true);
ASSERT_DEATH (release_assert (false), ".*Assertion \\(false\\) failed.*");
}

6
nano/core_test/block_store.cpp
View file

@ -1,6 +1,7 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/lmdbconfig.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/stats.hpp>
@ -1630,11 +1631,8 @@ TEST (block_store, final_vote)
ASSERT_EQ (store->final_vote.count (transaction), 0);
store->final_vote.put (transaction, qualified_root, nano::block_hash (2));
ASSERT_EQ (store->final_vote.count (transaction), 1);
// Clearing with incorrect root shouldn't remove
store->final_vote.clear (transaction, qualified_root.previous ());
ASSERT_EQ (store->final_vote.count (transaction), 1);
// Clearing with correct root should remove
store->final_vote.clear (transaction, qualified_root.root ());
store->final_vote.del (transaction, qualified_root);
ASSERT_EQ (store->final_vote.count (transaction), 0);
}
}

70
nano/core_test/bootstrap.cpp
View file

@ -27,11 +27,13 @@ nano::block_hash random_hash ()
}
}
/*
* account_sets
*/
TEST (account_sets, construction)
{
nano::test::system system;
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
}
@ -41,8 +43,6 @@ TEST (account_sets, empty_blocked)
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
ASSERT_FALSE (sets.blocked (account));
@ -53,10 +53,9 @@ TEST (account_sets, block)
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
sets.priority_up (account);
sets.block (account, random_hash ());
ASSERT_TRUE (sets.blocked (account));
}
@ -66,12 +65,12 @@ TEST (account_sets, unblock)
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
auto hash = random_hash ();
sets.priority_up (account);
sets.block (account, hash);
ASSERT_TRUE (sets.blocked (account));
sets.unblock (account, hash);
ASSERT_FALSE (sets.blocked (account));
}
@ -81,8 +80,6 @@ TEST (account_sets, priority_base)
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
ASSERT_EQ (0.0, sets.priority (account));
@ -93,32 +90,26 @@ TEST (account_sets, priority_blocked)
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
sets.block (account, random_hash ());
ASSERT_EQ (0.0, sets.priority (account));
}
// When account is unblocked, check that it retains it former priority
TEST (account_sets, priority_unblock_keep)
TEST (account_sets, priority_unblock)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
sets.priority_up (account);
sets.priority_up (account);
ASSERT_EQ (sets.priority (account), nano::bootstrap::account_sets::priority_initial + nano::bootstrap::account_sets::priority_increase);
ASSERT_EQ (sets.priority (account), nano::bootstrap::account_sets::priority_initial);
auto hash = random_hash ();
sets.block (account, hash);
ASSERT_EQ (0.0, sets.priority (account));
sets.unblock (account, hash);
ASSERT_EQ (sets.priority (account), nano::bootstrap::account_sets::priority_initial + nano::bootstrap::account_sets::priority_increase);
ASSERT_EQ (sets.priority (account), nano::bootstrap::account_sets::priority_initial);
}
TEST (account_sets, priority_up_down)
@ -126,37 +117,58 @@ TEST (account_sets, priority_up_down)
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
sets.priority_up (account);
ASSERT_EQ (sets.priority (account), nano::bootstrap::account_sets::priority_initial);
sets.priority_down (account);
ASSERT_EQ (sets.priority (account), nano::bootstrap::account_sets::priority_initial / nano::bootstrap::account_sets::priority_divide);
ASSERT_EQ (sets.priority (account), nano::bootstrap::account_sets::priority_initial);
}
TEST (account_sets, priority_down_sat)
TEST (account_sets, priority_down_empty)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
sets.priority_down (account);
ASSERT_EQ (0.0, sets.priority (account));
}
TEST (account_sets, priority_down_saturate)
{
nano::test::system system;
nano::account account{ 1 };
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
sets.priority_up (account);
ASSERT_EQ (sets.priority (account), nano::bootstrap::account_sets::priority_initial);
for (int n = 0; n < 1000; ++n)
{
sets.priority_down (account);
}
ASSERT_FALSE (sets.prioritized (account));
}
TEST (account_sets, priority_set)
{
nano::test::system system;
nano::account account{ 1 };
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
sets.priority_set (account, 10.0);
ASSERT_EQ (sets.priority (account), 10.0);
}
// Ensure priority value is bounded
TEST (account_sets, saturate_priority)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap::account_sets sets{ config, system.stats };
for (int n = 0; n < 1000; ++n)
@ -166,6 +178,10 @@ TEST (account_sets, saturate_priority)
ASSERT_EQ (sets.priority (account), nano::bootstrap::account_sets::priority_max);
}
/*
* bootstrap
*/
/**
* Tests the base case for returning
*/

1
nano/core_test/entry.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/files.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/memory.hpp>
#include <nano/secure/utility.hpp>

43
nano/core_test/ledger.cpp
View file

@ -5482,8 +5482,12 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
ASSERT_FALSE (rocksdb_store.confirmation_height.get (rocksdb_transaction, nano::dev::genesis_key.pub, confirmation_height_info));
ASSERT_EQ (confirmation_height_info.height, 2);
ASSERT_EQ (confirmation_height_info.frontier, send->hash ());
ASSERT_EQ (rocksdb_store.final_vote.get (rocksdb_transaction, nano::root (send->previous ())).size (), 1);
ASSERT_EQ (rocksdb_store.final_vote.get (rocksdb_transaction, nano::root (send->previous ()))[0], nano::block_hash (2));
ASSERT_TRUE (rocksdb_store.final_vote.get (rocksdb_transaction, send->qualified_root ()).has_value ());
ASSERT_EQ (rocksdb_store.final_vote.get (rocksdb_transaction, send->qualified_root ()).value (), nano::block_hash (2));
// Retry migration while rocksdb folder is still present
auto error_on_retry = ledger.migrate_lmdb_to_rocksdb (path);
ASSERT_EQ (error_on_retry, true);
}
TEST (ledger, is_send_genesis)
@ -5857,3 +5861,38 @@ TEST (ledger_transaction, write_wait_order)
// Signal to continue and drop the third transaction
latch3.count_down ();
}
TEST (ledger_transaction, multithreaded_interleaving)
{
nano::test::system system;
auto ctx = nano::test::ledger_empty ();
int constexpr num_threads = 2;
int constexpr num_iterations = 10;
int constexpr num_blocks = 10;
std::deque<std::thread> threads;
for (int i = 0; i < num_threads; ++i)
{
threads.emplace_back ([&] {
for (int n = 0; n < num_iterations; ++n)
{
auto tx = ctx.ledger ().tx_begin_write (nano::store::writer::testing);
for (unsigned k = 0; k < num_blocks; ++k)
{
ctx.store ().account.put (tx, nano::account{ k }, nano::account_info{});
}
for (unsigned k = 0; k < num_blocks; ++k)
{
ctx.store ().account.del (tx, nano::account{ k });
}
}
});
}
for (auto & thread : threads)
{
thread.join ();
}
}

31
nano/core_test/node.cpp
View file

@ -2157,6 +2157,13 @@ TEST (node, epoch_conflict_confirm)
nano::keypair key;
nano::keypair epoch_signer (nano::dev::genesis_key);
nano::state_block_builder builder;
// Node 1 is the voting node
// Send sends to an account we control: send -> open -> change
// Send2 sends to an account with public key of the open block
// Epoch open qualified root: (open, 0) on account with the same public key as the hash of the open block
// Epoch open and change have the same root!
auto send = builder.make_block ()
.account (nano::dev::genesis_key.pub)
.previous (nano::dev::genesis->hash ())
@ -2203,34 +2210,30 @@ TEST (node, epoch_conflict_confirm)
.work (*system.work.generate (open->hash ()))
.build ();
// Process initial blocks on node1
ASSERT_TRUE (nano::test::process (node1, { send, send2, open }));
// Process initial blocks
ASSERT_TRUE (nano::test::process (node0, nano::test::clone ({ send, send2, open })));
ASSERT_TRUE (nano::test::process (node1, nano::test::clone ({ send, send2, open })));
// Confirm open block in node1 to allow generating votes
nano::test::confirm (node1.ledger, open);
// Process initial blocks on node0
ASSERT_TRUE (nano::test::process (node0, { send, send2, open }));
// Process conflicting blocks on node 0 as blocks coming from live network
ASSERT_TRUE (nano::test::process_live (node0, { change, epoch_open }));
// Process conflicting blocks on nodes as blocks coming from live network
ASSERT_TRUE (nano::test::process_live (node0, nano::test::clone ({ change, epoch_open })));
ASSERT_TRUE (nano::test::process_live (node1, nano::test::clone ({ change, epoch_open })));
// Ensure blocks were propagated to both nodes
ASSERT_TIMELY (5s, nano::test::exists (node0, { change, epoch_open }));
ASSERT_TIMELY (5s, nano::test::exists (node1, { change, epoch_open }));
// Confirm initial blocks in node1 to allow generating votes later
ASSERT_TRUE (nano::test::start_elections (system, node1, { change, epoch_open, send2 }, true));
nano::test::confirm (node1, { change, epoch_open, send2 });
ASSERT_TIMELY (5s, nano::test::confirmed (node1, { change, epoch_open, send2 }));
// Start elections for node0 for conflicting change and epoch_open blocks (those two blocks have the same root)
// Start elections on node0 for conflicting change and epoch_open blocks (these two blocks have the same root)
ASSERT_TRUE (nano::test::activate (node0, { change, epoch_open }));
ASSERT_TIMELY (5s, nano::test::active (node0, { change, epoch_open }));
// Make node1 a representative
// Make node1 a representative so it can vote for both blocks
system.wallet (1)->insert_adhoc (nano::dev::genesis_key.prv);
// Ensure the elections for conflicting blocks have completed
// Ensure the elections for conflicting blocks have started
ASSERT_TIMELY (5s, nano::test::active (node0, { change, epoch_open }));
// Ensure both conflicting blocks were successfully processed and confirmed

113
nano/core_test/rate_limiting.cpp Normal file
View file

@ -0,0 +1,113 @@
#include <nano/lib/rate_limiting.hpp>
#include <nano/lib/utility.hpp>
#include <gtest/gtest.h>
#include <fstream>
#include <future>
using namespace std::chrono_literals;
TEST (rate, basic)
{
nano::rate::token_bucket bucket (10, 10);
// Initial burst
ASSERT_TRUE (bucket.try_consume (10));
ASSERT_FALSE (bucket.try_consume (10));
// With a fill rate of 10 tokens/sec, await 1/3 sec and get 3 tokens
std::this_thread::sleep_for (300ms);
ASSERT_TRUE (bucket.try_consume (3));
ASSERT_FALSE (bucket.try_consume (10));
// Allow time for the bucket to completely refill and do a full burst
std::this_thread::sleep_for (1s);
ASSERT_TRUE (bucket.try_consume (10));
ASSERT_EQ (bucket.largest_burst (), 10);
}
TEST (rate, network)
{
// For the purpose of the test, one token represents 1MB instead of one byte.
// Allow for 10 mb/s bursts (max bucket size), 5 mb/s long term rate
nano::rate::token_bucket bucket (10, 5);
// Initial burst of 10 mb/s over two calls
ASSERT_TRUE (bucket.try_consume (5));
ASSERT_EQ (bucket.largest_burst (), 5);
ASSERT_TRUE (bucket.try_consume (5));
ASSERT_EQ (bucket.largest_burst (), 10);
ASSERT_FALSE (bucket.try_consume (5));
// After 200 ms, the 5 mb/s fillrate means we have 1 mb available
std::this_thread::sleep_for (200ms);
ASSERT_TRUE (bucket.try_consume (1));
ASSERT_FALSE (bucket.try_consume (1));
}
TEST (rate, reset)
{
nano::rate::token_bucket bucket (0, 0);
// consume lots of tokens, buckets should be unlimited
ASSERT_TRUE (bucket.try_consume (1000000));
ASSERT_TRUE (bucket.try_consume (1000000));
// set bucket to be limited
bucket.reset (1000, 1000);
ASSERT_FALSE (bucket.try_consume (1001));
ASSERT_TRUE (bucket.try_consume (1000));
ASSERT_FALSE (bucket.try_consume (1000));
std::this_thread::sleep_for (2ms);
ASSERT_TRUE (bucket.try_consume (2));
// reduce the limit
bucket.reset (100, 100 * 1000);
ASSERT_FALSE (bucket.try_consume (101));
ASSERT_TRUE (bucket.try_consume (100));
std::this_thread::sleep_for (1ms);
ASSERT_TRUE (bucket.try_consume (100));
// increase the limit
bucket.reset (2000, 1);
ASSERT_FALSE (bucket.try_consume (2001));
ASSERT_TRUE (bucket.try_consume (2000));
// back to unlimited
bucket.reset (0, 0);
ASSERT_TRUE (bucket.try_consume (1000000));
ASSERT_TRUE (bucket.try_consume (1000000));
}
TEST (rate, unlimited)
{
nano::rate::token_bucket bucket (0, 0);
ASSERT_TRUE (bucket.try_consume (5));
ASSERT_EQ (bucket.largest_burst (), 5);
ASSERT_TRUE (bucket.try_consume (static_cast<size_t> (1e9)));
ASSERT_EQ (bucket.largest_burst (), static_cast<size_t> (1e9));
// With unlimited tokens, consuming always succeed
ASSERT_TRUE (bucket.try_consume (static_cast<size_t> (1e9)));
ASSERT_EQ (bucket.largest_burst (), static_cast<size_t> (1e9));
}
TEST (rate, busy_spin)
{
// Bucket should refill at a rate of 1 token per second
nano::rate::token_bucket bucket (1, 1);
// Run a very tight loop for 5 seconds + a bit of wiggle room
int counter = 0;
for (auto start = std::chrono::steady_clock::now (), now = start; now < start + 5500ms; now = std::chrono::steady_clock::now ())
{
if (bucket.try_consume ())
{
++counter;
}
}
// Bucket starts fully refilled, therefore we see 1 additional request
ASSERT_EQ (counter, 6);
}

105
nano/core_test/utility.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/files.hpp>
#include <nano/lib/optional_ptr.hpp>
#include <nano/lib/rate_limiting.hpp>
#include <nano/lib/relaxed_atomic.hpp>
@ -15,110 +16,6 @@
using namespace std::chrono_literals;
TEST (rate, basic)
{
nano::rate::token_bucket bucket (10, 10);
// Initial burst
ASSERT_TRUE (bucket.try_consume (10));
ASSERT_FALSE (bucket.try_consume (10));
// With a fill rate of 10 tokens/sec, await 1/3 sec and get 3 tokens
std::this_thread::sleep_for (300ms);
ASSERT_TRUE (bucket.try_consume (3));
ASSERT_FALSE (bucket.try_consume (10));
// Allow time for the bucket to completely refill and do a full burst
std::this_thread::sleep_for (1s);
ASSERT_TRUE (bucket.try_consume (10));
ASSERT_EQ (bucket.largest_burst (), 10);
}
TEST (rate, network)
{
// For the purpose of the test, one token represents 1MB instead of one byte.
// Allow for 10 mb/s bursts (max bucket size), 5 mb/s long term rate
nano::rate::token_bucket bucket (10, 5);
// Initial burst of 10 mb/s over two calls
ASSERT_TRUE (bucket.try_consume (5));
ASSERT_EQ (bucket.largest_burst (), 5);
ASSERT_TRUE (bucket.try_consume (5));
ASSERT_EQ (bucket.largest_burst (), 10);
ASSERT_FALSE (bucket.try_consume (5));
// After 200 ms, the 5 mb/s fillrate means we have 1 mb available
std::this_thread::sleep_for (200ms);
ASSERT_TRUE (bucket.try_consume (1));
ASSERT_FALSE (bucket.try_consume (1));
}
TEST (rate, reset)
{
nano::rate::token_bucket bucket (0, 0);
// consume lots of tokens, buckets should be unlimited
ASSERT_TRUE (bucket.try_consume (1000000));
ASSERT_TRUE (bucket.try_consume (1000000));
// set bucket to be limited
bucket.reset (1000, 1000);
ASSERT_FALSE (bucket.try_consume (1001));
ASSERT_TRUE (bucket.try_consume (1000));
ASSERT_FALSE (bucket.try_consume (1000));
std::this_thread::sleep_for (2ms);
ASSERT_TRUE (bucket.try_consume (2));
// reduce the limit
bucket.reset (100, 100 * 1000);
ASSERT_FALSE (bucket.try_consume (101));
ASSERT_TRUE (bucket.try_consume (100));
std::this_thread::sleep_for (1ms);
ASSERT_TRUE (bucket.try_consume (100));
// increase the limit
bucket.reset (2000, 1);
ASSERT_FALSE (bucket.try_consume (2001));
ASSERT_TRUE (bucket.try_consume (2000));
// back to unlimited
bucket.reset (0, 0);
ASSERT_TRUE (bucket.try_consume (1000000));
ASSERT_TRUE (bucket.try_consume (1000000));
}
TEST (rate, unlimited)
{
nano::rate::token_bucket bucket (0, 0);
ASSERT_TRUE (bucket.try_consume (5));
ASSERT_EQ (bucket.largest_burst (), 5);
ASSERT_TRUE (bucket.try_consume (static_cast<size_t> (1e9)));
ASSERT_EQ (bucket.largest_burst (), static_cast<size_t> (1e9));
// With unlimited tokens, consuming always succeed
ASSERT_TRUE (bucket.try_consume (static_cast<size_t> (1e9)));
ASSERT_EQ (bucket.largest_burst (), static_cast<size_t> (1e9));
}
TEST (rate, busy_spin)
{
// Bucket should refill at a rate of 1 token per second
nano::rate::token_bucket bucket (1, 1);
// Run a very tight loop for 5 seconds + a bit of wiggle room
int counter = 0;
for (auto start = std::chrono::steady_clock::now (), now = start; now < start + std::chrono::milliseconds{ 5500 }; now = std::chrono::steady_clock::now ())
{
if (bucket.try_consume ())
{
++counter;
}
}
// Bucket starts fully refilled, therefore we see 1 additional request
ASSERT_EQ (counter, 6);
}
TEST (optional_ptr, basic)
{
struct valtype

4
nano/lib/CMakeLists.txt
View file

@ -21,6 +21,8 @@ add_library(
${platform_sources}
asio.hpp
asio.cpp
assert.hpp
assert.cpp
block_sideband.hpp
block_sideband.cpp
block_type.hpp
@ -49,6 +51,8 @@ add_library(
epochs.hpp
errors.hpp
errors.cpp
files.hpp
files.cpp
fwd.hpp
id_dispenser.hpp
interval.hpp

49
nano/lib/assert.cpp Normal file
View file

@ -0,0 +1,49 @@
#include <nano/lib/assert.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/stacktrace.hpp>
#include <boost/dll/runtime_symbol_info.hpp>
#include <fstream>
#include <iostream>
/*
* Backing code for "release_assert" & "debug_assert", which are macros
*/
void assert_internal (char const * check_expr, char const * func, char const * file, unsigned int line, bool is_release_assert, std::string_view error_msg)
{
std::cerr << "Assertion (" << check_expr << ") failed\n"
<< func << "\n"
<< file << ":" << line << "\n";
if (!error_msg.empty ())
{
std::cerr << "Error: " << error_msg << "\n";
}
std::cerr << "\n";
// Output stack trace to cerr
auto backtrace_str = nano::generate_stacktrace ();
std::cerr << backtrace_str << std::endl;
// "abort" at the end of this function will go into any signal handlers (the daemon ones will generate a stack trace and load memory address files on non-Windows systems).
// As there is no async-signal-safe way to generate stacktraces on Windows it must be done before aborting
#ifdef _WIN32
{
// Try construct the stacktrace dump in the same folder as the running executable, otherwise use the current directory.
boost::system::error_code err;
auto running_executable_filepath = boost::dll::program_location (err);
std::string filename = is_release_assert ? "nano_node_backtrace_release_assert.txt" : "nano_node_backtrace_assert.txt";
std::string filepath = filename;
if (!err)
{
filepath = (running_executable_filepath.parent_path () / filename).string ();
}
std::ofstream file (filepath);
nano::set_secure_perm_file (filepath);
file << backtrace_str;
}
#endif
abort ();
}

33
nano/lib/assert.hpp Normal file
View file

@ -0,0 +1,33 @@
#pragma once
#include <boost/current_function.hpp>
#include <boost/preprocessor/facilities/empty.hpp>
#include <boost/preprocessor/facilities/overload.hpp>
#include <string_view>
[[noreturn]] void assert_internal (char const * check_expr, char const * func, char const * file, unsigned int line, bool is_release_assert, std::string_view error = "");
#define release_assert_1(check) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, true)
#define release_assert_2(check, error_msg) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, true, error_msg)
#if !BOOST_PP_VARIADICS_MSVC
#define release_assert(...) \
BOOST_PP_OVERLOAD (release_assert_, __VA_ARGS__) \
(__VA_ARGS__)
#else
#define release_assert(...) BOOST_PP_CAT (BOOST_PP_OVERLOAD (release_assert_, __VA_ARGS__) (__VA_ARGS__), BOOST_PP_EMPTY ())
#endif
#ifdef NDEBUG
#define debug_assert(...) (void)0
#else
#define debug_assert_1(check) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, false)
#define debug_assert_2(check, error_msg) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, false, error_msg)
#if !BOOST_PP_VARIADICS_MSVC
#define debug_assert(...) \
BOOST_PP_OVERLOAD (debug_assert_, __VA_ARGS__) \
(__VA_ARGS__)
#else
#define debug_assert(...) BOOST_PP_CAT (BOOST_PP_OVERLOAD (debug_assert_, __VA_ARGS__) (__VA_ARGS__), BOOST_PP_EMPTY ())
#endif
#endif

27
nano/lib/blocks.cpp
View file

@ -210,7 +210,7 @@ nano::block_hash nano::block::full_hash () const
nano::block_sideband const & nano::block::sideband () const
{
debug_assert (sideband_m.is_initialized ());
release_assert (sideband_m.is_initialized ());
return *sideband_m;
}
@ -569,6 +569,11 @@ nano::send_block::send_block (bool & error_a, boost::property_tree::ptree const
}
}
std::shared_ptr<nano::block> nano::send_block::clone () const
{
return std::make_shared<nano::send_block> (*this);
}
bool nano::send_block::operator== (nano::block const & other_a) const
{
return blocks_equal (*this, other_a);
@ -758,6 +763,11 @@ nano::open_block::open_block (bool & error_a, boost::property_tree::ptree const
}
}
std::shared_ptr<nano::block> nano::open_block::clone () const
{
return std::make_shared<nano::open_block> (*this);
}
void nano::open_block::generate_hash (blake2b_state & hash_a) const
{
hashables.hash (hash_a);
@ -1029,6 +1039,11 @@ nano::change_block::change_block (bool & error_a, boost::property_tree::ptree co
}
}
std::shared_ptr<nano::block> nano::change_block::clone () const
{
return std::make_shared<nano::change_block> (*this);
}
void nano::change_block::generate_hash (blake2b_state & hash_a) const
{
hashables.hash (hash_a);
@ -1326,6 +1341,11 @@ nano::state_block::state_block (bool & error_a, boost::property_tree::ptree cons
}
}
std::shared_ptr<nano::block> nano::state_block::clone () const
{
return std::make_shared<nano::state_block> (*this);
}
void nano::state_block::generate_hash (blake2b_state & hash_a) const
{
nano::uint256_union preamble (static_cast<uint64_t> (nano::block_type::state));
@ -1792,6 +1812,11 @@ nano::receive_block::receive_block (bool & error_a, boost::property_tree::ptree
}
}
std::shared_ptr<nano::block> nano::receive_block::clone () const
{
return std::make_shared<nano::receive_block> (*this);
}
void nano::receive_block::generate_hash (blake2b_state & hash_a) const
{
hashables.hash (hash_a);

8
nano/lib/blocks.hpp
View file

@ -23,6 +23,7 @@ namespace nano
class block
{
public:
virtual ~block () = default;
// Return a digest of the hashables in this block.
nano::block_hash const & hash () const;
// Return a digest of hashables and non-hashables in this block.
@ -46,10 +47,10 @@ public:
virtual nano::block_type type () const = 0;
virtual nano::signature const & block_signature () const = 0;
virtual void signature_set (nano::signature const &) = 0;
virtual ~block () = default;
virtual bool valid_predecessor (nano::block const &) const = 0;
static size_t size (nano::block_type);
virtual nano::work_version work_version () const;
virtual std::shared_ptr<nano::block> clone () const = 0;
// If there are any changes to the hashables, call this to update the cached hash
void refresh ();
bool is_send () const noexcept;
@ -138,6 +139,7 @@ public:
bool operator== (nano::block const &) const override;
bool operator== (nano::send_block const &) const;
bool valid_predecessor (nano::block const &) const override;
std::shared_ptr<nano::block> clone () const override;
send_hashables hashables;
nano::signature signature;
uint64_t work;
@ -192,6 +194,7 @@ public:
bool operator== (nano::block const &) const override;
bool operator== (nano::receive_block const &) const;
bool valid_predecessor (nano::block const &) const override;
std::shared_ptr<nano::block> clone () const override;
receive_hashables hashables;
nano::signature signature;
uint64_t work;
@ -247,6 +250,7 @@ public:
bool operator== (nano::block const &) const override;
bool operator== (nano::open_block const &) const;
bool valid_predecessor (nano::block const &) const override;
std::shared_ptr<nano::block> clone () const override;
nano::open_hashables hashables;
nano::signature signature;
uint64_t work;
@ -302,6 +306,7 @@ public:
bool operator== (nano::block const &) const override;
bool operator== (nano::change_block const &) const;
bool valid_predecessor (nano::block const &) const override;
std::shared_ptr<nano::block> clone () const override;
nano::change_hashables hashables;
nano::signature signature;
uint64_t work;
@ -368,6 +373,7 @@ public:
bool operator== (nano::block const &) const override;
bool operator== (nano::state_block const &) const;
bool valid_predecessor (nano::block const &) const override;
std::shared_ptr<nano::block> clone () const override;
nano::state_hashables hashables;
nano::signature signature;
uint64_t work;

85
nano/lib/files.cpp Normal file
View file

@ -0,0 +1,85 @@
#include <nano/lib/files.hpp>
#include <cstddef>
#include <cstring>
#include <fstream>
#include <iostream>
#include <limits>
#include <sstream>
#include <string_view>
#include <thread>
#ifndef _WIN32
#include <sys/resource.h>
#endif
std::size_t nano::get_file_descriptor_limit ()
{
std::size_t fd_limit = std::numeric_limits<std::size_t>::max ();
#ifndef _WIN32
rlimit limit{};
if (getrlimit (RLIMIT_NOFILE, &limit) == 0)
{
fd_limit = static_cast<std::size_t> (limit.rlim_cur);
}
#endif
return fd_limit;
}
void nano::set_file_descriptor_limit (std::size_t limit)
{
#ifndef _WIN32
rlimit fd_limit{};
if (-1 == getrlimit (RLIMIT_NOFILE, &fd_limit))
{
std::cerr << "WARNING: Unable to get current limits for the number of open file descriptors: " << std::strerror (errno);
return;
}
if (fd_limit.rlim_cur >= limit)
{
return;
}
fd_limit.rlim_cur = std::min (static_cast<rlim_t> (limit), fd_limit.rlim_max);
if (-1 == setrlimit (RLIMIT_NOFILE, &fd_limit))
{
std::cerr << "WARNING: Unable to set limits for the number of open file descriptors: " << std::strerror (errno);
return;
}
#endif
}
void nano::initialize_file_descriptor_limit ()
{
nano::set_file_descriptor_limit (DEFAULT_FILE_DESCRIPTOR_LIMIT);
auto limit = nano::get_file_descriptor_limit ();
if (limit < DEFAULT_FILE_DESCRIPTOR_LIMIT)
{
std::cerr << "WARNING: Current file descriptor limit of " << limit << " is lower than the " << DEFAULT_FILE_DESCRIPTOR_LIMIT << " recommended. Node was unable to change it." << std::endl;
}
}
void nano::remove_all_files_in_dir (std::filesystem::path const & dir)
{
for (auto & p : std::filesystem::directory_iterator (dir))
{
auto path = p.path ();
if (std::filesystem::is_regular_file (path))
{
std::filesystem::remove (path);
}
}
}
void nano::move_all_files_to_dir (std::filesystem::path const & from, std::filesystem::path const & to)
{
for (auto & p : std::filesystem::directory_iterator (from))
{
auto path = p.path ();
if (std::filesystem::is_regular_file (path))
{
std::filesystem::rename (path, to / path.filename ());
}
}
}

48
nano/lib/files.hpp Normal file
View file

@ -0,0 +1,48 @@
#pragma once
#include <filesystem>
namespace nano
{
/*
* Functions for managing filesystem permissions, platform specific
*/
void set_umask ();
void set_secure_perm_directory (std::filesystem::path const & path);
void set_secure_perm_directory (std::filesystem::path const & path, std::error_code & ec);
void set_secure_perm_file (std::filesystem::path const & path);
void set_secure_perm_file (std::filesystem::path const & path, std::error_code & ec);
/*
* Function to check if running Windows as an administrator
*/
bool is_windows_elevated ();
/*
* Function to check if the Windows Event log registry key exists
*/
bool event_log_reg_entry_exists ();
/*
* Create the load memory addresses for the executable and shared libraries.
*/
void create_load_memory_address_files ();
/**
* Some systems, especially in virtualized environments, may have very low file descriptor limits,
* causing the node to fail. This function attempts to query the limit and returns the value. If the
* limit cannot be queried, or running on a Windows system, this returns max-value of std::size_t.
* Increasing the limit programmatically can be done only for the soft limit, the hard one requiring
* super user permissions to modify.
*/
std::size_t get_file_descriptor_limit ();
void set_file_descriptor_limit (std::size_t limit);
/**
* This should be called from entry points. It sets the file descriptor limit to the maximum allowed and logs any errors.
*/
constexpr std::size_t DEFAULT_FILE_DESCRIPTOR_LIMIT = 16384;
void initialize_file_descriptor_limit ();
void remove_all_files_in_dir (std::filesystem::path const & dir);
void move_all_files_to_dir (std::filesystem::path const & from, std::filesystem::path const & to);
}

1
nano/lib/jsonconfig.cpp
View file

@ -1,4 +1,5 @@
#include <nano/boost/asio/ip/address_v6.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/jsonconfig.hpp>
#include <boost/property_tree/json_parser.hpp>

1
nano/lib/jsonconfig.hpp
View file

@ -7,6 +7,7 @@
#include <boost/lexical_cast.hpp>
#include <boost/property_tree/ptree.hpp>
#include <filesystem>
#include <fstream>
namespace boost

1
nano/lib/plat/default/debugging.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/files.hpp>
#include <nano/lib/utility.hpp>
void nano::create_load_memory_address_files ()

1
nano/lib/plat/linux/debugging.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/files.hpp>
#include <nano/lib/utility.hpp>
#include <cstring>

1
nano/lib/plat/posix/perms.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/files.hpp>
#include <nano/lib/utility.hpp>
#include <sys/stat.h>

1
nano/lib/plat/windows/perms.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/files.hpp>
#include <nano/lib/utility.hpp>
// clang-format off

1
nano/lib/plat/windows/priority.cpp
View file

@ -1,4 +1,5 @@
#include <windows.h>
namespace nano
{
void work_thread_reprioritize ()

21
nano/lib/rate_limiting.cpp
View file

@ -45,11 +45,6 @@ void nano::rate::token_bucket::refill ()
}
}
std::size_t nano::rate::token_bucket::largest_burst () const
{
return max_token_count - smallest_size;
}
void nano::rate::token_bucket::reset (std::size_t max_token_count_a, std::size_t refill_rate_a)
{
// A token count of 0 indicates unlimited capacity. We use 1e9 as
@ -63,6 +58,16 @@ void nano::rate::token_bucket::reset (std::size_t max_token_count_a, std::size_t
last_refill = std::chrono::steady_clock::now ();
}
std::size_t nano::rate::token_bucket::largest_burst () const
{
return max_token_count - smallest_size;
}
std::size_t nano::rate::token_bucket::size () const
{
return current_size;
}
/*
* rate_limiter
*/
@ -82,4 +87,10 @@ void nano::rate_limiter::reset (std::size_t limit_a, double burst_ratio_a)
{
nano::lock_guard<nano::mutex> guard{ mutex };
bucket.reset (static_cast<std::size_t> (limit_a * burst_ratio_a), limit_a);
}
std::size_t nano::rate_limiter::size () const
{
nano::lock_guard<nano::mutex> guard{ mutex };
return bucket.size ();
}

9
nano/lib/rate_limiting.hpp
View file

@ -38,12 +38,13 @@ public:
*/
bool try_consume (unsigned tokens_required = 1);
/** Returns the largest burst observed */
std::size_t largest_burst () const;
/** Update the max_token_count and/or refill_rate_a parameters */
void reset (std::size_t max_token_count, std::size_t refill_rate);
/** Returns the largest burst observed */
std::size_t largest_burst () const;
std::size_t size () const;
private:
void refill ();
@ -71,6 +72,8 @@ public:
bool should_pass (std::size_t buffer_size);
void reset (std::size_t limit, double burst_ratio = 1.0);
std::size_t size () const;
private:
nano::rate::token_bucket bucket;
mutable nano::mutex mutex;

24
nano/lib/stats_enums.hpp
View file

@ -13,14 +13,12 @@ enum class type
_invalid = 0, // Default value, should not be used
test,
traffic_tcp,
error,
message,
block,
ledger,
rollback,
network,
tcp_server,
vote,
vote_processor,
vote_processor_tier,
@ -31,11 +29,14 @@ enum class type
http_callback,
ipc,
tcp,
tcp_server,
tcp_channels,
tcp_channels_rejected,
tcp_channels_purge,
tcp_listener,
tcp_listener_rejected,
traffic_tcp,
traffic_tcp_type,
channel,
socket,
confirmation_height,
@ -69,6 +70,7 @@ enum class type
bootstrap_frontiers,
bootstrap_account_sets,
bootstrap_frontier_scan,
bootstrap_timeout,
bootstrap_server,
bootstrap_server_request,
bootstrap_server_overfill,
@ -144,6 +146,7 @@ enum class detail
retry,
prioritized,
pending,
sync,
requeued,
evicted,
@ -296,6 +299,9 @@ enum class detail
reachout_live,
reachout_cached,
// traffic
generic,
// tcp
tcp_write_drop,
tcp_write_no_socket_drop,
@ -454,13 +460,16 @@ enum class detail
loop_frontiers_processing,
duplicate_request,
invalid_response_type,
invalid_response,
timestamp_reset,
processing_frontiers,
frontiers_dropped,
sync_accounts,
prioritize,
prioritize_failed,
block,
block_failed,
unblock,
unblock_failed,
dependency_update,
@ -481,14 +490,17 @@ enum class detail
next_frontier,
blocking_insert,
blocking_erase_overflow,
blocking_overflow,
priority_insert,
priority_erase_by_threshold,
priority_erase_by_blocking,
priority_erase_overflow,
priority_set,
priority_unblocked,
erase_by_threshold,
erase_by_blocking,
priority_overflow,
deprioritize,
deprioritize_failed,
sync_dependencies,
dependency_synced,
request_blocks,
request_account_info,

2
nano/lib/thread_roles.cpp
View file

@ -106,7 +106,7 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
case nano::thread_role::name::bootstrap_database_scan:
thread_role_name_string = "Bootstrap db";
break;
case nano::thread_role::name::bootstrap_dependendy_walker:
case nano::thread_role::name::bootstrap_dependency_walker:
thread_role_name_string = "Bootstrap walkr";
break;
case nano::thread_role::name::bootstrap_frontier_scan:

2
nano/lib/thread_roles.hpp
View file

@ -41,7 +41,7 @@ enum class name
telemetry,
bootstrap,
bootstrap_database_scan,
bootstrap_dependendy_walker,
bootstrap_dependency_walker,
bootstrap_frontier_scan,
bootstrap_cleanup,
bootstrap_worker,

1
nano/lib/tomlconfig.cpp
View file

@ -1,4 +1,5 @@
#include <nano/boost/asio/ip/address_v6.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/tomlconfig.hpp>
nano::tomlconfig::tomlconfig () :

2
nano/lib/tomlconfig.hpp
View file

@ -6,6 +6,8 @@
#include <boost/lexical_cast.hpp>
#include <boost/optional.hpp>
#include <filesystem>
#include <cpptoml.h>
namespace boost

126
nano/lib/utility.cpp
View file

@ -1,133 +1,7 @@
#include <nano/lib/stacktrace.hpp>
#include <nano/lib/utility.hpp>
#include <boost/dll/runtime_symbol_info.hpp>
#include <boost/program_options.hpp>
#include <cstddef>
#include <fstream>
#include <iostream>
#include <limits>
#include <sstream>
#include <string_view>
#include <thread>
#ifndef _WIN32
#include <sys/resource.h>
#endif
std::size_t nano::get_file_descriptor_limit ()
{
std::size_t fd_limit = std::numeric_limits<std::size_t>::max ();
#ifndef _WIN32
rlimit limit{};
if (getrlimit (RLIMIT_NOFILE, &limit) == 0)
{
fd_limit = static_cast<std::size_t> (limit.rlim_cur);
}
#endif
return fd_limit;
}
void nano::set_file_descriptor_limit (std::size_t limit)
{
#ifndef _WIN32
rlimit fd_limit{};
if (-1 == getrlimit (RLIMIT_NOFILE, &fd_limit))
{
std::cerr << "WARNING: Unable to get current limits for the number of open file descriptors: " << std::strerror (errno);
return;
}
if (fd_limit.rlim_cur >= limit)
{
return;
}
fd_limit.rlim_cur = std::min (static_cast<rlim_t> (limit), fd_limit.rlim_max);
if (-1 == setrlimit (RLIMIT_NOFILE, &fd_limit))
{
std::cerr << "WARNING: Unable to set limits for the number of open file descriptors: " << std::strerror (errno);
return;
}
#endif
}
void nano::initialize_file_descriptor_limit ()
{
nano::set_file_descriptor_limit (DEFAULT_FILE_DESCRIPTOR_LIMIT);
auto limit = nano::get_file_descriptor_limit ();
if (limit < DEFAULT_FILE_DESCRIPTOR_LIMIT)
{
std::cerr << "WARNING: Current file descriptor limit of " << limit << " is lower than the " << DEFAULT_FILE_DESCRIPTOR_LIMIT << " recommended. Node was unable to change it." << std::endl;
}
}
void nano::remove_all_files_in_dir (std::filesystem::path const & dir)
{
for (auto & p : std::filesystem::directory_iterator (dir))
{
auto path = p.path ();
if (std::filesystem::is_regular_file (path))
{
std::filesystem::remove (path);
}
}
}
void nano::move_all_files_to_dir (std::filesystem::path const & from, std::filesystem::path const & to)
{
for (auto & p : std::filesystem::directory_iterator (from))
{
auto path = p.path ();
if (std::filesystem::is_regular_file (path))
{
std::filesystem::rename (path, to / path.filename ());
}
}
}
/*
* Backing code for "release_assert" & "debug_assert", which are macros
*/
void assert_internal (char const * check_expr, char const * func, char const * file, unsigned int line, bool is_release_assert, std::string_view error_msg)
{
std::cerr << "Assertion (" << check_expr << ") failed\n"
<< func << "\n"
<< file << ":" << line << "\n";
if (!error_msg.empty ())
{
std::cerr << "Error: " << error_msg << "\n";
}
std::cerr << "\n";
// Output stack trace to cerr
auto backtrace_str = nano::generate_stacktrace ();
std::cerr << backtrace_str << std::endl;
// "abort" at the end of this function will go into any signal handlers (the daemon ones will generate a stack trace and load memory address files on non-Windows systems).
// As there is no async-signal-safe way to generate stacktraces on Windows it must be done before aborting
#ifdef _WIN32
{
// Try construct the stacktrace dump in the same folder as the running executable, otherwise use the current directory.
boost::system::error_code err;
auto running_executable_filepath = boost::dll::program_location (err);
std::string filename = is_release_assert ? "nano_node_backtrace_release_assert.txt" : "nano_node_backtrace_assert.txt";
std::string filepath = filename;
if (!err)
{
filepath = (running_executable_filepath.parent_path () / filename).string ();
}
std::ofstream file (filepath);
nano::set_secure_perm_file (filepath);
file << backtrace_str;
}
#endif
abort ();
}
// Issue #3748
void nano::sort_options_description (const boost::program_options::options_description & source, boost::program_options::options_description & target)
{

75
nano/lib/utility.hpp
View file

@ -1,17 +1,12 @@
#pragma once
#include <nano/lib/assert.hpp>
#include <nano/lib/container_info.hpp>
#include <nano/lib/locks.hpp>
#include <boost/current_function.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/preprocessor/facilities/empty.hpp>
#include <boost/preprocessor/facilities/overload.hpp>
#include <cassert>
#include <filesystem>
#include <functional>
#include <mutex>
#include <sstream>
#include <vector>
@ -28,79 +23,11 @@ namespace program_options
}
}
[[noreturn]] void assert_internal (char const * check_expr, char const * func, char const * file, unsigned int line, bool is_release_assert, std::string_view error = "");
#define release_assert_1(check) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, true)
#define release_assert_2(check, error_msg) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, true, error_msg)
#if !BOOST_PP_VARIADICS_MSVC
#define release_assert(...) \
BOOST_PP_OVERLOAD (release_assert_, __VA_ARGS__) \
(__VA_ARGS__)
#else
#define release_assert(...) BOOST_PP_CAT (BOOST_PP_OVERLOAD (release_assert_, __VA_ARGS__) (__VA_ARGS__), BOOST_PP_EMPTY ())
#endif
#ifdef NDEBUG
#define debug_assert(...) (void)0
#else
#define debug_assert_1(check) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, false)
#define debug_assert_2(check, error_msg) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, false, error_msg)
#if !BOOST_PP_VARIADICS_MSVC
#define debug_assert(...) \
BOOST_PP_OVERLOAD (debug_assert_, __VA_ARGS__) \
(__VA_ARGS__)
#else
#define debug_assert(...) BOOST_PP_CAT (BOOST_PP_OVERLOAD (debug_assert_, __VA_ARGS__) (__VA_ARGS__), BOOST_PP_EMPTY ())
#endif
#endif
namespace nano
{
// Lower priority of calling work generating thread
void work_thread_reprioritize ();
/*
* Functions for managing filesystem permissions, platform specific
*/
void set_umask ();
void set_secure_perm_directory (std::filesystem::path const & path);
void set_secure_perm_directory (std::filesystem::path const & path, std::error_code & ec);
void set_secure_perm_file (std::filesystem::path const & path);
void set_secure_perm_file (std::filesystem::path const & path, std::error_code & ec);
/*
* Function to check if running Windows as an administrator
*/
bool is_windows_elevated ();
/*
* Function to check if the Windows Event log registry key exists
*/
bool event_log_reg_entry_exists ();
/*
* Create the load memory addresses for the executable and shared libraries.
*/
void create_load_memory_address_files ();
/**
* Some systems, especially in virtualized environments, may have very low file descriptor limits,
* causing the node to fail. This function attempts to query the limit and returns the value. If the
* limit cannot be queried, or running on a Windows system, this returns max-value of std::size_t.
* Increasing the limit programmatically can be done only for the soft limit, the hard one requiring
* super user permissions to modify.
*/
std::size_t get_file_descriptor_limit ();
void set_file_descriptor_limit (std::size_t limit);
/**
* This should be called from entry points. It sets the file descriptor limit to the maximum allowed and logs any errors.
*/
constexpr std::size_t DEFAULT_FILE_DESCRIPTOR_LIMIT = 16384;
void initialize_file_descriptor_limit ();
void remove_all_files_in_dir (std::filesystem::path const & dir);
void move_all_files_to_dir (std::filesystem::path const & from, std::filesystem::path const & to);
template <class InputIt, class OutputIt, class Pred, class Func>
void transform_if (InputIt first, InputIt last, OutputIt dest, Pred pred, Func transform)
{

1
nano/nano_node/daemon.cpp
View file

@ -1,4 +1,5 @@
#include <nano/boost/process/child.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/signal_manager.hpp>
#include <nano/lib/stacktrace.hpp>
#include <nano/lib/thread_runner.hpp>

1
nano/nano_node/entry.cpp
View file

@ -2,6 +2,7 @@
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/cli.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/utility.hpp>
#include <nano/lib/work_version.hpp>

1
nano/nano_rpc/entry.cpp
View file

@ -1,5 +1,6 @@
#include <nano/lib/cli.hpp>
#include <nano/lib/errors.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/signal_manager.hpp>
#include <nano/lib/thread_runner.hpp>

1
nano/nano_wallet/entry.cpp
View file

@ -2,6 +2,7 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/cli.hpp>
#include <nano/lib/errors.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/rpcconfig.hpp>
#include <nano/lib/thread_runner.hpp>

2
nano/node/CMakeLists.txt
View file

@ -166,6 +166,8 @@ add_library(
transport/tcp_server.cpp
transport/tcp_socket.hpp
transport/tcp_socket.cpp
transport/traffic_type.hpp
transport/traffic_type.cpp
transport/transport.hpp
transport/transport.cpp
unchecked_map.cpp

8
nano/node/bandwidth_limiter.cpp
View file

@ -41,6 +41,14 @@ void nano::bandwidth_limiter::reset (std::size_t limit, double burst_ratio, nano
limiter.reset (limit, burst_ratio);
}
nano::container_info nano::bandwidth_limiter::container_info () const
{
nano::container_info info;
info.put ("generic", limiter_generic.size ());
info.put ("bootstrap", limiter_bootstrap.size ());
return info;
}
/*
* bandwidth_limiter_config
*/

2
nano/node/bandwidth_limiter.hpp
View file

@ -37,6 +37,8 @@ public:
*/
void reset (std::size_t limit, double burst_ratio, nano::transport::traffic_type type = nano::transport::traffic_type::generic);
nano::container_info container_info () const;
private:
/**
* Returns reference to limiter corresponding to the limit type

86
nano/node/bootstrap/account_sets.cpp
View file

@ -30,11 +30,11 @@ void nano::bootstrap::account_sets::priority_up (nano::account const & account)
{
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::prioritize);
auto iter = priorities.get<tag_account> ().find (account);
if (iter != priorities.get<tag_account> ().end ())
if (auto it = priorities.get<tag_account> ().find (account); it != priorities.get<tag_account> ().end ())
{
priorities.get<tag_account> ().modify (iter, [] (auto & val) {
priorities.get<tag_account> ().modify (it, [] (auto & val) {
val.priority = std::min ((val.priority + account_sets::priority_increase), account_sets::priority_max);
val.fails = 0;
});
}
else
@ -57,21 +57,19 @@ void nano::bootstrap::account_sets::priority_down (nano::account const & account
return;
}
auto iter = priorities.get<tag_account> ().find (account);
if (iter != priorities.get<tag_account> ().end ())
if (auto it = priorities.get<tag_account> ().find (account); it != priorities.get<tag_account> ().end ())
{
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::deprioritize);
auto priority_new = iter->priority / account_sets::priority_divide;
if (priority_new <= account_sets::priority_cutoff)
if (it->fails >= account_sets::max_fails || it->fails >= it->priority)
{
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_erase_by_threshold);
priorities.get<tag_account> ().erase (iter);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::erase_by_threshold);
priorities.get<tag_account> ().erase (it);
}
else
{
priorities.get<tag_account> ().modify (iter, [priority_new] (auto & val) {
val.priority = priority_new;
priorities.get<tag_account> ().modify (it, [] (auto & val) {
val.fails += 1;
});
}
}
@ -81,7 +79,7 @@ void nano::bootstrap::account_sets::priority_down (nano::account const & account
}
}
void nano::bootstrap::account_sets::priority_set (nano::account const & account)
void nano::bootstrap::account_sets::priority_set (nano::account const & account, double priority)
{
if (account.is_zero ())
{
@ -90,11 +88,10 @@ void nano::bootstrap::account_sets::priority_set (nano::account const & account)
if (!blocked (account))
{
auto iter = priorities.get<tag_account> ().find (account);
if (iter == priorities.get<tag_account> ().end ())
if (!priorities.get<tag_account> ().contains (account))
{
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_insert);
priorities.get<tag_account> ().insert ({ account, account_sets::priority_initial });
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_set);
priorities.get<tag_account> ().insert ({ account, priority });
trim_overflow ();
}
}
@ -108,18 +105,20 @@ void nano::bootstrap::account_sets::block (nano::account const & account, nano::
{
debug_assert (!account.is_zero ());
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::block);
auto erased = priorities.get<tag_account> ().erase (account);
if (erased > 0)
{
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::erase_by_blocking);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::block);
auto existing = priorities.get<tag_account> ().find (account);
auto entry = (existing == priorities.get<tag_account> ().end ()) ? priority_entry{ account, 0 } : *existing;
priorities.get<tag_account> ().erase (account);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_erase_by_blocking);
blocking.get<tag_account> ().insert ({ entry, dependency });
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::blocking_insert);
trim_overflow ();
debug_assert (blocking.get<tag_account> ().count (account) == 0);
blocking.get<tag_account> ().insert ({ account, dependency });
trim_overflow ();
}
else
{
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::block_failed);
}
}
void nano::bootstrap::account_sets::unblock (nano::account const & account, std::optional<nano::block_hash> const & hash)
@ -134,19 +133,11 @@ void nano::bootstrap::account_sets::unblock (nano::account const & account, std:
if (existing != blocking.get<tag_account> ().end () && (!hash || existing->dependency == *hash))
{
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::unblock);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_unblocked);
debug_assert (priorities.get<tag_account> ().count (account) == 0);
if (!existing->original_entry.account.is_zero ())
{
debug_assert (existing->original_entry.account == account);
priorities.get<tag_account> ().insert (existing->original_entry);
}
else
{
priorities.get<tag_account> ().insert ({ account, account_sets::priority_initial });
}
priorities.get<tag_account> ().insert ({ account, account_sets::priority_initial });
blocking.get<tag_account> ().erase (account);
trim_overflow ();
}
else
@ -212,17 +203,17 @@ void nano::bootstrap::account_sets::dependency_update (nano::block_hash const &
void nano::bootstrap::account_sets::trim_overflow ()
{
while (priorities.size () > config.priorities_max)
while (!priorities.empty () && priorities.size () > config.priorities_max)
{
// Erase the oldest entry
priorities.pop_front ();
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_erase_overflow);
// Erase the lowest priority entry
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_overflow);
priorities.get<tag_priority> ().erase (std::prev (priorities.get<tag_priority> ().end ()));
}
while (blocking.size () > config.blocking_max)
while (!blocking.empty () && blocking.size () > config.blocking_max)
{
// Erase the oldest entry
// Erase the lowest priority entry
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::blocking_overflow);
blocking.pop_front ();
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::blocking_erase_overflow);
}
}
@ -275,6 +266,8 @@ nano::block_hash nano::bootstrap::account_sets::next_blocking (std::function<boo
void nano::bootstrap::account_sets::sync_dependencies ()
{
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::sync_dependencies);
// Sample all accounts with a known dependency account (> account 0)
auto begin = blocking.get<tag_dependency_account> ().upper_bound (nano::account{ 0 });
auto end = blocking.get<tag_dependency_account> ().end ();
@ -290,7 +283,7 @@ void nano::bootstrap::account_sets::sync_dependencies ()
if (!blocked (entry.dependency_account) && !prioritized (entry.dependency_account))
{
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::sync_dependencies);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::dependency_synced);
priority_set (entry.dependency_account);
}
}
@ -332,8 +325,7 @@ double nano::bootstrap::account_sets::priority (nano::account const & account) c
{
if (!blocked (account))
{
auto existing = priorities.get<tag_account> ().find (account);
if (existing != priorities.get<tag_account> ().end ())
if (auto existing = priorities.get<tag_account> ().find (account); existing != priorities.get<tag_account> ().end ())
{
return existing->priority;
}

35
nano/node/bootstrap/account_sets.hpp
View file

@ -24,6 +24,14 @@ namespace bootstrap
/** This class tracks accounts various account sets which are shared among the multiple bootstrap threads */
class account_sets
{
public: // Constants
static double constexpr priority_initial = 2.0;
static double constexpr priority_increase = 2.0;
static double constexpr priority_divide = 2.0;
static double constexpr priority_max = 128.0;
static double constexpr priority_cutoff = 0.15;
static unsigned constexpr max_fails = 3;
public:
account_sets (account_sets_config const &, nano::stats &);
@ -38,7 +46,7 @@ namespace bootstrap
* Current implementation divides priority by 2.0f and saturates down to 1.0f.
*/
void priority_down (nano::account const & account);
void priority_set (nano::account const & account);
void priority_set (nano::account const & account, double priority = priority_initial);
void block (nano::account const & account, nano::block_hash const & dependency);
void unblock (nano::account const & account, std::optional<nano::block_hash> const & hash = std::nullopt);
@ -86,27 +94,17 @@ namespace bootstrap
{
nano::account account;
double priority;
id_t id{ generate_id () }; // Uniformly distributed, used for random querying
unsigned fails{ 0 };
std::chrono::steady_clock::time_point timestamp{};
id_t id{ generate_id () }; // Uniformly distributed, used for random querying
};
struct blocking_entry
{
priority_entry original_entry;
nano::account account;
nano::block_hash dependency;
nano::account dependency_account{ 0 };
id_t id{ generate_id () }; // Uniformly distributed, used for random querying
nano::account account () const
{
return original_entry.account;
}
double priority () const
{
return original_entry.priority;
}
};
// clang-format off
@ -135,7 +133,7 @@ namespace bootstrap
mi::indexed_by<
mi::sequenced<mi::tag<tag_sequenced>>,
mi::ordered_unique<mi::tag<tag_account>,
mi::const_mem_fun<blocking_entry, nano::account, &blocking_entry::account>>,
mi::member<blocking_entry, nano::account, &blocking_entry::account>>,
mi::ordered_non_unique<mi::tag<tag_dependency>,
mi::member<blocking_entry, nano::block_hash, &blocking_entry::dependency>>,
mi::ordered_non_unique<mi::tag<tag_dependency_account>,
@ -148,13 +146,6 @@ namespace bootstrap
ordered_priorities priorities;
ordered_blocking blocking;
public: // Constants
static double constexpr priority_initial = 2.0;
static double constexpr priority_increase = 2.0;
static double constexpr priority_divide = 2.0;
static double constexpr priority_max = 128.0;
static double constexpr priority_cutoff = 0.15;
public:
using info_t = std::tuple<decltype (blocking), decltype (priorities)>; // <blocking, priorities>
info_t info () const;

2
nano/node/bootstrap/bootstrap_config.hpp
View file

@ -53,7 +53,7 @@ public:
std::size_t frontier_rate_limit{ 8 };
std::size_t database_warmup_ratio{ 10 };
std::size_t max_pull_count{ nano::bootstrap_server::max_blocks };
std::chrono::milliseconds request_timeout{ 1000 * 5 };
std::chrono::milliseconds request_timeout{ 1000 * 15 };
std::size_t throttle_coefficient{ 8 * 1024 };
std::chrono::milliseconds throttle_wait{ 100 };
std::size_t block_processor_threshold{ 1000 };

71
nano/node/bootstrap/bootstrap_service.cpp
View file

@ -60,7 +60,7 @@ nano::bootstrap_service::~bootstrap_service ()
debug_assert (!database_thread.joinable ());
debug_assert (!dependencies_thread.joinable ());
debug_assert (!frontiers_thread.joinable ());
debug_assert (!timeout_thread.joinable ());
debug_assert (!cleanup_thread.joinable ());
debug_assert (!workers.alive ());
}
@ -70,7 +70,7 @@ void nano::bootstrap_service::start ()
debug_assert (!database_thread.joinable ());
debug_assert (!dependencies_thread.joinable ());
debug_assert (!frontiers_thread.joinable ());
debug_assert (!timeout_thread.joinable ());
debug_assert (!cleanup_thread.joinable ());
if (!config.enable)
{
@ -99,7 +99,7 @@ void nano::bootstrap_service::start ()
if (config.enable_dependency_walker)
{
dependencies_thread = std::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::bootstrap_dependendy_walker);
nano::thread_role::set (nano::thread_role::name::bootstrap_dependency_walker);
run_dependencies ();
});
}
@ -112,7 +112,7 @@ void nano::bootstrap_service::start ()
});
}
timeout_thread = std::thread ([this] () {
cleanup_thread = std::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::bootstrap_cleanup);
run_timeouts ();
});
@ -130,7 +130,7 @@ void nano::bootstrap_service::stop ()
nano::join_or_pass (database_thread);
nano::join_or_pass (dependencies_thread);
nano::join_or_pass (frontiers_thread);
nano::join_or_pass (timeout_thread);
nano::join_or_pass (cleanup_thread);
workers.stop ();
}
@ -418,7 +418,7 @@ nano::block_hash nano::bootstrap_service::next_blocking ()
debug_assert (!mutex.try_lock ());
auto blocking = accounts.next_blocking ([this] (nano::block_hash const & hash) {
return count_tags (hash, query_source::blocking) == 0;
return count_tags (hash, query_source::dependencies) == 0;
});
if (blocking.is_zero ())
{
@ -590,7 +590,7 @@ void nano::bootstrap_service::run_database ()
}
}
void nano::bootstrap_service::run_one_blocking ()
void nano::bootstrap_service::run_one_dependency ()
{
// No need to wait for blockprocessor, as we are not processing blocks
auto channel = wait_channel ();
@ -603,7 +603,7 @@ void nano::bootstrap_service::run_one_blocking ()
{
return;
}
request_info (blocking, channel, query_source::blocking);
request_info (blocking, channel, query_source::dependencies);
}
void nano::bootstrap_service::run_dependencies ()
@ -613,7 +613,7 @@ void nano::bootstrap_service::run_dependencies ()
{
lock.unlock ();
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::loop_dependencies);
run_one_blocking ();
run_one_dependency ();
lock.lock ();
}
}
@ -659,7 +659,7 @@ void nano::bootstrap_service::cleanup_and_sync ()
{
debug_assert (!mutex.try_lock ());
scoring.sync (network.list ());
scoring.sync (network.list (/* all */ 0, network_constants.bootstrap_protocol_version_min));
scoring.timeout ();
throttle.resize (compute_throttle_size ());
@ -673,8 +673,9 @@ void nano::bootstrap_service::cleanup_and_sync ()
while (!tags_by_order.empty () && should_timeout (tags_by_order.front ()))
{
auto tag = tags_by_order.front ();
tags_by_order.pop_front ();
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::timeout);
stats.inc (nano::stat::type::bootstrap_timeout, to_stat_detail (tag.type));
tags_by_order.pop_front ();
}
if (sync_dependencies_interval.elapsed (60s))
@ -746,17 +747,25 @@ void nano::bootstrap_service::process (nano::asc_pull_ack const & message, std::
stats.inc (nano::stat::type::bootstrap_reply, to_stat_detail (tag.type));
stats.sample (nano::stat::sample::bootstrap_tag_duration, nano::log::milliseconds_delta (tag.timestamp), { 0, config.request_timeout.count () });
scoring.received_message (channel);
lock.unlock ();
// Process the response payload
std::visit ([this, &tag] (auto && request) { return process (request, tag); }, message.payload);
bool ok = std::visit ([this, &tag] (auto && request) { return process (request, tag); }, message.payload);
if (ok)
{
lock.lock ();
scoring.received_message (channel);
lock.unlock ();
}
else
{
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::invalid_response);
}
condition.notify_all ();
}
void nano::bootstrap_service::process (const nano::asc_pull_ack::blocks_payload & response, const async_tag & tag)
bool nano::bootstrap_service::process (const nano::asc_pull_ack::blocks_payload & response, const async_tag & tag)
{
debug_assert (tag.type == query_type::blocks_by_hash || tag.type == query_type::blocks_by_account);
@ -824,9 +833,11 @@ void nano::bootstrap_service::process (const nano::asc_pull_ack::blocks_payload
}
break;
}
return result != verify_result::invalid;
}
void nano::bootstrap_service::process (const nano::asc_pull_ack::account_info_payload & response, const async_tag & tag)
bool nano::bootstrap_service::process (const nano::asc_pull_ack::account_info_payload & response, const async_tag & tag)
{
debug_assert (tag.type == query_type::account_info_by_hash);
debug_assert (!tag.hash.is_zero ());
@ -834,7 +845,7 @@ void nano::bootstrap_service::process (const nano::asc_pull_ack::account_info_pa
if (response.account.is_zero ())
{
stats.inc (nano::stat::type::bootstrap_process, nano::stat::detail::account_info_empty);
return;
return true; // OK, but nothing to do
}
stats.inc (nano::stat::type::bootstrap_process, nano::stat::detail::account_info);
@ -843,11 +854,13 @@ void nano::bootstrap_service::process (const nano::asc_pull_ack::account_info_pa
{
nano::lock_guard<nano::mutex> lock{ mutex };
accounts.dependency_update (tag.hash, response.account);
accounts.priority_set (response.account);
accounts.priority_set (response.account, nano::bootstrap::account_sets::priority_cutoff); // Use the lowest possible priority here
}
return true; // OK, no way to verify the response
}
void nano::bootstrap_service::process (const nano::asc_pull_ack::frontiers_payload & response, const async_tag & tag)
bool nano::bootstrap_service::process (const nano::asc_pull_ack::frontiers_payload & response, const async_tag & tag)
{
debug_assert (tag.type == query_type::frontiers);
debug_assert (!tag.start.is_zero ());
@ -855,7 +868,7 @@ void nano::bootstrap_service::process (const nano::asc_pull_ack::frontiers_paylo
if (response.frontiers.empty ())
{
stats.inc (nano::stat::type::bootstrap_process, nano::stat::detail::frontiers_empty);
return;
return true; // OK, but nothing to do
}
stats.inc (nano::stat::type::bootstrap_process, nano::stat::detail::frontiers);
@ -897,12 +910,15 @@ void nano::bootstrap_service::process (const nano::asc_pull_ack::frontiers_paylo
}
break;
}
return result != verify_result::invalid;
}
void nano::bootstrap_service::process (const nano::empty_payload & response, const async_tag & tag)
bool nano::bootstrap_service::process (const nano::empty_payload & response, const async_tag & tag)
{
stats.inc (nano::stat::type::bootstrap_process, nano::stat::detail::empty);
debug_assert (false, "empty payload"); // Should not happen
return false; // Invalid
}
void nano::bootstrap_service::process_frontiers (std::deque<std::pair<nano::account, nano::block_hash>> const & frontiers)
@ -981,7 +997,8 @@ void nano::bootstrap_service::process_frontiers (std::deque<std::pair<nano::acco
for (auto const & account : result)
{
accounts.priority_set (account);
// Use the lowest possible priority here
accounts.priority_set (account, nano::bootstrap::account_sets::priority_cutoff);
}
}
@ -1091,6 +1108,14 @@ nano::container_info nano::bootstrap_service::container_info () const
{
nano::lock_guard<nano::mutex> lock{ mutex };
auto collect_limiters = [this] () {
nano::container_info info;
info.put ("total", limiter.size ());
info.put ("database", database_limiter.size ());
info.put ("frontiers", frontiers_limiter.size ());
return info;
};
nano::container_info info;
info.put ("tags", tags);
info.put ("throttle", throttle.size ());
@ -1099,6 +1124,8 @@ nano::container_info nano::bootstrap_service::container_info () const
info.add ("database_scan", database_scan.container_info ());
info.add ("frontiers", frontiers.container_info ());
info.add ("workers", workers.container_info ());
info.add ("peers", scoring.container_info ());
info.add ("limiters", collect_limiters ());
return info;
}

17
nano/node/bootstrap/bootstrap_service.hpp
View file

@ -76,7 +76,7 @@ public: // Tag
invalid,
priority,
database,
blocking,
dependencies,
frontiers,
};
@ -104,9 +104,9 @@ private:
void run_database ();
void run_one_database (bool should_throttle);
void run_dependencies ();
void run_one_blocking ();
void run_one_frontier ();
void run_one_dependency ();
void run_frontiers ();
void run_one_frontier ();
void run_timeouts ();
void cleanup_and_sync ();
@ -134,10 +134,10 @@ private:
bool request_frontiers (nano::account, std::shared_ptr<nano::transport::channel> const &, query_source);
bool send (std::shared_ptr<nano::transport::channel> const &, async_tag tag);
void process (nano::asc_pull_ack::blocks_payload const & response, async_tag const & tag);
void process (nano::asc_pull_ack::account_info_payload const & response, async_tag const & tag);
void process (nano::asc_pull_ack::frontiers_payload const & response, async_tag const & tag);
void process (nano::empty_payload const & response, async_tag const & tag);
bool process (nano::asc_pull_ack::blocks_payload const & response, async_tag const & tag);
bool process (nano::asc_pull_ack::account_info_payload const & response, async_tag const & tag);
bool process (nano::asc_pull_ack::frontiers_payload const & response, async_tag const & tag);
bool process (nano::empty_payload const & response, async_tag const & tag);
void process_frontiers (std::deque<std::pair<nano::account, nano::block_hash>> const & frontiers);
@ -194,6 +194,7 @@ private:
// Requests for accounts from database have much lower hitrate and could introduce strain on the network
// A separate (lower) limiter ensures that we always reserve resources for querying accounts from priority queue
nano::rate_limiter database_limiter;
// Rate limiter for frontier requests
nano::rate_limiter frontiers_limiter;
nano::interval sync_dependencies_interval;
@ -205,7 +206,7 @@ private:
std::thread database_thread;
std::thread dependencies_thread;
std::thread frontiers_thread;
std::thread timeout_thread;
std::thread cleanup_thread;
nano::thread_pool workers;
nano::random_generator_mt rng;

64
nano/node/bootstrap/peer_scoring.cpp
View file

@ -12,7 +12,17 @@ nano::bootstrap::peer_scoring::peer_scoring (bootstrap_config const & config_a,
{
}
bool nano::bootstrap::peer_scoring::try_send_message (std::shared_ptr<nano::transport::channel> channel)
bool nano::bootstrap::peer_scoring::limit_exceeded (std::shared_ptr<nano::transport::channel> const & channel) const
{
auto & index = scoring.get<tag_channel> ();
if (auto existing = index.find (channel.get ()); existing != index.end ())
{
return existing->outstanding >= config.channel_limit;
}
return false;
}
bool nano::bootstrap::peer_scoring::try_send_message (std::shared_ptr<nano::transport::channel> const & channel)
{
auto & index = scoring.get<tag_channel> ();
auto existing = index.find (channel.get ());
@ -38,11 +48,10 @@ bool nano::bootstrap::peer_scoring::try_send_message (std::shared_ptr<nano::tran
return false;
}
void nano::bootstrap::peer_scoring::received_message (std::shared_ptr<nano::transport::channel> channel)
void nano::bootstrap::peer_scoring::received_message (std::shared_ptr<nano::transport::channel> const & channel)
{
auto & index = scoring.get<tag_channel> ();
auto existing = index.find (channel.get ());
if (existing != index.end ())
if (auto existing = index.find (channel.get ()); existing != index.end ())
{
if (existing->outstanding > 1)
{
@ -57,17 +66,13 @@ void nano::bootstrap::peer_scoring::received_message (std::shared_ptr<nano::tran
std::shared_ptr<nano::transport::channel> nano::bootstrap::peer_scoring::channel ()
{
auto & index = scoring.get<tag_outstanding> ();
for (auto const & score : index)
for (auto const & channel : channels)
{
if (auto channel = score.shared ())
if (!channel->max (nano::transport::traffic_type::bootstrap))
{
if (!channel->max ())
if (!try_send_message (channel))
{
if (!try_send_message (channel))
{
return channel;
}
return channel;
}
}
}
@ -79,11 +84,16 @@ std::size_t nano::bootstrap::peer_scoring::size () const
return scoring.size ();
}
std::size_t nano::bootstrap::peer_scoring::available () const
{
return std::count_if (channels.begin (), channels.end (), [this] (auto const & channel) {
return !limit_exceeded (channel);
});
}
void nano::bootstrap::peer_scoring::timeout ()
{
auto & index = scoring.get<tag_channel> ();
erase_if (index, [] (auto const & score) {
erase_if (scoring, [] (auto const & score) {
if (auto channel = score.shared ())
{
if (channel->alive ())
@ -104,20 +114,16 @@ void nano::bootstrap::peer_scoring::timeout ()
void nano::bootstrap::peer_scoring::sync (std::deque<std::shared_ptr<nano::transport::channel>> const & list)
{
auto & index = scoring.get<tag_channel> ();
for (auto const & channel : list)
{
if (channel->get_network_version () >= network_constants.bootstrap_protocol_version_min)
{
if (index.find (channel.get ()) == index.end ())
{
if (!channel->max (nano::transport::traffic_type::bootstrap))
{
index.emplace (channel, 1, 1, 0);
}
}
}
}
channels = list;
}
nano::container_info nano::bootstrap::peer_scoring::container_info () const
{
nano::container_info info;
info.put ("scores", size ());
info.put ("available", available ());
info.put ("channels", channels.size ());
return info;
}
/*

23
nano/node/bootstrap/peer_scoring.hpp
View file

@ -23,14 +23,23 @@ namespace bootstrap
peer_scoring (bootstrap_config const &, nano::network_constants const &);
// Returns true if channel limit has been exceeded
bool try_send_message (std::shared_ptr<nano::transport::channel> channel);
void received_message (std::shared_ptr<nano::transport::channel> channel);
bool limit_exceeded (std::shared_ptr<nano::transport::channel> const & channel) const;
bool try_send_message (std::shared_ptr<nano::transport::channel> const & channel);
void received_message (std::shared_ptr<nano::transport::channel> const & channel);
std::shared_ptr<nano::transport::channel> channel ();
[[nodiscard]] std::size_t size () const;
// Synchronize channels with the network, passed channels should be shuffled
void sync (std::deque<std::shared_ptr<nano::transport::channel>> const & list);
// Cleans up scores for closed channels
// Decays scores which become inaccurate over time due to message drops
void timeout ();
void sync (std::deque<std::shared_ptr<nano::transport::channel>> const & list);
std::size_t size () const;
std::size_t available () const;
nano::container_info container_info () const;
private:
bootstrap_config const & config;
@ -71,14 +80,16 @@ namespace bootstrap
// Indexes scores by the number of outstanding requests in ascending order
class tag_outstanding {};
using scoring_t = boost::multi_index_container<peer_score,
using ordered_scoring = boost::multi_index_container<peer_score,
mi::indexed_by<
mi::hashed_unique<mi::tag<tag_channel>,
mi::member<peer_score, nano::transport::channel *, &peer_score::channel_ptr>>,
mi::ordered_non_unique<mi::tag<tag_outstanding>,
mi::member<peer_score, uint64_t, &peer_score::outstanding>>>>;
// clang-format on
scoring_t scoring;
ordered_scoring scoring;
std::deque<std::shared_ptr<nano::transport::channel>> channels;
};
}
}

5
nano/node/cli.cpp
View file

@ -1,5 +1,6 @@
#include <nano/lib/blocks.hpp>
#include <nano/lib/cli.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/tomlconfig.hpp>
#include <nano/node/cli.hpp>
#include <nano/node/daemonconfig.hpp>
@ -646,10 +647,10 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
{
auto root_str = root_it->second.as<std::string> ();
auto transaction (node.node->store.tx_begin_write ());
nano::root root;
nano::qualified_root root;
if (!root.decode_hex (root_str))
{
node.node->store.final_vote.clear (transaction, root);
node.node->store.final_vote.del (transaction, root);
std::cout << "Successfully cleared final votes" << std::endl;
}
else

2
nano/node/json_handler.cpp
View file

@ -5182,7 +5182,7 @@ void nano::json_handler::debug_bootstrap_priority_info ()
boost::property_tree::ptree response_blocking;
for (auto const & entry : blocking)
{
const auto account = entry.account ();
const auto account = entry.account;
const auto dependency = entry.dependency;
response_blocking.put (account.to_account (), dependency.to_string ());

2
nano/node/json_handler.hpp
View file

@ -81,8 +81,6 @@ public:
void key_create ();
void key_expand ();
void ledger ();
void mnano_to_raw (nano::uint128_t = nano::nano_ratio);
void mnano_from_raw (nano::uint128_t = nano::nano_ratio);
void nano_to_raw ();
void raw_to_nano ();
void node_id ();

2
nano/node/node.cpp
View file

@ -1,5 +1,6 @@
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/stream.hpp>
#include <nano/lib/thread_pool.hpp>
#include <nano/lib/thread_runner.hpp>
@ -1191,6 +1192,7 @@ nano::container_info nano::node::container_info () const
info.add ("local_block_broadcaster", local_block_broadcaster.container_info ());
info.add ("rep_tiers", rep_tiers.container_info ());
info.add ("message_processor", message_processor.container_info ());
info.add ("bandwidth", outbound_limiter.container_info ());
return info;
}

1
nano/node/node_wrapper.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/files.hpp>
#include <nano/node/daemonconfig.hpp>
#include <nano/node/node.hpp>
#include <nano/node/node_wrapper.hpp>

72
nano/node/request_aggregator.cpp
View file

@ -12,6 +12,7 @@
#include <nano/node/wallet.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
#include <nano/secure/ledger_set_confirmed.hpp>
#include <nano/store/component.hpp>
nano::request_aggregator::request_aggregator (request_aggregator_config const & config_a, nano::node & node_a, nano::stats & stats_a, nano::vote_generator & generator_a, nano::vote_generator & final_generator_a, nano::local_vote_history & history_a, nano::ledger & ledger_a, nano::wallets & wallets_a, nano::vote_router & vote_router_a) :
@ -208,68 +209,45 @@ void nano::request_aggregator::erase_duplicates (std::vector<std::pair<nano::blo
requests_a.end ());
}
// This filters candidates for vote generation, the final decision and necessary checks are also performed by the vote generator
auto nano::request_aggregator::aggregate (nano::secure::transaction const & transaction, request_type const & requests_a, std::shared_ptr<nano::transport::channel> const & channel_a) const -> aggregate_result
{
std::vector<std::shared_ptr<nano::block>> to_generate;
std::vector<std::shared_ptr<nano::block>> to_generate_final;
for (auto const & [hash, root] : requests_a)
{
bool generate_final_vote (false);
std::shared_ptr<nano::block> block;
// Ledger by hash
std::shared_ptr<nano::block> block = ledger.any.block_get (transaction, hash);
// 2. Final votes
auto final_vote_hashes (ledger.store.final_vote.get (transaction, root));
if (!final_vote_hashes.empty ())
{
generate_final_vote = true;
block = ledger.any.block_get (transaction, final_vote_hashes[0]);
// Allow same root vote
if (block != nullptr && final_vote_hashes.size () > 1)
{
// WTF? This shouldn't be done like this
to_generate_final.push_back (block);
block = ledger.any.block_get (transaction, final_vote_hashes[1]);
debug_assert (final_vote_hashes.size () == 2);
}
}
// 4. Ledger by hash
if (block == nullptr)
{
block = ledger.any.block_get (transaction, hash);
// Confirmation status. Generate final votes for confirmed
if (block != nullptr)
{
nano::confirmation_height_info confirmation_height_info;
ledger.store.confirmation_height.get (transaction, block->account (), confirmation_height_info);
generate_final_vote = (confirmation_height_info.height >= block->sideband ().height);
}
}
// 5. Ledger by root
if (block == nullptr && !root.is_zero ())
// Ledger by root
if (!block && !root.is_zero ())
{
// Search for block root
auto successor = ledger.any.block_successor (transaction, root.as_block_hash ());
if (successor)
if (auto successor = ledger.any.block_successor (transaction, root.as_block_hash ()))
{
auto successor_block = ledger.any.block_get (transaction, successor.value ());
release_assert (successor_block != nullptr);
block = std::move (successor_block);
// Confirmation status. Generate final votes for confirmed successor
if (block != nullptr)
{
nano::confirmation_height_info confirmation_height_info;
ledger.store.confirmation_height.get (transaction, block->account (), confirmation_height_info);
generate_final_vote = (confirmation_height_info.height >= block->sideband ().height);
}
block = ledger.any.block_get (transaction, successor.value ());
release_assert (block);
}
}
auto should_generate_final_vote = [&] (auto const & block) {
release_assert (block);
// Check if final vote is set for this block
if (auto final_hash = ledger.store.final_vote.get (transaction, block->qualified_root ()))
{
return final_hash == block->hash ();
}
// If the final vote is not set, generate vote if the block is confirmed
else
{
return ledger.confirmed.block_exists (transaction, block->hash ());
}
};
if (block)
{
if (generate_final_vote)
if (should_generate_final_vote (block))
{
to_generate_final.push_back (block);
stats.inc (nano::stat::type::requests, nano::stat::detail::requests_final);

20
nano/node/transport/tcp_socket.cpp
View file

@ -186,17 +186,18 @@ void nano::transport::tcp_socket::write_queued_messages ()
return;
}
auto next = send_queue.pop ();
if (!next)
auto maybe_next = send_queue.pop ();
if (!maybe_next)
{
return;
}
auto const & [next, type] = *maybe_next;
set_default_timeout ();
write_in_progress = true;
nano::async_write (raw_socket, next->buffer,
boost::asio::bind_executor (strand, [this_l = shared_from_this (), next /* `next` object keeps buffer in scope */] (boost::system::error_code ec, std::size_t size) {
nano::async_write (raw_socket, next.buffer,
boost::asio::bind_executor (strand, [this_l = shared_from_this (), next /* `next` object keeps buffer in scope */, type] (boost::system::error_code ec, std::size_t size) {
debug_assert (this_l->strand.running_in_this_thread ());
auto node_l = this_l->node_w.lock ();
@ -214,12 +215,13 @@ void nano::transport::tcp_socket::write_queued_messages ()
else
{
node_l->stats.add (nano::stat::type::traffic_tcp, nano::stat::detail::all, nano::stat::dir::out, size, /* aggregate all */ true);
node_l->stats.add (nano::stat::type::traffic_tcp_type, to_stat_detail (type), nano::stat::dir::out, size);
this_l->set_last_completion ();
}
if (next->callback)
if (next.callback)
{
next->callback (ec, size);
next.callback (ec, size);
}
if (!ec)
@ -436,17 +438,17 @@ bool nano::transport::socket_queue::insert (const buffer_t & buffer, callback_t
return false; // Not queued
}
std::optional<nano::transport::socket_queue::entry> nano::transport::socket_queue::pop ()
auto nano::transport::socket_queue::pop () -> std::optional<result_t>
{
nano::lock_guard<nano::mutex> guard{ mutex };
auto try_pop = [this] (nano::transport::traffic_type type) -> std::optional<entry> {
auto try_pop = [this] (nano::transport::traffic_type type) -> std::optional<result_t> {
auto & que = queues[type];
if (!que.empty ())
{
auto item = que.front ();
que.pop ();
return item;
return std::make_pair (item, type);
}
return std::nullopt;
};

4
nano/node/transport/tcp_socket.hpp
View file

@ -42,10 +42,12 @@ public:
};
public:
using result_t = std::pair<entry, nano::transport::traffic_type>;
explicit socket_queue (std::size_t max_size);
bool insert (buffer_t const &, callback_t, nano::transport::traffic_type);
std::optional<entry> pop ();
std::optional<result_t> pop ();
void clear ();
std::size_t size (nano::transport::traffic_type) const;
bool empty () const;

7
nano/node/transport/traffic_type.cpp Normal file
View file

@ -0,0 +1,7 @@
#include <nano/lib/enum_util.hpp>
#include <nano/node/transport/traffic_type.hpp>
nano::stat::detail nano::transport::to_stat_detail (nano::transport::traffic_type type)
{
return nano::enum_util::cast<nano::stat::detail> (type);
}

4
nano/node/transport/traffic_type.hpp
View file

@ -1,5 +1,7 @@
#pragma once
#include <nano/lib/stats.hpp>
namespace nano::transport
{
/**
@ -10,4 +12,6 @@ enum class traffic_type
generic,
bootstrap, // Ascending bootstrap (asc_pull_ack, asc_pull_req) traffic
};
nano::stat::detail to_stat_detail (traffic_type);
}

31
nano/node/vote_generator.cpp
View file

@ -76,19 +76,20 @@ bool nano::vote_generator::should_vote (transaction_variant_t const & transactio
void nano::vote_generator::start ()
{
debug_assert (!thread.joinable ());
thread = std::thread ([this] () { run (); });
thread = std::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::voting);
run ();
});
vote_generation_queue.start ();
}
void nano::vote_generator::stop ()
{
vote_generation_queue.stop ();
nano::unique_lock<nano::mutex> lock{ mutex };
stopped = true;
lock.unlock ();
{
nano::lock_guard<nano::mutex> lock{ mutex };
stopped = true;
}
condition.notify_all ();
if (thread.joinable ())
@ -284,16 +285,22 @@ void nano::vote_generator::broadcast_action (std::shared_ptr<nano::vote> const &
void nano::vote_generator::run ()
{
nano::thread_role::set (nano::thread_role::name::voting);
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
condition.wait_for (lock, config.vote_generator_delay, [this] () { return broadcast_predicate () || !requests.empty (); });
condition.wait_for (lock, config.vote_generator_delay, [this] () {
return stopped || broadcast_predicate () || !requests.empty ();
});
if (stopped)
{
return;
}
if (broadcast_predicate ())
{
broadcast (lock);
next_broadcast = std::chrono::steady_clock::now () + std::chrono::milliseconds (config.vote_generator_delay);
next_broadcast = std::chrono::steady_clock::now () + config.vote_generator_delay;
}
if (!requests.empty ())
@ -307,11 +314,13 @@ void nano::vote_generator::run ()
bool nano::vote_generator::broadcast_predicate () const
{
debug_assert (!mutex.try_lock ());
if (candidates.size () >= nano::network::confirm_ack_hashes_max)
{
return true;
}
if (candidates.size () > 0 && std::chrono::steady_clock::now () > next_broadcast)
if (!candidates.empty () && std::chrono::steady_clock::now () > next_broadcast)
{
return true;
}

1
nano/node/wallet.cpp
View file

@ -1,5 +1,6 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/utility.hpp>
#include <nano/lib/work_version.hpp>

1
nano/qt_test/entry.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/files.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/memory.hpp>

1
nano/rpc_test/entry.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/files.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/memory.hpp>
#include <nano/node/endpoint.hpp>

8
nano/secure/ledger.cpp
View file

@ -1,5 +1,6 @@
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/stats.hpp>
@ -1280,7 +1281,12 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p
boost::system::error_code error_chmod;
nano::set_secure_perm_directory (data_path_a, error_chmod);
auto rockdb_data_path = data_path_a / "rocksdb";
std::filesystem::remove_all (rockdb_data_path);
if (std::filesystem::exists (rockdb_data_path))
{
logger.error (nano::log::type::ledger, "Existing RocksDb folder found in '{}'. Please remove it and try again.", rockdb_data_path.string ());
return true;
}
auto error (false);

15
nano/secure/transaction.hpp
View file

@ -29,17 +29,18 @@ public:
virtual operator const nano::store::transaction & () const = 0;
};
class write_transaction : public transaction
class write_transaction final : public transaction
{
nano::store::write_guard guard; // Guard should be released after the transaction
nano::store::write_transaction txn;
nano::store::write_guard guard;
std::chrono::steady_clock::time_point start;
public:
explicit write_transaction (nano::store::write_transaction && txn, nano::store::write_guard && guard) noexcept :
txn{ std::move (txn) },
guard{ std::move (guard) }
explicit write_transaction (nano::store::write_transaction && txn_a, nano::store::write_guard && guard_a) noexcept :
guard{ std::move (guard_a) },
txn{ std::move (txn_a) }
{
debug_assert (guard.is_owned ());
start = std::chrono::steady_clock::now ();
}
@ -97,7 +98,7 @@ public:
}
};
class read_transaction : public transaction
class read_transaction final : public transaction
{
nano::store::read_transaction txn;
@ -140,4 +141,4 @@ public:
return txn;
}
};
} // namespace nano::secure
}

1
nano/slow_test/entry.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/files.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/memory.hpp>

5
nano/store/final_vote.hpp
View file

@ -22,10 +22,9 @@ public:
public:
virtual bool put (store::write_transaction const & transaction_a, nano::qualified_root const & root_a, nano::block_hash const & hash_a) = 0;
virtual std::vector<nano::block_hash> get (store::transaction const & transaction_a, nano::root const & root_a) = 0;
virtual void del (store::write_transaction const & transaction_a, nano::root const & root_a) = 0;
virtual std::optional<nano::block_hash> get (store::transaction const & transaction_a, nano::qualified_root const & qualified_root_a) = 0;
virtual void del (store::write_transaction const & transaction_a, nano::qualified_root const & root_a) = 0;
virtual size_t count (store::transaction const & transaction_a) const = 0;
virtual void clear (store::write_transaction const &, nano::root const &) = 0;
virtual void clear (store::write_transaction const &) = 0;
virtual iterator begin (store::transaction const & transaction_a, nano::qualified_root const & root_a) const = 0;
virtual iterator begin (store::transaction const & transaction_a) const = 0;

33
nano/store/lmdb/final_vote.cpp
View file

@ -23,30 +23,22 @@ bool nano::store::lmdb::final_vote::put (store::write_transaction const & transa
return result;
}
std::vector<nano::block_hash> nano::store::lmdb::final_vote::get (store::transaction const & transaction, nano::root const & root_a)
std::optional<nano::block_hash> nano::store::lmdb::final_vote::get (store::transaction const & transaction, nano::qualified_root const & qualified_root_a)
{
std::vector<nano::block_hash> result;
nano::qualified_root key_start{ root_a.raw, 0 };
for (auto i = begin (transaction, key_start), n = end (transaction); i != n && nano::qualified_root{ i->first }.root () == root_a; ++i)
nano::store::lmdb::db_val result;
auto status = store.get (transaction, tables::final_votes, qualified_root_a, result);
std::optional<nano::block_hash> final_vote_hash;
if (store.success (status))
{
result.push_back (i->second);
final_vote_hash = static_cast<nano::block_hash> (result);
}
return result;
return final_vote_hash;
}
void nano::store::lmdb::final_vote::del (store::write_transaction const & transaction, nano::root const & root)
void nano::store::lmdb::final_vote::del (store::write_transaction const & transaction, nano::qualified_root const & root)
{
std::vector<nano::qualified_root> final_vote_qualified_roots;
for (auto i = begin (transaction, nano::qualified_root{ root.raw, 0 }), n = end (transaction); i != n && nano::qualified_root{ i->first }.root () == root; ++i)
{
final_vote_qualified_roots.push_back (i->first);
}
for (auto & final_vote_qualified_root : final_vote_qualified_roots)
{
auto status = store.del (transaction, tables::final_votes, final_vote_qualified_root);
store.release_assert_success (status);
}
auto status = store.del (transaction, tables::final_votes, root);
store.release_assert_success (status);
}
size_t nano::store::lmdb::final_vote::count (store::transaction const & transaction_a) const
@ -54,11 +46,6 @@ size_t nano::store::lmdb::final_vote::count (store::transaction const & transact
return store.count (transaction_a, tables::final_votes);
}
void nano::store::lmdb::final_vote::clear (store::write_transaction const & transaction_a, nano::root const & root_a)
{
del (transaction_a, root_a);
}
void nano::store::lmdb::final_vote::clear (store::write_transaction const & transaction_a)
{
store.drop (transaction_a, nano::tables::final_votes);

5
nano/store/lmdb/final_vote.hpp
View file

@ -18,10 +18,9 @@ private:
public:
explicit final_vote (nano::store::lmdb::component & store);
bool put (store::write_transaction const & transaction_a, nano::qualified_root const & root_a, nano::block_hash const & hash_a) override;
std::vector<nano::block_hash> get (store::transaction const & transaction_a, nano::root const & root_a) override;
void del (store::write_transaction const & transaction_a, nano::root const & root_a) override;
std::optional<nano::block_hash> get (store::transaction const & transaction_a, nano::qualified_root const & qualified_root_a) override;
void del (store::write_transaction const & transaction_a, nano::qualified_root const & root_a) override;
size_t count (store::transaction const & transaction_a) const override;
void clear (store::write_transaction const & transaction_a, nano::root const & root_a) override;
void clear (store::write_transaction const & transaction_a) override;
iterator begin (store::transaction const & transaction_a, nano::qualified_root const & root_a) const override;
iterator begin (store::transaction const & transaction_a) const override;

1
nano/store/lmdb/lmdb_env.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/files.hpp>
#include <nano/lib/utility.hpp>
#include <nano/store/lmdb/lmdb_env.hpp>

33
nano/store/rocksdb/final_vote.cpp
View file

@ -24,30 +24,22 @@ bool nano::store::rocksdb::final_vote::put (store::write_transaction const & tra
return result;
}
std::vector<nano::block_hash> nano::store::rocksdb::final_vote::get (store::transaction const & transaction, nano::root const & root_a)
std::optional<nano::block_hash> nano::store::rocksdb::final_vote::get (store::transaction const & transaction, nano::qualified_root const & qualified_root_a)
{
std::vector<nano::block_hash> result;
nano::qualified_root key_start{ root_a.raw, 0 };
for (auto i = begin (transaction, key_start), n = end (transaction); i != n && nano::qualified_root{ i->first }.root () == root_a; ++i)
nano::store::rocksdb::db_val result;
auto status = store.get (transaction, tables::final_votes, qualified_root_a, result);
std::optional<nano::block_hash> final_vote_hash;
if (store.success (status))
{
result.push_back (i->second);
final_vote_hash = static_cast<nano::block_hash> (result);
}
return result;
return final_vote_hash;
}
void nano::store::rocksdb::final_vote::del (store::write_transaction const & transaction, nano::root const & root)
void nano::store::rocksdb::final_vote::del (store::write_transaction const & transaction, nano::qualified_root const & root)
{
std::vector<nano::qualified_root> final_vote_qualified_roots;
for (auto i = begin (transaction, nano::qualified_root{ root.raw, 0 }), n = end (transaction); i != n && nano::qualified_root{ i->first }.root () == root; ++i)
{
final_vote_qualified_roots.push_back (i->first);
}
for (auto & final_vote_qualified_root : final_vote_qualified_roots)
{
auto status = store.del (transaction, tables::final_votes, final_vote_qualified_root);
store.release_assert_success (status);
}
auto status = store.del (transaction, tables::final_votes, root);
store.release_assert_success (status);
}
size_t nano::store::rocksdb::final_vote::count (store::transaction const & transaction_a) const
@ -55,11 +47,6 @@ size_t nano::store::rocksdb::final_vote::count (store::transaction const & trans
return store.count (transaction_a, tables::final_votes);
}
void nano::store::rocksdb::final_vote::clear (store::write_transaction const & transaction_a, nano::root const & root_a)
{
del (transaction_a, root_a);
}
void nano::store::rocksdb::final_vote::clear (store::write_transaction const & transaction_a)
{
store.drop (transaction_a, nano::tables::final_votes);

5
nano/store/rocksdb/final_vote.hpp
View file

@ -16,10 +16,9 @@ private:
public:
explicit final_vote (nano::store::rocksdb::component & store);
bool put (store::write_transaction const & transaction_a, nano::qualified_root const & root_a, nano::block_hash const & hash_a) override;
std::vector<nano::block_hash> get (store::transaction const & transaction_a, nano::root const & root_a) override;
void del (store::write_transaction const & transaction_a, nano::root const & root_a) override;
std::optional<nano::block_hash> get (store::transaction const & transaction_a, nano::qualified_root const & qualified_root_a) override;
void del (store::write_transaction const & transaction_a, nano::qualified_root const & root_a) override;
size_t count (store::transaction const & transaction_a) const override;
void clear (store::write_transaction const & transaction_a, nano::root const & root_a) override;
void clear (store::write_transaction const & transaction_a) override;
iterator begin (store::transaction const & transaction_a, nano::qualified_root const & root_a) const override;
iterator begin (store::transaction const & transaction_a) const override;

1
nano/store/rocksdb/rocksdb.cpp
View file

@ -1,5 +1,6 @@
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/files.hpp>
#include <nano/lib/rocksdbconfig.hpp>
#include <nano/store/rocksdb/iterator.hpp>
#include <nano/store/rocksdb/rocksdb.hpp>

24
nano/store/write_queue.cpp
View file

@ -66,7 +66,9 @@ nano::store::write_guard nano::store::write_queue::wait (writer writer)
bool nano::store::write_queue::contains (writer writer) const
{
nano::lock_guard<nano::mutex> guard{ mutex };
return std::find (queue.cbegin (), queue.cend (), writer) != queue.cend ();
return std::any_of (queue.cbegin (), queue.cend (), [writer] (auto const & item) {
return item.first == writer;
});
}
void nano::store::write_queue::pop ()
@ -83,17 +85,19 @@ void nano::store::write_queue::acquire (writer writer)
{
nano::unique_lock<nano::mutex> lock{ mutex };
// There should be no duplicates in the queue
debug_assert (std::none_of (queue.cbegin (), queue.cend (), [writer] (auto const & item) { return item == writer; }));
// There should be no duplicates in the queue (exception is testing)
debug_assert (std::none_of (queue.cbegin (), queue.cend (), [writer] (auto const & item) {
return item.first == writer;
})
|| writer == writer::testing);
auto const id = next++;
// Add writer to the end of the queue if it's not already waiting
auto exists = std::find (queue.cbegin (), queue.cend (), writer) != queue.cend ();
if (!exists)
{
queue.push_back (writer);
}
queue.push_back ({ writer, id });
condition.wait (lock, [&] () { return queue.front () == writer; });
// Wait until we are at the front of the queue
condition.wait (lock, [&] () { return queue.front ().second == id; });
}
void nano::store::write_queue::release (writer writer)
@ -101,7 +105,7 @@ void nano::store::write_queue::release (writer writer)
{
nano::lock_guard<nano::mutex> guard{ mutex };
release_assert (!queue.empty ());
release_assert (queue.front () == writer);
release_assert (queue.front ().first == writer);
queue.pop_front ();
}
condition.notify_all ();

4
nano/store/write_queue.hpp
View file

@ -70,7 +70,9 @@ private:
void release (writer writer);
private:
std::deque<writer> queue;
uint64_t next{ 0 };
using entry = std::pair<writer, uint64_t>; // uint64_t is a unique id for each write_guard
std::deque<entry> queue;
mutable nano::mutex mutex;
nano::condition_variable condition;

12
nano/test_common/testutil.cpp
View file

@ -124,6 +124,11 @@ bool nano::test::exists (nano::node & node, std::vector<std::shared_ptr<nano::bl
return exists (node, blocks_to_hashes (blocks));
}
void nano::test::confirm (nano::node & node, std::vector<std::shared_ptr<nano::block>> const blocks)
{
confirm (node.ledger, blocks);
}
void nano::test::confirm (nano::ledger & ledger, std::vector<std::shared_ptr<nano::block>> const blocks)
{
for (auto const block : blocks)
@ -237,6 +242,13 @@ std::vector<nano::block_hash> nano::test::blocks_to_hashes (std::vector<std::sha
return hashes;
}
std::vector<std::shared_ptr<nano::block>> nano::test::clone (std::vector<std::shared_ptr<nano::block>> blocks)
{
std::vector<std::shared_ptr<nano::block>> clones;
std::transform (blocks.begin (), blocks.end (), std::back_inserter (clones), [] (auto & block) { return block->clone (); });
return clones;
}
std::shared_ptr<nano::transport::fake::channel> nano::test::fake_channel (nano::node & node, nano::account node_id)
{
auto channel = std::make_shared<nano::transport::fake::channel> (node);

5
nano/test_common/testutil.hpp
View file

@ -329,6 +329,7 @@ namespace test
void confirm (nano::ledger & ledger, std::vector<std::shared_ptr<nano::block>> const blocks);
void confirm (nano::ledger & ledger, std::shared_ptr<nano::block> const block);
void confirm (nano::ledger & ledger, nano::block_hash const & hash);
void confirm (nano::node & node, std::vector<std::shared_ptr<nano::block>> const blocks);
/*
* Convenience function to check whether *all* of the hashes exists in node ledger or in the pruned table.
* @return true if all blocks are fully processed and inserted in the ledger, false otherwise
@ -389,6 +390,10 @@ namespace test
* Converts list of blocks to list of hashes
*/
std::vector<nano::block_hash> blocks_to_hashes (std::vector<std::shared_ptr<nano::block>> blocks);
/*
* Clones list of blocks
*/
std::vector<std::shared_ptr<nano::block>> clone (std::vector<std::shared_ptr<nano::block>> blocks);
/*
* Creates a new fake channel associated with `node`
*/