dn724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 1

Merge branch 'develop' into remove-process-confirmed

Browse source 
# Conflicts:
#	nano/lib/stats_enums.hpp
#	nano/node/node.hpp
This commit is contained in:
Piotr Wójcik 2024-11-01 16:07:01 +01:00
commit bcb5159530
216 changed files with 3204 additions and 9060 deletions
Download patch file Download diff file Expand all files Collapse all files

2
CMakeLists.txt
View file

@ -35,6 +35,8 @@ endif()
if(MSVC)
add_definitions(/MP)
add_definitions(
-D_SILENCE_STDEXT_ARR_ITERS_DEPRECATION_WARNING) # Suppress iterator warning
endif()
set(CPACK_PACKAGE_VENDOR "Nano Currency")

2
nano/core_test/CMakeLists.txt
View file

@ -10,7 +10,6 @@ add_executable(
block_store.cpp
blockprocessor.cpp
bootstrap.cpp
bootstrap_ascending.cpp
bootstrap_server.cpp
cli.cpp
confirmation_solicitor.cpp
@ -41,6 +40,7 @@ add_executable(
optimistic_scheduler.cpp
processing_queue.cpp
processor_service.cpp
random.cpp
random_pool.cpp
rep_crawler.cpp
receivable.cpp

22
nano/core_test/active_elections.cpp
View file

@ -6,11 +6,13 @@
#include <nano/node/scheduler/component.hpp>
#include <nano/node/scheduler/manual.hpp>
#include <nano/node/scheduler/priority.hpp>
#include <nano/node/transport/fake.hpp>
#include <nano/node/transport/inproc.hpp>
#include <nano/node/vote_router.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
#include <nano/secure/ledger_set_confirmed.hpp>
#include <nano/secure/vote.hpp>
#include <nano/test_common/chains.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>
@ -121,8 +123,9 @@ TEST (active_elections, confirm_frontier)
nano::node_flags node_flags;
node_flags.disable_request_loop = true;
node_flags.disable_ongoing_bootstrap = true;
node_flags.disable_ascending_bootstrap = true;
auto & node1 = *system.add_node (node_flags);
nano::node_config node_config;
node_config.bootstrap.enable = false;
auto & node1 = *system.add_node (node_config, node_flags);
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
// we cannot use the same block instance on 2 different nodes, so make a copy
@ -134,10 +137,11 @@ TEST (active_elections, confirm_frontier)
// The rep crawler would otherwise request confirmations in order to find representatives
nano::node_flags node_flags2;
node_flags2.disable_ongoing_bootstrap = true;
node_flags2.disable_ascending_bootstrap = true;
node_flags2.disable_rep_crawler = true;
nano::node_config node_config2;
node_config2.bootstrap.enable = false;
// start node2 later so that we do not get the gossip traffic
auto & node2 = *system.add_node (node_flags2);
auto & node2 = *system.add_node (node_config2, node_flags2);
// Add representative to disabled rep crawler
auto peers (node2.network.random_set (1));
@ -434,8 +438,8 @@ TEST (inactive_votes_cache, election_start)
nano::test::system system;
nano::node_config node_config = system.default_config ();
node_config.backlog_population.enable = false;
node_config.priority_scheduler.enabled = false;
node_config.optimistic_scheduler.enabled = false;
node_config.priority_scheduler.enable = false;
node_config.optimistic_scheduler.enable = false;
auto & node = *system.add_node (node_config);
nano::block_hash latest (node.latest (nano::dev::genesis_key.pub));
nano::keypair key1, key2;
@ -1334,7 +1338,7 @@ TEST (active_elections, limit_vote_hinted_elections)
nano::node_config config = system.default_config ();
const int aec_limit = 10;
config.backlog_population.enable = false;
config.optimistic_scheduler.enabled = false;
config.optimistic_scheduler.enable = false;
config.active_elections.size = aec_limit;
config.active_elections.hinted_limit_percentage = 10; // Should give us a limit of 1 hinted election
auto & node = *system.add_node (config);
@ -1442,10 +1446,10 @@ TEST (active_elections, broadcast_block_on_activation)
nano::node_config config1 = system.default_config ();
// Deactivates elections on both nodes.
config1.active_elections.size = 0;
config1.bootstrap_ascending.enable = false;
config1.bootstrap.enable = false;
nano::node_config config2 = system.default_config ();
config2.active_elections.size = 0;
config2.bootstrap_ascending.enable = false;
config2.bootstrap.enable = false;
nano::node_flags flags;
// Disables bootstrap listener to make sure the block won't be shared by this channel.
flags.disable_bootstrap_listener = true;

5
nano/core_test/block.cpp
View file

@ -1,6 +1,9 @@
#include <nano/lib/block_uniquer.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/stream.hpp>
#include <nano/node/common.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/node/endpoint.hpp>
#include <nano/node/messages.hpp>
#include <nano/test_common/testutil.hpp>
#include <gtest/gtest.h>

3
nano/core_test/block_store.cpp
View file

@ -1,11 +1,12 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/lmdbconfig.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/stats.hpp>
#include <nano/lib/utility.hpp>
#include <nano/lib/work.hpp>
#include <nano/node/common.hpp>
#include <nano/node/endpoint.hpp>
#include <nano/node/make_store.hpp>
#include <nano/secure/common.hpp>
#include <nano/secure/ledger.hpp>

2459
nano/core_test/bootstrap.cpp
View file

File diff suppressed because it is too large Load diff

486
nano/core_test/bootstrap_ascending.cpp
View file

@ -1,486 +0,0 @@
#include <nano/lib/blocks.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/stats.hpp>
#include <nano/lib/tomlconfig.hpp>
#include <nano/node/bootstrap_ascending/database_scan.hpp>
#include <nano/node/bootstrap_ascending/service.hpp>
#include <nano/node/make_store.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
#include <nano/test_common/ledger_context.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>
#include <gtest/gtest.h>
#include <sstream>
using namespace std::chrono_literals;
namespace
{
nano::block_hash random_hash ()
{
nano::block_hash random_hash;
nano::random_pool::generate_block (random_hash.bytes.data (), random_hash.bytes.size ());
return random_hash;
}
}
TEST (account_sets, construction)
{
nano::test::system system;
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap_ascending::account_sets sets{ config, system.stats };
}
TEST (account_sets, empty_blocked)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap_ascending::account_sets sets{ config, system.stats };
ASSERT_FALSE (sets.blocked (account));
}
TEST (account_sets, block)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap_ascending::account_sets sets{ config, system.stats };
sets.block (account, random_hash ());
ASSERT_TRUE (sets.blocked (account));
}
TEST (account_sets, unblock)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap_ascending::account_sets sets{ config, system.stats };
auto hash = random_hash ();
sets.block (account, hash);
sets.unblock (account, hash);
ASSERT_FALSE (sets.blocked (account));
}
TEST (account_sets, priority_base)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap_ascending::account_sets sets{ config, system.stats };
ASSERT_EQ (0.0, sets.priority (account));
}
TEST (account_sets, priority_blocked)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap_ascending::account_sets sets{ config, system.stats };
sets.block (account, random_hash ());
ASSERT_EQ (0.0, sets.priority (account));
}
// When account is unblocked, check that it retains it former priority
TEST (account_sets, priority_unblock_keep)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap_ascending::account_sets sets{ config, system.stats };
sets.priority_up (account);
sets.priority_up (account);
ASSERT_EQ (sets.priority (account), nano::bootstrap_ascending::account_sets::priority_initial + nano::bootstrap_ascending::account_sets::priority_increase);
auto hash = random_hash ();
sets.block (account, hash);
ASSERT_EQ (0.0, sets.priority (account));
sets.unblock (account, hash);
ASSERT_EQ (sets.priority (account), nano::bootstrap_ascending::account_sets::priority_initial + nano::bootstrap_ascending::account_sets::priority_increase);
}
TEST (account_sets, priority_up_down)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap_ascending::account_sets sets{ config, system.stats };
sets.priority_up (account);
ASSERT_EQ (sets.priority (account), nano::bootstrap_ascending::account_sets::priority_initial);
sets.priority_down (account);
ASSERT_EQ (sets.priority (account), nano::bootstrap_ascending::account_sets::priority_initial / nano::bootstrap_ascending::account_sets::priority_divide);
}
TEST (account_sets, priority_down_sat)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap_ascending::account_sets sets{ config, system.stats };
sets.priority_down (account);
ASSERT_EQ (0.0, sets.priority (account));
}
// Ensure priority value is bounded
TEST (account_sets, saturate_priority)
{
nano::test::system system;
nano::account account{ 1 };
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::account_sets_config config;
nano::bootstrap_ascending::account_sets sets{ config, system.stats };
for (int n = 0; n < 1000; ++n)
{
sets.priority_up (account);
}
ASSERT_EQ (sets.priority (account), nano::bootstrap_ascending::account_sets::priority_max);
}
/**
* Tests the base case for returning
*/
TEST (bootstrap_ascending, account_base)
{
nano::node_flags flags;
nano::test::system system{ 1, nano::transport::transport_type::tcp, flags };
auto & node0 = *system.nodes[0];
nano::state_block_builder builder;
auto send1 = builder.make_block ()
.account (nano::dev::genesis_key.pub)
.previous (nano::dev::genesis->hash ())
.representative (nano::dev::genesis_key.pub)
.link (0)
.balance (nano::dev::constants.genesis_amount - 1)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (nano::dev::genesis->hash ()))
.build ();
ASSERT_EQ (nano::block_status::progress, node0.process (send1));
auto & node1 = *system.add_node (flags);
ASSERT_TIMELY (5s, node1.block (send1->hash ()) != nullptr);
}
/**
* Tests that bootstrap_ascending will return multiple new blocks in-order
*/
TEST (bootstrap_ascending, account_inductive)
{
nano::node_flags flags;
nano::test::system system{ 1, nano::transport::transport_type::tcp, flags };
auto & node0 = *system.nodes[0];
nano::state_block_builder builder;
auto send1 = builder.make_block ()
.account (nano::dev::genesis_key.pub)
.previous (nano::dev::genesis->hash ())
.representative (nano::dev::genesis_key.pub)
.link (0)
.balance (nano::dev::constants.genesis_amount - 1)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (nano::dev::genesis->hash ()))
.build ();
auto send2 = builder.make_block ()
.account (nano::dev::genesis_key.pub)
.previous (send1->hash ())
.representative (nano::dev::genesis_key.pub)
.link (0)
.balance (nano::dev::constants.genesis_amount - 2)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (send1->hash ()))
.build ();
// std::cerr << "Genesis: " << nano::dev::genesis->hash ().to_string () << std::endl;
// std::cerr << "Send1: " << send1->hash ().to_string () << std::endl;
// std::cerr << "Send2: " << send2->hash ().to_string () << std::endl;
ASSERT_EQ (nano::block_status::progress, node0.process (send1));
ASSERT_EQ (nano::block_status::progress, node0.process (send2));
auto & node1 = *system.add_node (flags);
ASSERT_TIMELY (50s, node1.block (send2->hash ()) != nullptr);
}
/**
* Tests that bootstrap_ascending will return multiple new blocks in-order
*/
TEST (bootstrap_ascending, trace_base)
{
nano::node_flags flags;
flags.disable_legacy_bootstrap = true;
nano::test::system system{ 1, nano::transport::transport_type::tcp, flags };
auto & node0 = *system.nodes[0];
nano::keypair key;
nano::state_block_builder builder;
auto send1 = builder.make_block ()
.account (nano::dev::genesis_key.pub)
.previous (nano::dev::genesis->hash ())
.representative (nano::dev::genesis_key.pub)
.link (key.pub)
.balance (nano::dev::constants.genesis_amount - 1)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (nano::dev::genesis->hash ()))
.build ();
auto receive1 = builder.make_block ()
.account (key.pub)
.previous (0)
.representative (nano::dev::genesis_key.pub)
.link (send1->hash ())
.balance (1)
.sign (key.prv, key.pub)
.work (*system.work.generate (key.pub))
.build ();
// std::cerr << "Genesis key: " << nano::dev::genesis_key.pub.to_account () << std::endl;
// std::cerr << "Key: " << key.pub.to_account () << std::endl;
// std::cerr << "Genesis: " << nano::dev::genesis->hash ().to_string () << std::endl;
// std::cerr << "send1: " << send1->hash ().to_string () << std::endl;
// std::cerr << "receive1: " << receive1->hash ().to_string () << std::endl;
auto & node1 = *system.add_node ();
// std::cerr << "--------------- Start ---------------\n";
ASSERT_EQ (nano::block_status::progress, node0.process (send1));
ASSERT_EQ (nano::block_status::progress, node0.process (receive1));
ASSERT_EQ (node1.ledger.any.receivable_end (), node1.ledger.any.receivable_upper_bound (node1.ledger.tx_begin_read (), key.pub, 0));
// std::cerr << "node0: " << node0.network.endpoint () << std::endl;
// std::cerr << "node1: " << node1.network.endpoint () << std::endl;
ASSERT_TIMELY (10s, node1.block (receive1->hash ()) != nullptr);
}
TEST (bootstrap_ascending, pending_database_scanner)
{
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
// Prepare pending sends from genesis
// 1 account with 1 pending
// 1 account with 21 pendings
// 2 accounts with 1 pending each
std::deque<std::shared_ptr<nano::block>> blocks;
nano::keypair key1, key2, key3, key4;
{
nano::state_block_builder builder;
auto source = nano::dev::genesis_key;
auto latest = nano::dev::genesis->hash ();
auto balance = nano::dev::genesis->balance ().number ();
// 1 account with 1 pending
{
auto send = builder.make_block ()
.account (source.pub)
.previous (latest)
.representative (source.pub)
.link (key1.pub)
.balance (balance - 1)
.sign (source.prv, source.pub)
.work (*pool.generate (latest))
.build ();
latest = send->hash ();
balance = send->balance_field ().value ().number ();
blocks.push_back (send);
}
// 1 account with 21 pendings
for (int i = 0; i < 21; ++i)
{
auto send = builder.make_block ()
.account (source.pub)
.previous (latest)
.representative (source.pub)
.link (key2.pub)
.balance (balance - 1)
.sign (source.prv, source.pub)
.work (*pool.generate (latest))
.build ();
latest = send->hash ();
balance = send->balance_field ().value ().number ();
blocks.push_back (send);
}
// 2 accounts with 1 pending each
{
auto send = builder.make_block ()
.account (source.pub)
.previous (latest)
.representative (source.pub)
.link (key3.pub)
.balance (balance - 1)
.sign (source.prv, source.pub)
.work (*pool.generate (latest))
.build ();
latest = send->hash ();
balance = send->balance_field ().value ().number ();
blocks.push_back (send);
}
{
auto send = builder.make_block ()
.account (source.pub)
.previous (latest)
.representative (source.pub)
.link (key4.pub)
.balance (balance - 1)
.sign (source.prv, source.pub)
.work (*pool.generate (latest))
.build ();
latest = send->hash ();
balance = send->balance_field ().value ().number ();
blocks.push_back (send);
}
}
nano::test::ledger_context ctx{ std::move (blocks) };
// Single batch
{
nano::bootstrap_ascending::pending_database_iterator scanner{ ctx.ledger () };
auto transaction = ctx.store ().tx_begin_read ();
auto accounts = scanner.next_batch (transaction, 256);
// Check that account set contains all keys
ASSERT_EQ (accounts.size (), 4);
ASSERT_TRUE (std::find (accounts.begin (), accounts.end (), key1.pub) != accounts.end ());
ASSERT_TRUE (std::find (accounts.begin (), accounts.end (), key2.pub) != accounts.end ());
ASSERT_TRUE (std::find (accounts.begin (), accounts.end (), key3.pub) != accounts.end ());
ASSERT_TRUE (std::find (accounts.begin (), accounts.end (), key4.pub) != accounts.end ());
ASSERT_EQ (scanner.completed, 1);
}
// Multi batch
{
nano::bootstrap_ascending::pending_database_iterator scanner{ ctx.ledger () };
auto transaction = ctx.store ().tx_begin_read ();
// Request accounts in multiple batches
auto accounts1 = scanner.next_batch (transaction, 2);
auto accounts2 = scanner.next_batch (transaction, 1);
auto accounts3 = scanner.next_batch (transaction, 1);
ASSERT_EQ (accounts1.size (), 2);
ASSERT_EQ (accounts2.size (), 1);
ASSERT_EQ (accounts3.size (), 1);
std::deque<nano::account> accounts;
accounts.insert (accounts.end (), accounts1.begin (), accounts1.end ());
accounts.insert (accounts.end (), accounts2.begin (), accounts2.end ());
accounts.insert (accounts.end (), accounts3.begin (), accounts3.end ());
// Check that account set contains all keys
ASSERT_EQ (accounts.size (), 4);
ASSERT_TRUE (std::find (accounts.begin (), accounts.end (), key1.pub) != accounts.end ());
ASSERT_TRUE (std::find (accounts.begin (), accounts.end (), key2.pub) != accounts.end ());
ASSERT_TRUE (std::find (accounts.begin (), accounts.end (), key3.pub) != accounts.end ());
ASSERT_TRUE (std::find (accounts.begin (), accounts.end (), key4.pub) != accounts.end ());
ASSERT_EQ (scanner.completed, 1);
}
}
TEST (bootstrap_ascending, account_database_scanner)
{
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
size_t const count = 4;
// Prepare some accounts
std::deque<std::shared_ptr<nano::block>> blocks;
std::deque<nano::keypair> keys;
{
nano::state_block_builder builder;
auto source = nano::dev::genesis_key;
auto latest = nano::dev::genesis->hash ();
auto balance = nano::dev::genesis->balance ().number ();
for (int i = 0; i < count; ++i)
{
nano::keypair key;
auto send = builder.make_block ()
.account (source.pub)
.previous (latest)
.representative (source.pub)
.link (key.pub)
.balance (balance - 1)
.sign (source.prv, source.pub)
.work (*pool.generate (latest))
.build ();
auto open = builder.make_block ()
.account (key.pub)
.previous (0)
.representative (key.pub)
.link (send->hash ())
.balance (1)
.sign (key.prv, key.pub)
.work (*pool.generate (key.pub))
.build ();
latest = send->hash ();
balance = send->balance_field ().value ().number ();
blocks.push_back (send);
blocks.push_back (open);
keys.push_back (key);
}
}
nano::test::ledger_context ctx{ std::move (blocks) };
// Single batch
{
nano::bootstrap_ascending::account_database_iterator scanner{ ctx.ledger () };
auto transaction = ctx.store ().tx_begin_read ();
auto accounts = scanner.next_batch (transaction, 256);
// Check that account set contains all keys
ASSERT_EQ (accounts.size (), keys.size () + 1); // +1 for genesis
for (auto const & key : keys)
{
ASSERT_TRUE (std::find (accounts.begin (), accounts.end (), key.pub) != accounts.end ());
}
ASSERT_EQ (scanner.completed, 1);
}
// Multi batch
{
nano::bootstrap_ascending::account_database_iterator scanner{ ctx.ledger () };
auto transaction = ctx.store ().tx_begin_read ();
// Request accounts in multiple batches
auto accounts1 = scanner.next_batch (transaction, 2);
auto accounts2 = scanner.next_batch (transaction, 2);
auto accounts3 = scanner.next_batch (transaction, 1);
ASSERT_EQ (accounts1.size (), 2);
ASSERT_EQ (accounts2.size (), 2);
ASSERT_EQ (accounts3.size (), 1);
std::deque<nano::account> accounts;
accounts.insert (accounts.end (), accounts1.begin (), accounts1.end ());
accounts.insert (accounts.end (), accounts2.begin (), accounts2.end ());
accounts.insert (accounts.end (), accounts3.begin (), accounts3.end ());
// Check that account set contains all keys
ASSERT_EQ (accounts.size (), keys.size () + 1); // +1 for genesis
for (auto const & key : keys)
{
ASSERT_TRUE (std::find (accounts.begin (), accounts.end (), key.pub) != accounts.end ());
}
ASSERT_EQ (scanner.completed, 1);
}
}

1
nano/core_test/bootstrap_server.cpp
View file

@ -1,4 +1,5 @@
#include <nano/lib/blocks.hpp>
#include <nano/node/transport/fake.hpp>
#include <nano/test_common/chains.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>

5
nano/core_test/confirming_set.cpp
View file

@ -144,11 +144,10 @@ TEST (confirmation_callback, observer_callbacks)
TEST (confirmation_callback, confirmed_history)
{
nano::test::system system;
nano::node_flags node_flags;
node_flags.disable_ascending_bootstrap = true;
nano::node_config node_config = system.default_config ();
node_config.backlog_population.enable = false;
auto node = system.add_node (node_config, node_flags);
node_config.bootstrap.enable = false;
auto node = system.add_node (node_config);
nano::block_hash latest (node->latest (nano::dev::genesis_key.pub));

1
nano/core_test/conflicts.cpp
View file

@ -4,6 +4,7 @@
#include <nano/node/scheduler/component.hpp>
#include <nano/node/scheduler/priority.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/vote.hpp>
#include <nano/test_common/chains.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>

2
nano/core_test/difficulty.cpp
View file

@ -1,8 +1,10 @@
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/config.hpp>
#include <nano/lib/epoch.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/work.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/secure/common.hpp>
#include <nano/test_common/testutil.hpp>

1
nano/core_test/distributed_work.cpp
View file

@ -1,4 +1,5 @@
#include <nano/core_test/fakes/work_peer.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>

2
nano/core_test/entry.cpp
View file

@ -1,5 +1,5 @@
#include <nano/lib/logging.hpp>
#include <nano/node/common.hpp>
#include <nano/lib/memory.hpp>
#include <nano/secure/utility.hpp>
#include <gtest/gtest.h>

1
nano/core_test/fair_queue.cpp
View file

@ -1,4 +1,5 @@
#include <nano/node/fair_queue.hpp>
#include <nano/node/transport/fake.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>

3
nano/core_test/fakes/work_peer.hpp
View file

@ -4,7 +4,8 @@
#include <nano/lib/locks.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/work.hpp>
#include <nano/node/common.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/node/endpoint.hpp>
#include <boost/asio.hpp>
#include <boost/asio/steady_timer.hpp>

1
nano/core_test/ledger.cpp
View file

@ -12,6 +12,7 @@
#include <nano/node/vote_router.hpp>
#include <nano/secure/ledger_set_any.hpp>
#include <nano/secure/ledger_set_confirmed.hpp>
#include <nano/secure/vote.hpp>
#include <nano/store/rocksdb/rocksdb.hpp>
#include <nano/test_common/ledger_context.hpp>
#include <nano/test_common/make_store.hpp>

1
nano/core_test/memory_pool.cpp
View file

@ -2,6 +2,7 @@
#include <nano/lib/memory.hpp>
#include <nano/node/active_elections.hpp>
#include <nano/secure/common.hpp>
#include <nano/secure/vote.hpp>
#include <gtest/gtest.h>

3
nano/core_test/message.cpp
View file

@ -1,8 +1,9 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/stream.hpp>
#include <nano/node/common.hpp>
#include <nano/node/endpoint.hpp>
#include <nano/node/network.hpp>
#include <nano/secure/vote.hpp>
#include <nano/test_common/testutil.hpp>
#include <gtest/gtest.h>

1
nano/core_test/message_deserializer.cpp
View file

@ -1,5 +1,6 @@
#include <nano/lib/blocks.hpp>
#include <nano/node/transport/message_deserializer.hpp>
#include <nano/secure/vote.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>

3
nano/core_test/network.cpp
View file

@ -4,6 +4,7 @@
#include <nano/node/nodeconfig.hpp>
#include <nano/node/scheduler/component.hpp>
#include <nano/node/scheduler/priority.hpp>
#include <nano/node/transport/fake.hpp>
#include <nano/node/transport/inproc.hpp>
#include <nano/node/transport/tcp_listener.hpp>
#include <nano/node/transport/tcp_socket.hpp>
@ -265,8 +266,6 @@ TEST (network, send_valid_publish)
nano::test::system system (2, type, node_flags);
auto & node1 (*system.nodes[0]);
auto & node2 (*system.nodes[1]);
node1.bootstrap_initiator.stop ();
node2.bootstrap_initiator.stop ();
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
nano::keypair key2;
system.wallet (1)->insert_adhoc (key2.prv);

5
nano/core_test/network_filter.cpp
View file

@ -1,7 +1,8 @@
#include <nano/lib/blocks.hpp>
#include <nano/lib/network_filter.hpp>
#include <nano/lib/stream.hpp>
#include <nano/node/common.hpp>
#include <nano/node/endpoint.hpp>
#include <nano/node/messages.hpp>
#include <nano/secure/common.hpp>
#include <nano/test_common/testutil.hpp>
@ -163,4 +164,4 @@ TEST (network_filter, expire)
ASSERT_FALSE (filter.check (2)); // Entry with epoch 1 should be expired
ASSERT_FALSE (filter.apply (2)); // Entry with epoch 1 should be replaced
}
}

164
nano/core_test/node.cpp
View file

@ -1,6 +1,7 @@
#include <nano/lib/blocks.hpp>
#include <nano/lib/config.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/node/active_elections.hpp>
#include <nano/node/confirming_set.hpp>
#include <nano/node/election.hpp>
@ -19,6 +20,7 @@
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
#include <nano/secure/ledger_set_confirmed.hpp>
#include <nano/secure/vote.hpp>
#include <nano/test_common/network.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>
@ -278,7 +280,6 @@ TEST (node, auto_bootstrap)
system.nodes.push_back (node1);
ASSERT_NE (nullptr, nano::test::establish_tcp (system, *node1, node0->network.endpoint ()));
ASSERT_TIMELY_EQ (10s, node1->balance (key2.pub), node0->config.receive_minimum.number ());
ASSERT_TIMELY (10s, !node1->bootstrap_initiator.in_progress ());
ASSERT_TRUE (node1->block_or_pruned_exists (send1->hash ()));
// Wait block receive
ASSERT_TIMELY_EQ (5s, node1->ledger.block_count (), 3);
@ -307,27 +308,6 @@ TEST (node, auto_bootstrap_reverse)
ASSERT_TIMELY_EQ (10s, node1->balance (key2.pub), node0->config.receive_minimum.number ());
}
TEST (node, auto_bootstrap_age)
{
nano::test::system system;
nano::node_config config (system.get_available_port ());
config.backlog_population.enable = false;
nano::node_flags node_flags;
node_flags.disable_bootstrap_bulk_push_client = true;
node_flags.disable_lazy_bootstrap = true;
node_flags.bootstrap_interval = 1;
auto node0 = system.add_node (config, node_flags);
auto node1 (std::make_shared<nano::node> (system.io_ctx, system.get_available_port (), nano::unique_path (), system.work, node_flags));
ASSERT_FALSE (node1->init_error ());
node1->start ();
system.nodes.push_back (node1);
ASSERT_NE (nullptr, nano::test::establish_tcp (system, *node1, node0->network.endpoint ()));
// 4 bootstraps with frontiers age
ASSERT_TIMELY (10s, node0->stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate_legacy_age, nano::stat::dir::out) >= 3);
// More attempts with frontiers age
ASSERT_GE (node0->stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate_legacy_age, nano::stat::dir::out), node0->stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate, nano::stat::dir::out));
}
TEST (node, merge_peers)
{
nano::test::system system (1);
@ -722,6 +702,7 @@ TEST (node, fork_multi_flip)
node_config.backlog_population.enable = false;
auto & node1 (*system.add_node (node_config, node_flags, type));
node_config.peering_port = system.get_available_port ();
node_config.bootstrap.account_sets.cooldown = 100ms; // Reduce cooldown to speed up fork resolution
auto & node2 (*system.add_node (node_config, node_flags, type));
ASSERT_EQ (1, node1.network.size ());
nano::keypair key1;
@ -756,9 +737,9 @@ TEST (node, fork_multi_flip)
auto election = nano::test::start_election (system, node2, send2->hash ());
ASSERT_NE (nullptr, election);
ASSERT_TIMELY (5s, election->contains (send1->hash ()));
ASSERT_TIMELY (10s, election->contains (send1->hash ()));
nano::test::confirm (node1.ledger, send1);
ASSERT_TIMELY (5s, node2.block_or_pruned_exists (send1->hash ()));
ASSERT_TIMELY (10s, node2.block_or_pruned_exists (send1->hash ()));
ASSERT_TRUE (nano::test::block_or_pruned_none_exists (node2, { send2, send3 }));
auto winner = *election->tally ().begin ();
ASSERT_EQ (*send1, *winner.second);
@ -770,15 +751,16 @@ TEST (node, fork_multi_flip)
TEST (node, fork_bootstrap_flip)
{
nano::test::system system;
nano::node_config config0{ system.get_available_port () };
config0.backlog_population.enable = false;
nano::node_config config1{ system.get_available_port () };
config1.backlog_population.enable = false;
nano::node_flags node_flags;
node_flags.disable_bootstrap_bulk_push_client = true;
node_flags.disable_lazy_bootstrap = true;
auto & node1 = *system.add_node (config0, node_flags);
auto & node1 = *system.add_node (config1, node_flags);
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
nano::node_config config1 (system.get_available_port ());
auto & node2 = *system.make_disconnected_node (config1, node_flags);
nano::node_config config2 (system.get_available_port ());
config2.bootstrap.account_sets.cooldown = 100ms; // Reduce cooldown to speed up fork resolution
auto & node2 = *system.make_disconnected_node (config2, node_flags);
nano::block_hash latest = node1.latest (nano::dev::genesis_key.pub);
nano::keypair key1;
nano::send_block_builder builder;
@ -801,8 +783,8 @@ TEST (node, fork_bootstrap_flip)
ASSERT_EQ (nano::block_status::progress, node1.ledger.process (node1.ledger.tx_begin_write (), send1));
ASSERT_EQ (nano::block_status::progress, node2.ledger.process (node2.ledger.tx_begin_write (), send2));
nano::test::confirm (node1.ledger, send1);
ASSERT_TIMELY (1s, node1.ledger.any.block_exists (node1.ledger.tx_begin_read (), send1->hash ()));
ASSERT_TIMELY (1s, node2.ledger.any.block_exists (node2.ledger.tx_begin_read (), send2->hash ()));
ASSERT_TIMELY (5s, node1.ledger.any.block_exists (node1.ledger.tx_begin_read (), send1->hash ()));
ASSERT_TIMELY (5s, node2.ledger.any.block_exists (node2.ledger.tx_begin_read (), send2->hash ()));
// Additionally add new peer to confirm & replace bootstrap block
node2.network.merge_peer (node1.network.endpoint ());
@ -1107,7 +1089,6 @@ TEST (node, DISABLED_fork_stale)
nano::test::system system2 (1);
auto & node1 (*system1.nodes[0]);
auto & node2 (*system2.nodes[0]);
node2.bootstrap_initiator.bootstrap (node1.network.endpoint (), false);
auto channel = nano::test::establish_tcp (system1, node2, node1.network.endpoint ());
auto vote = std::make_shared<nano::vote> (nano::dev::genesis_key.pub, nano::dev::genesis_key.prv, 0, 0, std::vector<nano::block_hash> ());
@ -1162,7 +1143,6 @@ TEST (node, DISABLED_fork_stale)
node1.process_active (send2);
node2.process_active (send1);
node2.process_active (send2);
node2.bootstrap_initiator.bootstrap (node1.network.endpoint (), false);
while (node2.block (send1->hash ()) == nullptr)
{
system1.poll ();
@ -1376,8 +1356,6 @@ TEST (node, DISABLED_bootstrap_no_publish)
auto transaction = node0->ledger.tx_begin_write ();
ASSERT_EQ (nano::block_status::progress, node0->ledger.process (transaction, send0));
}
ASSERT_FALSE (node1->bootstrap_initiator.in_progress ());
node1->bootstrap_initiator.bootstrap (node0->network.endpoint (), false);
ASSERT_TRUE (node1->active.empty ());
system1.deadline_set (10s);
while (node1->block (send0->hash ()) == nullptr)
@ -1391,51 +1369,6 @@ TEST (node, DISABLED_bootstrap_no_publish)
}
}
// Check that an outgoing bootstrap request can push blocks
// Test disabled because it's failing intermittently.
// PR in which it got disabled: https://github.com/nanocurrency/nano-node/pull/3512
// Issue for investigating it: https://github.com/nanocurrency/nano-node/issues/3515
TEST (node, DISABLED_bootstrap_bulk_push)
{
nano::test::system system;
nano::test::system system0;
nano::test::system system1;
nano::node_config config0 (system.get_available_port ());
config0.backlog_population.enable = false;
auto node0 (system0.add_node (config0));
nano::node_config config1 (system.get_available_port ());
config1.backlog_population.enable = false;
auto node1 (system1.add_node (config1));
nano::keypair key0;
// node0 knows about send0 but node1 doesn't.
auto send0 = nano::send_block_builder ()
.previous (nano::dev::genesis->hash ())
.destination (key0.pub)
.balance (500)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*node0->work_generate_blocking (nano::dev::genesis->hash ()))
.build ();
ASSERT_EQ (nano::block_status::progress, node0->process (send0));
ASSERT_FALSE (node0->bootstrap_initiator.in_progress ());
ASSERT_FALSE (node1->bootstrap_initiator.in_progress ());
ASSERT_TRUE (node1->active.empty ());
node0->bootstrap_initiator.bootstrap (node1->network.endpoint (), false);
system1.deadline_set (10s);
while (node1->block (send0->hash ()) == nullptr)
{
ASSERT_NO_ERROR (system0.poll ());
ASSERT_NO_ERROR (system1.poll ());
}
// since this uses bulk_push, the new block should be republished
system1.deadline_set (10s);
while (node1->active.empty ())
{
ASSERT_NO_ERROR (system0.poll ());
ASSERT_NO_ERROR (system1.poll ());
}
}
// Bootstrapping a forked open block should succeed.
TEST (node, bootstrap_fork_open)
{
@ -1486,8 +1419,6 @@ TEST (node, bootstrap_fork_open)
ASSERT_EQ (nano::block_status::progress, node1->process (open1));
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
ASSERT_FALSE (node1->block_or_pruned_exists (open0->hash ()));
ASSERT_FALSE (node1->bootstrap_initiator.in_progress ());
node1->bootstrap_initiator.bootstrap (node0->network.endpoint (), false);
ASSERT_TIMELY (1s, node1->active.empty ());
ASSERT_TIMELY (10s, !node1->block_or_pruned_exists (open1->hash ()) && node1->block_or_pruned_exists (open0->hash ()));
}
@ -1495,12 +1426,10 @@ TEST (node, bootstrap_fork_open)
// Unconfirmed blocks from bootstrap should be confirmed
TEST (node, bootstrap_confirm_frontiers)
{
// create 2 separate systems, the 2 system do not interact with each other automatically
nano::test::system system0 (1);
nano::test::system system1 (1);
auto node0 = system0.nodes[0];
auto node1 = system1.nodes[0];
system0.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
nano::test::system system;
auto node0 = system.add_node ();
auto node1 = system.add_node ();
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
nano::keypair key0;
// create block to send 500 raw from genesis to key0 and save into node0 ledger without immediately triggering an election
@ -1512,25 +1441,7 @@ TEST (node, bootstrap_confirm_frontiers)
.work (*node0->work_generate_blocking (nano::dev::genesis->hash ()))
.build ();
ASSERT_EQ (nano::block_status::progress, node0->process (send0));
// each system only has one node, so there should be no bootstrapping going on
ASSERT_FALSE (node0->bootstrap_initiator.in_progress ());
ASSERT_FALSE (node1->bootstrap_initiator.in_progress ());
ASSERT_TRUE (node1->active.empty ());
// create a bootstrap connection from node1 to node0
// this also has the side effect of adding node0 to node1's list of peers, which will trigger realtime connections too
node1->bootstrap_initiator.bootstrap (node0->network.endpoint ());
// Wait until the block is confirmed on node1. Poll more than usual because we are polling
// on 2 different systems at once and in sequence and there might be strange timing effects.
system0.deadline_set (10s);
system1.deadline_set (10s);
while (!node1->ledger.confirmed.block_exists_or_pruned (node1->ledger.tx_begin_read (), send0->hash ()))
{
ASSERT_NO_ERROR (system0.poll (std::chrono::milliseconds (1)));
ASSERT_NO_ERROR (system1.poll (std::chrono::milliseconds (1)));
}
ASSERT_TIMELY (10s, node1->block_confirmed (send0->hash ()));
}
// Test that if we create a block that isn't confirmed, the bootstrapping processes sync the missing block.
@ -1644,37 +1555,6 @@ TEST (node, balance_observer)
}
}
TEST (node, bootstrap_connection_scaling)
{
nano::test::system system (1);
auto & node1 (*system.nodes[0]);
ASSERT_EQ (34, node1.bootstrap_initiator.connections->target_connections (5000, 1));
ASSERT_EQ (4, node1.bootstrap_initiator.connections->target_connections (0, 1));
ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (50000, 1));
ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (10000000000, 1));
ASSERT_EQ (32, node1.bootstrap_initiator.connections->target_connections (5000, 0));
ASSERT_EQ (1, node1.bootstrap_initiator.connections->target_connections (0, 0));
ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (50000, 0));
ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (10000000000, 0));
ASSERT_EQ (36, node1.bootstrap_initiator.connections->target_connections (5000, 2));
ASSERT_EQ (8, node1.bootstrap_initiator.connections->target_connections (0, 2));
ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (50000, 2));
ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (10000000000, 2));
node1.config.bootstrap_connections = 128;
ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (0, 1));
ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (50000, 1));
ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (0, 2));
ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (50000, 2));
node1.config.bootstrap_connections_max = 256;
ASSERT_EQ (128, node1.bootstrap_initiator.connections->target_connections (0, 1));
ASSERT_EQ (256, node1.bootstrap_initiator.connections->target_connections (50000, 1));
ASSERT_EQ (256, node1.bootstrap_initiator.connections->target_connections (0, 2));
ASSERT_EQ (256, node1.bootstrap_initiator.connections->target_connections (50000, 2));
node1.config.bootstrap_connections_max = 0;
ASSERT_EQ (1, node1.bootstrap_initiator.connections->target_connections (0, 1));
ASSERT_EQ (1, node1.bootstrap_initiator.connections->target_connections (50000, 1));
}
TEST (node, online_reps)
{
nano::test::system system (1);
@ -3760,9 +3640,9 @@ TEST (node, local_block_broadcast)
// Disable active elections to prevent the block from being broadcasted by the election
auto node_config = system.default_config ();
node_config.priority_scheduler.enabled = false;
node_config.hinted_scheduler.enabled = false;
node_config.optimistic_scheduler.enabled = false;
node_config.priority_scheduler.enable = false;
node_config.hinted_scheduler.enable = false;
node_config.optimistic_scheduler.enable = false;
node_config.local_block_broadcaster.rebroadcast_interval = 1s;
auto & node1 = *system.add_node (node_config);
auto & node2 = *system.make_disconnected_node ();
@ -3814,4 +3694,4 @@ TEST (node, container_info)
// This should just execute, sanitizers will catch any problems
ASSERT_NO_THROW (node1.container_info ());
ASSERT_NO_THROW (node2.container_info ());
}
}

3
nano/core_test/numbers.cpp
View file

@ -1,4 +1,5 @@
#include <nano/lib/numbers.hpp>
#include <nano/lib/numbers_templ.hpp>
#include <nano/secure/common.hpp>
#include <gtest/gtest.h>
@ -647,4 +648,4 @@ TEST (uint512_union, hash)
ASSERT_NE (h (x1), h (x2));
}
}
}
}

3
nano/core_test/random.cpp Normal file
View file

@ -0,0 +1,3 @@
#include <nano/lib/random.hpp>
#include <gtest/gtest.h>

3
nano/core_test/rep_crawler.cpp
View file

@ -6,6 +6,7 @@
#include <nano/node/transport/fake.hpp>
#include <nano/node/transport/inproc.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/vote.hpp>
#include <nano/test_common/chains.hpp>
#include <nano/test_common/network.hpp>
#include <nano/test_common/system.hpp>
@ -327,4 +328,4 @@ TEST (rep_crawler, ignore_rebroadcasted)
};
ASSERT_NEVER (1s, tick () || node1.rep_crawler.representative_count () > 0);
}
}

1
nano/core_test/request_aggregator.cpp
View file

@ -5,6 +5,7 @@
#include <nano/node/election.hpp>
#include <nano/node/local_vote_history.hpp>
#include <nano/node/request_aggregator.hpp>
#include <nano/node/transport/fake.hpp>
#include <nano/node/transport/inproc.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_confirmed.hpp>

1
nano/core_test/system.cpp
View file

@ -1,5 +1,6 @@
#include <nano/lib/blocks.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/node/transport/inproc.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>

1
nano/core_test/telemetry.cpp
View file

@ -1,4 +1,5 @@
#include <nano/node/telemetry.hpp>
#include <nano/node/transport/fake.hpp>
#include <nano/test_common/network.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/telemetry.hpp>

10
nano/core_test/throttle.cpp
View file

@ -1,16 +1,16 @@
#include <nano/node/bootstrap_ascending/throttle.hpp>
#include <nano/node/bootstrap/throttle.hpp>
#include <gtest/gtest.h>
TEST (throttle, construction)
{
nano::bootstrap_ascending::throttle throttle{ 2 };
nano::bootstrap::throttle throttle{ 2 };
ASSERT_FALSE (throttle.throttled ());
}
TEST (throttle, throttled)
{
nano::bootstrap_ascending::throttle throttle{ 2 };
nano::bootstrap::throttle throttle{ 2 };
throttle.add (false);
ASSERT_FALSE (throttle.throttled ());
throttle.add (false);
@ -19,7 +19,7 @@ TEST (throttle, throttled)
TEST (throttle, resize_up)
{
nano::bootstrap_ascending::throttle throttle{ 2 };
nano::bootstrap::throttle throttle{ 2 };
throttle.add (false);
throttle.resize (4);
ASSERT_FALSE (throttle.throttled ());
@ -29,7 +29,7 @@ TEST (throttle, resize_up)
TEST (throttle, resize_down)
{
nano::bootstrap_ascending::throttle throttle{ 4 };
nano::bootstrap::throttle throttle{ 4 };
throttle.add (false);
ASSERT_FALSE (throttle.throttled ());
throttle.resize (2);

68
nano/core_test/toml.cpp
View file

@ -117,7 +117,7 @@ TEST (toml, daemon_config_deserialize_defaults)
ss << R"toml(
[node]
[node.backlog_population]
[node.bootstrap_ascending]
[node.bootstrap]
[node.bootstrap_server]
[node.block_processor]
[node.diagnostics.txn_tracking]
@ -244,11 +244,11 @@ TEST (toml, daemon_config_deserialize_defaults)
ASSERT_EQ (conf.node.rocksdb_config.read_cache, defaults.node.rocksdb_config.read_cache);
ASSERT_EQ (conf.node.rocksdb_config.write_cache, defaults.node.rocksdb_config.write_cache);
ASSERT_EQ (conf.node.optimistic_scheduler.enabled, defaults.node.optimistic_scheduler.enabled);
ASSERT_EQ (conf.node.optimistic_scheduler.enable, defaults.node.optimistic_scheduler.enable);
ASSERT_EQ (conf.node.optimistic_scheduler.gap_threshold, defaults.node.optimistic_scheduler.gap_threshold);
ASSERT_EQ (conf.node.optimistic_scheduler.max_size, defaults.node.optimistic_scheduler.max_size);
ASSERT_EQ (conf.node.hinted_scheduler.enabled, defaults.node.hinted_scheduler.enabled);
ASSERT_EQ (conf.node.hinted_scheduler.enable, defaults.node.hinted_scheduler.enable);
ASSERT_EQ (conf.node.hinted_scheduler.hinting_threshold_percent, defaults.node.hinted_scheduler.hinting_threshold_percent);
ASSERT_EQ (conf.node.hinted_scheduler.check_interval.count (), defaults.node.hinted_scheduler.check_interval.count ());
ASSERT_EQ (conf.node.hinted_scheduler.block_cooldown.count (), defaults.node.hinted_scheduler.block_cooldown.count ());
@ -269,18 +269,18 @@ TEST (toml, daemon_config_deserialize_defaults)
ASSERT_EQ (conf.node.vote_processor.threads, defaults.node.vote_processor.threads);
ASSERT_EQ (conf.node.vote_processor.batch_size, defaults.node.vote_processor.batch_size);
ASSERT_EQ (conf.node.bootstrap_ascending.enable, defaults.node.bootstrap_ascending.enable);
ASSERT_EQ (conf.node.bootstrap_ascending.enable_database_scan, defaults.node.bootstrap_ascending.enable_database_scan);
ASSERT_EQ (conf.node.bootstrap_ascending.enable_dependency_walker, defaults.node.bootstrap_ascending.enable_dependency_walker);
ASSERT_EQ (conf.node.bootstrap_ascending.channel_limit, defaults.node.bootstrap_ascending.channel_limit);
ASSERT_EQ (conf.node.bootstrap_ascending.database_rate_limit, defaults.node.bootstrap_ascending.database_rate_limit);
ASSERT_EQ (conf.node.bootstrap_ascending.database_warmup_ratio, defaults.node.bootstrap_ascending.database_warmup_ratio);
ASSERT_EQ (conf.node.bootstrap_ascending.max_pull_count, defaults.node.bootstrap_ascending.max_pull_count);
ASSERT_EQ (conf.node.bootstrap_ascending.request_timeout, defaults.node.bootstrap_ascending.request_timeout);
ASSERT_EQ (conf.node.bootstrap_ascending.throttle_coefficient, defaults.node.bootstrap_ascending.throttle_coefficient);
ASSERT_EQ (conf.node.bootstrap_ascending.throttle_wait, defaults.node.bootstrap_ascending.throttle_wait);
ASSERT_EQ (conf.node.bootstrap_ascending.block_processor_threshold, defaults.node.bootstrap_ascending.block_processor_threshold);
ASSERT_EQ (conf.node.bootstrap_ascending.max_requests, defaults.node.bootstrap_ascending.max_requests);
ASSERT_EQ (conf.node.bootstrap.enable, defaults.node.bootstrap.enable);
ASSERT_EQ (conf.node.bootstrap.enable_database_scan, defaults.node.bootstrap.enable_database_scan);
ASSERT_EQ (conf.node.bootstrap.enable_dependency_walker, defaults.node.bootstrap.enable_dependency_walker);
ASSERT_EQ (conf.node.bootstrap.channel_limit, defaults.node.bootstrap.channel_limit);
ASSERT_EQ (conf.node.bootstrap.database_rate_limit, defaults.node.bootstrap.database_rate_limit);
ASSERT_EQ (conf.node.bootstrap.database_warmup_ratio, defaults.node.bootstrap.database_warmup_ratio);
ASSERT_EQ (conf.node.bootstrap.max_pull_count, defaults.node.bootstrap.max_pull_count);
ASSERT_EQ (conf.node.bootstrap.request_timeout, defaults.node.bootstrap.request_timeout);
ASSERT_EQ (conf.node.bootstrap.throttle_coefficient, defaults.node.bootstrap.throttle_coefficient);
ASSERT_EQ (conf.node.bootstrap.throttle_wait, defaults.node.bootstrap.throttle_wait);
ASSERT_EQ (conf.node.bootstrap.block_processor_threshold, defaults.node.bootstrap.block_processor_threshold);
ASSERT_EQ (conf.node.bootstrap.max_requests, defaults.node.bootstrap.max_requests);
ASSERT_EQ (conf.node.bootstrap_server.max_queue, defaults.node.bootstrap_server.max_queue);
ASSERT_EQ (conf.node.bootstrap_server.threads, defaults.node.bootstrap_server.threads);
@ -597,9 +597,10 @@ TEST (toml, daemon_config_deserialize_no_defaults)
threads = 999
batch_size = 999
[node.bootstrap_ascending]
[node.bootstrap]
enable = false
enable_database_scan = false
enable_frontier_scan = false
enable_database_scan = true
enable_dependency_walker = false
channel_limit = 999
database_rate_limit = 999
@ -751,11 +752,11 @@ TEST (toml, daemon_config_deserialize_no_defaults)
ASSERT_NE (conf.node.rocksdb_config.read_cache, defaults.node.rocksdb_config.read_cache);
ASSERT_NE (conf.node.rocksdb_config.write_cache, defaults.node.rocksdb_config.write_cache);
ASSERT_NE (conf.node.optimistic_scheduler.enabled, defaults.node.optimistic_scheduler.enabled);
ASSERT_NE (conf.node.optimistic_scheduler.enable, defaults.node.optimistic_scheduler.enable);
ASSERT_NE (conf.node.optimistic_scheduler.gap_threshold, defaults.node.optimistic_scheduler.gap_threshold);
ASSERT_NE (conf.node.optimistic_scheduler.max_size, defaults.node.optimistic_scheduler.max_size);
ASSERT_NE (conf.node.hinted_scheduler.enabled, defaults.node.hinted_scheduler.enabled);
ASSERT_NE (conf.node.hinted_scheduler.enable, defaults.node.hinted_scheduler.enable);
ASSERT_NE (conf.node.hinted_scheduler.hinting_threshold_percent, defaults.node.hinted_scheduler.hinting_threshold_percent);
ASSERT_NE (conf.node.hinted_scheduler.check_interval.count (), defaults.node.hinted_scheduler.check_interval.count ());
ASSERT_NE (conf.node.hinted_scheduler.block_cooldown.count (), defaults.node.hinted_scheduler.block_cooldown.count ());
@ -776,18 +777,19 @@ TEST (toml, daemon_config_deserialize_no_defaults)
ASSERT_NE (conf.node.vote_processor.threads, defaults.node.vote_processor.threads);
ASSERT_NE (conf.node.vote_processor.batch_size, defaults.node.vote_processor.batch_size);
ASSERT_NE (conf.node.bootstrap_ascending.enable, defaults.node.bootstrap_ascending.enable);
ASSERT_NE (conf.node.bootstrap_ascending.enable_database_scan, defaults.node.bootstrap_ascending.enable_database_scan);
ASSERT_NE (conf.node.bootstrap_ascending.enable_dependency_walker, defaults.node.bootstrap_ascending.enable_dependency_walker);
ASSERT_NE (conf.node.bootstrap_ascending.channel_limit, defaults.node.bootstrap_ascending.channel_limit);
ASSERT_NE (conf.node.bootstrap_ascending.database_rate_limit, defaults.node.bootstrap_ascending.database_rate_limit);
ASSERT_NE (conf.node.bootstrap_ascending.database_warmup_ratio, defaults.node.bootstrap_ascending.database_warmup_ratio);
ASSERT_NE (conf.node.bootstrap_ascending.max_pull_count, defaults.node.bootstrap_ascending.max_pull_count);
ASSERT_NE (conf.node.bootstrap_ascending.request_timeout, defaults.node.bootstrap_ascending.request_timeout);
ASSERT_NE (conf.node.bootstrap_ascending.throttle_coefficient, defaults.node.bootstrap_ascending.throttle_coefficient);
ASSERT_NE (conf.node.bootstrap_ascending.throttle_wait, defaults.node.bootstrap_ascending.throttle_wait);
ASSERT_NE (conf.node.bootstrap_ascending.block_processor_threshold, defaults.node.bootstrap_ascending.block_processor_threshold);
ASSERT_NE (conf.node.bootstrap_ascending.max_requests, defaults.node.bootstrap_ascending.max_requests);
ASSERT_NE (conf.node.bootstrap.enable, defaults.node.bootstrap.enable);
ASSERT_NE (conf.node.bootstrap.enable_database_scan, defaults.node.bootstrap.enable_database_scan);
ASSERT_NE (conf.node.bootstrap.enable_frontier_scan, defaults.node.bootstrap.enable_frontier_scan);
ASSERT_NE (conf.node.bootstrap.enable_dependency_walker, defaults.node.bootstrap.enable_dependency_walker);
ASSERT_NE (conf.node.bootstrap.channel_limit, defaults.node.bootstrap.channel_limit);
ASSERT_NE (conf.node.bootstrap.database_rate_limit, defaults.node.bootstrap.database_rate_limit);
ASSERT_NE (conf.node.bootstrap.database_warmup_ratio, defaults.node.bootstrap.database_warmup_ratio);
ASSERT_NE (conf.node.bootstrap.max_pull_count, defaults.node.bootstrap.max_pull_count);
ASSERT_NE (conf.node.bootstrap.request_timeout, defaults.node.bootstrap.request_timeout);
ASSERT_NE (conf.node.bootstrap.throttle_coefficient, defaults.node.bootstrap.throttle_coefficient);
ASSERT_NE (conf.node.bootstrap.throttle_wait, defaults.node.bootstrap.throttle_wait);
ASSERT_NE (conf.node.bootstrap.block_processor_threshold, defaults.node.bootstrap.block_processor_threshold);
ASSERT_NE (conf.node.bootstrap.max_requests, defaults.node.bootstrap.max_requests);
ASSERT_NE (conf.node.bootstrap_server.max_queue, defaults.node.bootstrap_server.max_queue);
ASSERT_NE (conf.node.bootstrap_server.threads, defaults.node.bootstrap_server.threads);
@ -1079,7 +1081,7 @@ TEST (toml, merge_config_files)
[node]
active_elections.size = 999
# background_threads = 7777
[node.bootstrap_ascending]
[node.bootstrap]
block_processor_threshold = 33333
old_entry = 34
)toml";
@ -1103,6 +1105,6 @@ TEST (toml, merge_config_files)
ASSERT_NE (merged_config.node.active_elections.size, default_config.node.active_elections.size);
ASSERT_EQ (merged_config.node.active_elections.size, 999);
ASSERT_NE (merged_config.node.background_threads, 7777);
ASSERT_EQ (merged_config.node.bootstrap_ascending.block_processor_threshold, 33333);
ASSERT_EQ (merged_config.node.bootstrap.block_processor_threshold, 33333);
ASSERT_TRUE (merged_config_string.find ("old_entry") == std::string::npos);
}

6
nano/core_test/vote_processor.cpp
View file

@ -2,10 +2,12 @@
#include <nano/lib/jsonconfig.hpp>
#include <nano/node/active_elections.hpp>
#include <nano/node/election.hpp>
#include <nano/node/transport/fake.hpp>
#include <nano/node/transport/inproc.hpp>
#include <nano/node/vote_processor.hpp>
#include <nano/node/vote_router.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/vote.hpp>
#include <nano/test_common/chains.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>
@ -20,8 +22,8 @@ TEST (vote_processor, codes)
auto node_config = system.default_config ();
// Disable all election schedulers
node_config.backlog_population.enable = false;
node_config.hinted_scheduler.enabled = false;
node_config.optimistic_scheduler.enabled = false;
node_config.hinted_scheduler.enable = false;
node_config.optimistic_scheduler.enable = false;
auto & node = *system.add_node (node_config);
auto blocks = nano::test::setup_chain (system, node, 1, nano::dev::genesis_key, false);

3
nano/core_test/voting.cpp
View file

@ -1,9 +1,10 @@
#include <nano/lib/blocks.hpp>
#include <nano/node/common.hpp>
#include <nano/node/endpoint.hpp>
#include <nano/node/local_vote_history.hpp>
#include <nano/node/vote_generator.hpp>
#include <nano/node/vote_spacing.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/vote.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>

1
nano/core_test/wallet.cpp
View file

@ -1,6 +1,7 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/node/active_elections.hpp>
#include <nano/node/election.hpp>
#include <nano/secure/ledger.hpp>

117
nano/core_test/websocket.cpp
View file

@ -1,5 +1,7 @@
#include <nano/core_test/fakes/websocket_client.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/jsonconfig.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/node/active_elections.hpp>
#include <nano/node/telemetry.hpp>
#include <nano/node/transport/fake.hpp>
@ -840,121 +842,6 @@ TEST (websocket, work)
ASSERT_EQ (contents.get<std::string> ("reason"), "");
}
// Test client subscribing to notifications for bootstrap
TEST (websocket, bootstrap)
{
nano::test::system system;
nano::node_config config = system.default_config ();
config.websocket_config.enabled = true;
config.websocket_config.port = system.get_available_port ();
auto node1 (system.add_node (config));
ASSERT_EQ (0, node1->websocket.server->subscriber_count (nano::websocket::topic::bootstrap));
// Subscribe to bootstrap and wait for response asynchronously
std::atomic<bool> ack_ready{ false };
auto task = ([&ack_ready, config, &node1] () {
fake_websocket_client client (node1->websocket.server->listening_port ());
client.send_message (R"json({"action": "subscribe", "topic": "bootstrap", "ack": true})json");
client.await_ack ();
ack_ready = true;
EXPECT_EQ (1, node1->websocket.server->subscriber_count (nano::websocket::topic::bootstrap));
return client.get_response ();
});
auto future = std::async (std::launch::async, task);
// Wait for acknowledge
ASSERT_TIMELY (5s, ack_ready);
// Start bootstrap attempt
node1->bootstrap_initiator.bootstrap (true, "123abc");
ASSERT_TIMELY_EQ (5s, nullptr, node1->bootstrap_initiator.current_attempt ());
// Wait for the bootstrap notification
ASSERT_TIMELY_EQ (5s, future.wait_for (0s), std::future_status::ready);
// Check the bootstrap notification message
auto response = future.get ();
ASSERT_TRUE (response);
std::stringstream stream;
stream << response;
boost::property_tree::ptree event;
boost::property_tree::read_json (stream, event);
ASSERT_EQ (event.get<std::string> ("topic"), "bootstrap");
auto & contents = event.get_child ("message");
ASSERT_EQ (contents.get<std::string> ("reason"), "started");
ASSERT_EQ (contents.get<std::string> ("id"), "123abc");
ASSERT_EQ (contents.get<std::string> ("mode"), "legacy");
// Wait for bootstrap finish
ASSERT_TIMELY (5s, !node1->bootstrap_initiator.in_progress ());
}
TEST (websocket, bootstrap_exited)
{
nano::test::system system;
nano::node_config config = system.default_config ();
config.websocket_config.enabled = true;
config.websocket_config.port = system.get_available_port ();
auto node1 (system.add_node (config));
// Start bootstrap, exit after subscription
std::atomic<bool> bootstrap_started{ false };
nano::test::counted_completion subscribed_completion (1);
std::thread bootstrap_thread ([node1, &system, &bootstrap_started, &subscribed_completion] () {
std::shared_ptr<nano::bootstrap_attempt> attempt;
while (attempt == nullptr)
{
std::this_thread::sleep_for (50ms);
node1->bootstrap_initiator.bootstrap (true, "123abc");
attempt = node1->bootstrap_initiator.current_attempt ();
}
ASSERT_NE (nullptr, attempt);
bootstrap_started = true;
EXPECT_FALSE (subscribed_completion.await_count_for (5s));
});
// Wait for bootstrap start
ASSERT_TIMELY (5s, bootstrap_started);
// Subscribe to bootstrap and wait for response asynchronously
std::atomic<bool> ack_ready{ false };
auto task = ([&ack_ready, config, &node1] () {
fake_websocket_client client (node1->websocket.server->listening_port ());
client.send_message (R"json({"action": "subscribe", "topic": "bootstrap", "ack": true})json");
client.await_ack ();
ack_ready = true;
EXPECT_EQ (1, node1->websocket.server->subscriber_count (nano::websocket::topic::bootstrap));
return client.get_response ();
});
auto future = std::async (std::launch::async, task);
// Wait for acknowledge
ASSERT_TIMELY (5s, ack_ready);
// Wait for the bootstrap notification
subscribed_completion.increment ();
bootstrap_thread.join ();
ASSERT_TIMELY_EQ (5s, future.wait_for (0s), std::future_status::ready);
// Check the bootstrap notification message
auto response = future.get ();
ASSERT_TRUE (response);
std::stringstream stream;
stream << response;
boost::property_tree::ptree event;
boost::property_tree::read_json (stream, event);
ASSERT_EQ (event.get<std::string> ("topic"), "bootstrap");
auto & contents = event.get_child ("message");
ASSERT_EQ (contents.get<std::string> ("reason"), "exited");
ASSERT_EQ (contents.get<std::string> ("id"), "123abc");
ASSERT_EQ (contents.get<std::string> ("mode"), "legacy");
ASSERT_EQ (contents.get<unsigned> ("total_blocks"), 0U);
ASSERT_LT (contents.get<unsigned> ("duration"), 15000U);
}
// Tests sending keepalive
TEST (websocket, ws_keepalive)
{

1
nano/core_test/work_pool.cpp
View file

@ -3,6 +3,7 @@
#include <nano/lib/logging.hpp>
#include <nano/lib/timer.hpp>
#include <nano/lib/work.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/node/openclconfig.hpp>
#include <nano/node/openclwork.hpp>
#include <nano/secure/common.hpp>

8
nano/lib/CMakeLists.txt
View file

@ -33,6 +33,7 @@ add_library(
char_traits.hpp
cli.hpp
cli.cpp
common.hpp
config.hpp
config.cpp
configbase.hpp
@ -44,8 +45,11 @@ add_library(
env.cpp
epoch.hpp
epoch.cpp
epochs.cpp
epochs.hpp
errors.hpp
errors.cpp
fwd.hpp
id_dispenser.hpp
interval.hpp
ipc.hpp
@ -69,6 +73,7 @@ add_library(
network_filter.cpp
numbers.hpp
numbers.cpp
numbers_templ.hpp
object_stream.hpp
object_stream.cpp
object_stream_adapters.hpp
@ -110,7 +115,8 @@ add_library(
walletconfig.hpp
walletconfig.cpp
work.hpp
work.cpp)
work.cpp
work_version.hpp)
include_directories(${CMAKE_SOURCE_DIR}/submodules)
include_directories(

216
nano/lib/block_sideband.cpp
View file

@ -0,0 +1,216 @@
#include <nano/lib/block_sideband.hpp>
#include <nano/lib/block_type.hpp>
#include <nano/lib/object_stream.hpp>
#include <nano/lib/stream.hpp>
#include <bitset>
/*
* block_details
*/
nano::block_details::block_details (nano::epoch const epoch_a, bool const is_send_a, bool const is_receive_a, bool const is_epoch_a) :
epoch (epoch_a), is_send (is_send_a), is_receive (is_receive_a), is_epoch (is_epoch_a)
{
}
bool nano::block_details::operator== (nano::block_details const & other_a) const
{
return epoch == other_a.epoch && is_send == other_a.is_send && is_receive == other_a.is_receive && is_epoch == other_a.is_epoch;
}
uint8_t nano::block_details::packed () const
{
std::bitset<8> result (static_cast<uint8_t> (epoch));
result.set (7, is_send);
result.set (6, is_receive);
result.set (5, is_epoch);
return static_cast<uint8_t> (result.to_ulong ());
}
void nano::block_details::unpack (uint8_t details_a)
{
constexpr std::bitset<8> epoch_mask{ 0b00011111 };
auto as_bitset = static_cast<std::bitset<8>> (details_a);
is_send = as_bitset.test (7);
is_receive = as_bitset.test (6);
is_epoch = as_bitset.test (5);
epoch = static_cast<nano::epoch> ((as_bitset & epoch_mask).to_ulong ());
}
void nano::block_details::serialize (nano::stream & stream_a) const
{
nano::write (stream_a, packed ());
}
bool nano::block_details::deserialize (nano::stream & stream_a)
{
bool result (false);
try
{
uint8_t packed{ 0 };
nano::read (stream_a, packed);
unpack (packed);
}
catch (std::runtime_error &)
{
result = true;
}
return result;
}
void nano::block_details::operator() (nano::object_stream & obs) const
{
obs.write ("epoch", epoch);
obs.write ("is_send", is_send);
obs.write ("is_receive", is_receive);
obs.write ("is_epoch", is_epoch);
}
std::string nano::state_subtype (nano::block_details const details_a)
{
debug_assert (details_a.is_epoch + details_a.is_receive + details_a.is_send <= 1);
if (details_a.is_send)
{
return "send";
}
else if (details_a.is_receive)
{
return "receive";
}
else if (details_a.is_epoch)
{
return "epoch";
}
else
{
return "change";
}
}
/*
* block_sideband
*/
nano::block_sideband::block_sideband (nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::block_details const & details_a, nano::epoch const source_epoch_a) :
successor (successor_a),
account (account_a),
balance (balance_a),
height (height_a),
timestamp (timestamp_a),
details (details_a),
source_epoch (source_epoch_a)
{
}
nano::block_sideband::block_sideband (nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::epoch const epoch_a, bool const is_send, bool const is_receive, bool const is_epoch, nano::epoch const source_epoch_a) :
successor (successor_a),
account (account_a),
balance (balance_a),
height (height_a),
timestamp (timestamp_a),
details (epoch_a, is_send, is_receive, is_epoch),
source_epoch (source_epoch_a)
{
}
size_t nano::block_sideband::size (nano::block_type type_a)
{
size_t result (0);
result += sizeof (successor);
if (type_a != nano::block_type::state && type_a != nano::block_type::open)
{
result += sizeof (account);
}
if (type_a != nano::block_type::open)
{
result += sizeof (height);
}
if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open)
{
result += sizeof (balance);
}
result += sizeof (timestamp);
if (type_a == nano::block_type::state)
{
static_assert (sizeof (nano::epoch) == nano::block_details::size (), "block_details is larger than the epoch enum");
result += nano::block_details::size () + sizeof (nano::epoch);
}
return result;
}
void nano::block_sideband::serialize (nano::stream & stream_a, nano::block_type type_a) const
{
nano::write (stream_a, successor.bytes);
if (type_a != nano::block_type::state && type_a != nano::block_type::open)
{
nano::write (stream_a, account.bytes);
}
if (type_a != nano::block_type::open)
{
nano::write (stream_a, boost::endian::native_to_big (height));
}
if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open)
{
nano::write (stream_a, balance.bytes);
}
nano::write (stream_a, boost::endian::native_to_big (timestamp));
if (type_a == nano::block_type::state)
{
details.serialize (stream_a);
nano::write (stream_a, static_cast<uint8_t> (source_epoch));
}
}
bool nano::block_sideband::deserialize (nano::stream & stream_a, nano::block_type type_a)
{
bool result (false);
try
{
nano::read (stream_a, successor.bytes);
if (type_a != nano::block_type::state && type_a != nano::block_type::open)
{
nano::read (stream_a, account.bytes);
}
if (type_a != nano::block_type::open)
{
nano::read (stream_a, height);
boost::endian::big_to_native_inplace (height);
}
else
{
height = 1;
}
if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open)
{
nano::read (stream_a, balance.bytes);
}
nano::read (stream_a, timestamp);
boost::endian::big_to_native_inplace (timestamp);
if (type_a == nano::block_type::state)
{
result = details.deserialize (stream_a);
uint8_t source_epoch_uint8_t{ 0 };
nano::read (stream_a, source_epoch_uint8_t);
source_epoch = static_cast<nano::epoch> (source_epoch_uint8_t);
}
}
catch (std::runtime_error &)
{
result = true;
}
return result;
}
void nano::block_sideband::operator() (nano::object_stream & obs) const
{
obs.write ("successor", successor);
obs.write ("account", account);
obs.write ("balance", balance);
obs.write ("height", height);
obs.write ("timestamp", timestamp);
obs.write ("source_epoch", source_epoch);
obs.write ("details", details);
}

8
nano/lib/block_sideband.hpp
View file

@ -1,19 +1,13 @@
#pragma once
#include <nano/lib/block_type.hpp>
#include <nano/lib/epoch.hpp>
#include <nano/lib/fwd.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/stream.hpp>
#include <nano/lib/timer.hpp>
#include <cstdint>
#include <memory>
namespace nano
{
class object_stream;
}
namespace nano
{
class block_details

5
nano/lib/block_type.cpp
View file

@ -5,8 +5,3 @@ std::string_view nano::to_string (nano::block_type type)
{
return nano::enum_util::name (type);
}
void nano::serialize_block_type (nano::stream & stream, const nano::block_type & type)
{
nano::write (stream, type);
}

6
nano/lib/block_type.hpp
View file

@ -1,7 +1,5 @@
#pragma once
#include <nano/lib/stream.hpp>
#include <cstdint>
#include <string>
@ -19,8 +17,4 @@ enum class block_type : uint8_t
};
std::string_view to_string (block_type);
/**
* Serialize block type as an 8-bit value
*/
void serialize_block_type (nano::stream &, nano::block_type const &);
} // namespace nano

1
nano/lib/block_uniquer.hpp
View file

@ -1,6 +1,7 @@
#pragma once
#include <nano/lib/numbers.hpp>
#include <nano/lib/numbers_templ.hpp>
#include <nano/lib/uniquer.hpp>
namespace nano

216
nano/lib/blocks.cpp
View file

@ -1,9 +1,13 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/block_type.hpp>
#include <nano/lib/block_uniquer.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/enum_util.hpp>
#include <nano/lib/memory.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/stream.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/secure/common.hpp>
#include <boost/endian/conversion.hpp>
@ -1582,7 +1586,7 @@ std::shared_ptr<nano::block> nano::deserialize_block_json (boost::property_tree:
void nano::serialize_block (nano::stream & stream_a, nano::block const & block_a)
{
nano::serialize_block_type (stream_a, block_a.type ());
nano::write (stream_a, block_a.type ());
block_a.serialize (stream_a);
}
@ -1908,213 +1912,3 @@ void nano::receive_block::operator() (nano::object_stream & obs) const
obs.write ("signature", signature);
obs.write ("work", work);
}
/*
* block_details
*/
nano::block_details::block_details (nano::epoch const epoch_a, bool const is_send_a, bool const is_receive_a, bool const is_epoch_a) :
epoch (epoch_a), is_send (is_send_a), is_receive (is_receive_a), is_epoch (is_epoch_a)
{
}
bool nano::block_details::operator== (nano::block_details const & other_a) const
{
return epoch == other_a.epoch && is_send == other_a.is_send && is_receive == other_a.is_receive && is_epoch == other_a.is_epoch;
}
uint8_t nano::block_details::packed () const
{
std::bitset<8> result (static_cast<uint8_t> (epoch));
result.set (7, is_send);
result.set (6, is_receive);
result.set (5, is_epoch);
return static_cast<uint8_t> (result.to_ulong ());
}
void nano::block_details::unpack (uint8_t details_a)
{
constexpr std::bitset<8> epoch_mask{ 0b00011111 };
auto as_bitset = static_cast<std::bitset<8>> (details_a);
is_send = as_bitset.test (7);
is_receive = as_bitset.test (6);
is_epoch = as_bitset.test (5);
epoch = static_cast<nano::epoch> ((as_bitset & epoch_mask).to_ulong ());
}
void nano::block_details::serialize (nano::stream & stream_a) const
{
nano::write (stream_a, packed ());
}
bool nano::block_details::deserialize (nano::stream & stream_a)
{
bool result (false);
try
{
uint8_t packed{ 0 };
nano::read (stream_a, packed);
unpack (packed);
}
catch (std::runtime_error &)
{
result = true;
}
return result;
}
void nano::block_details::operator() (nano::object_stream & obs) const
{
obs.write ("epoch", epoch);
obs.write ("is_send", is_send);
obs.write ("is_receive", is_receive);
obs.write ("is_epoch", is_epoch);
}
std::string nano::state_subtype (nano::block_details const details_a)
{
debug_assert (details_a.is_epoch + details_a.is_receive + details_a.is_send <= 1);
if (details_a.is_send)
{
return "send";
}
else if (details_a.is_receive)
{
return "receive";
}
else if (details_a.is_epoch)
{
return "epoch";
}
else
{
return "change";
}
}
/*
* block_sideband
*/
nano::block_sideband::block_sideband (nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::block_details const & details_a, nano::epoch const source_epoch_a) :
successor (successor_a),
account (account_a),
balance (balance_a),
height (height_a),
timestamp (timestamp_a),
details (details_a),
source_epoch (source_epoch_a)
{
}
nano::block_sideband::block_sideband (nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::epoch const epoch_a, bool const is_send, bool const is_receive, bool const is_epoch, nano::epoch const source_epoch_a) :
successor (successor_a),
account (account_a),
balance (balance_a),
height (height_a),
timestamp (timestamp_a),
details (epoch_a, is_send, is_receive, is_epoch),
source_epoch (source_epoch_a)
{
}
size_t nano::block_sideband::size (nano::block_type type_a)
{
size_t result (0);
result += sizeof (successor);
if (type_a != nano::block_type::state && type_a != nano::block_type::open)
{
result += sizeof (account);
}
if (type_a != nano::block_type::open)
{
result += sizeof (height);
}
if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open)
{
result += sizeof (balance);
}
result += sizeof (timestamp);
if (type_a == nano::block_type::state)
{
static_assert (sizeof (nano::epoch) == nano::block_details::size (), "block_details is larger than the epoch enum");
result += nano::block_details::size () + sizeof (nano::epoch);
}
return result;
}
void nano::block_sideband::serialize (nano::stream & stream_a, nano::block_type type_a) const
{
nano::write (stream_a, successor.bytes);
if (type_a != nano::block_type::state && type_a != nano::block_type::open)
{
nano::write (stream_a, account.bytes);
}
if (type_a != nano::block_type::open)
{
nano::write (stream_a, boost::endian::native_to_big (height));
}
if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open)
{
nano::write (stream_a, balance.bytes);
}
nano::write (stream_a, boost::endian::native_to_big (timestamp));
if (type_a == nano::block_type::state)
{
details.serialize (stream_a);
nano::write (stream_a, static_cast<uint8_t> (source_epoch));
}
}
bool nano::block_sideband::deserialize (nano::stream & stream_a, nano::block_type type_a)
{
bool result (false);
try
{
nano::read (stream_a, successor.bytes);
if (type_a != nano::block_type::state && type_a != nano::block_type::open)
{
nano::read (stream_a, account.bytes);
}
if (type_a != nano::block_type::open)
{
nano::read (stream_a, height);
boost::endian::big_to_native_inplace (height);
}
else
{
height = 1;
}
if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open)
{
nano::read (stream_a, balance.bytes);
}
nano::read (stream_a, timestamp);
boost::endian::big_to_native_inplace (timestamp);
if (type_a == nano::block_type::state)
{
result = details.deserialize (stream_a);
uint8_t source_epoch_uint8_t{ 0 };
nano::read (stream_a, source_epoch_uint8_t);
source_epoch = static_cast<nano::epoch> (source_epoch_uint8_t);
}
}
catch (std::runtime_error &)
{
result = true;
}
return result;
}
void nano::block_sideband::operator() (nano::object_stream & obs) const
{
obs.write ("successor", successor);
obs.write ("account", account);
obs.write ("balance", balance);
obs.write ("height", height);
obs.write ("timestamp", timestamp);
obs.write ("source_epoch", source_epoch);
obs.write ("details", details);
}

13
nano/lib/blocks.hpp
View file

@ -1,24 +1,25 @@
#pragma once
#include <nano/lib/block_sideband.hpp>
#include <nano/lib/block_uniquer.hpp>
#include <nano/lib/config.hpp>
#include <nano/lib/epoch.hpp>
#include <nano/lib/errors.hpp>
#include <nano/lib/fwd.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/optional_ptr.hpp>
#include <nano/lib/stream.hpp>
#include <boost/property_tree/ptree_fwd.hpp>
#include <optional>
typedef struct blake2b_state__ blake2b_state;
namespace nano
{
class block_visitor;
class mutable_block_visitor;
class object_stream;
using block_uniquer = uniquer<nano::uint256_union, nano::block>;
}
namespace nano
{
class block
{
public:

11
nano/lib/common.hpp
View file

@ -1,9 +1,14 @@
#pragma once
#include <nano/boost/asio/ip/tcp.hpp>
namespace boost::asio::ip
{
class tcp;
template <typename InternetProtocol>
class basic_endpoint;
}
namespace nano
{
using endpoint = boost::asio::ip::tcp::endpoint;
using endpoint = boost::asio::ip::basic_endpoint<boost::asio::ip::tcp>;
using tcp_endpoint = endpoint;
}
}

2
nano/lib/config.cpp
View file

@ -1,8 +1,10 @@
#include <nano/crypto/blake2/blake2.h>
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/config.hpp>
#include <nano/lib/env.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/work_version.hpp>
#include <boost/format.hpp>
#include <boost/lexical_cast.hpp>

6
nano/lib/config.hpp
View file

@ -105,15 +105,11 @@ enum class networks : uint16_t
std::string_view to_string (nano::networks);
enum class work_version
{
unspecified,
work_1
};
enum class block_type : uint8_t;
class root;
class block;
class block_details;
enum class work_version;
class work_thresholds
{

37
nano/lib/epoch.cpp
View file

@ -1,43 +1,6 @@
#include <nano/lib/epoch.hpp>
#include <nano/lib/utility.hpp>
#include <algorithm>
nano::link const & nano::epochs::link (nano::epoch epoch_a) const
{
return epochs_m.at (epoch_a).link;
}
bool nano::epochs::is_epoch_link (nano::link const & link_a) const
{
return std::any_of (epochs_m.begin (), epochs_m.end (), [&link_a] (auto const & item_a) { return item_a.second.link == link_a; });
}
nano::public_key const & nano::epochs::signer (nano::epoch epoch_a) const
{
return epochs_m.at (epoch_a).signer;
}
nano::epoch nano::epochs::epoch (nano::link const & link_a) const
{
auto existing (std::find_if (epochs_m.begin (), epochs_m.end (), [&link_a] (auto const & item_a) { return item_a.second.link == link_a; }));
debug_assert (existing != epochs_m.end ());
return existing->first;
}
void nano::epochs::add (nano::epoch epoch_a, nano::public_key const & signer_a, nano::link const & link_a)
{
debug_assert (epochs_m.find (epoch_a) == epochs_m.end ());
epochs_m[epoch_a] = { signer_a, link_a };
}
bool nano::epochs::is_sequential (nano::epoch epoch_a, nano::epoch new_epoch_a)
{
auto head_epoch = std::underlying_type_t<nano::epoch> (epoch_a);
bool is_valid_epoch (head_epoch >= std::underlying_type_t<nano::epoch> (nano::epoch::epoch_0));
return is_valid_epoch && (std::underlying_type_t<nano::epoch> (new_epoch_a) == (head_epoch + 1));
}
std::underlying_type_t<nano::epoch> nano::normalized_epoch (nano::epoch epoch_a)
{
// Currently assumes that the epoch versions in the enum are sequential.

42
nano/lib/epoch.hpp
View file

@ -1,9 +1,8 @@
#pragma once
#include <nano/lib/numbers.hpp>
#include <cstdint>
#include <functional>
#include <type_traits>
#include <unordered_map>
namespace nano
{
@ -31,42 +30,7 @@ struct hash<::nano::epoch>
{
std::size_t operator() (::nano::epoch const & epoch_a) const
{
std::hash<std::underlying_type_t<::nano::epoch>> hash;
return hash (static_cast<std::underlying_type_t<::nano::epoch>> (epoch_a));
return std::underlying_type_t<::nano::epoch> (epoch_a);
}
};
}
namespace nano
{
class epoch_info
{
public:
nano::public_key signer;
nano::link link;
};
class epochs
{
public:
/** Returns true if link matches one of the released epoch links.
* WARNING: just because a legal block contains an epoch link, it does not mean it is an epoch block.
* A legal block containing an epoch link can easily be constructed by sending to an address identical
* to one of the epoch links.
* Epoch blocks follow the following rules and a block must satisfy them all to be a true epoch block:
* epoch blocks are always state blocks
* epoch blocks never change the balance of an account
* epoch blocks always have a link field that starts with the ascii bytes "epoch v1 block" or "epoch v2 block" (and possibly others in the future)
* epoch blocks never change the representative
* epoch blocks are not signed by the account key, they are signed either by genesis or by special epoch keys
*/
bool is_epoch_link (nano::link const & link_a) const;
nano::link const & link (nano::epoch epoch_a) const;
nano::public_key const & signer (nano::epoch epoch_a) const;
nano::epoch epoch (nano::link const & link_a) const;
void add (nano::epoch epoch_a, nano::public_key const & signer_a, nano::link const & link_a);
/** Checks that new_epoch is 1 version higher than epoch */
static bool is_sequential (nano::epoch epoch_a, nano::epoch new_epoch_a);
private:
std::unordered_map<nano::epoch, nano::epoch_info> epochs_m;
};
}

39
nano/lib/epochs.cpp Normal file
View file

@ -0,0 +1,39 @@
#include <nano/lib/epochs.hpp>
#include <nano/lib/utility.hpp>
#include <algorithm>
nano::link const & nano::epochs::link (nano::epoch epoch_a) const
{
return epochs_m.at (epoch_a).link;
}
bool nano::epochs::is_epoch_link (nano::link const & link_a) const
{
return std::any_of (epochs_m.begin (), epochs_m.end (), [&link_a] (auto const & item_a) { return item_a.second.link == link_a; });
}
nano::public_key const & nano::epochs::signer (nano::epoch epoch_a) const
{
return epochs_m.at (epoch_a).signer;
}
nano::epoch nano::epochs::epoch (nano::link const & link_a) const
{
auto existing (std::find_if (epochs_m.begin (), epochs_m.end (), [&link_a] (auto const & item_a) { return item_a.second.link == link_a; }));
debug_assert (existing != epochs_m.end ());
return existing->first;
}
void nano::epochs::add (nano::epoch epoch_a, nano::public_key const & signer_a, nano::link const & link_a)
{
debug_assert (epochs_m.find (epoch_a) == epochs_m.end ());
epochs_m[epoch_a] = { signer_a, link_a };
}
bool nano::epochs::is_sequential (nano::epoch epoch_a, nano::epoch new_epoch_a)
{
auto head_epoch = std::underlying_type_t<nano::epoch> (epoch_a);
bool is_valid_epoch (head_epoch >= std::underlying_type_t<nano::epoch> (nano::epoch::epoch_0));
return is_valid_epoch && (std::underlying_type_t<nano::epoch> (new_epoch_a) == (head_epoch + 1));
}

42
nano/lib/epochs.hpp Normal file
View file

@ -0,0 +1,42 @@
#pragma once
#include <nano/lib/epoch.hpp>
#include <nano/lib/numbers.hpp>
#include <cstdint>
#include <unordered_map>
namespace nano
{
class epoch_info
{
public:
nano::public_key signer;
nano::link link;
};
class epochs
{
public:
/** Returns true if link matches one of the released epoch links.
* WARNING: just because a legal block contains an epoch link, it does not mean it is an epoch block.
* A legal block containing an epoch link can easily be constructed by sending to an address identical
* to one of the epoch links.
* Epoch blocks follow the following rules and a block must satisfy them all to be a true epoch block:
* epoch blocks are always state blocks
* epoch blocks never change the balance of an account
* epoch blocks always have a link field that starts with the ascii bytes "epoch v1 block" or "epoch v2 block" (and possibly others in the future)
* epoch blocks never change the representative
* epoch blocks are not signed by the account key, they are signed either by genesis or by special epoch keys
*/
bool is_epoch_link (nano::link const & link_a) const;
nano::link const & link (nano::epoch epoch_a) const;
nano::public_key const & signer (nano::epoch epoch_a) const;
nano::epoch epoch (nano::link const & link_a) const;
void add (nano::epoch epoch_a, nano::public_key const & signer_a, nano::link const & link_a);
/** Checks that new_epoch is 1 version higher than epoch */
static bool is_sequential (nano::epoch epoch_a, nano::epoch new_epoch_a);
private:
std::unordered_map<nano::epoch, nano::epoch_info> epochs_m;
};
}

31
nano/lib/fwd.hpp Normal file
View file

@ -0,0 +1,31 @@
#pragma once
#include <cstdint>
#include <iosfwd>
struct uint8_char_traits;
namespace nano
{
class block;
enum class block_type : uint8_t;
class block_visitor;
class container_info;
enum class epoch : uint8_t;
class jsonconfig;
class mutable_block_visitor;
class network_constants;
class object_stream;
class thread_pool;
class tomlconfig;
template <typename Key, typename Value>
class uniquer;
enum class work_version;
using stream = std::basic_streambuf<uint8_t, uint8_char_traits>;
}
namespace nano::stat
{
enum class detail;
enum class dir;
}

10
nano/lib/memory.hpp
View file

@ -48,6 +48,16 @@ private:
std::vector<std::function<void ()>> cleanup_funcs;
};
/** Helper guard which contains all the necessary purge (remove all memory even if used) functions */
class node_singleton_memory_pool_purge_guard
{
public:
node_singleton_memory_pool_purge_guard ();
private:
nano::cleanup_guard cleanup_guard;
};
template <typename T, typename... Args>
std::shared_ptr<T> make_shared (Args &&... args)
{

1
nano/lib/network_filter.hpp
View file

@ -1,6 +1,7 @@
#pragma once
#include <nano/lib/locks.hpp>
#include <nano/lib/numbers.hpp>
#include <cryptopp/seckey.h>

178
nano/lib/numbers.hpp
View file

@ -1,6 +1,6 @@
#pragma once
#include <boost/functional/hash.hpp>
#include <boost/functional/hash_fwd.hpp>
#include <boost/multiprecision/cpp_int.hpp>
#include <array>
@ -516,185 +516,53 @@ namespace difficulty
namespace std
{
template <>
struct hash<::nano::uint128_union>
{
size_t operator() (::nano::uint128_union const & value) const noexcept
{
return value.qwords[0] + value.qwords[1];
}
};
struct hash<::nano::uint128_union>;
template <>
struct hash<::nano::uint256_union>
{
size_t operator() (::nano::uint256_union const & value) const noexcept
{
return value.qwords[0] + value.qwords[1] + value.qwords[2] + value.qwords[3];
}
};
struct hash<::nano::uint256_union>;
template <>
struct hash<::nano::public_key>
{
size_t operator() (::nano::public_key const & value) const noexcept
{
return hash<::nano::uint256_union>{}(value);
}
};
struct hash<::nano::public_key>;
template <>
struct hash<::nano::block_hash>
{
size_t operator() (::nano::block_hash const & value) const noexcept
{
return hash<::nano::uint256_union>{}(value);
}
};
struct hash<::nano::block_hash>;
template <>
struct hash<::nano::hash_or_account>
{
size_t operator() (::nano::hash_or_account const & value) const noexcept
{
return hash<::nano::block_hash>{}(value.as_block_hash ());
}
};
struct hash<::nano::hash_or_account>;
template <>
struct hash<::nano::root>
{
size_t operator() (::nano::root const & value) const noexcept
{
return hash<::nano::hash_or_account>{}(value);
}
};
struct hash<::nano::root>;
template <>
struct hash<::nano::link>
{
size_t operator() (::nano::link const & value) const noexcept
{
return hash<::nano::hash_or_account>{}(value);
}
};
struct hash<::nano::link>;
template <>
struct hash<::nano::raw_key>
{
size_t operator() (::nano::raw_key const & value) const noexcept
{
return hash<::nano::uint256_union>{}(value);
}
};
struct hash<::nano::raw_key>;
template <>
struct hash<::nano::wallet_id>
{
size_t operator() (::nano::wallet_id const & value) const noexcept
{
return hash<::nano::uint256_union>{}(value);
}
};
struct hash<::nano::wallet_id>;
template <>
struct hash<::nano::uint512_union>
{
size_t operator() (::nano::uint512_union const & value) const noexcept
{
return hash<::nano::uint256_union>{}(value.uint256s[0]) + hash<::nano::uint256_union> () (value.uint256s[1]);
}
};
struct hash<::nano::uint512_union>;
template <>
struct hash<::nano::qualified_root>
{
size_t operator() (::nano::qualified_root const & value) const noexcept
{
return hash<::nano::uint512_union>{}(value);
}
};
struct hash<::nano::qualified_root>;
}
namespace boost
{
template <>
struct hash<::nano::uint128_union>
{
size_t operator() (::nano::uint128_union const & value) const noexcept
{
return std::hash<::nano::uint128_union> () (value);
}
};
struct hash<::nano::uint128_union>;
template <>
struct hash<::nano::uint256_union>
{
size_t operator() (::nano::uint256_union const & value) const noexcept
{
return std::hash<::nano::uint256_union> () (value);
}
};
struct hash<::nano::uint256_union>;
template <>
struct hash<::nano::public_key>
{
size_t operator() (::nano::public_key const & value) const noexcept
{
return std::hash<::nano::public_key> () (value);
}
};
struct hash<::nano::public_key>;
template <>
struct hash<::nano::block_hash>
{
size_t operator() (::nano::block_hash const & value) const noexcept
{
return std::hash<::nano::block_hash> () (value);
}
};
struct hash<::nano::block_hash>;
template <>
struct hash<::nano::hash_or_account>
{
size_t operator() (::nano::hash_or_account const & value) const noexcept
{
return std::hash<::nano::hash_or_account> () (value);
}
};
struct hash<::nano::hash_or_account>;
template <>
struct hash<::nano::root>
{
size_t operator() (::nano::root const & value) const noexcept
{
return std::hash<::nano::root> () (value);
}
};
struct hash<::nano::root>;
template <>
struct hash<::nano::link>
{
size_t operator() (::nano::link const & value) const noexcept
{
return std::hash<::nano::link> () (value);
}
};
struct hash<::nano::link>;
template <>
struct hash<::nano::raw_key>
{
size_t operator() (::nano::raw_key const & value) const noexcept
{
return std::hash<::nano::raw_key> () (value);
}
};
struct hash<::nano::raw_key>;
template <>
struct hash<::nano::wallet_id>
{
size_t operator() (::nano::wallet_id const & value) const noexcept
{
return std::hash<::nano::wallet_id> () (value);
}
};
struct hash<::nano::wallet_id>;
template <>
struct hash<::nano::uint512_union>
{
size_t operator() (::nano::uint512_union const & value) const noexcept
{
return std::hash<::nano::uint512_union> () (value);
}
};
struct hash<::nano::uint512_union>;
template <>
struct hash<::nano::qualified_root>
{
size_t operator() (::nano::qualified_root const & value) const noexcept
{
return std::hash<::nano::qualified_root> () (value);
}
};
struct hash<::nano::qualified_root>;
}
/*

189
nano/lib/numbers_templ.hpp Normal file
View file

@ -0,0 +1,189 @@
#pragma once
#include <nano/lib/numbers.hpp>
#include <boost/functional/hash.hpp>
namespace std
{
template <>
struct hash<::nano::uint128_union>
{
size_t operator() (::nano::uint128_union const & value) const noexcept
{
return value.qwords[0] + value.qwords[1];
}
};
template <>
struct hash<::nano::uint256_union>
{
size_t operator() (::nano::uint256_union const & value) const noexcept
{
return value.qwords[0] + value.qwords[1] + value.qwords[2] + value.qwords[3];
}
};
template <>
struct hash<::nano::public_key>
{
size_t operator() (::nano::public_key const & value) const noexcept
{
return hash<::nano::uint256_union>{}(value);
}
};
template <>
struct hash<::nano::block_hash>
{
size_t operator() (::nano::block_hash const & value) const noexcept
{
return hash<::nano::uint256_union>{}(value);
}
};
template <>
struct hash<::nano::hash_or_account>
{
size_t operator() (::nano::hash_or_account const & value) const noexcept
{
return hash<::nano::block_hash>{}(value.as_block_hash ());
}
};
template <>
struct hash<::nano::root>
{
size_t operator() (::nano::root const & value) const noexcept
{
return hash<::nano::hash_or_account>{}(value);
}
};
template <>
struct hash<::nano::link>
{
size_t operator() (::nano::link const & value) const noexcept
{
return hash<::nano::hash_or_account>{}(value);
}
};
template <>
struct hash<::nano::raw_key>
{
size_t operator() (::nano::raw_key const & value) const noexcept
{
return hash<::nano::uint256_union>{}(value);
}
};
template <>
struct hash<::nano::wallet_id>
{
size_t operator() (::nano::wallet_id const & value) const noexcept
{
return hash<::nano::uint256_union>{}(value);
}
};
template <>
struct hash<::nano::uint512_union>
{
size_t operator() (::nano::uint512_union const & value) const noexcept
{
return hash<::nano::uint256_union>{}(value.uint256s[0]) + hash<::nano::uint256_union> () (value.uint256s[1]);
}
};
template <>
struct hash<::nano::qualified_root>
{
size_t operator() (::nano::qualified_root const & value) const noexcept
{
return hash<::nano::uint512_union>{}(value);
}
};
}
namespace boost
{
template <>
struct hash<::nano::uint128_union>
{
size_t operator() (::nano::uint128_union const & value) const noexcept
{
return std::hash<::nano::uint128_union> () (value);
}
};
template <>
struct hash<::nano::uint256_union>
{
size_t operator() (::nano::uint256_union const & value) const noexcept
{
return std::hash<::nano::uint256_union> () (value);
}
};
template <>
struct hash<::nano::public_key>
{
size_t operator() (::nano::public_key const & value) const noexcept
{
return std::hash<::nano::public_key> () (value);
}
};
template <>
struct hash<::nano::block_hash>
{
size_t operator() (::nano::block_hash const & value) const noexcept
{
return std::hash<::nano::block_hash> () (value);
}
};
template <>
struct hash<::nano::hash_or_account>
{
size_t operator() (::nano::hash_or_account const & value) const noexcept
{
return std::hash<::nano::hash_or_account> () (value);
}
};
template <>
struct hash<::nano::root>
{
size_t operator() (::nano::root const & value) const noexcept
{
return std::hash<::nano::root> () (value);
}
};
template <>
struct hash<::nano::link>
{
size_t operator() (::nano::link const & value) const noexcept
{
return std::hash<::nano::link> () (value);
}
};
template <>
struct hash<::nano::raw_key>
{
size_t operator() (::nano::raw_key const & value) const noexcept
{
return std::hash<::nano::raw_key> () (value);
}
};
template <>
struct hash<::nano::wallet_id>
{
size_t operator() (::nano::wallet_id const & value) const noexcept
{
return std::hash<::nano::wallet_id> () (value);
}
};
template <>
struct hash<::nano::uint512_union>
{
size_t operator() (::nano::uint512_union const & value) const noexcept
{
return std::hash<::nano::uint512_union> () (value);
}
};
template <>
struct hash<::nano::qualified_root>
{
size_t operator() (::nano::qualified_root const & value) const noexcept
{
return std::hash<::nano::qualified_root> () (value);
}
};
}

2
nano/lib/optional_ptr.hpp
View file

@ -1,7 +1,5 @@
#pragma once
#include <nano/lib/utility.hpp>
#include <cstddef>
#include <memory>

29
nano/lib/random.hpp
View file

@ -1,5 +1,6 @@
#pragma once
#include <mutex>
#include <random>
namespace nano
@ -28,4 +29,32 @@ private:
std::random_device device;
std::default_random_engine rng{ device () };
};
/**
* Not safe for any crypto related code, use for non-crypto PRNG only.
* Thread safe.
*/
class random_generator_mt final
{
public:
/// Generate a random number in the range [min, max)
auto random (auto min, auto max)
{
release_assert (min < max);
std::lock_guard<std::mutex> lock{ mutex };
std::uniform_int_distribution<decltype (min)> dist (min, max - 1);
return dist (rng);
}
/// Generate a random number in the range [0, max)
auto random (auto max)
{
return random (decltype (max){ 0 }, max);
}
private:
std::random_device device;
std::default_random_engine rng{ device () };
std::mutex mutex;
};
}

8
nano/lib/rate_limiting.hpp
View file

@ -1,5 +1,7 @@
#pragma once
#include <nano/lib/locks.hpp>
#include <algorithm>
#include <chrono>
#include <mutex>
@ -64,13 +66,13 @@ class rate_limiter final
{
public:
// initialize with limit 0 = unbounded
rate_limiter (std::size_t limit, double burst_ratio);
rate_limiter (std::size_t limit, double burst_ratio = 1.0);
bool should_pass (std::size_t buffer_size);
void reset (std::size_t limit, double burst_ratio);
void reset (std::size_t limit, double burst_ratio = 1.0);
private:
nano::rate::token_bucket bucket;
mutable nano::mutex mutex;
};
}
}

43
nano/lib/stats_enums.hpp
View file

@ -19,7 +19,6 @@ enum class type
block,
ledger,
rollback,
bootstrap,
network,
tcp_server,
vote,
@ -58,13 +57,18 @@ enum class type
blockprocessor_source,
blockprocessor_result,
blockprocessor_overfill,
bootstrap_ascending,
bootstrap_ascending_accounts,
bootstrap_ascending_verify,
bootstrap_ascending_process,
bootstrap_ascending_request,
bootstrap_ascending_reply,
bootstrap_ascending_next,
bootstrap,
bootstrap_verify,
bootstrap_verify_blocks,
bootstrap_verify_frontiers,
bootstrap_process,
bootstrap_request,
bootstrap_request_blocks,
bootstrap_reply,
bootstrap_next,
bootstrap_frontiers,
bootstrap_account_sets,
bootstrap_frontier_scan,
bootstrap_server,
bootstrap_server_request,
bootstrap_server_overfill,
@ -118,6 +122,7 @@ enum class detail
inserted,
erased,
request,
request_failed,
broadcast,
cleanup,
top,
@ -137,6 +142,8 @@ enum class detail
empty,
done,
retry,
prioritized,
pending,
requeued,
evicted,
@ -432,7 +439,7 @@ enum class detail
missing_cookie,
invalid_genesis,
// bootstrap ascending
// bootstrap
missing_tag,
reply,
throttled,
@ -440,13 +447,17 @@ enum class detail
timeout,
nothing_new,
account_info_empty,
frontiers_empty,
loop_database,
loop_dependencies,
loop_frontiers,
loop_frontiers_processing,
duplicate_request,
invalid_response_type,
timestamp_reset,
processing_frontiers,
frontiers_dropped,
// bootstrap ascending accounts
prioritize,
prioritize_failed,
block,
@ -455,11 +466,19 @@ enum class detail
dependency_update,
dependency_update_failed,
done_range,
done_empty,
next_by_requests,
next_by_timestamp,
advance,
advance_failed,
next_none,
next_priority,
next_database,
next_blocking,
next_dependency,
next_frontier,
blocking_insert,
blocking_erase_overflow,
@ -474,9 +493,13 @@ enum class detail
request_blocks,
request_account_info,
safe,
base,
// active
started_hinted,
started_optimistic,
// rep_crawler
channel_dead,
query_target_failed,

22
nano/lib/thread_roles.cpp
View file

@ -73,9 +73,6 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
case nano::thread_role::name::worker:
thread_role_name_string = "Worker";
break;
case nano::thread_role::name::bootstrap_worker:
thread_role_name_string = "Bootstrap work";
break;
case nano::thread_role::name::wallet_worker:
thread_role_name_string = "Wallet work";
break;
@ -103,8 +100,23 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
case nano::thread_role::name::vote_generator_queue:
thread_role_name_string = "Voting que";
break;
case nano::thread_role::name::ascending_bootstrap:
thread_role_name_string = "Bootstrap asc";
case nano::thread_role::name::bootstrap:
thread_role_name_string = "Bootstrap";
break;
case nano::thread_role::name::bootstrap_database_scan:
thread_role_name_string = "Bootstrap db";
break;
case nano::thread_role::name::bootstrap_dependendy_walker:
thread_role_name_string = "Bootstrap walkr";
break;
case nano::thread_role::name::bootstrap_frontier_scan:
thread_role_name_string = "Bootstrap front";
break;
case nano::thread_role::name::bootstrap_cleanup:
thread_role_name_string = "Bootstrap clean";
break;
case nano::thread_role::name::bootstrap_worker:
thread_role_name_string = "Bootstrap work";
break;
case nano::thread_role::name::bootstrap_server:
thread_role_name_string = "Bootstrap serv";

12
nano/lib/thread_roles.hpp
View file

@ -29,7 +29,6 @@ enum class name
confirmation_height,
confirmation_height_notifications,
worker,
bootstrap_worker,
wallet_worker,
election_worker,
request_aggregator,
@ -39,11 +38,14 @@ enum class name
unchecked,
backlog_population,
vote_generator_queue,
bootstrap_server,
telemetry,
ascending_bootstrap,
bootstrap_server_requests,
bootstrap_server_responses,
bootstrap,
bootstrap_database_scan,
bootstrap_dependendy_walker,
bootstrap_frontier_scan,
bootstrap_cleanup,
bootstrap_worker,
bootstrap_server,
scheduler_hinted,
scheduler_manual,
scheduler_optimistic,

5
nano/lib/work.cpp
View file

@ -5,6 +5,7 @@
#include <nano/lib/thread_roles.hpp>
#include <nano/lib/threading.hpp>
#include <nano/lib/work.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/node/xorshift.hpp>
#include <future>
@ -41,7 +42,7 @@ nano::work_pool::work_pool (nano::network_constants & network_constants, unsigne
}
for (auto i (0u); i < count; ++i)
{
threads.emplace_back (nano::thread_attributes::get_default (), [this, i] () {
threads.emplace_back ([this, i] () {
nano::thread_role::set (nano::thread_role::name::work);
nano::work_thread_reprioritize ();
loop (i);
@ -240,4 +241,4 @@ nano::container_info nano::work_pool::container_info () const
info.put ("pending", pending);
info.add ("work_observers", work_observers.container_info ());
return info;
}
}

4
nano/lib/work.hpp
View file

@ -8,10 +8,10 @@
#include <nano/node/openclwork.hpp>
#include <boost/optional.hpp>
#include <boost/thread/thread.hpp>
#include <atomic>
#include <memory>
#include <thread>
namespace nano
{
@ -54,7 +54,7 @@ public:
nano::network_constants & network_constants;
std::atomic<int> ticket;
bool done;
std::vector<boost::thread> threads;
std::vector<std::thread> threads;
std::list<nano::work_item> pending;
mutable nano::mutex mutex{ mutex_identifier (mutexes::work_pool) };
nano::condition_variable producer_condition;

10
nano/lib/work_version.hpp Normal file
View file

@ -0,0 +1,10 @@
#pragma once
namespace nano
{
enum class work_version
{
unspecified,
work_1
};
}

3
nano/nano_node/entry.cpp
View file

@ -1,8 +1,10 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/cli.hpp>
#include <nano/lib/thread_runner.hpp>
#include <nano/lib/utility.hpp>
#include <nano/lib/work_version.hpp>
#include <nano/nano_node/daemon.hpp>
#include <nano/node/active_elections.hpp>
#include <nano/node/cli.hpp>
@ -15,6 +17,7 @@
#include <nano/node/transport/inproc.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
#include <nano/secure/vote.hpp>
#include <nano/store/pending.hpp>
#include <boost/dll/runtime_symbol_info.hpp>

53
nano/node/CMakeLists.txt
View file

@ -22,43 +22,27 @@ add_library(
bandwidth_limiter.cpp
blockprocessor.hpp
blockprocessor.cpp
bootstrap/block_deserializer.hpp
bootstrap/block_deserializer.cpp
bootstrap/bootstrap_attempt.hpp
bootstrap/bootstrap_attempt.cpp
bootstrap/bootstrap_bulk_pull.hpp
bootstrap/bootstrap_bulk_pull.cpp
bootstrap/bootstrap_bulk_push.hpp
bootstrap/bootstrap_bulk_push.cpp
bootstrap_weights_beta.hpp
bootstrap_weights_live.hpp
bootstrap/account_sets.hpp
bootstrap/account_sets.cpp
bootstrap/bootstrap_config.hpp
bootstrap/bootstrap_config.cpp
bootstrap/bootstrap_connections.hpp
bootstrap/bootstrap_connections.cpp
bootstrap/bootstrap_frontier.hpp
bootstrap/bootstrap_frontier.cpp
bootstrap/bootstrap_lazy.hpp
bootstrap/bootstrap_lazy.cpp
bootstrap/bootstrap_legacy.hpp
bootstrap/bootstrap_legacy.cpp
bootstrap/bootstrap.hpp
bootstrap/bootstrap.cpp
bootstrap/bootstrap_server.hpp
bootstrap/bootstrap_server.cpp
bootstrap_ascending/common.hpp
bootstrap_ascending/throttle.hpp
bootstrap_ascending/throttle.cpp
bootstrap_ascending/account_sets.hpp
bootstrap_ascending/account_sets.cpp
bootstrap_ascending/database_scan.hpp
bootstrap_ascending/database_scan.cpp
bootstrap_ascending/peer_scoring.hpp
bootstrap_ascending/peer_scoring.cpp
bootstrap_ascending/service.hpp
bootstrap_ascending/service.cpp
bootstrap/bootstrap_service.hpp
bootstrap/bootstrap_service.cpp
bootstrap/common.hpp
bootstrap/throttle.hpp
bootstrap/throttle.cpp
bootstrap/database_scan.hpp
bootstrap/database_scan.cpp
bootstrap/frontier_scan.hpp
bootstrap/frontier_scan.cpp
bootstrap/peer_scoring.hpp
bootstrap/peer_scoring.cpp
cli.hpp
cli.cpp
common.hpp
common.cpp
confirming_set.hpp
confirming_set.cpp
confirmation_solicitor.hpp
@ -74,9 +58,13 @@ add_library(
election_behavior.hpp
election_insertion_result.hpp
election_status.hpp
endpoint.cpp
endpoint.hpp
endpoint_templ.hpp
epoch_upgrader.hpp
epoch_upgrader.cpp
fair_queue.hpp
fwd.hpp
ipc/action_handler.hpp
ipc/action_handler.cpp
ipc/flatbuffers_handler.hpp
@ -157,12 +145,15 @@ add_library(
scheduler/priority.cpp
telemetry.hpp
telemetry.cpp
transport/block_deserializer.hpp
transport/block_deserializer.cpp
transport/channel.hpp
transport/channel.cpp
transport/tcp_channel.hpp
transport/tcp_channel.cpp
transport/fake.hpp
transport/fake.cpp
transport/fwd.hpp
transport/inproc.hpp
transport/inproc.cpp
transport/message_deserializer.hpp

3
nano/node/active_elections.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/enum_util.hpp>
#include <nano/lib/numbers.hpp>
@ -632,4 +633,4 @@ nano::stat::type nano::to_stat_type (nano::election_state state)
nano::stat::detail nano::to_stat_detail (nano::election_status_type type)
{
return nano::enum_util::cast<nano::stat::detail> (type);
}
}

2
nano/node/bandwidth_limiter.hpp
View file

@ -50,4 +50,4 @@ private:
nano::rate_limiter limiter_generic;
nano::rate_limiter limiter_bootstrap;
};
}
}

3
nano/node/blockprocessor.cpp
View file

@ -1,3 +1,4 @@
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/enum_util.hpp>
#include <nano/lib/threading.hpp>
@ -517,4 +518,4 @@ std::string_view nano::to_string (nano::block_source source)
nano::stat::detail nano::to_stat_detail (nano::block_source type)
{
return nano::enum_util::cast<nano::stat::detail> (type);
}
}

86
nano/node/bootstrap_ascending/account_sets.cpp → nano/node/bootstrap/account_sets.cpp
View file

@ -1,7 +1,9 @@
#include <nano/lib/stats.hpp>
#include <nano/lib/utility.hpp>
#include <nano/node/bootstrap/account_sets.hpp>
#include <nano/node/bootstrap/bootstrap_config.hpp>
#include <nano/node/bootstrap_ascending/account_sets.hpp>
#include <boost/range/iterator_range.hpp>
#include <algorithm>
#include <memory>
@ -11,13 +13,13 @@
* account_sets
*/
nano::bootstrap_ascending::account_sets::account_sets (nano::account_sets_config const & config_a, nano::stats & stats_a) :
nano::bootstrap::account_sets::account_sets (nano::account_sets_config const & config_a, nano::stats & stats_a) :
config{ config_a },
stats{ stats_a }
{
}
void nano::bootstrap_ascending::account_sets::priority_up (nano::account const & account)
void nano::bootstrap::account_sets::priority_up (nano::account const & account)
{
if (account.is_zero ())
{
@ -26,7 +28,7 @@ void nano::bootstrap_ascending::account_sets::priority_up (nano::account const &
if (!blocked (account))
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::prioritize);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::prioritize);
auto iter = priorities.get<tag_account> ().find (account);
if (iter != priorities.get<tag_account> ().end ())
@ -37,18 +39,18 @@ void nano::bootstrap_ascending::account_sets::priority_up (nano::account const &
}
else
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::priority_insert);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_insert);
priorities.get<tag_account> ().insert ({ account, account_sets::priority_initial });
trim_overflow ();
}
}
else
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::prioritize_failed);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::prioritize_failed);
}
}
void nano::bootstrap_ascending::account_sets::priority_down (nano::account const & account)
void nano::bootstrap::account_sets::priority_down (nano::account const & account)
{
if (account.is_zero ())
{
@ -58,12 +60,12 @@ void nano::bootstrap_ascending::account_sets::priority_down (nano::account const
auto iter = priorities.get<tag_account> ().find (account);
if (iter != priorities.get<tag_account> ().end ())
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::deprioritize);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::deprioritize);
auto priority_new = iter->priority / account_sets::priority_divide;
if (priority_new <= account_sets::priority_cutoff)
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::priority_erase_by_threshold);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_erase_by_threshold);
priorities.get<tag_account> ().erase (iter);
}
else
@ -75,11 +77,11 @@ void nano::bootstrap_ascending::account_sets::priority_down (nano::account const
}
else
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::deprioritize_failed);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::deprioritize_failed);
}
}
void nano::bootstrap_ascending::account_sets::priority_set (nano::account const & account)
void nano::bootstrap::account_sets::priority_set (nano::account const & account)
{
if (account.is_zero ())
{
@ -91,36 +93,36 @@ void nano::bootstrap_ascending::account_sets::priority_set (nano::account const
auto iter = priorities.get<tag_account> ().find (account);
if (iter == priorities.get<tag_account> ().end ())
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::priority_insert);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_insert);
priorities.get<tag_account> ().insert ({ account, account_sets::priority_initial });
trim_overflow ();
}
}
else
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::prioritize_failed);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::prioritize_failed);
}
}
void nano::bootstrap_ascending::account_sets::block (nano::account const & account, nano::block_hash const & dependency)
void nano::bootstrap::account_sets::block (nano::account const & account, nano::block_hash const & dependency)
{
debug_assert (!account.is_zero ());
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::block);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::block);
auto existing = priorities.get<tag_account> ().find (account);
auto entry = (existing == priorities.get<tag_account> ().end ()) ? priority_entry{ account, 0 } : *existing;
priorities.get<tag_account> ().erase (account);
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::priority_erase_by_blocking);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_erase_by_blocking);
blocking.get<tag_account> ().insert ({ entry, dependency });
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::blocking_insert);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::blocking_insert);
trim_overflow ();
}
void nano::bootstrap_ascending::account_sets::unblock (nano::account const & account, std::optional<nano::block_hash> const & hash)
void nano::bootstrap::account_sets::unblock (nano::account const & account, std::optional<nano::block_hash> const & hash)
{
if (account.is_zero ())
{
@ -131,7 +133,7 @@ void nano::bootstrap_ascending::account_sets::unblock (nano::account const & acc
auto existing = blocking.get<tag_account> ().find (account);
if (existing != blocking.get<tag_account> ().end () && (!hash || existing->dependency == *hash))
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::unblock);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::unblock);
debug_assert (priorities.get<tag_account> ().count (account) == 0);
if (!existing->original_entry.account.is_zero ())
@ -149,11 +151,11 @@ void nano::bootstrap_ascending::account_sets::unblock (nano::account const & acc
}
else
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::unblock_failed);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::unblock_failed);
}
}
void nano::bootstrap_ascending::account_sets::timestamp_set (const nano::account & account)
void nano::bootstrap::account_sets::timestamp_set (const nano::account & account)
{
debug_assert (!account.is_zero ());
@ -166,7 +168,7 @@ void nano::bootstrap_ascending::account_sets::timestamp_set (const nano::account
}
}
void nano::bootstrap_ascending::account_sets::timestamp_reset (const nano::account & account)
void nano::bootstrap::account_sets::timestamp_reset (const nano::account & account)
{
debug_assert (!account.is_zero ());
@ -179,7 +181,7 @@ void nano::bootstrap_ascending::account_sets::timestamp_reset (const nano::accou
}
}
void nano::bootstrap_ascending::account_sets::dependency_update (nano::block_hash const & hash, nano::account const & dependency_account)
void nano::bootstrap::account_sets::dependency_update (nano::block_hash const & hash, nano::account const & dependency_account)
{
debug_assert (!dependency_account.is_zero ());
@ -190,7 +192,7 @@ void nano::bootstrap_ascending::account_sets::dependency_update (nano::block_has
{
if (it->dependency_account != dependency_account)
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::dependency_update);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::dependency_update);
blocking.get<tag_dependency> ().modify (it++, [dependency_account] (auto & entry) {
entry.dependency_account = dependency_account;
@ -204,27 +206,27 @@ void nano::bootstrap_ascending::account_sets::dependency_update (nano::block_has
}
else
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::dependency_update_failed);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::dependency_update_failed);
}
}
void nano::bootstrap_ascending::account_sets::trim_overflow ()
void nano::bootstrap::account_sets::trim_overflow ()
{
while (priorities.size () > config.priorities_max)
{
// Erase the oldest entry
priorities.pop_front ();
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::priority_erase_overflow);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_erase_overflow);
}
while (blocking.size () > config.blocking_max)
{
// Erase the oldest entry
blocking.pop_front ();
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::blocking_erase_overflow);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::blocking_erase_overflow);
}
}
nano::account nano::bootstrap_ascending::account_sets::next_priority (std::function<bool (nano::account const &)> const & filter)
nano::account nano::bootstrap::account_sets::next_priority (std::function<bool (nano::account const &)> const & filter)
{
if (priorities.empty ())
{
@ -249,7 +251,7 @@ nano::account nano::bootstrap_ascending::account_sets::next_priority (std::funct
return { 0 };
}
nano::block_hash nano::bootstrap_ascending::account_sets::next_blocking (std::function<bool (nano::block_hash const &)> const & filter)
nano::block_hash nano::bootstrap::account_sets::next_blocking (std::function<bool (nano::block_hash const &)> const & filter)
{
if (blocking.empty ())
{
@ -271,7 +273,7 @@ nano::block_hash nano::bootstrap_ascending::account_sets::next_blocking (std::fu
return { 0 };
}
void nano::bootstrap_ascending::account_sets::sync_dependencies ()
void nano::bootstrap::account_sets::sync_dependencies ()
{
// Sample all accounts with a known dependency account (> account 0)
auto begin = blocking.get<tag_dependency_account> ().upper_bound (nano::account{ 0 });
@ -288,7 +290,7 @@ void nano::bootstrap_ascending::account_sets::sync_dependencies ()
if (!blocked (entry.dependency_account) && !prioritized (entry.dependency_account))
{
stats.inc (nano::stat::type::bootstrap_ascending_accounts, nano::stat::detail::sync_dependencies);
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::sync_dependencies);
priority_set (entry.dependency_account);
}
}
@ -296,37 +298,37 @@ void nano::bootstrap_ascending::account_sets::sync_dependencies ()
trim_overflow ();
}
bool nano::bootstrap_ascending::account_sets::blocked (nano::account const & account) const
bool nano::bootstrap::account_sets::blocked (nano::account const & account) const
{
return blocking.get<tag_account> ().contains (account);
}
bool nano::bootstrap_ascending::account_sets::prioritized (nano::account const & account) const
bool nano::bootstrap::account_sets::prioritized (nano::account const & account) const
{
return priorities.get<tag_account> ().contains (account);
}
std::size_t nano::bootstrap_ascending::account_sets::priority_size () const
std::size_t nano::bootstrap::account_sets::priority_size () const
{
return priorities.size ();
}
std::size_t nano::bootstrap_ascending::account_sets::blocked_size () const
std::size_t nano::bootstrap::account_sets::blocked_size () const
{
return blocking.size ();
}
bool nano::bootstrap_ascending::account_sets::priority_half_full () const
bool nano::bootstrap::account_sets::priority_half_full () const
{
return priorities.size () > config.priorities_max / 2;
}
bool nano::bootstrap_ascending::account_sets::blocked_half_full () const
bool nano::bootstrap::account_sets::blocked_half_full () const
{
return blocking.size () > config.blocking_max / 2;
}
double nano::bootstrap_ascending::account_sets::priority (nano::account const & account) const
double nano::bootstrap::account_sets::priority (nano::account const & account) const
{
if (!blocked (account))
{
@ -339,12 +341,12 @@ double nano::bootstrap_ascending::account_sets::priority (nano::account const &
return 0.0;
}
auto nano::bootstrap_ascending::account_sets::info () const -> nano::bootstrap_ascending::account_sets::info_t
auto nano::bootstrap::account_sets::info () const -> nano::bootstrap::account_sets::info_t
{
return { blocking, priorities };
}
nano::container_info nano::bootstrap_ascending::account_sets::container_info () const
nano::container_info nano::bootstrap::account_sets::container_info () const
{
// Count blocking entries with their dependency account unknown
auto blocking_unknown = blocking.get<tag_dependency_account> ().count (nano::account{ 0 });
@ -354,4 +356,4 @@ nano::container_info nano::bootstrap_ascending::account_sets::container_info ()
info.put ("blocking", blocking);
info.put ("blocking_unknown", blocking_unknown);
return info;
}
}

9
nano/node/bootstrap_ascending/account_sets.hpp → nano/node/bootstrap/account_sets.hpp
View file

@ -2,7 +2,8 @@
#include <nano/lib/numbers.hpp>
#include <nano/node/bootstrap/bootstrap_config.hpp>
#include <nano/node/bootstrap_ascending/common.hpp>
#include <nano/node/bootstrap/common.hpp>
#include <nano/node/fwd.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/mem_fun.hpp>
@ -18,9 +19,7 @@ namespace mi = boost::multi_index;
namespace nano
{
class stats;
namespace bootstrap_ascending
namespace bootstrap
{
/** This class tracks accounts various account sets which are shared among the multiple bootstrap threads */
class account_sets
@ -161,4 +160,4 @@ namespace bootstrap_ascending
info_t info () const;
};
}
}
}

48
nano/node/bootstrap/block_deserializer.hpp
View file

@ -1,48 +0,0 @@
#pragma once
#include <nano/lib/block_type.hpp>
#include <nano/node/transport/fwd.hpp>
#include <boost/system/error_code.hpp>
#include <memory>
#include <vector>
namespace nano
{
class block;
namespace bootstrap
{
/**
* Class to read a block-type byte followed by a serialised block from a stream.
* It is typically used to read a series of block-types and blocks terminated by a not-a-block type.
*/
class block_deserializer : public std::enable_shared_from_this<nano::bootstrap::block_deserializer>
{
public:
using callback_type = std::function<void (boost::system::error_code, std::shared_ptr<nano::block>)>;
block_deserializer ();
/**
* Read a type-prefixed block from 'socket' and pass the result, or an error, to 'callback'
* A normal end to series of blocks is a marked by return no error and a nullptr for block.
*/
void read (nano::transport::tcp_socket & socket, callback_type const && callback);
private:
/**
* Called by read method on receipt of a block type byte.
* The type byte will be in the read_buffer.
*/
void received_type (nano::transport::tcp_socket & socket, callback_type const && callback);
/**
* Called by received_type when a block is received, it parses the block and calls the callback.
*/
void received_block (nano::block_type type, callback_type const && callback);
std::shared_ptr<std::vector<uint8_t>> read_buffer;
};
}
}

390
nano/node/bootstrap/bootstrap.cpp
View file

@ -1,390 +0,0 @@
#include <nano/node/bootstrap/bootstrap.hpp>
#include <nano/node/bootstrap/bootstrap_lazy.hpp>
#include <nano/node/bootstrap/bootstrap_legacy.hpp>
#include <nano/node/common.hpp>
#include <nano/node/node.hpp>
#include <algorithm>
#include <memory>
nano::bootstrap_initiator::bootstrap_initiator (nano::node & node_a) :
node (node_a)
{
connections = std::make_shared<nano::bootstrap_connections> (node);
bootstrap_initiator_threads.push_back (boost::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::bootstrap_connections);
connections->run ();
}));
for (std::size_t i = 0; i < node.config.bootstrap_initiator_threads; ++i)
{
bootstrap_initiator_threads.push_back (boost::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::bootstrap_initiator);
run_bootstrap ();
}));
}
}
nano::bootstrap_initiator::~bootstrap_initiator ()
{
stop ();
}
void nano::bootstrap_initiator::bootstrap (bool force, std::string id_a, uint32_t const frontiers_age_a, nano::account const & start_account_a)
{
if (force)
{
stop_attempts ();
}
nano::unique_lock<nano::mutex> lock{ mutex };
if (!stopped && find_attempt (nano::bootstrap_mode::legacy) == nullptr)
{
node.stats.inc (nano::stat::type::bootstrap, frontiers_age_a == std::numeric_limits<uint32_t>::max () ? nano::stat::detail::initiate : nano::stat::detail::initiate_legacy_age, nano::stat::dir::out);
auto legacy_attempt (std::make_shared<nano::bootstrap_attempt_legacy> (node.shared (), attempts.incremental++, id_a, frontiers_age_a, start_account_a));
attempts_list.push_back (legacy_attempt);
attempts.add (legacy_attempt);
lock.unlock ();
condition.notify_all ();
}
}
void nano::bootstrap_initiator::bootstrap (nano::endpoint const & endpoint_a, bool add_to_peers, std::string id_a)
{
if (add_to_peers)
{
if (!node.flags.disable_tcp_realtime)
{
node.network.merge_peer (nano::transport::map_endpoint_to_v6 (endpoint_a));
}
}
if (!stopped)
{
stop_attempts ();
node.stats.inc (nano::stat::type::bootstrap, nano::stat::detail::initiate, nano::stat::dir::out);
nano::lock_guard<nano::mutex> lock{ mutex };
auto legacy_attempt (std::make_shared<nano::bootstrap_attempt_legacy> (node.shared (), attempts.incremental++, id_a, std::numeric_limits<uint32_t>::max (), 0));
attempts_list.push_back (legacy_attempt);
attempts.add (legacy_attempt);
if (!node.network.excluded_peers.check (nano::transport::map_endpoint_to_tcp (endpoint_a)))
{
connections->add_connection (endpoint_a);
}
}
condition.notify_all ();
}
bool nano::bootstrap_initiator::bootstrap_lazy (nano::hash_or_account const & hash_or_account_a, bool force, std::string id_a)
{
bool key_inserted (false);
auto lazy_attempt (current_lazy_attempt ());
if (lazy_attempt == nullptr || force)
{
if (force)
{
stop_attempts ();
}
node.stats.inc (nano::stat::type::bootstrap, nano::stat::detail::initiate_lazy, nano::stat::dir::out);
nano::lock_guard<nano::mutex> lock{ mutex };
if (!stopped && find_attempt (nano::bootstrap_mode::lazy) == nullptr)
{
lazy_attempt = std::make_shared<nano::bootstrap_attempt_lazy> (node.shared (), attempts.incremental++, id_a.empty () ? hash_or_account_a.to_string () : id_a);
attempts_list.push_back (lazy_attempt);
attempts.add (lazy_attempt);
key_inserted = lazy_attempt->lazy_start (hash_or_account_a);
}
}
else
{
key_inserted = lazy_attempt->lazy_start (hash_or_account_a);
}
condition.notify_all ();
return key_inserted;
}
void nano::bootstrap_initiator::bootstrap_wallet (std::deque<nano::account> & accounts_a)
{
debug_assert (!accounts_a.empty ());
auto wallet_attempt (current_wallet_attempt ());
node.stats.inc (nano::stat::type::bootstrap, nano::stat::detail::initiate_wallet_lazy, nano::stat::dir::out);
if (wallet_attempt == nullptr)
{
nano::lock_guard<nano::mutex> lock{ mutex };
std::string id (!accounts_a.empty () ? accounts_a[0].to_account () : "");
wallet_attempt = std::make_shared<nano::bootstrap_attempt_wallet> (node.shared (), attempts.incremental++, id);
attempts_list.push_back (wallet_attempt);
attempts.add (wallet_attempt);
wallet_attempt->wallet_start (accounts_a);
}
else
{
wallet_attempt->wallet_start (accounts_a);
}
condition.notify_all ();
}
void nano::bootstrap_initiator::run_bootstrap ()
{
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
if (has_new_attempts ())
{
auto attempt (new_attempt ());
lock.unlock ();
if (attempt != nullptr)
{
attempt->run ();
remove_attempt (attempt);
}
lock.lock ();
}
else
{
condition.wait (lock);
}
}
}
void nano::bootstrap_initiator::lazy_requeue (nano::block_hash const & hash_a, nano::block_hash const & previous_a)
{
auto lazy_attempt (current_lazy_attempt ());
if (lazy_attempt != nullptr)
{
lazy_attempt->lazy_requeue (hash_a, previous_a);
}
}
void nano::bootstrap_initiator::add_observer (std::function<void (bool)> const & observer_a)
{
nano::lock_guard<nano::mutex> lock{ observers_mutex };
observers.push_back (observer_a);
}
bool nano::bootstrap_initiator::in_progress ()
{
nano::lock_guard<nano::mutex> lock{ mutex };
return !attempts_list.empty ();
}
void nano::bootstrap_initiator::block_processed (store::transaction const & tx, nano::block_status const & result, nano::block const & block)
{
nano::lock_guard<nano::mutex> lock{ mutex };
for (auto & i : attempts_list)
{
i->block_processed (tx, result, block);
}
}
std::shared_ptr<nano::bootstrap_attempt> nano::bootstrap_initiator::find_attempt (nano::bootstrap_mode mode_a)
{
for (auto & i : attempts_list)
{
if (i->mode == mode_a)
{
return i;
}
}
return nullptr;
}
void nano::bootstrap_initiator::remove_attempt (std::shared_ptr<nano::bootstrap_attempt> attempt_a)
{
nano::unique_lock<nano::mutex> lock{ mutex };
auto attempt (std::find (attempts_list.begin (), attempts_list.end (), attempt_a));
if (attempt != attempts_list.end ())
{
auto attempt_ptr (*attempt);
attempts.remove (attempt_ptr->incremental_id);
attempts_list.erase (attempt);
debug_assert (attempts.size () == attempts_list.size ());
lock.unlock ();
attempt_ptr->stop ();
}
else
{
lock.unlock ();
}
condition.notify_all ();
}
std::shared_ptr<nano::bootstrap_attempt> nano::bootstrap_initiator::new_attempt ()
{
for (auto & i : attempts_list)
{
if (!i->started.exchange (true))
{
return i;
}
}
return nullptr;
}
bool nano::bootstrap_initiator::has_new_attempts ()
{
for (auto & i : attempts_list)
{
if (!i->started)
{
return true;
}
}
return false;
}
std::shared_ptr<nano::bootstrap_attempt> nano::bootstrap_initiator::current_attempt ()
{
nano::lock_guard<nano::mutex> lock{ mutex };
return find_attempt (nano::bootstrap_mode::legacy);
}
std::shared_ptr<nano::bootstrap_attempt_lazy> nano::bootstrap_initiator::current_lazy_attempt ()
{
nano::lock_guard<nano::mutex> lock{ mutex };
return std::dynamic_pointer_cast<nano::bootstrap_attempt_lazy> (find_attempt (nano::bootstrap_mode::lazy));
}
std::shared_ptr<nano::bootstrap_attempt_wallet> nano::bootstrap_initiator::current_wallet_attempt ()
{
nano::lock_guard<nano::mutex> lock{ mutex };
return std::dynamic_pointer_cast<nano::bootstrap_attempt_wallet> (find_attempt (nano::bootstrap_mode::wallet_lazy));
}
void nano::bootstrap_initiator::stop_attempts ()
{
nano::unique_lock<nano::mutex> lock{ mutex };
std::vector<std::shared_ptr<nano::bootstrap_attempt>> copy_attempts;
copy_attempts.swap (attempts_list);
attempts.clear ();
lock.unlock ();
for (auto & i : copy_attempts)
{
i->stop ();
}
}
void nano::bootstrap_initiator::stop ()
{
if (!stopped.exchange (true))
{
stop_attempts ();
connections->stop ();
condition.notify_all ();
for (auto & thread : bootstrap_initiator_threads)
{
if (thread.joinable ())
{
thread.join ();
}
}
}
}
void nano::bootstrap_initiator::notify_listeners (bool in_progress_a)
{
nano::lock_guard<nano::mutex> lock{ observers_mutex };
for (auto & i : observers)
{
i (in_progress_a);
}
}
nano::container_info nano::bootstrap_initiator::container_info () const
{
nano::container_info info;
{
nano::lock_guard<nano::mutex> guard{ observers_mutex };
info.put ("observers", observers.size ());
}
{
nano::lock_guard<nano::mutex> guard{ cache.pulls_cache_mutex };
info.put ("pulls_cache", cache.cache.size ());
}
return info;
}
void nano::pulls_cache::add (nano::pull_info const & pull_a)
{
if (pull_a.processed > 500)
{
nano::lock_guard<nano::mutex> guard{ pulls_cache_mutex };
// Clean old pull
if (cache.size () > cache_size_max)
{
cache.erase (cache.begin ());
}
debug_assert (cache.size () <= cache_size_max);
nano::uint512_union head_512 (pull_a.account_or_head.as_union (), pull_a.head_original);
auto existing (cache.get<account_head_tag> ().find (head_512));
if (existing == cache.get<account_head_tag> ().end ())
{
// Insert new pull
auto inserted (cache.emplace (nano::cached_pulls{ std::chrono::steady_clock::now (), head_512, pull_a.head }));
(void)inserted;
debug_assert (inserted.second);
}
else
{
// Update existing pull
cache.get<account_head_tag> ().modify (existing, [pull_a] (nano::cached_pulls & cache_a) {
cache_a.time = std::chrono::steady_clock::now ();
cache_a.new_head = pull_a.head;
});
}
}
}
void nano::pulls_cache::update_pull (nano::pull_info & pull_a)
{
nano::lock_guard<nano::mutex> guard{ pulls_cache_mutex };
nano::uint512_union head_512 (pull_a.account_or_head.as_union (), pull_a.head_original);
auto existing (cache.get<account_head_tag> ().find (head_512));
if (existing != cache.get<account_head_tag> ().end ())
{
pull_a.head = existing->new_head;
}
}
void nano::pulls_cache::remove (nano::pull_info const & pull_a)
{
nano::lock_guard<nano::mutex> guard{ pulls_cache_mutex };
nano::uint512_union head_512 (pull_a.account_or_head.as_union (), pull_a.head_original);
cache.get<account_head_tag> ().erase (head_512);
}
void nano::bootstrap_attempts::add (std::shared_ptr<nano::bootstrap_attempt> attempt_a)
{
nano::lock_guard<nano::mutex> lock{ bootstrap_attempts_mutex };
attempts.emplace (attempt_a->incremental_id, attempt_a);
}
void nano::bootstrap_attempts::remove (uint64_t incremental_id_a)
{
nano::lock_guard<nano::mutex> lock{ bootstrap_attempts_mutex };
attempts.erase (incremental_id_a);
}
void nano::bootstrap_attempts::clear ()
{
nano::lock_guard<nano::mutex> lock{ bootstrap_attempts_mutex };
attempts.clear ();
}
std::shared_ptr<nano::bootstrap_attempt> nano::bootstrap_attempts::find (uint64_t incremental_id_a)
{
nano::lock_guard<nano::mutex> lock{ bootstrap_attempts_mutex };
auto find_attempt (attempts.find (incremental_id_a));
if (find_attempt != attempts.end ())
{
return find_attempt->second;
}
else
{
return nullptr;
}
}
std::size_t nano::bootstrap_attempts::size ()
{
nano::lock_guard<nano::mutex> lock{ bootstrap_attempts_mutex };
return attempts.size ();
}

157
nano/node/bootstrap/bootstrap.hpp
View file

@ -1,157 +0,0 @@
#pragma once
#include <nano/node/bootstrap/bootstrap_connections.hpp>
#include <nano/node/common.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index_container.hpp>
#include <boost/thread/thread.hpp>
#include <atomic>
namespace mi = boost::multi_index;
namespace nano::store
{
class transaction;
}
namespace nano
{
class node;
class bootstrap_connections;
namespace transport
{
class tcp_channel;
}
enum class bootstrap_mode
{
legacy,
lazy,
wallet_lazy,
ascending
};
enum class sync_result
{
success,
error,
fork
};
class cached_pulls final
{
public:
std::chrono::steady_clock::time_point time;
nano::uint512_union account_head;
nano::block_hash new_head;
};
class pulls_cache final
{
public:
void add (nano::pull_info const &);
void update_pull (nano::pull_info &);
void remove (nano::pull_info const &);
mutable nano::mutex pulls_cache_mutex;
class account_head_tag
{
};
// clang-format off
boost::multi_index_container<nano::cached_pulls,
mi::indexed_by<
mi::ordered_non_unique<
mi::member<nano::cached_pulls, std::chrono::steady_clock::time_point, &nano::cached_pulls::time>>,
mi::hashed_unique<mi::tag<account_head_tag>,
mi::member<nano::cached_pulls, nano::uint512_union, &nano::cached_pulls::account_head>>>>
cache;
// clang-format on
constexpr static std::size_t cache_size_max = 10000;
};
/**
* Container for bootstrap sessions that are active. Owned by bootstrap_initiator.
*/
class bootstrap_attempts final
{
public:
void add (std::shared_ptr<nano::bootstrap_attempt>);
void remove (uint64_t);
void clear ();
std::shared_ptr<nano::bootstrap_attempt> find (uint64_t);
std::size_t size ();
std::atomic<uint64_t> incremental{ 0 };
nano::mutex bootstrap_attempts_mutex;
std::map<uint64_t, std::shared_ptr<nano::bootstrap_attempt>> attempts;
};
class bootstrap_attempt_lazy;
class bootstrap_attempt_wallet;
/**
* Client side portion to initiate bootstrap sessions. Prevents multiple legacy-type bootstrap sessions from being started at the same time. Does permit
* lazy/wallet bootstrap sessions to overlap with legacy sessions.
*/
class bootstrap_initiator final
{
public:
explicit bootstrap_initiator (nano::node &);
~bootstrap_initiator ();
void bootstrap (nano::endpoint const &, bool add_to_peers = true, std::string id_a = "");
void bootstrap (bool force = false, std::string id_a = "", uint32_t const frontiers_age_a = std::numeric_limits<uint32_t>::max (), nano::account const & start_account_a = nano::account{});
bool bootstrap_lazy (nano::hash_or_account const &, bool force = false, std::string id_a = "");
void bootstrap_wallet (std::deque<nano::account> &);
void run_bootstrap ();
void lazy_requeue (nano::block_hash const &, nano::block_hash const &);
void notify_listeners (bool);
void add_observer (std::function<void (bool)> const &);
bool in_progress ();
void block_processed (store::transaction const & tx, nano::block_status const & result, nano::block const & block);
std::shared_ptr<nano::bootstrap_connections> connections;
std::shared_ptr<nano::bootstrap_attempt> new_attempt ();
bool has_new_attempts ();
void remove_attempt (std::shared_ptr<nano::bootstrap_attempt>);
std::shared_ptr<nano::bootstrap_attempt> current_attempt ();
std::shared_ptr<nano::bootstrap_attempt_lazy> current_lazy_attempt ();
std::shared_ptr<nano::bootstrap_attempt_wallet> current_wallet_attempt ();
nano::pulls_cache cache;
nano::bootstrap_attempts attempts;
void stop ();
nano::container_info container_info () const;
private:
nano::node & node;
std::shared_ptr<nano::bootstrap_attempt> find_attempt (nano::bootstrap_mode);
void stop_attempts ();
std::vector<std::shared_ptr<nano::bootstrap_attempt>> attempts_list;
std::atomic<bool> stopped{ false };
mutable nano::mutex mutex;
nano::condition_variable condition;
mutable nano::mutex observers_mutex;
std::vector<std::function<void (bool)>> observers;
std::vector<boost::thread> bootstrap_initiator_threads;
};
/**
* Defines the numeric values for the bootstrap feature.
*/
class bootstrap_limits final
{
public:
static constexpr double bootstrap_connection_scale_target_blocks = 10000.0;
static constexpr double bootstrap_connection_warmup_time_sec = 5.0;
static constexpr double bootstrap_minimum_blocks_per_sec = 10.0;
static constexpr double bootstrap_minimum_elapsed_seconds_blockrate = 0.02;
static constexpr double bootstrap_minimum_frontier_blocks_per_sec = 1000.0;
static constexpr double bootstrap_minimum_termination_time_sec = 30.0;
static constexpr unsigned bootstrap_max_new_connections = 32;
static constexpr unsigned requeued_pulls_limit = 256;
static constexpr unsigned requeued_pulls_limit_dev = 1;
static constexpr unsigned requeued_pulls_processed_blocks_factor = 4096;
static constexpr uint64_t pull_count_per_check = 8 * 1024;
static constexpr unsigned bulk_push_cost_limit = 200;
static constexpr std::chrono::seconds lazy_flush_delay_sec = std::chrono::seconds (5);
static constexpr uint64_t lazy_batch_pull_count_resize_blocks_limit = 4 * 1024 * 1024;
static constexpr double lazy_batch_pull_count_resize_ratio = 2.0;
static constexpr std::size_t lazy_blocks_restart_limit = 1024 * 1024;
};
}

145
nano/node/bootstrap/bootstrap_attempt.cpp
View file

@ -1,145 +0,0 @@
#include <nano/lib/blocks.hpp>
#include <nano/node/bootstrap/bootstrap.hpp>
#include <nano/node/bootstrap/bootstrap_attempt.hpp>
#include <nano/node/bootstrap/bootstrap_bulk_push.hpp>
#include <nano/node/node.hpp>
#include <nano/node/websocket.hpp>
#include <nano/secure/ledger.hpp>
#include <boost/format.hpp>
constexpr unsigned nano::bootstrap_limits::requeued_pulls_limit;
constexpr unsigned nano::bootstrap_limits::requeued_pulls_limit_dev;
nano::bootstrap_attempt::bootstrap_attempt (std::shared_ptr<nano::node> const & node_a, nano::bootstrap_mode mode_a, uint64_t incremental_id_a, std::string id_a) :
node (node_a),
incremental_id (incremental_id_a),
id (id_a),
mode (mode_a)
{
if (id.empty ())
{
id = nano::hardened_constants::get ().random_128.to_string ();
}
node_a->logger.debug (nano::log::type::bootstrap, "Starting bootstrap attempt with ID: {} (mode: {})", mode_text (), id);
node_a->bootstrap_initiator.notify_listeners (true);
if (node_a->websocket.server)
{
nano::websocket::message_builder builder;
node_a->websocket.server->broadcast (builder.bootstrap_started (id, mode_text ()));
}
}
nano::bootstrap_attempt::~bootstrap_attempt ()
{
auto node = this->node.lock ();
if (!node)
{
return;
}
node->logger.debug (nano::log::type::bootstrap, "Exiting bootstrap attempt with ID: {} (mode: {})", mode_text (), id);
node->bootstrap_initiator.notify_listeners (false);
if (node->websocket.server)
{
nano::websocket::message_builder builder;
node->websocket.server->broadcast (builder.bootstrap_exited (id, mode_text (), attempt_start, total_blocks));
}
}
bool nano::bootstrap_attempt::should_log ()
{
nano::lock_guard<nano::mutex> guard{ next_log_mutex };
auto result (false);
auto now (std::chrono::steady_clock::now ());
if (next_log < now)
{
result = true;
next_log = now + std::chrono::seconds (15);
}
return result;
}
bool nano::bootstrap_attempt::still_pulling ()
{
debug_assert (!mutex.try_lock ());
auto running (!stopped);
auto still_pulling (pulling > 0);
return running && still_pulling;
}
void nano::bootstrap_attempt::pull_started ()
{
{
nano::lock_guard<nano::mutex> guard{ mutex };
++pulling;
}
condition.notify_all ();
}
void nano::bootstrap_attempt::pull_finished ()
{
{
nano::lock_guard<nano::mutex> guard{ mutex };
--pulling;
}
condition.notify_all ();
}
void nano::bootstrap_attempt::stop ()
{
{
nano::lock_guard<nano::mutex> lock{ mutex };
stopped = true;
}
condition.notify_all ();
auto node_l = node.lock ();
if (!node_l)
{
return;
}
node_l->bootstrap_initiator.connections->clear_pulls (incremental_id);
}
char const * nano::bootstrap_attempt::mode_text ()
{
switch (mode)
{
case nano::bootstrap_mode::legacy:
return "legacy";
case nano::bootstrap_mode::lazy:
return "lazy";
case nano::bootstrap_mode::wallet_lazy:
return "wallet_lazy";
case nano::bootstrap_mode::ascending:
return "ascending";
}
return "unknown";
}
bool nano::bootstrap_attempt::process_block (std::shared_ptr<nano::block> const & block_a, nano::account const & known_account_a, uint64_t pull_blocks_processed, nano::bulk_pull::count_t max_blocks, bool block_expected, unsigned retry_limit)
{
auto node_l = node.lock ();
if (!node_l)
{
return true;
}
bool stop_pull (false);
// If block already exists in the ledger, then we can avoid next part of long account chain
if (pull_blocks_processed % nano::bootstrap_limits::pull_count_per_check == 0 && node_l->block_or_pruned_exists (block_a->hash ()))
{
stop_pull = true;
}
else
{
node_l->block_processor.add (block_a, nano::block_source::bootstrap_legacy);
}
return stop_pull;
}
void nano::bootstrap_attempt::block_processed (store::transaction const & tx, nano::block_status const & result, nano::block const & block)
{
}

54
nano/node/bootstrap/bootstrap_attempt.hpp
View file

@ -1,54 +0,0 @@
#pragma once
#include <nano/node/bootstrap/bootstrap.hpp>
#include <atomic>
#include <future>
namespace nano::store
{
class transaction;
}
namespace nano
{
class node;
class frontier_req_client;
class bulk_push_client;
/**
* Polymorphic base class for bootstrap sessions.
*/
class bootstrap_attempt : public std::enable_shared_from_this<bootstrap_attempt>
{
public:
explicit bootstrap_attempt (std::shared_ptr<nano::node> const & node_a, nano::bootstrap_mode mode_a, uint64_t incremental_id_a, std::string id_a);
virtual ~bootstrap_attempt ();
virtual void run () = 0;
virtual void stop ();
bool still_pulling ();
void pull_started ();
void pull_finished ();
bool should_log ();
char const * mode_text ();
virtual bool process_block (std::shared_ptr<nano::block> const &, nano::account const &, uint64_t, nano::bulk_pull::count_t, bool, unsigned);
virtual void get_information (boost::property_tree::ptree &) = 0;
virtual void block_processed (store::transaction const & tx, nano::block_status const & result, nano::block const & block);
nano::mutex next_log_mutex;
std::chrono::steady_clock::time_point next_log{ std::chrono::steady_clock::now () };
std::atomic<unsigned> pulling{ 0 };
std::weak_ptr<nano::node> node;
std::atomic<uint64_t> total_blocks{ 0 };
std::atomic<unsigned> requeued_pulls{ 0 };
std::atomic<bool> started{ false };
std::atomic<bool> stopped{ false };
uint64_t incremental_id{ 0 };
std::string id;
std::chrono::steady_clock::time_point attempt_start{ std::chrono::steady_clock::now () };
std::atomic<bool> frontiers_received{ false };
nano::bootstrap_mode mode;
nano::mutex mutex;
nano::condition_variable condition;
};
}

912
nano/node/bootstrap/bootstrap_bulk_pull.cpp
View file

@ -1,912 +0,0 @@
#include <nano/lib/blocks.hpp>
#include <nano/node/bootstrap/block_deserializer.hpp>
#include <nano/node/bootstrap/bootstrap.hpp>
#include <nano/node/bootstrap/bootstrap_bulk_pull.hpp>
#include <nano/node/bootstrap/bootstrap_connections.hpp>
#include <nano/node/bootstrap/bootstrap_lazy.hpp>
#include <nano/node/node.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
#include <boost/format.hpp>
nano::pull_info::pull_info (nano::hash_or_account const & account_or_head_a, nano::block_hash const & head_a, nano::block_hash const & end_a, uint64_t bootstrap_id_a, count_t count_a, unsigned retry_limit_a) :
account_or_head (account_or_head_a),
head (head_a),
head_original (head_a),
end (end_a),
count (count_a),
retry_limit (retry_limit_a),
bootstrap_id (bootstrap_id_a)
{
}
nano::bulk_pull_client::bulk_pull_client (std::shared_ptr<nano::bootstrap_client> const & connection_a, std::shared_ptr<nano::bootstrap_attempt> const & attempt_a, nano::pull_info const & pull_a) :
connection{ connection_a },
attempt{ attempt_a },
pull{ pull_a },
block_deserializer{ std::make_shared<nano::bootstrap::block_deserializer> () }
{
attempt->condition.notify_all ();
}
nano::bulk_pull_client::~bulk_pull_client ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
/* If received end block is not expected end block
Or if given start and end blocks are from different chains (i.e. forked node or malicious node) */
if (expected != pull.end && !expected.is_zero ())
{
pull.head = expected;
if (attempt->mode != nano::bootstrap_mode::legacy)
{
pull.account_or_head = expected;
}
pull.processed += pull_blocks - unexpected_count;
node->bootstrap_initiator.connections->requeue_pull (pull, network_error);
node->logger.debug (nano::log::type::bulk_pull_client, "Bulk pull end block is not expected {} for account {} or head block {}", pull.end.to_string (), pull.account_or_head.to_account (), pull.account_or_head.to_string ());
}
else
{
node->bootstrap_initiator.cache.remove (pull);
}
attempt->pull_finished ();
}
void nano::bulk_pull_client::request ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
debug_assert (!pull.head.is_zero () || pull.retry_limit <= node->network_params.bootstrap.lazy_retry_limit);
expected = pull.head;
nano::bulk_pull req{ node->network_params.network };
if (pull.head == pull.head_original && pull.attempts % 4 < 3)
{
// Account for new pulls
req.start = pull.account_or_head;
}
else
{
// Head for cached pulls or accounts with public key equal to existing block hash (25% of attempts)
req.start = pull.head;
}
req.end = pull.end;
req.count = pull.count;
req.set_count_present (pull.count != 0);
node->logger.trace (nano::log::type::bulk_pull_client, nano::log::detail::requesting_account_or_head,
nano::log::arg{ "account_or_head", pull.account_or_head },
nano::log::arg{ "channel", connection->channel });
if (attempt->should_log ())
{
node->logger.debug (nano::log::type::bulk_pull_client, "Accounts in pull queue: {}", attempt->pulling.load ());
}
auto this_l (shared_from_this ());
connection->channel->send (
req, [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
auto node = this_l->connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
this_l->throttled_receive_block ();
}
else
{
node->logger.debug (nano::log::type::bulk_pull_client, "Error sending bulk pull request to: {} ({})", this_l->connection->channel->to_string (), ec.message ());
node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_request_failure, nano::stat::dir::in);
}
},
nano::transport::buffer_drop_policy::no_limiter_drop);
}
void nano::bulk_pull_client::throttled_receive_block ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
debug_assert (!network_error);
if (node->block_processor.size (nano::block_source::bootstrap_legacy) < 1024 && !node->block_processor.flushing)
{
receive_block ();
}
else
{
auto this_l (shared_from_this ());
node->workers.post_delayed (std::chrono::seconds (1), [this_l] () {
if (!this_l->connection->pending_stop && !this_l->attempt->stopped)
{
this_l->throttled_receive_block ();
}
});
}
}
void nano::bulk_pull_client::receive_block ()
{
block_deserializer->read (*connection->socket, [this_l = shared_from_this ()] (boost::system::error_code ec, std::shared_ptr<nano::block> block) {
this_l->received_block (ec, block);
});
}
void nano::bulk_pull_client::received_block (boost::system::error_code ec, std::shared_ptr<nano::block> block)
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (ec)
{
network_error = true;
return;
}
if (block == nullptr)
{
// Avoid re-using slow peers, or peers that sent the wrong blocks.
if (!connection->pending_stop && (expected == pull.end || (pull.count != 0 && pull.count == pull_blocks)))
{
node->bootstrap_initiator.connections->pool_connection (connection);
}
return;
}
if (node->network_params.work.validate_entry (*block))
{
node->logger.debug (nano::log::type::bulk_pull_client, "Insufficient work for bulk pull block: {}", block->hash ().to_string ());
node->stats.inc (nano::stat::type::error, nano::stat::detail::insufficient_work);
return;
}
auto hash = block->hash ();
node->logger.trace (nano::log::type::bulk_pull_client, nano::log::detail::pulled_block, nano::log::arg{ "block", block });
// Is block expected?
bool block_expected (false);
// Unconfirmed head is used only for lazy destinations if legacy bootstrap is not available, see nano::bootstrap_attempt::lazy_destinations_increment (...)
bool unconfirmed_account_head = node->flags.disable_legacy_bootstrap && pull_blocks == 0 && pull.retry_limit <= node->network_params.bootstrap.lazy_retry_limit && (expected == pull.account_or_head.as_block_hash ()) && (block->account_field ().value_or (0) == pull.account_or_head.as_account ());
if (hash == expected || unconfirmed_account_head)
{
expected = block->previous ();
block_expected = true;
}
else
{
unexpected_count++;
}
if (pull_blocks == 0 && block_expected)
{
known_account = block->account_field ().value_or (0);
}
if (connection->block_count++ == 0)
{
connection->set_start_time (std::chrono::steady_clock::now ());
}
attempt->total_blocks++;
pull_blocks++;
bool stop_pull (attempt->process_block (block, known_account, pull_blocks, pull.count, block_expected, pull.retry_limit));
if (!stop_pull && !connection->hard_stop.load ())
{
/* Process block in lazy pull if not stopped
Stop usual pull request with unexpected block & more than 16k blocks processed
to prevent spam */
if (attempt->mode != nano::bootstrap_mode::legacy || unexpected_count < 16384)
{
throttled_receive_block ();
}
}
else if (!stop_pull && block_expected)
{
node->bootstrap_initiator.connections->pool_connection (connection);
}
}
nano::bulk_pull_account_client::bulk_pull_account_client (std::shared_ptr<nano::bootstrap_client> const & connection_a, std::shared_ptr<nano::bootstrap_attempt_wallet> const & attempt_a, nano::account const & account_a) :
connection (connection_a),
attempt (attempt_a),
account (account_a),
pull_blocks (0)
{
attempt->condition.notify_all ();
}
nano::bulk_pull_account_client::~bulk_pull_account_client ()
{
attempt->pull_finished ();
}
void nano::bulk_pull_account_client::request ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
nano::bulk_pull_account req{ node->network_params.network };
req.account = account;
req.minimum_amount = node->config.receive_minimum;
req.flags = nano::bulk_pull_account_flags::pending_hash_and_amount;
node->logger.trace (nano::log::type::bulk_pull_account_client, nano::log::detail::requesting_pending,
nano::log::arg{ "account", req.account.to_account () }, // TODO: Convert to lazy eval
nano::log::arg{ "connection", connection->channel });
if (attempt->should_log ())
{
node->logger.debug (nano::log::type::bulk_pull_account_client, "Accounts in pull queue: {}", attempt->wallet_size ());
}
auto this_l (shared_from_this ());
connection->channel->send (
req, [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
auto node = this_l->connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
this_l->receive_pending ();
}
else
{
node->logger.debug (nano::log::type::bulk_pull_account_client, "Error starting bulk pull request to: {} ({})", this_l->connection->channel->to_string (), ec.message ());
node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_error_starting_request, nano::stat::dir::in);
this_l->attempt->requeue_pending (this_l->account);
}
},
nano::transport::buffer_drop_policy::no_limiter_drop);
}
void nano::bulk_pull_account_client::receive_pending ()
{
auto this_l (shared_from_this ());
std::size_t size_l (sizeof (nano::uint256_union) + sizeof (nano::uint128_union));
connection->socket->async_read (connection->receive_buffer, size_l, [this_l, size_l] (boost::system::error_code const & ec, std::size_t size_a) {
auto node = this_l->connection->node.lock ();
if (!node)
{
return;
}
// An issue with asio is that sometimes, instead of reporting a bad file descriptor during disconnect,
// we simply get a size of 0.
if (size_a == size_l)
{
if (!ec)
{
nano::block_hash pending;
nano::bufferstream frontier_stream (this_l->connection->receive_buffer->data (), sizeof (nano::uint256_union));
auto error1 (nano::try_read (frontier_stream, pending));
(void)error1;
debug_assert (!error1);
nano::amount balance;
nano::bufferstream balance_stream (this_l->connection->receive_buffer->data () + sizeof (nano::uint256_union), sizeof (nano::uint128_union));
auto error2 (nano::try_read (balance_stream, balance));
(void)error2;
debug_assert (!error2);
if (this_l->pull_blocks == 0 || !pending.is_zero ())
{
if (this_l->pull_blocks == 0 || balance.number () >= node->config.receive_minimum.number ())
{
this_l->pull_blocks++;
{
if (!pending.is_zero ())
{
if (!node->block_or_pruned_exists (pending))
{
node->bootstrap_initiator.bootstrap_lazy (pending, false);
}
}
}
this_l->receive_pending ();
}
else
{
this_l->attempt->requeue_pending (this_l->account);
}
}
else
{
node->bootstrap_initiator.connections->pool_connection (this_l->connection);
}
}
else
{
node->logger.debug (nano::log::type::bulk_pull_account_client, "Error while receiving bulk pull account frontier: {}", ec.message ());
this_l->attempt->requeue_pending (this_l->account);
}
}
else
{
node->logger.debug (nano::log::type::bulk_pull_account_client, "Invalid size: Expected {}, got: {}", size_l, size_a);
this_l->attempt->requeue_pending (this_l->account);
}
});
}
/**
* Handle a request for the pull of all blocks associated with an account
* The account is supplied as the "start" member, and the final block to
* send is the "end" member. The "start" member may also be a block
* hash, in which case the that hash is used as the start of a chain
* to send. To determine if "start" is interpreted as an account or
* hash, the ledger is checked to see if the block specified exists,
* if not then it is interpreted as an account.
*
* Additionally, if "start" is specified as a block hash the range
* is inclusive of that block hash, that is the range will be:
* [start, end); In the case that a block hash is not specified the
* range will be exclusive of the frontier for that account with
* a range of (frontier, end)
*/
void nano::bulk_pull_server::set_current_end ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
include_start = false;
debug_assert (request != nullptr);
auto transaction = node->ledger.tx_begin_read ();
if (!node->ledger.any.block_exists (transaction, request->end))
{
node->logger.debug (nano::log::type::bulk_pull_server, "Bulk pull end block doesn't exist: {}, sending everything", request->end.to_string ());
request->end.clear ();
}
if (node->ledger.any.block_exists (transaction, request->start.as_block_hash ()))
{
node->logger.debug (nano::log::type::bulk_pull_server, "Bulk pull request for block hash: {}", request->start.to_string ());
current = ascending () ? node->ledger.any.block_successor (transaction, request->start.as_block_hash ()).value_or (0) : request->start.as_block_hash ();
include_start = true;
}
else
{
auto info = node->ledger.any.account_get (transaction, request->start.as_account ());
if (!info)
{
node->logger.debug (nano::log::type::bulk_pull_server, "Request for unknown account: {}", request->start.to_account ());
current = request->end;
}
else
{
current = ascending () ? info->open_block : info->head;
if (!request->end.is_zero ())
{
auto account (node->ledger.any.block_account (transaction, request->end));
if (account.value_or (0) != request->start.as_account ())
{
node->logger.debug (nano::log::type::bulk_pull_server, "Request for block that is not on account chain: {} not on {}", request->end.to_string (), request->start.to_account ());
current = request->end;
}
}
}
}
sent_count = 0;
if (request->is_count_present ())
{
max_count = request->count;
}
else
{
max_count = 0;
}
}
void nano::bulk_pull_server::send_next ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
auto block = get_next ();
if (block != nullptr)
{
node->logger.trace (nano::log::type::bulk_pull_server, nano::log::detail::sending_block,
nano::log::arg{ "block", block },
nano::log::arg{ "socket", connection->socket });
std::vector<uint8_t> send_buffer;
{
nano::vectorstream stream (send_buffer);
nano::serialize_block (stream, *block);
}
connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l = shared_from_this ()] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->sent_action (ec, size_a);
});
}
else
{
send_finished ();
}
}
std::shared_ptr<nano::block> nano::bulk_pull_server::get_next ()
{
auto node = connection->node.lock ();
if (!node)
{
return nullptr;
}
std::shared_ptr<nano::block> result;
bool send_current = false, set_current_to_end = false;
/*
* Determine if we should reply with a block
*
* If our cursor is on the final block, we should signal that we
* are done by returning a null result.
*
* Unless we are including the "start" member and this is the
* start member, then include it anyway.
*/
if (current != request->end)
{
send_current = true;
}
else if (current == request->end && include_start == true)
{
send_current = true;
/*
* We also need to ensure that the next time
* are invoked that we return a null result
*/
set_current_to_end = true;
}
/*
* Account for how many blocks we have provided. If this
* exceeds the requested maximum, return an empty object
* to signal the end of results
*/
if (max_count != 0 && sent_count >= max_count)
{
send_current = false;
}
if (send_current)
{
result = node->block (current);
if (result != nullptr && set_current_to_end == false)
{
auto next = ascending () ? result->sideband ().successor : result->previous ();
if (!next.is_zero ())
{
current = next;
}
else
{
current = request->end;
}
}
else
{
current = request->end;
}
sent_count++;
}
/*
* Once we have processed "get_next()" once our cursor is no longer on
* the "start" member, so this flag is not relevant is always false.
*/
include_start = false;
return result;
}
void nano::bulk_pull_server::sent_action (boost::system::error_code const & ec, std::size_t size_a)
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
node->bootstrap_workers.post ([this_l = shared_from_this ()] () {
this_l->send_next ();
});
}
else
{
node->logger.debug (nano::log::type::bulk_pull_server, "Unable to bulk send block: {}", ec.message ());
}
}
void nano::bulk_pull_server::send_finished ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
nano::shared_const_buffer send_buffer (static_cast<uint8_t> (nano::block_type::not_a_block));
auto this_l (shared_from_this ());
node->logger.debug (nano::log::type::bulk_pull_server, "Bulk sending finished");
connection->socket->async_write (send_buffer, [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->no_block_sent (ec, size_a);
});
}
void nano::bulk_pull_server::no_block_sent (boost::system::error_code const & ec, std::size_t size_a)
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
debug_assert (size_a == 1);
connection->start ();
}
else
{
node->logger.debug (nano::log::type::bulk_pull_server, "Unable to bulk send not-a-block: {}", ec.message ());
}
}
bool nano::bulk_pull_server::ascending () const
{
return request->header.bulk_pull_ascending ();
}
nano::bulk_pull_server::bulk_pull_server (std::shared_ptr<nano::transport::tcp_server> const & connection_a, std::unique_ptr<nano::bulk_pull> request_a) :
connection (connection_a),
request (std::move (request_a))
{
set_current_end ();
}
/**
* Bulk pull blocks related to an account
*/
void nano::bulk_pull_account_server::set_params ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
debug_assert (request != nullptr);
/*
* Parse the flags
*/
invalid_request = false;
pending_include_address = false;
pending_address_only = false;
if (request->flags == nano::bulk_pull_account_flags::pending_address_only)
{
pending_address_only = true;
}
else if (request->flags == nano::bulk_pull_account_flags::pending_hash_amount_and_address)
{
/**
** This is the same as "pending_hash_and_amount" but with the
** sending address appended, for UI purposes mainly.
**/
pending_include_address = true;
}
else if (request->flags == nano::bulk_pull_account_flags::pending_hash_and_amount)
{
/** The defaults are set above **/
}
else
{
node->logger.debug (nano::log::type::bulk_pull_account_server, "Invalid bulk_pull_account flags supplied: {}", static_cast<uint8_t> (request->flags));
invalid_request = true;
return;
}
/*
* Initialize the current item from the requested account
*/
current_key.account = request->account;
current_key.hash = 0;
}
void nano::bulk_pull_account_server::send_frontier ()
{
/*
* This function is really the entry point into this class,
* so handle the invalid_request case by terminating the
* request without any response
*/
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (!invalid_request)
{
auto stream_transaction = node->ledger.tx_begin_read ();
// Get account balance and frontier block hash
auto account_frontier_hash (node->ledger.any.account_head (stream_transaction, request->account));
auto account_frontier_balance_int (node->ledger.any.account_balance (stream_transaction, request->account).value_or (0));
nano::uint128_union account_frontier_balance (account_frontier_balance_int);
// Write the frontier block hash and balance into a buffer
std::vector<uint8_t> send_buffer;
{
nano::vectorstream output_stream (send_buffer);
write (output_stream, account_frontier_hash.bytes);
write (output_stream, account_frontier_balance.bytes);
}
// Send the buffer to the requestor
auto this_l (shared_from_this ());
connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->sent_action (ec, size_a);
});
}
}
void nano::bulk_pull_account_server::send_next_block ()
{
/*
* Get the next item from the queue, it is a tuple with the key (which
* contains the account and hash) and data (which contains the amount)
*/
auto node = connection->node.lock ();
if (!node)
{
return;
}
auto block_data (get_next ());
auto block_info_key (block_data.first.get ());
auto block_info (block_data.second.get ());
if (block_info_key != nullptr)
{
/*
* If we have a new item, emit it to the socket
*/
std::vector<uint8_t> send_buffer;
if (pending_address_only)
{
node->logger.trace (nano::log::type::bulk_pull_account_server, nano::log::detail::sending_pending,
nano::log::arg{ "pending", block_info->source });
nano::vectorstream output_stream (send_buffer);
write (output_stream, block_info->source.bytes);
}
else
{
node->logger.trace (nano::log::type::bulk_pull_account_server, nano::log::detail::sending_block,
nano::log::arg{ "block", block_info_key->hash });
nano::vectorstream output_stream (send_buffer);
write (output_stream, block_info_key->hash.bytes);
write (output_stream, block_info->amount.bytes);
if (pending_include_address)
{
/**
** Write the source address as well, if requested
**/
write (output_stream, block_info->source.bytes);
}
}
auto this_l (shared_from_this ());
connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->sent_action (ec, size_a);
});
}
else
{
/*
* Otherwise, finalize the connection
*/
node->logger.debug (nano::log::type::bulk_pull_account_server, "Done sending blocks");
send_finished ();
}
}
std::pair<std::unique_ptr<nano::pending_key>, std::unique_ptr<nano::pending_info>> nano::bulk_pull_account_server::get_next ()
{
auto node = connection->node.lock ();
if (!node)
{
return { nullptr, nullptr };
}
std::pair<std::unique_ptr<nano::pending_key>, std::unique_ptr<nano::pending_info>> result;
while (true)
{
/*
* For each iteration of this loop, establish and then
* destroy a database transaction, to avoid locking the
* database for a prolonged period.
*/
auto tx = node->ledger.tx_begin_read ();
auto & ledger = node->ledger;
auto stream = ledger.any.receivable_upper_bound (tx, current_key.account, current_key.hash);
if (stream == ledger.any.receivable_end ())
{
break;
}
auto const & [key, info] = *stream;
current_key = key;
/*
* Skip entries where the amount is less than the requested
* minimum
*/
if (info.amount < request->minimum_amount)
{
continue;
}
/*
* If the pending_address_only flag is set, de-duplicate the
* responses. The responses are the address of the sender,
* so they are part of the pending table's information
* and not key, so we have to de-duplicate them manually.
*/
if (pending_address_only)
{
if (!deduplication.insert (info.source).second)
{
/*
* If the deduplication map gets too
* large, clear it out. This may
* result in some duplicates getting
* sent to the client, but we do not
* want to commit too much memory
*/
if (deduplication.size () > 4096)
{
deduplication.clear ();
}
continue;
}
}
result.first = std::make_unique<nano::pending_key> (key);
result.second = std::make_unique<nano::pending_info> (info);
break;
}
return result;
}
void nano::bulk_pull_account_server::sent_action (boost::system::error_code const & ec, std::size_t size_a)
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
node->bootstrap_workers.post ([this_l = shared_from_this ()] () {
this_l->send_next_block ();
});
}
else
{
node->logger.debug (nano::log::type::bulk_pull_account_server, "Unable to bulk send block: {}", ec.message ());
}
}
void nano::bulk_pull_account_server::send_finished ()
{
/*
* The "bulk_pull_account" final sequence is a final block of all
* zeros. If we are sending only account public keys (with the
* "pending_address_only" flag) then it will be 256-bits of zeros,
* otherwise it will be either 384-bits of zeros (if the
* "pending_include_address" flag is not set) or 640-bits of zeros
* (if that flag is set).
*/
auto node = connection->node.lock ();
if (!node)
{
return;
}
std::vector<uint8_t> send_buffer;
{
nano::vectorstream output_stream (send_buffer);
nano::uint256_union account_zero (0);
nano::uint128_union balance_zero (0);
write (output_stream, account_zero.bytes);
if (!pending_address_only)
{
write (output_stream, balance_zero.bytes);
if (pending_include_address)
{
write (output_stream, account_zero.bytes);
}
}
}
node->logger.debug (nano::log::type::bulk_pull_account_server, "Bulk sending for an account finished");
auto this_l (shared_from_this ());
connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->complete (ec, size_a);
});
}
void nano::bulk_pull_account_server::complete (boost::system::error_code const & ec, std::size_t size_a)
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
if (pending_address_only)
{
debug_assert (size_a == 32);
}
else
{
if (pending_include_address)
{
debug_assert (size_a == 80);
}
else
{
debug_assert (size_a == 48);
}
}
connection->start ();
}
else
{
node->logger.debug (nano::log::type::bulk_pull_account_server, "Unable to pending-as-zero: {}", ec.message ());
}
}
nano::bulk_pull_account_server::bulk_pull_account_server (std::shared_ptr<nano::transport::tcp_server> const & connection_a, std::unique_ptr<nano::bulk_pull_account> request_a) :
connection (connection_a),
request (std::move (request_a)),
current_key (0, 0)
{
/*
* Setup the streaming response for the first call to "send_frontier" and "send_next_block"
*/
set_params ();
}

140
nano/node/bootstrap/bootstrap_bulk_pull.hpp
View file

@ -1,140 +0,0 @@
#pragma once
#include <nano/node/messages.hpp>
#include <nano/node/transport/tcp_socket.hpp>
#include <nano/secure/pending_info.hpp>
#include <unordered_set>
namespace nano
{
class bootstrap_attempt;
namespace transport
{
class tcp_server;
}
namespace bootstrap
{
class block_deserializer;
};
class pull_info
{
public:
using count_t = nano::bulk_pull::count_t;
pull_info () = default;
pull_info (nano::hash_or_account const &, nano::block_hash const &, nano::block_hash const &, uint64_t, count_t = 0, unsigned = 16);
nano::hash_or_account account_or_head{ 0 };
nano::block_hash head{ 0 };
nano::block_hash head_original{ 0 };
nano::block_hash end{ 0 };
count_t count{ 0 };
unsigned attempts{ 0 };
uint64_t processed{ 0 };
unsigned retry_limit{ 0 };
uint64_t bootstrap_id{ 0 };
};
class bootstrap_client;
/**
* Client side of a bulk_pull request. Created when the bootstrap_attempt wants to make a bulk_pull request to the remote side.
*/
class bulk_pull_client final : public std::enable_shared_from_this<nano::bulk_pull_client>
{
public:
bulk_pull_client (std::shared_ptr<nano::bootstrap_client> const &, std::shared_ptr<nano::bootstrap_attempt> const &, nano::pull_info const &);
~bulk_pull_client ();
void request ();
void receive_block ();
void throttled_receive_block ();
void received_block (boost::system::error_code ec, std::shared_ptr<nano::block> block);
nano::block_hash first ();
std::shared_ptr<nano::bootstrap_client> connection;
std::shared_ptr<nano::bootstrap_attempt> attempt;
bool network_error{ false };
private:
/**
* Tracks the next block expected to be received starting with the block hash that was expected and followed by previous blocks for this account chain
*/
nano::block_hash expected{ 0 };
/**
* Tracks the account number for this account chain
* Used when an account chain has a mix between state blocks and legacy blocks which do not encode the account number in the block
* 0 if the account is unknown
*/
nano::account known_account{ 0 };
/**
* Original pull request
*/
nano::pull_info pull;
/**
* Tracks the number of blocks successfully deserialized
*/
uint64_t pull_blocks{ 0 };
/**
* Tracks the number of times an unexpected block was received
*/
uint64_t unexpected_count{ 0 };
std::shared_ptr<nano::bootstrap::block_deserializer> block_deserializer;
};
class bootstrap_attempt_wallet;
class bulk_pull_account_client final : public std::enable_shared_from_this<nano::bulk_pull_account_client>
{
public:
bulk_pull_account_client (std::shared_ptr<nano::bootstrap_client> const &, std::shared_ptr<nano::bootstrap_attempt_wallet> const &, nano::account const &);
~bulk_pull_account_client ();
void request ();
void receive_pending ();
std::shared_ptr<nano::bootstrap_client> connection;
std::shared_ptr<nano::bootstrap_attempt_wallet> attempt;
nano::account account;
uint64_t pull_blocks;
};
class bulk_pull;
/**
* Server side of a bulk_pull request. Created when tcp_server receives a bulk_pull message and is exited after the contents
* have been sent. If the 'start' in the bulk_pull message is an account, send blocks for that account down to 'end'. If the 'start'
* is a block hash, send blocks for that chain down to 'end'. If end doesn't exist, send all accounts in the chain.
*/
class bulk_pull_server final : public std::enable_shared_from_this<nano::bulk_pull_server>
{
public:
bulk_pull_server (std::shared_ptr<nano::transport::tcp_server> const &, std::unique_ptr<nano::bulk_pull>);
void set_current_end ();
std::shared_ptr<nano::block> get_next ();
void send_next ();
void sent_action (boost::system::error_code const &, std::size_t);
void send_finished ();
void no_block_sent (boost::system::error_code const &, std::size_t);
bool ascending () const;
std::shared_ptr<nano::transport::tcp_server> connection;
std::unique_ptr<nano::bulk_pull> request;
nano::block_hash current;
bool include_start;
nano::bulk_pull::count_t max_count;
nano::bulk_pull::count_t sent_count;
};
class bulk_pull_account;
class bulk_pull_account_server final : public std::enable_shared_from_this<nano::bulk_pull_account_server>
{
public:
bulk_pull_account_server (std::shared_ptr<nano::transport::tcp_server> const &, std::unique_ptr<nano::bulk_pull_account>);
void set_params ();
std::pair<std::unique_ptr<nano::pending_key>, std::unique_ptr<nano::pending_info>> get_next ();
void send_frontier ();
void send_next_block ();
void sent_action (boost::system::error_code const &, std::size_t);
void send_finished ();
void complete (boost::system::error_code const &, std::size_t);
std::shared_ptr<nano::transport::tcp_server> connection;
std::unique_ptr<nano::bulk_pull_account> request;
std::unordered_set<nano::uint256_union> deduplication;
nano::pending_key current_key;
bool pending_address_only;
bool pending_include_address;
bool invalid_request;
};
}

279
nano/node/bootstrap/bootstrap_bulk_push.cpp
View file

@ -1,279 +0,0 @@
#include <nano/lib/blocks.hpp>
#include <nano/node/bootstrap/bootstrap_attempt.hpp>
#include <nano/node/bootstrap/bootstrap_bulk_push.hpp>
#include <nano/node/bootstrap/bootstrap_legacy.hpp>
#include <nano/node/node.hpp>
#include <boost/format.hpp>
nano::bulk_push_client::bulk_push_client (std::shared_ptr<nano::bootstrap_client> const & connection_a, std::shared_ptr<nano::bootstrap_attempt_legacy> const & attempt_a) :
connection (connection_a),
attempt (attempt_a)
{
}
nano::bulk_push_client::~bulk_push_client ()
{
}
void nano::bulk_push_client::start ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
nano::bulk_push message{ node->network_params.network };
auto this_l (shared_from_this ());
connection->channel->send (
message, [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
auto node = this_l->connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
this_l->push ();
}
else
{
node->logger.debug (nano::log::type::bulk_push_client, "Unable to send bulk push request: {}", ec.message ());
}
},
nano::transport::buffer_drop_policy::no_limiter_drop);
}
void nano::bulk_push_client::push ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
std::shared_ptr<nano::block> block;
bool finished (false);
while (block == nullptr && !finished)
{
if (current_target.first.is_zero () || current_target.first == current_target.second)
{
finished = attempt->request_bulk_push_target (current_target);
}
if (!finished)
{
block = node->block (current_target.first);
if (block == nullptr)
{
current_target.first = nano::block_hash (0);
}
else
{
node->logger.debug (nano::log::type::bulk_push_client, "Bulk pushing range: [{}:{}]", current_target.first.to_string (), current_target.second.to_string ());
}
}
}
if (finished)
{
send_finished ();
}
else
{
current_target.first = block->previous ();
push_block (*block);
}
}
void nano::bulk_push_client::send_finished ()
{
nano::shared_const_buffer buffer (static_cast<uint8_t> (nano::block_type::not_a_block));
auto this_l (shared_from_this ());
connection->channel->send_buffer (buffer, [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
try
{
this_l->promise.set_value (false);
}
catch (std::future_error &)
{
}
});
}
void nano::bulk_push_client::push_block (nano::block const & block_a)
{
std::vector<uint8_t> buffer;
{
nano::vectorstream stream (buffer);
nano::serialize_block (stream, block_a);
}
auto this_l (shared_from_this ());
connection->channel->send_buffer (nano::shared_const_buffer (std::move (buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
auto node = this_l->connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
this_l->push ();
}
else
{
node->logger.debug (nano::log::type::bulk_push_client, "Error sending block during bulk push: {}", ec.message ());
}
});
}
nano::bulk_push_server::bulk_push_server (std::shared_ptr<nano::transport::tcp_server> const & connection_a) :
receive_buffer (std::make_shared<std::vector<uint8_t>> ()),
connection (connection_a)
{
receive_buffer->resize (256);
}
void nano::bulk_push_server::throttled_receive ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (node->block_processor.size (nano::block_source::bootstrap_legacy) < 1024)
{
receive ();
}
else
{
auto this_l (shared_from_this ());
node->workers.post_delayed (std::chrono::seconds (1), [this_l] () {
if (!this_l->connection->stopped)
{
this_l->throttled_receive ();
}
});
}
}
void nano::bulk_push_server::receive ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (node->bootstrap_initiator.in_progress ())
{
node->logger.debug (nano::log::type::bulk_push_server, "Aborting bulk push because a bootstrap attempt is in progress");
}
else
{
auto this_l (shared_from_this ());
connection->socket->async_read (receive_buffer, 1, [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
auto node = this_l->connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
this_l->received_type ();
}
else
{
node->logger.debug (nano::log::type::bulk_push_server, "Error receiving block type: {}", ec.message ());
}
});
}
}
void nano::bulk_push_server::received_type ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
auto this_l (shared_from_this ());
nano::block_type type (static_cast<nano::block_type> (receive_buffer->data ()[0]));
switch (type)
{
case nano::block_type::send:
{
node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::send, nano::stat::dir::in);
connection->socket->async_read (receive_buffer, nano::send_block::size, [this_l, type] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->received_block (ec, size_a, type);
});
break;
}
case nano::block_type::receive:
{
node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::receive, nano::stat::dir::in);
connection->socket->async_read (receive_buffer, nano::receive_block::size, [this_l, type] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->received_block (ec, size_a, type);
});
break;
}
case nano::block_type::open:
{
node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::open, nano::stat::dir::in);
connection->socket->async_read (receive_buffer, nano::open_block::size, [this_l, type] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->received_block (ec, size_a, type);
});
break;
}
case nano::block_type::change:
{
node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::change, nano::stat::dir::in);
connection->socket->async_read (receive_buffer, nano::change_block::size, [this_l, type] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->received_block (ec, size_a, type);
});
break;
}
case nano::block_type::state:
{
node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::state_block, nano::stat::dir::in);
connection->socket->async_read (receive_buffer, nano::state_block::size, [this_l, type] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->received_block (ec, size_a, type);
});
break;
}
case nano::block_type::not_a_block:
{
connection->start ();
break;
}
default:
{
node->logger.debug (nano::log::type::bulk_push_server, "Unknown type received as block type");
break;
}
}
}
void nano::bulk_push_server::received_block (boost::system::error_code const & ec, std::size_t size_a, nano::block_type type_a)
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
nano::bufferstream stream (receive_buffer->data (), size_a);
auto block (nano::deserialize_block (stream, type_a));
if (block != nullptr)
{
if (node->network_params.work.validate_entry (*block))
{
node->logger.debug (nano::log::type::bulk_push_server, "Insufficient work for bulk push block: {}", block->hash ().to_string ());
node->stats.inc (nano::stat::type::error, nano::stat::detail::insufficient_work);
return;
}
node->process_active (std::move (block));
throttled_receive ();
}
else
{
node->logger.debug (nano::log::type::bulk_push_server, "Error deserializing block received from pull request");
}
}
}

48
nano/node/bootstrap/bootstrap_bulk_push.hpp
View file

@ -1,48 +0,0 @@
#pragma once
#include <nano/node/common.hpp>
#include <future>
namespace nano
{
class bootstrap_attempt_legacy;
class bootstrap_client;
namespace transport
{
class tcp_server;
}
/**
* Client side of a bulk_push request. Sends a sequence of blocks the other side did not report in their frontier_req response.
*/
class bulk_push_client final : public std::enable_shared_from_this<nano::bulk_push_client>
{
public:
explicit bulk_push_client (std::shared_ptr<nano::bootstrap_client> const &, std::shared_ptr<nano::bootstrap_attempt_legacy> const &);
~bulk_push_client ();
void start ();
void push ();
void push_block (nano::block const &);
void send_finished ();
std::shared_ptr<nano::bootstrap_client> connection;
std::shared_ptr<nano::bootstrap_attempt_legacy> attempt;
std::promise<bool> promise;
std::pair<nano::block_hash, nano::block_hash> current_target;
};
/**
* Server side of a bulk_push request. Receives blocks and puts them in the block processor to be processed.
*/
class bulk_push_server final : public std::enable_shared_from_this<nano::bulk_push_server>
{
public:
explicit bulk_push_server (std::shared_ptr<nano::transport::tcp_server> const &);
void throttled_receive ();
void receive ();
void received_type ();
void received_block (boost::system::error_code const &, std::size_t, nano::block_type);
std::shared_ptr<std::vector<uint8_t>> receive_buffer;
std::shared_ptr<nano::transport::tcp_server> connection;
};
}

20
nano/node/bootstrap/bootstrap_config.cpp
View file

@ -26,16 +26,18 @@ nano::error nano::account_sets_config::serialize (nano::tomlconfig & toml) const
}
/*
* bootstrap_ascending_config
* bootstrap_config
*/
nano::error nano::bootstrap_ascending_config::deserialize (nano::tomlconfig & toml)
nano::error nano::bootstrap_config::deserialize (nano::tomlconfig & toml)
{
toml.get ("enable", enable);
toml.get ("enable_database_scan", enable_database_scan);
toml.get ("enable_dependency_walker", enable_dependency_walker);
toml.get ("enable_frontier_scan", enable_frontier_scan);
toml.get ("channel_limit", channel_limit);
toml.get ("rate_limit", rate_limit);
toml.get ("database_rate_limit", database_rate_limit);
toml.get ("database_warmup_ratio", database_warmup_ratio);
toml.get ("max_pull_count", max_pull_count);
@ -44,6 +46,7 @@ nano::error nano::bootstrap_ascending_config::deserialize (nano::tomlconfig & to
toml.get_duration ("throttle_wait", throttle_wait);
toml.get ("block_processor_threshold", block_processor_threshold);
toml.get ("max_requests", max_requests);
toml.get ("optimistic_request_percentage", optimistic_request_percentage);
if (toml.has_key ("account_sets"))
{
@ -54,21 +57,24 @@ nano::error nano::bootstrap_ascending_config::deserialize (nano::tomlconfig & to
return toml.get_error ();
}
nano::error nano::bootstrap_ascending_config::serialize (nano::tomlconfig & toml) const
nano::error nano::bootstrap_config::serialize (nano::tomlconfig & toml) const
{
toml.put ("enable", enable, "Enable or disable the ascending bootstrap. Disabling it is not recommended and will prevent the node from syncing.\ntype:bool");
toml.put ("enable", enable, "Enable or disable the bootstrap. Disabling it is not recommended and will prevent the node from syncing.\ntype:bool");
toml.put ("enable_database_scan", enable_database_scan, "Enable or disable the 'database scan` strategy for the ascending bootstrap.\ntype:bool");
toml.put ("enable_dependency_walker", enable_dependency_walker, "Enable or disable the 'dependency walker` strategy for the ascending bootstrap.\ntype:bool");
toml.put ("enable_frontier_scan", enable_frontier_scan, "Enable or disable the 'frontier scan` strategy for the ascending bootstrap.\ntype:bool");
toml.put ("channel_limit", channel_limit, "Maximum number of un-responded requests per channel.\nNote: changing to unlimited (0) is not recommended.\ntype:uint64");
toml.put ("rate_limit", rate_limit, "Rate limit on requests.\nNote: changing to unlimited (0) is not recommended as this operation competes for resources with realtime traffic.\ntype:uint64");
toml.put ("database_rate_limit", database_rate_limit, "Rate limit on scanning accounts and pending entries from database.\nNote: changing to unlimited (0) is not recommended as this operation competes for resources on querying the database.\ntype:uint64");
toml.put ("database_warmup_ratio", database_warmup_ratio, "Ratio of the database rate limit to use for the initial warmup.\ntype:uint64");
toml.put ("max_pull_count", max_pull_count, "Maximum number of requested blocks for ascending bootstrap request.\ntype:uint64");
toml.put ("request_timeout", request_timeout.count (), "Timeout in milliseconds for incoming ascending bootstrap messages to be processed.\ntype:milliseconds");
toml.put ("max_pull_count", max_pull_count, "Maximum number of requested blocks for bootstrap request.\ntype:uint64");
toml.put ("request_timeout", request_timeout.count (), "Timeout in milliseconds for incoming bootstrap messages to be processed.\ntype:milliseconds");
toml.put ("throttle_coefficient", throttle_coefficient, "Scales the number of samples to track for bootstrap throttling.\ntype:uint64");
toml.put ("throttle_wait", throttle_wait.count (), "Length of time to wait between requests when throttled.\ntype:milliseconds");
toml.put ("block_processor_threshold", block_processor_threshold, "Ascending bootstrap will wait while block processor has more than this many blocks queued.\ntype:uint64");
toml.put ("block_processor_threshold", block_processor_threshold, "Bootstrap will wait while block processor has more than this many blocks queued.\ntype:uint64");
toml.put ("max_requests", max_requests, "Maximum total number of in flight requests.\ntype:uint64");
toml.put ("optimistic_request_percentage", optimistic_request_percentage, "Percentage of requests that will be optimistic. Optimistic requests start from the (possibly unconfirmed) account frontier and are vulnerable to bootstrap poisoning. Safe requests start from the confirmed frontier and given enough time will eventually resolve forks.\ntype:uint64");
nano::tomlconfig account_sets_l;
account_sets.serialize (account_sets_l);

28
nano/node/bootstrap/bootstrap_config.hpp
View file

@ -8,7 +8,6 @@ namespace nano
{
class tomlconfig;
// TODO: This should be moved next to `account_sets` class
class account_sets_config final
{
public:
@ -22,8 +21,19 @@ public:
std::chrono::milliseconds cooldown{ 1000 * 3 };
};
// TODO: This should be moved next to `bootstrap_ascending` class
class bootstrap_ascending_config final
class frontier_scan_config final
{
public:
// TODO: Serialize & deserialize
unsigned head_parallelistm{ 128 };
unsigned consideration_count{ 4 };
std::size_t candidates{ 1000 };
std::chrono::milliseconds cooldown{ 1000 * 5 };
std::size_t max_pending{ 16 };
};
class bootstrap_config final
{
public:
nano::error deserialize (nano::tomlconfig & toml);
@ -31,12 +41,16 @@ public:
public:
bool enable{ true };
bool enable_database_scan{ true };
bool enable_scan{ true };
bool enable_database_scan{ false };
bool enable_dependency_walker{ true };
bool enable_frontier_scan{ true };
// Maximum number of un-responded requests per channel, should be lower or equal to bootstrap server max queue size
std::size_t channel_limit{ 16 };
std::size_t database_rate_limit{ 256 };
std::size_t rate_limit{ 500 };
std::size_t database_rate_limit{ 250 };
std::size_t frontier_rate_limit{ 8 };
std::size_t database_warmup_ratio{ 10 };
std::size_t max_pull_count{ nano::bootstrap_server::max_blocks };
std::chrono::milliseconds request_timeout{ 1000 * 5 };
@ -44,7 +58,9 @@ public:
std::chrono::milliseconds throttle_wait{ 100 };
std::size_t block_processor_threshold{ 1000 };
std::size_t max_requests{ 1024 };
unsigned optimistic_request_percentage{ 75 };
nano::account_sets_config account_sets;
account_sets_config account_sets;
frontier_scan_config frontier_scan;
};
}

489
nano/node/bootstrap/bootstrap_connections.cpp
View file

@ -1,489 +0,0 @@
#include <nano/node/bootstrap/bootstrap.hpp>
#include <nano/node/bootstrap/bootstrap_attempt.hpp>
#include <nano/node/bootstrap/bootstrap_connections.hpp>
#include <nano/node/bootstrap/bootstrap_lazy.hpp>
#include <nano/node/common.hpp>
#include <nano/node/node.hpp>
#include <boost/format.hpp>
#include <memory>
constexpr double nano::bootstrap_limits::bootstrap_connection_scale_target_blocks;
constexpr double nano::bootstrap_limits::bootstrap_minimum_blocks_per_sec;
constexpr double nano::bootstrap_limits::bootstrap_minimum_termination_time_sec;
constexpr unsigned nano::bootstrap_limits::bootstrap_max_new_connections;
constexpr unsigned nano::bootstrap_limits::requeued_pulls_processed_blocks_factor;
nano::bootstrap_client::bootstrap_client (std::shared_ptr<nano::node> const & node_a, std::shared_ptr<nano::transport::tcp_channel> const & channel_a, std::shared_ptr<nano::transport::tcp_socket> const & socket_a) :
node (node_a),
channel (channel_a),
socket (socket_a),
receive_buffer (std::make_shared<std::vector<uint8_t>> ()),
start_time_m (std::chrono::steady_clock::now ())
{
++node_a->bootstrap_initiator.connections->connections_count;
receive_buffer->resize (256);
channel->update_endpoints ();
}
nano::bootstrap_client::~bootstrap_client ()
{
auto node = this->node.lock ();
if (!node)
{
return;
}
--node->bootstrap_initiator.connections->connections_count;
}
double nano::bootstrap_client::sample_block_rate ()
{
auto elapsed = std::max (elapsed_seconds (), nano::bootstrap_limits::bootstrap_minimum_elapsed_seconds_blockrate);
block_rate = static_cast<double> (block_count.load ()) / elapsed;
return block_rate;
}
void nano::bootstrap_client::set_start_time (std::chrono::steady_clock::time_point start_time_a)
{
nano::lock_guard<nano::mutex> guard{ start_time_mutex };
start_time_m = start_time_a;
}
double nano::bootstrap_client::elapsed_seconds () const
{
nano::lock_guard<nano::mutex> guard{ start_time_mutex };
return std::chrono::duration_cast<std::chrono::duration<double>> (std::chrono::steady_clock::now () - start_time_m).count ();
}
void nano::bootstrap_client::stop (bool force)
{
pending_stop = true;
if (force)
{
hard_stop = true;
}
}
nano::bootstrap_connections::bootstrap_connections (nano::node & node_a) :
node (node_a)
{
}
std::shared_ptr<nano::bootstrap_client> nano::bootstrap_connections::connection (std::shared_ptr<nano::bootstrap_attempt> const & attempt_a, bool use_front_connection)
{
nano::unique_lock<nano::mutex> lock{ mutex };
condition.wait (lock, [&stopped = stopped, &idle = idle, &new_connections_empty = new_connections_empty] { return stopped || !idle.empty () || new_connections_empty; });
std::shared_ptr<nano::bootstrap_client> result;
if (!stopped && !idle.empty ())
{
if (!use_front_connection)
{
result = idle.back ();
idle.pop_back ();
}
else
{
result = idle.front ();
idle.pop_front ();
}
}
if (result == nullptr && connections_count == 0 && new_connections_empty && attempt_a != nullptr)
{
node.logger.debug (nano::log::type::bootstrap, "Bootstrap attempt stopped because there are no peers");
lock.unlock ();
attempt_a->stop ();
}
return result;
}
void nano::bootstrap_connections::pool_connection (std::shared_ptr<nano::bootstrap_client> const & client_a, bool new_client, bool push_front)
{
nano::unique_lock<nano::mutex> lock{ mutex };
auto const & socket_l = client_a->socket;
if (!stopped && !client_a->pending_stop && !node.network.excluded_peers.check (client_a->channel->get_remote_endpoint ()))
{
socket_l->set_timeout (node.network_params.network.idle_timeout);
// Push into idle deque
if (!push_front)
{
idle.push_back (client_a);
}
else
{
idle.push_front (client_a);
}
if (new_client)
{
clients.push_back (client_a);
}
}
else
{
socket_l->close ();
}
lock.unlock ();
condition.notify_all ();
}
void nano::bootstrap_connections::add_connection (nano::endpoint const & endpoint_a)
{
connect_client (nano::tcp_endpoint (endpoint_a.address (), endpoint_a.port ()), true);
}
std::shared_ptr<nano::bootstrap_client> nano::bootstrap_connections::find_connection (nano::tcp_endpoint const & endpoint_a)
{
nano::lock_guard<nano::mutex> lock{ mutex };
std::shared_ptr<nano::bootstrap_client> result;
for (auto i (idle.begin ()), end (idle.end ()); i != end && !stopped; ++i)
{
if ((*i)->channel->get_remote_endpoint () == endpoint_a)
{
result = *i;
idle.erase (i);
break;
}
}
return result;
}
void nano::bootstrap_connections::connect_client (nano::tcp_endpoint const & endpoint_a, bool push_front)
{
++connections_count;
auto socket (std::make_shared<nano::transport::tcp_socket> (node));
auto this_l (shared_from_this ());
socket->async_connect (endpoint_a,
[this_l, socket, endpoint_a, push_front] (boost::system::error_code const & ec) {
if (!ec)
{
this_l->node.logger.debug (nano::log::type::bootstrap, "Connection established to: {}", nano::util::to_str (endpoint_a));
auto client (std::make_shared<nano::bootstrap_client> (this_l->node.shared (), std::make_shared<nano::transport::tcp_channel> (*this_l->node.shared (), socket), socket));
this_l->pool_connection (client, true, push_front);
}
else
{
switch (ec.value ())
{
default:
this_l->node.logger.debug (nano::log::type::bootstrap, "Error initiating bootstrap connection to: {} ({})", nano::util::to_str (endpoint_a), ec.message ());
break;
case boost::system::errc::connection_refused:
case boost::system::errc::operation_canceled:
case boost::system::errc::timed_out:
case 995: // Windows The I/O operation has been aborted because of either a thread exit or an application request
case 10061: // Windows No connection could be made because the target machine actively refused it
break;
}
}
--this_l->connections_count;
});
}
unsigned nano::bootstrap_connections::target_connections (std::size_t pulls_remaining, std::size_t attempts_count) const
{
auto const attempts_factor = nano::narrow_cast<unsigned> (node.config.bootstrap_connections * attempts_count);
if (attempts_factor >= node.config.bootstrap_connections_max)
{
return std::max (1U, node.config.bootstrap_connections_max);
}
// Only scale up to bootstrap_connections_max for large pulls.
double step_scale = std::min (1.0, std::max (0.0, (double)pulls_remaining / nano::bootstrap_limits::bootstrap_connection_scale_target_blocks));
double target = (double)attempts_factor + (double)(node.config.bootstrap_connections_max - attempts_factor) * step_scale;
return std::max (1U, (unsigned)(target + 0.5f));
}
struct block_rate_cmp
{
bool operator() (std::shared_ptr<nano::bootstrap_client> const & lhs, std::shared_ptr<nano::bootstrap_client> const & rhs) const
{
return lhs->block_rate > rhs->block_rate;
}
};
void nano::bootstrap_connections::populate_connections (bool repeat)
{
double rate_sum = 0.0;
std::size_t num_pulls = 0;
std::size_t attempts_count = node.bootstrap_initiator.attempts.size ();
std::priority_queue<std::shared_ptr<nano::bootstrap_client>, std::vector<std::shared_ptr<nano::bootstrap_client>>, block_rate_cmp> sorted_connections;
std::unordered_set<nano::tcp_endpoint> endpoints;
{
nano::unique_lock<nano::mutex> lock{ mutex };
num_pulls = pulls.size ();
std::deque<std::weak_ptr<nano::bootstrap_client>> new_clients;
for (auto & c : clients)
{
if (auto client = c.lock ())
{
new_clients.push_back (client);
endpoints.insert (client->socket->remote_endpoint ());
double elapsed_sec = client->elapsed_seconds ();
auto blocks_per_sec = client->sample_block_rate ();
rate_sum += blocks_per_sec;
if (client->elapsed_seconds () > nano::bootstrap_limits::bootstrap_connection_warmup_time_sec && client->block_count > 0)
{
sorted_connections.push (client);
}
// Force-stop the slowest peers, since they can take the whole bootstrap hostage by dribbling out blocks on the last remaining pull.
// This is ~1.5kilobits/sec.
if (elapsed_sec > nano::bootstrap_limits::bootstrap_minimum_termination_time_sec && blocks_per_sec < nano::bootstrap_limits::bootstrap_minimum_blocks_per_sec)
{
node.logger.debug (nano::log::type::bootstrap, "Stopping slow peer {} (elapsed sec {} > {} and {} blocks per second < {})",
client->channel->to_string (),
elapsed_sec,
nano::bootstrap_limits::bootstrap_minimum_termination_time_sec,
blocks_per_sec,
nano::bootstrap_limits::bootstrap_minimum_blocks_per_sec);
client->stop (true);
new_clients.pop_back ();
}
}
}
// Cleanup expired clients
clients.swap (new_clients);
}
auto target = target_connections (num_pulls, attempts_count);
// We only want to drop slow peers when more than 2/3 are active. 2/3 because 1/2 is too aggressive, and 100% rarely happens.
// Probably needs more tuning.
if (sorted_connections.size () >= (target * 2) / 3 && target >= 4)
{
// 4 -> 1, 8 -> 2, 16 -> 4, arbitrary, but seems to work well.
auto drop = (int)roundf (sqrtf ((float)target - 2.0f));
node.logger.debug (nano::log::type::bootstrap, "Dropping {} bulk pull peers, target connections {}", drop, target);
for (int i = 0; i < drop; i++)
{
auto client = sorted_connections.top ();
node.logger.debug (nano::log::type::bootstrap, "Dropping peer with block rate {} and block count {} ({})",
client->block_rate.load (),
client->block_count.load (),
client->channel->to_string ());
client->stop (false);
sorted_connections.pop ();
}
}
node.logger.debug (nano::log::type::bootstrap, "Bulk pull connections: {}, rate: {} blocks/sec, bootstrap attempts {}, remaining pulls: {}",
connections_count.load (),
(int)rate_sum,
attempts_count,
num_pulls);
if (connections_count < target && (attempts_count != 0 || new_connections_empty) && !stopped)
{
auto delta = std::min ((target - connections_count) * 2, nano::bootstrap_limits::bootstrap_max_new_connections);
// TODO - tune this better
// Not many peers respond, need to try to make more connections than we need.
for (auto i = 0u; i < delta; i++)
{
auto endpoint (node.network.bootstrap_peer ()); // Legacy bootstrap is compatible with older version of protocol
if (endpoint != nano::tcp_endpoint (boost::asio::ip::address_v6::any (), 0) && (node.flags.allow_bootstrap_peers_duplicates || endpoints.find (endpoint) == endpoints.end ()) && !node.network.excluded_peers.check (endpoint))
{
connect_client (endpoint);
endpoints.insert (endpoint);
nano::lock_guard<nano::mutex> lock{ mutex };
new_connections_empty = false;
}
else if (connections_count == 0)
{
{
nano::lock_guard<nano::mutex> lock{ mutex };
new_connections_empty = true;
}
condition.notify_all ();
}
}
}
if (!stopped && repeat)
{
std::weak_ptr<nano::bootstrap_connections> this_w (shared_from_this ());
node.workers.post_delayed (std::chrono::seconds (1), [this_w] () {
if (auto this_l = this_w.lock ())
{
this_l->populate_connections ();
}
});
}
}
void nano::bootstrap_connections::start_populate_connections ()
{
if (!populate_connections_started.exchange (true))
{
populate_connections ();
}
}
void nano::bootstrap_connections::add_pull (nano::pull_info const & pull_a)
{
nano::pull_info pull (pull_a);
node.bootstrap_initiator.cache.update_pull (pull);
{
nano::lock_guard<nano::mutex> lock{ mutex };
pulls.push_back (pull);
}
condition.notify_all ();
}
void nano::bootstrap_connections::request_pull (nano::unique_lock<nano::mutex> & lock_a)
{
lock_a.unlock ();
auto connection_l (connection ());
lock_a.lock ();
if (connection_l != nullptr && !pulls.empty ())
{
std::shared_ptr<nano::bootstrap_attempt> attempt_l;
nano::pull_info pull;
// Search pulls with existing attempts
while (attempt_l == nullptr && !pulls.empty ())
{
pull = pulls.front ();
pulls.pop_front ();
attempt_l = node.bootstrap_initiator.attempts.find (pull.bootstrap_id);
// Check if lazy pull is obsolete (head was processed or head is 0 for destinations requests)
if (auto lazy = std::dynamic_pointer_cast<nano::bootstrap_attempt_lazy> (attempt_l))
{
if (!pull.head.is_zero () && lazy->lazy_processed_or_exists (pull.head))
{
attempt_l->pull_finished ();
attempt_l = nullptr;
}
}
}
if (attempt_l != nullptr)
{
// The bulk_pull_client destructor attempt to requeue_pull which can cause a deadlock if this is the last reference
// Dispatch request in an external thread in case it needs to be destroyed
node.background ([connection_l, attempt_l, pull] () {
auto client (std::make_shared<nano::bulk_pull_client> (connection_l, attempt_l, pull));
client->request ();
});
}
}
else if (connection_l != nullptr)
{
// Reuse connection if pulls deque become empty
lock_a.unlock ();
pool_connection (connection_l);
lock_a.lock ();
}
}
void nano::bootstrap_connections::requeue_pull (nano::pull_info const & pull_a, bool network_error)
{
auto pull (pull_a);
if (!network_error)
{
++pull.attempts;
}
auto attempt_l (node.bootstrap_initiator.attempts.find (pull.bootstrap_id));
if (attempt_l != nullptr)
{
auto lazy = std::dynamic_pointer_cast<nano::bootstrap_attempt_lazy> (attempt_l);
++attempt_l->requeued_pulls;
if (lazy)
{
pull.count = lazy->lazy_batch_size ();
}
if (attempt_l->mode == nano::bootstrap_mode::legacy && (pull.attempts < pull.retry_limit + (pull.processed / nano::bootstrap_limits::requeued_pulls_processed_blocks_factor)))
{
{
nano::lock_guard<nano::mutex> lock{ mutex };
pulls.push_front (pull);
}
attempt_l->pull_started ();
condition.notify_all ();
}
else if (lazy && (pull.attempts <= pull.retry_limit + (pull.processed / node.network_params.bootstrap.lazy_max_pull_blocks)))
{
debug_assert (pull.account_or_head.as_block_hash () == pull.head);
if (!lazy->lazy_processed_or_exists (pull.account_or_head.as_block_hash ()))
{
{
nano::lock_guard<nano::mutex> lock{ mutex };
pulls.push_back (pull);
}
attempt_l->pull_started ();
condition.notify_all ();
}
}
else
{
node.stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in);
node.logger.debug (nano::log::type::bootstrap, "Failed to pull account {} or head block {} down to {} after {} attempts and {} blocks processed",
pull.account_or_head.to_account (),
pull.account_or_head.to_string (),
pull.end.to_string (),
pull.attempts,
pull.processed);
if (lazy && pull.processed > 0)
{
lazy->lazy_add (pull);
}
else if (attempt_l->mode == nano::bootstrap_mode::legacy)
{
node.bootstrap_initiator.cache.add (pull);
}
}
}
}
void nano::bootstrap_connections::clear_pulls (uint64_t bootstrap_id_a)
{
{
nano::lock_guard<nano::mutex> lock{ mutex };
erase_if (pulls, [bootstrap_id_a] (auto const & pull) {
return pull.bootstrap_id == bootstrap_id_a;
});
}
condition.notify_all ();
}
void nano::bootstrap_connections::run ()
{
start_populate_connections ();
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
if (!pulls.empty ())
{
request_pull (lock);
}
else
{
condition.wait (lock);
}
}
stopped = true;
lock.unlock ();
condition.notify_all ();
}
void nano::bootstrap_connections::stop ()
{
nano::unique_lock<nano::mutex> lock{ mutex };
stopped = true;
lock.unlock ();
condition.notify_all ();
lock.lock ();
for (auto const & i : clients)
{
if (auto client = i.lock ())
{
client->socket->close ();
}
}
clients.clear ();
idle.clear ();
}

81
nano/node/bootstrap/bootstrap_connections.hpp
View file

@ -1,81 +0,0 @@
#pragma once
#include <nano/node/bootstrap/bootstrap_bulk_pull.hpp>
#include <nano/node/common.hpp>
#include <nano/node/transport/tcp_socket.hpp>
#include <atomic>
namespace nano
{
class node;
namespace transport
{
class tcp_channel;
}
class bootstrap_attempt;
class bootstrap_connections;
class frontier_req_client;
class pull_info;
/**
* Owns the client side of the bootstrap connection.
*/
class bootstrap_client final : public std::enable_shared_from_this<bootstrap_client>
{
public:
bootstrap_client (std::shared_ptr<nano::node> const & node_a, std::shared_ptr<nano::transport::tcp_channel> const & channel_a, std::shared_ptr<nano::transport::tcp_socket> const & socket_a);
~bootstrap_client ();
void stop (bool force);
double sample_block_rate ();
double elapsed_seconds () const;
void set_start_time (std::chrono::steady_clock::time_point start_time_a);
std::weak_ptr<nano::node> node;
std::shared_ptr<nano::transport::tcp_channel> channel;
std::shared_ptr<nano::transport::tcp_socket> socket;
std::shared_ptr<std::vector<uint8_t>> receive_buffer;
std::atomic<uint64_t> block_count{ 0 };
std::atomic<double> block_rate{ 0 };
std::atomic<bool> pending_stop{ false };
std::atomic<bool> hard_stop{ false };
private:
mutable nano::mutex start_time_mutex;
std::chrono::steady_clock::time_point start_time_m;
};
/**
* Container for bootstrap_client objects. Owned by bootstrap_initiator which pools open connections and makes them available
* for use by different bootstrap sessions.
*/
class bootstrap_connections final : public std::enable_shared_from_this<bootstrap_connections>
{
public:
explicit bootstrap_connections (nano::node & node_a);
std::shared_ptr<nano::bootstrap_client> connection (std::shared_ptr<nano::bootstrap_attempt> const & attempt_a = nullptr, bool use_front_connection = false);
void pool_connection (std::shared_ptr<nano::bootstrap_client> const & client_a, bool new_client = false, bool push_front = false);
void add_connection (nano::endpoint const & endpoint_a);
std::shared_ptr<nano::bootstrap_client> find_connection (nano::tcp_endpoint const & endpoint_a);
void connect_client (nano::tcp_endpoint const & endpoint_a, bool push_front = false);
unsigned target_connections (std::size_t pulls_remaining, std::size_t attempts_count) const;
void populate_connections (bool repeat = true);
void start_populate_connections ();
void add_pull (nano::pull_info const & pull_a);
void request_pull (nano::unique_lock<nano::mutex> & lock_a);
void requeue_pull (nano::pull_info const & pull_a, bool network_error = false);
void clear_pulls (uint64_t);
void run ();
void stop ();
std::deque<std::weak_ptr<nano::bootstrap_client>> clients;
std::atomic<unsigned> connections_count{ 0 };
nano::node & node;
std::deque<std::shared_ptr<nano::bootstrap_client>> idle;
std::deque<nano::pull_info> pulls;
std::atomic<bool> populate_connections_started{ false };
std::atomic<bool> new_connections_empty{ false };
std::atomic<bool> stopped{ false };
nano::mutex mutex;
nano::condition_variable condition;
};
}

424
nano/node/bootstrap/bootstrap_frontier.cpp
View file

@ -1,424 +0,0 @@
#include <nano/node/bootstrap/bootstrap_attempt.hpp>
#include <nano/node/bootstrap/bootstrap_frontier.hpp>
#include <nano/node/bootstrap/bootstrap_legacy.hpp>
#include <nano/node/node.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
#include <boost/format.hpp>
constexpr double nano::bootstrap_limits::bootstrap_connection_warmup_time_sec;
constexpr double nano::bootstrap_limits::bootstrap_minimum_elapsed_seconds_blockrate;
constexpr double nano::bootstrap_limits::bootstrap_minimum_frontier_blocks_per_sec;
constexpr unsigned nano::bootstrap_limits::bulk_push_cost_limit;
constexpr std::size_t nano::frontier_req_client::size_frontier;
void nano::frontier_req_client::run (nano::account const & start_account_a, uint32_t const frontiers_age_a, uint32_t const count_a)
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
nano::frontier_req request{ node->network_params.network };
request.start = (start_account_a.is_zero () || start_account_a.number () == std::numeric_limits<nano::uint256_t>::max ()) ? start_account_a.number () : start_account_a.number () + 1;
request.age = frontiers_age_a;
request.count = count_a;
current = start_account_a;
frontiers_age = frontiers_age_a;
count_limit = count_a;
next (); // Load accounts from disk
auto this_l (shared_from_this ());
connection->channel->send (
request, [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
auto node = this_l->connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
this_l->receive_frontier ();
}
else
{
node->logger.debug (nano::log::type::frontier_req_client, "Error while sending bootstrap request: {}", ec.message ());
}
},
nano::transport::buffer_drop_policy::no_limiter_drop);
}
nano::frontier_req_client::frontier_req_client (std::shared_ptr<nano::bootstrap_client> const & connection_a, std::shared_ptr<nano::bootstrap_attempt_legacy> const & attempt_a) :
connection (connection_a),
attempt (attempt_a),
count (0),
bulk_push_cost (0)
{
}
void nano::frontier_req_client::receive_frontier ()
{
auto this_l (shared_from_this ());
connection->socket->async_read (connection->receive_buffer, nano::frontier_req_client::size_frontier, [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
auto node = this_l->connection->node.lock ();
if (!node)
{
return;
}
// An issue with asio is that sometimes, instead of reporting a bad file descriptor during disconnect,
// we simply get a size of 0.
if (size_a == nano::frontier_req_client::size_frontier)
{
node->bootstrap_workers.post ([this_l, ec, size_a] () {
this_l->received_frontier (ec, size_a);
});
}
else
{
node->logger.debug (nano::log::type::frontier_req_client, "Invalid size: expected {}, got {}", nano::frontier_req_client::size_frontier, size_a);
}
});
}
bool nano::frontier_req_client::bulk_push_available ()
{
return bulk_push_cost < nano::bootstrap_limits::bulk_push_cost_limit && frontiers_age == std::numeric_limits<decltype (frontiers_age)>::max ();
}
void nano::frontier_req_client::unsynced (nano::block_hash const & head, nano::block_hash const & end)
{
if (bulk_push_available ())
{
attempt->add_bulk_push_target (head, end);
if (end.is_zero ())
{
bulk_push_cost += 2;
}
else
{
bulk_push_cost += 1;
}
}
}
void nano::frontier_req_client::received_frontier (boost::system::error_code const & ec, std::size_t size_a)
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
debug_assert (size_a == nano::frontier_req_client::size_frontier);
nano::account account;
nano::bufferstream account_stream (connection->receive_buffer->data (), sizeof (account));
auto error1 (nano::try_read (account_stream, account));
(void)error1;
debug_assert (!error1);
nano::block_hash latest;
nano::bufferstream latest_stream (connection->receive_buffer->data () + sizeof (account), sizeof (latest));
auto error2 (nano::try_read (latest_stream, latest));
(void)error2;
debug_assert (!error2);
if (count == 0)
{
start_time = std::chrono::steady_clock::now ();
}
++count;
std::chrono::duration<double> time_span = std::chrono::duration_cast<std::chrono::duration<double>> (std::chrono::steady_clock::now () - start_time);
double elapsed_sec = std::max (time_span.count (), nano::bootstrap_limits::bootstrap_minimum_elapsed_seconds_blockrate);
double blocks_per_sec = static_cast<double> (count) / elapsed_sec;
double age_factor = (frontiers_age == std::numeric_limits<decltype (frontiers_age)>::max ()) ? 1.0 : 1.5; // Allow slower frontiers receive for requests with age
if (elapsed_sec > nano::bootstrap_limits::bootstrap_connection_warmup_time_sec && blocks_per_sec * age_factor < nano::bootstrap_limits::bootstrap_minimum_frontier_blocks_per_sec)
{
node->logger.debug (nano::log::type::frontier_req_client, "Aborting frontier req because it was too slow: {} frontiers per second, last {}", blocks_per_sec, account.to_account ());
promise.set_value (true);
return;
}
if (attempt->should_log ())
{
node->logger.debug (nano::log::type::frontier_req_client, "Received {} frontiers from {}", count, connection->channel->to_string ());
}
if (!account.is_zero () && count <= count_limit)
{
last_account = account;
while (!current.is_zero () && current < account)
{
// We know about an account they don't.
unsynced (frontier, 0);
next ();
}
if (!current.is_zero ())
{
if (account == current)
{
if (latest == frontier)
{
// In sync
}
else
{
if (node->block_or_pruned_exists (latest))
{
// We know about a block they don't.
unsynced (frontier, latest);
}
else
{
attempt->add_frontier (nano::pull_info (account, latest, frontier, attempt->incremental_id, 0, node->network_params.bootstrap.frontier_retry_limit));
// Either we're behind or there's a fork we differ on
// Either way, bulk pushing will probably not be effective
bulk_push_cost += 5;
}
}
next ();
}
else
{
debug_assert (account < current);
attempt->add_frontier (nano::pull_info (account, latest, nano::block_hash (0), attempt->incremental_id, 0, node->network_params.bootstrap.frontier_retry_limit));
}
}
else
{
attempt->add_frontier (nano::pull_info (account, latest, nano::block_hash (0), attempt->incremental_id, 0, node->network_params.bootstrap.frontier_retry_limit));
}
receive_frontier ();
}
else
{
if (count <= count_limit)
{
while (!current.is_zero () && bulk_push_available ())
{
// We know about an account they don't.
unsynced (frontier, 0);
next ();
}
// Prevent new frontier_req requests
attempt->set_start_account (std::numeric_limits<nano::uint256_t>::max ());
node->logger.debug (nano::log::type::frontier_req_client, "Bulk push cost: {}", bulk_push_cost);
}
else
{
// Set last processed account as new start target
attempt->set_start_account (last_account);
}
node->bootstrap_initiator.connections->pool_connection (connection);
try
{
promise.set_value (false);
}
catch (std::future_error &)
{
}
}
}
else
{
node->logger.debug (nano::log::type::frontier_req_client, "Error while receiving frontier: {}", ec.message ());
}
}
void nano::frontier_req_client::next ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
// Filling accounts deque to prevent often read transactions
if (accounts.empty ())
{
std::size_t max_size (128);
auto transaction (node->store.tx_begin_read ());
for (auto i (node->store.account.begin (transaction, current.number () + 1)), n (node->store.account.end (transaction)); i != n && accounts.size () != max_size; ++i)
{
nano::account_info const & info (i->second);
nano::account const & account (i->first);
accounts.emplace_back (account, info.head);
}
/* If loop breaks before max_size, then accounts_end () is reached. Add empty record */
if (accounts.size () != max_size)
{
accounts.emplace_back (nano::account{}, nano::block_hash (0));
}
}
// Retrieving accounts from deque
auto const & account_pair (accounts.front ());
current = account_pair.first;
frontier = account_pair.second;
accounts.pop_front ();
}
nano::frontier_req_server::frontier_req_server (std::shared_ptr<nano::transport::tcp_server> const & connection_a, std::unique_ptr<nano::frontier_req> request_a) :
connection (connection_a),
current (request_a->start.number () - 1),
frontier (0),
request (std::move (request_a)),
count (0)
{
next ();
}
void nano::frontier_req_server::send_next ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (!current.is_zero () && count < request->count)
{
node->logger.trace (nano::log::type::frontier_req_server, nano::log::detail::sending_frontier,
nano::log::arg{ "account", current.to_account () }, // TODO: Convert to lazy eval
nano::log::arg{ "frontier", frontier },
nano::log::arg{ "socket", connection->socket });
std::vector<uint8_t> send_buffer;
{
nano::vectorstream stream (send_buffer);
write (stream, current.bytes);
write (stream, frontier.bytes);
debug_assert (!current.is_zero ());
debug_assert (!frontier.is_zero ());
}
auto this_l (shared_from_this ());
next ();
connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->sent_action (ec, size_a);
});
}
else
{
send_finished ();
}
}
void nano::frontier_req_server::send_finished ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
std::vector<uint8_t> send_buffer;
{
nano::vectorstream stream (send_buffer);
nano::uint256_union zero (0);
write (stream, zero.bytes);
write (stream, zero.bytes);
}
node->logger.debug (nano::log::type::frontier_req_server, "Frontier sending finished");
auto this_l (shared_from_this ());
connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) {
this_l->no_block_sent (ec, size_a);
});
}
void nano::frontier_req_server::no_block_sent (boost::system::error_code const & ec, std::size_t size_a)
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
connection->start ();
}
else
{
node->logger.debug (nano::log::type::frontier_req_server, "Error sending frontier finish: {}", ec.message ());
}
}
void nano::frontier_req_server::sent_action (boost::system::error_code const & ec, std::size_t size_a)
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
if (!ec)
{
count++;
node->bootstrap_workers.post ([this_l = shared_from_this ()] () {
this_l->send_next ();
});
}
else
{
node->logger.debug (nano::log::type::frontier_req_server, "Error sending frontier pair: {}", ec.message ());
}
}
void nano::frontier_req_server::next ()
{
auto node = connection->node.lock ();
if (!node)
{
return;
}
// Filling accounts deque to prevent often read transactions
if (accounts.empty ())
{
auto now (nano::seconds_since_epoch ());
bool disable_age_filter (request->age == std::numeric_limits<decltype (request->age)>::max ());
std::size_t max_size (128);
auto transaction (node->store.tx_begin_read ());
if (!send_confirmed ())
{
for (auto i (node->store.account.begin (transaction, current.number () + 1)), n (node->store.account.end (transaction)); i != n && accounts.size () != max_size; ++i)
{
nano::account_info const & info (i->second);
if (disable_age_filter || (now - info.modified) <= request->age)
{
nano::account const & account (i->first);
accounts.emplace_back (account, info.head);
}
}
}
else
{
for (auto i (node->store.confirmation_height.begin (transaction, current.number () + 1)), n (node->store.confirmation_height.end (transaction)); i != n && accounts.size () != max_size; ++i)
{
nano::confirmation_height_info const & info (i->second);
nano::block_hash const & confirmed_frontier (info.frontier);
if (!confirmed_frontier.is_zero ())
{
nano::account const & account (i->first);
accounts.emplace_back (account, confirmed_frontier);
}
}
}
/* If loop breaks before max_size, then accounts_end () is reached. Add empty record to finish frontier_req_server */
if (accounts.size () != max_size)
{
accounts.emplace_back (nano::account{}, nano::block_hash (0));
}
}
// Retrieving accounts from deque
auto const & account_pair (accounts.front ());
current = account_pair.first;
frontier = account_pair.second;
accounts.pop_front ();
}
bool nano::frontier_req_server::send_confirmed ()
{
return request->header.frontier_req_is_only_confirmed_present ();
}

69
nano/node/bootstrap/bootstrap_frontier.hpp
View file

@ -1,69 +0,0 @@
#pragma once
#include <nano/lib/numbers.hpp>
#include <deque>
#include <future>
#include <memory>
namespace nano
{
class bootstrap_attempt_legacy;
class bootstrap_client;
namespace transport
{
class tcp_server;
}
/**
* Client side of a frontier request. Created to send and listen for frontier sequences from the server.
*/
class frontier_req_client final : public std::enable_shared_from_this<nano::frontier_req_client>
{
public:
explicit frontier_req_client (std::shared_ptr<nano::bootstrap_client> const &, std::shared_ptr<nano::bootstrap_attempt_legacy> const &);
void run (nano::account const & start_account_a, uint32_t const frontiers_age_a, uint32_t const count_a);
void receive_frontier ();
void received_frontier (boost::system::error_code const &, std::size_t);
bool bulk_push_available ();
void unsynced (nano::block_hash const &, nano::block_hash const &);
void next ();
std::shared_ptr<nano::bootstrap_client> connection;
std::shared_ptr<nano::bootstrap_attempt_legacy> attempt;
nano::account current;
nano::block_hash frontier;
unsigned count;
nano::account last_account{ std::numeric_limits<nano::uint256_t>::max () }; // Using last possible account stop further frontier requests
std::chrono::steady_clock::time_point start_time;
std::promise<bool> promise;
/** A very rough estimate of the cost of `bulk_push`ing missing blocks */
uint64_t bulk_push_cost;
std::deque<std::pair<nano::account, nano::block_hash>> accounts;
uint32_t frontiers_age{ std::numeric_limits<uint32_t>::max () };
uint32_t count_limit{ std::numeric_limits<uint32_t>::max () };
static std::size_t constexpr size_frontier = sizeof (nano::account) + sizeof (nano::block_hash);
};
class frontier_req;
/**
* Server side of a frontier request. Created when a tcp_server receives a frontier_req message and exited when end-of-list is reached.
*/
class frontier_req_server final : public std::enable_shared_from_this<nano::frontier_req_server>
{
public:
frontier_req_server (std::shared_ptr<nano::transport::tcp_server> const &, std::unique_ptr<nano::frontier_req>);
void send_next ();
void sent_action (boost::system::error_code const &, std::size_t);
void send_finished ();
void no_block_sent (boost::system::error_code const &, std::size_t);
void next ();
bool send_confirmed ();
std::shared_ptr<nano::transport::tcp_server> connection;
nano::account current;
nano::block_hash frontier;
std::unique_ptr<nano::frontier_req> request;
std::size_t count;
std::deque<std::pair<nano::account, nano::block_hash>> accounts;
};
}

634
nano/node/bootstrap/bootstrap_lazy.cpp
View file

@ -1,634 +0,0 @@
#include <nano/lib/blocks.hpp>
#include <nano/node/bootstrap/bootstrap.hpp>
#include <nano/node/bootstrap/bootstrap_lazy.hpp>
#include <nano/node/common.hpp>
#include <nano/node/node.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
#include <boost/format.hpp>
#include <algorithm>
constexpr std::chrono::seconds nano::bootstrap_limits::lazy_flush_delay_sec;
constexpr uint64_t nano::bootstrap_limits::lazy_batch_pull_count_resize_blocks_limit;
constexpr double nano::bootstrap_limits::lazy_batch_pull_count_resize_ratio;
constexpr std::size_t nano::bootstrap_limits::lazy_blocks_restart_limit;
nano::bootstrap_attempt_lazy::bootstrap_attempt_lazy (std::shared_ptr<nano::node> const & node_a, uint64_t incremental_id_a, std::string const & id_a) :
nano::bootstrap_attempt (node_a, nano::bootstrap_mode::lazy, incremental_id_a, id_a)
{
node_a->bootstrap_initiator.notify_listeners (true);
}
nano::bootstrap_attempt_lazy::~bootstrap_attempt_lazy ()
{
auto node = this->node.lock ();
if (!node)
{
return;
}
debug_assert (lazy_blocks.size () == lazy_blocks_count);
node->bootstrap_initiator.notify_listeners (false);
}
bool nano::bootstrap_attempt_lazy::lazy_start (nano::hash_or_account const & hash_or_account_a)
{
auto node = this->node.lock ();
if (!node)
{
return false;
}
nano::unique_lock<nano::mutex> lock{ mutex };
bool inserted (false);
// Add start blocks, limit 1024 (4k with disabled legacy bootstrap)
std::size_t max_keys (node->flags.disable_legacy_bootstrap ? 4 * 1024 : 1024);
if (lazy_keys.size () < max_keys && lazy_keys.find (hash_or_account_a.as_block_hash ()) == lazy_keys.end () && !lazy_blocks_processed (hash_or_account_a.as_block_hash ()))
{
lazy_keys.insert (hash_or_account_a.as_block_hash ());
lazy_pulls.emplace_back (hash_or_account_a, node->network_params.bootstrap.lazy_retry_limit);
lock.unlock ();
condition.notify_all ();
inserted = true;
}
return inserted;
}
void nano::bootstrap_attempt_lazy::lazy_add (nano::hash_or_account const & hash_or_account_a, unsigned retry_limit)
{
// Add only unknown blocks
debug_assert (!mutex.try_lock ());
if (!lazy_blocks_processed (hash_or_account_a.as_block_hash ()))
{
lazy_pulls.emplace_back (hash_or_account_a, retry_limit);
}
}
void nano::bootstrap_attempt_lazy::lazy_add (nano::pull_info const & pull_a)
{
debug_assert (pull_a.account_or_head.as_block_hash () == pull_a.head);
nano::lock_guard<nano::mutex> lock{ mutex };
lazy_add (pull_a.account_or_head, pull_a.retry_limit);
}
void nano::bootstrap_attempt_lazy::lazy_requeue (nano::block_hash const & hash_a, nano::block_hash const & previous_a)
{
auto node = this->node.lock ();
if (!node)
{
return;
}
nano::unique_lock<nano::mutex> lock{ mutex };
// Add only known blocks
if (lazy_blocks_processed (hash_a))
{
lazy_blocks_erase (hash_a);
lock.unlock ();
node->bootstrap_initiator.connections->requeue_pull (nano::pull_info (hash_a, hash_a, previous_a, incremental_id, static_cast<nano::pull_info::count_t> (1), node->network_params.bootstrap.lazy_destinations_retry_limit));
}
}
uint32_t nano::bootstrap_attempt_lazy::lazy_batch_size ()
{
auto node = this->node.lock ();
if (!node)
{
return 0;
}
auto result (node->network_params.bootstrap.lazy_max_pull_blocks);
if (total_blocks > nano::bootstrap_limits::lazy_batch_pull_count_resize_blocks_limit && lazy_blocks_count != 0)
{
auto lazy_blocks_ratio (static_cast<double> (total_blocks / lazy_blocks_count));
if (lazy_blocks_ratio > nano::bootstrap_limits::lazy_batch_pull_count_resize_ratio)
{
// Increasing blocks ratio weight as more important (^3). Small batch count should lower blocks ratio below target
double lazy_blocks_factor (std::pow (lazy_blocks_ratio / nano::bootstrap_limits::lazy_batch_pull_count_resize_ratio, 3.0));
// Decreasing total block count weight as less important (sqrt)
double total_blocks_factor (std::sqrt (total_blocks / nano::bootstrap_limits::lazy_batch_pull_count_resize_blocks_limit));
uint32_t batch_count_min (node->network_params.bootstrap.lazy_max_pull_blocks / static_cast<uint32_t> (lazy_blocks_factor * total_blocks_factor));
result = std::max (node->network_params.bootstrap.lazy_min_pull_blocks, batch_count_min);
}
}
return result;
}
void nano::bootstrap_attempt_lazy::lazy_pull_flush (nano::unique_lock<nano::mutex> & lock_a)
{
auto node = this->node.lock ();
if (!node)
{
return;
}
static std::size_t const max_pulls (static_cast<std::size_t> (nano::bootstrap_limits::bootstrap_connection_scale_target_blocks) * 3);
if (pulling < max_pulls)
{
debug_assert (node->network_params.bootstrap.lazy_max_pull_blocks <= std::numeric_limits<nano::pull_info::count_t>::max ());
nano::pull_info::count_t batch_count (lazy_batch_size ());
uint64_t read_count (0);
std::size_t count (0);
auto transaction = node->ledger.tx_begin_read ();
while (!lazy_pulls.empty () && count < max_pulls)
{
auto pull_start (lazy_pulls.front ());
lazy_pulls.pop_front ();
// Recheck if block was already processed
if (!lazy_blocks_processed (pull_start.first.as_block_hash ()) && !node->ledger.any.block_exists_or_pruned (transaction, pull_start.first.as_block_hash ()))
{
lock_a.unlock ();
node->bootstrap_initiator.connections->add_pull (nano::pull_info (pull_start.first, pull_start.first.as_block_hash (), nano::block_hash (0), incremental_id, batch_count, pull_start.second));
++pulling;
++count;
lock_a.lock ();
}
// We don't want to open read transactions for too long
++read_count;
if (read_count % batch_read_size == 0)
{
lock_a.unlock ();
transaction.refresh ();
lock_a.lock ();
}
}
}
}
bool nano::bootstrap_attempt_lazy::lazy_finished ()
{
auto node = this->node.lock ();
if (!node)
{
return true;
}
debug_assert (!mutex.try_lock ());
if (stopped)
{
return true;
}
bool result (true);
uint64_t read_count (0);
auto transaction = node->ledger.tx_begin_read ();
for (auto it (lazy_keys.begin ()), end (lazy_keys.end ()); it != end && !stopped;)
{
if (node->ledger.any.block_exists_or_pruned (transaction, *it))
{
it = lazy_keys.erase (it);
}
else
{
result = false;
break;
// No need to increment `it` as we break above.
}
// We don't want to open read transactions for too long
++read_count;
if (read_count % batch_read_size == 0)
{
transaction.refresh ();
}
}
// Finish lazy bootstrap without lazy pulls (in combination with still_pulling ())
if (!result && lazy_pulls.empty () && lazy_state_backlog.empty ())
{
result = true;
}
return result;
}
bool nano::bootstrap_attempt_lazy::lazy_has_expired () const
{
auto node = this->node.lock ();
if (!node)
{
return true;
}
bool result (false);
// Max 30 minutes run with enabled legacy bootstrap
static std::chrono::minutes const max_lazy_time (node->flags.disable_legacy_bootstrap ? 7 * 24 * 60 : 30);
if (std::chrono::steady_clock::now () - lazy_start_time >= max_lazy_time)
{
result = true;
}
else if (!node->flags.disable_legacy_bootstrap && lazy_blocks_count > nano::bootstrap_limits::lazy_blocks_restart_limit)
{
result = true;
}
return result;
}
void nano::bootstrap_attempt_lazy::run ()
{
auto node = this->node.lock ();
if (!node)
{
return;
}
debug_assert (started);
debug_assert (!node->flags.disable_lazy_bootstrap);
node->bootstrap_initiator.connections->populate_connections (false);
lazy_start_time = std::chrono::steady_clock::now ();
nano::unique_lock<nano::mutex> lock{ mutex };
while ((still_pulling () || !lazy_finished ()) && !lazy_has_expired ())
{
unsigned iterations (0);
while (still_pulling () && !lazy_has_expired ())
{
condition.wait (lock, [this, &stopped = stopped, &pulling = pulling, &lazy_pulls = lazy_pulls] { return stopped || pulling == 0 || (pulling < nano::bootstrap_limits::bootstrap_connection_scale_target_blocks && !lazy_pulls.empty ()) || lazy_has_expired (); });
++iterations;
// Flushing lazy pulls
lazy_pull_flush (lock);
// Start backlog cleanup
if (iterations % 100 == 0)
{
lazy_backlog_cleanup ();
}
}
// Flushing lazy pulls
lazy_pull_flush (lock);
// Check if some blocks required for backlog were processed. Start destinations check
if (pulling == 0)
{
lazy_backlog_cleanup ();
lazy_pull_flush (lock);
}
}
if (!stopped)
{
node->logger.debug (nano::log::type::bootstrap_lazy, "Completed lazy pulls");
}
if (lazy_has_expired ())
{
node->logger.debug (nano::log::type::bootstrap_lazy, "Lazy bootstrap attempt ID {} expired", id);
}
lock.unlock ();
stop ();
condition.notify_all ();
}
bool nano::bootstrap_attempt_lazy::process_block (std::shared_ptr<nano::block> const & block_a, nano::account const & known_account_a, uint64_t pull_blocks_processed, nano::bulk_pull::count_t max_blocks, bool block_expected, unsigned retry_limit)
{
bool stop_pull (false);
if (block_expected)
{
stop_pull = process_block_lazy (block_a, known_account_a, pull_blocks_processed, max_blocks, retry_limit);
}
else
{
// Drop connection with unexpected block for lazy bootstrap
stop_pull = true;
}
return stop_pull;
}
bool nano::bootstrap_attempt_lazy::process_block_lazy (std::shared_ptr<nano::block> const & block_a, nano::account const & known_account_a, uint64_t pull_blocks_processed, nano::bulk_pull::count_t max_blocks, unsigned retry_limit)
{
auto node = this->node.lock ();
if (!node)
{
return true;
}
bool stop_pull (false);
auto hash (block_a->hash ());
nano::unique_lock<nano::mutex> lock{ mutex };
// Processing new blocks
if (!lazy_blocks_processed (hash))
{
// Search for new dependencies
if (block_a->source_field () && !node->block_or_pruned_exists (block_a->source_field ().value ()) && block_a->source_field ().value () != node->network_params.ledger.genesis->account ().as_union ())
{
lazy_add (block_a->source_field ().value (), retry_limit);
}
else if (block_a->type () == nano::block_type::state)
{
lazy_block_state (block_a, retry_limit);
}
lazy_blocks_insert (hash);
// Adding lazy balances for first processed block in pull
if (pull_blocks_processed == 1 && (block_a->type () == nano::block_type::state || block_a->type () == nano::block_type::send))
{
lazy_balances.emplace (hash, block_a->balance_field ().value ().number ());
}
// Clearing lazy balances for previous block
if (!block_a->previous ().is_zero () && lazy_balances.find (block_a->previous ()) != lazy_balances.end ())
{
lazy_balances.erase (block_a->previous ());
}
lazy_block_state_backlog_check (block_a, hash);
lock.unlock ();
node->block_processor.add (block_a, nano::block_source::bootstrap_legacy);
}
// Force drop lazy bootstrap connection for long bulk_pull
if (pull_blocks_processed > max_blocks)
{
stop_pull = true;
}
return stop_pull;
}
void nano::bootstrap_attempt_lazy::lazy_block_state (std::shared_ptr<nano::block> const & block_a, unsigned retry_limit)
{
auto node = this->node.lock ();
if (!node)
{
return;
}
std::shared_ptr<nano::state_block> block_l (std::static_pointer_cast<nano::state_block> (block_a));
if (block_l != nullptr)
{
auto transaction = node->ledger.tx_begin_read ();
nano::uint128_t balance (block_l->hashables.balance.number ());
auto const & link (block_l->hashables.link);
// If link is not epoch link or 0. And if block from link is unknown
if (!link.is_zero () && !node->ledger.is_epoch_link (link) && !lazy_blocks_processed (link.as_block_hash ()) && !node->ledger.any.block_exists_or_pruned (transaction, link.as_block_hash ()))
{
auto const & previous (block_l->hashables.previous);
// If state block previous is 0 then source block required
if (previous.is_zero ())
{
lazy_add (link, retry_limit);
}
// In other cases previous block balance required to find out subtype of state block
else if (node->ledger.any.block_exists_or_pruned (transaction, previous))
{
auto previous_balance = node->ledger.any.block_balance (transaction, previous);
if (previous_balance)
{
if (previous_balance.value ().number () <= balance)
{
lazy_add (link, retry_limit);
}
}
// Else ignore pruned blocks
}
// Search balance of already processed previous blocks
else if (lazy_blocks_processed (previous))
{
auto previous_balance (lazy_balances.find (previous));
if (previous_balance != lazy_balances.end ())
{
if (previous_balance->second <= balance)
{
lazy_add (link, retry_limit);
}
lazy_balances.erase (previous_balance);
}
}
// Insert in backlog state blocks if previous wasn't already processed
else
{
lazy_state_backlog.emplace (previous, nano::lazy_state_backlog_item{ link, balance, retry_limit });
}
}
}
}
void nano::bootstrap_attempt_lazy::lazy_block_state_backlog_check (std::shared_ptr<nano::block> const & block_a, nano::block_hash const & hash_a)
{
auto node = this->node.lock ();
if (!node)
{
return;
}
// Search unknown state blocks balances
auto find_state (lazy_state_backlog.find (hash_a));
if (find_state != lazy_state_backlog.end ())
{
auto next_block (find_state->second);
// Retrieve balance for previous state & send blocks
if (block_a->type () == nano::block_type::state || block_a->type () == nano::block_type::send)
{
if (block_a->balance_field ().value ().number () <= next_block.balance) // balance
{
lazy_add (next_block.link, next_block.retry_limit); // link
}
}
// Assumption for other legacy block types
else if (lazy_undefined_links.find (next_block.link.as_block_hash ()) == lazy_undefined_links.end ())
{
lazy_add (next_block.link, node->network_params.bootstrap.lazy_retry_limit); // Head is not confirmed. It can be account or hash or non-existing
lazy_undefined_links.insert (next_block.link.as_block_hash ());
}
lazy_state_backlog.erase (find_state);
}
}
void nano::bootstrap_attempt_lazy::lazy_backlog_cleanup ()
{
auto node = this->node.lock ();
if (!node)
{
return;
}
uint64_t read_count (0);
auto transaction = node->ledger.tx_begin_read ();
for (auto it (lazy_state_backlog.begin ()), end (lazy_state_backlog.end ()); it != end && !stopped;)
{
if (node->ledger.any.block_exists_or_pruned (transaction, it->first))
{
auto next_block (it->second);
auto balance = node->ledger.any.block_balance (transaction, it->first);
if (balance)
{
if (balance.value ().number () <= next_block.balance) // balance
{
lazy_add (next_block.link, next_block.retry_limit); // link
}
}
else
{
lazy_add (next_block.link, node->network_params.bootstrap.lazy_retry_limit); // Not confirmed
}
it = lazy_state_backlog.erase (it);
}
else
{
lazy_add (it->first, it->second.retry_limit);
++it;
}
// We don't want to open read transactions for too long
++read_count;
if (read_count % batch_read_size == 0)
{
transaction.refresh ();
}
}
}
void nano::bootstrap_attempt_lazy::lazy_blocks_insert (nano::block_hash const & hash_a)
{
debug_assert (!mutex.try_lock ());
auto inserted (lazy_blocks.insert (std::hash<::nano::block_hash> () (hash_a)));
if (inserted.second)
{
++lazy_blocks_count;
debug_assert (lazy_blocks_count > 0);
}
}
void nano::bootstrap_attempt_lazy::lazy_blocks_erase (nano::block_hash const & hash_a)
{
debug_assert (!mutex.try_lock ());
auto erased (lazy_blocks.erase (std::hash<::nano::block_hash> () (hash_a)));
if (erased)
{
--lazy_blocks_count;
debug_assert (lazy_blocks_count != std::numeric_limits<std::size_t>::max ());
}
}
bool nano::bootstrap_attempt_lazy::lazy_blocks_processed (nano::block_hash const & hash_a)
{
return lazy_blocks.find (std::hash<::nano::block_hash> () (hash_a)) != lazy_blocks.end ();
}
bool nano::bootstrap_attempt_lazy::lazy_processed_or_exists (nano::block_hash const & hash_a)
{
auto node = this->node.lock ();
if (!node)
{
return true;
}
bool result (false);
nano::unique_lock<nano::mutex> lock{ mutex };
if (lazy_blocks_processed (hash_a))
{
result = true;
}
else
{
lock.unlock ();
if (node->block_or_pruned_exists (hash_a))
{
result = true;
}
}
return result;
}
void nano::bootstrap_attempt_lazy::get_information (boost::property_tree::ptree & tree_a)
{
nano::lock_guard<nano::mutex> lock{ mutex };
tree_a.put ("lazy_blocks", std::to_string (lazy_blocks.size ()));
tree_a.put ("lazy_state_backlog", std::to_string (lazy_state_backlog.size ()));
tree_a.put ("lazy_balances", std::to_string (lazy_balances.size ()));
tree_a.put ("lazy_undefined_links", std::to_string (lazy_undefined_links.size ()));
tree_a.put ("lazy_pulls", std::to_string (lazy_pulls.size ()));
tree_a.put ("lazy_keys", std::to_string (lazy_keys.size ()));
if (!lazy_keys.empty ())
{
tree_a.put ("lazy_key_1", (*(lazy_keys.begin ())).to_string ());
}
}
nano::bootstrap_attempt_wallet::bootstrap_attempt_wallet (std::shared_ptr<nano::node> const & node_a, uint64_t incremental_id_a, std::string id_a) :
nano::bootstrap_attempt (node_a, nano::bootstrap_mode::wallet_lazy, incremental_id_a, id_a)
{
node_a->bootstrap_initiator.notify_listeners (true);
}
nano::bootstrap_attempt_wallet::~bootstrap_attempt_wallet ()
{
auto node = this->node.lock ();
if (!node)
{
return;
}
node->bootstrap_initiator.notify_listeners (false);
}
void nano::bootstrap_attempt_wallet::request_pending (nano::unique_lock<nano::mutex> & lock_a)
{
auto node = this->node.lock ();
if (!node)
{
return;
}
lock_a.unlock ();
auto connection_l (node->bootstrap_initiator.connections->connection (shared_from_this ()));
lock_a.lock ();
if (connection_l && !stopped)
{
auto account (wallet_accounts.front ());
wallet_accounts.pop_front ();
++pulling;
auto this_l = std::dynamic_pointer_cast<nano::bootstrap_attempt_wallet> (shared_from_this ());
// The bulk_pull_account_client destructor attempt to requeue_pull which can cause a deadlock if this is the last reference
// Dispatch request in an external thread in case it needs to be destroyed
node->background ([connection_l, this_l, account] () {
auto client (std::make_shared<nano::bulk_pull_account_client> (connection_l, this_l, account));
client->request ();
});
}
}
void nano::bootstrap_attempt_wallet::requeue_pending (nano::account const & account_a)
{
auto account (account_a);
{
nano::lock_guard<nano::mutex> lock{ mutex };
wallet_accounts.push_front (account);
}
condition.notify_all ();
}
void nano::bootstrap_attempt_wallet::wallet_start (std::deque<nano::account> & accounts_a)
{
{
nano::lock_guard<nano::mutex> lock{ mutex };
wallet_accounts.swap (accounts_a);
}
condition.notify_all ();
}
bool nano::bootstrap_attempt_wallet::wallet_finished ()
{
debug_assert (!mutex.try_lock ());
auto running (!stopped);
auto more_accounts (!wallet_accounts.empty ());
auto still_pulling (pulling > 0);
return running && (more_accounts || still_pulling);
}
void nano::bootstrap_attempt_wallet::run ()
{
auto node = this->node.lock ();
if (!node)
{
return;
}
debug_assert (started);
debug_assert (!node->flags.disable_wallet_bootstrap);
node->bootstrap_initiator.connections->populate_connections (false);
auto start_time (std::chrono::steady_clock::now ());
auto max_time (std::chrono::minutes (10));
nano::unique_lock<nano::mutex> lock{ mutex };
while (wallet_finished () && std::chrono::steady_clock::now () - start_time < max_time)
{
if (!wallet_accounts.empty ())
{
request_pending (lock);
}
else
{
condition.wait_for (lock, std::chrono::seconds (1));
}
}
if (!stopped)
{
node->logger.info (nano::log::type::bootstrap_lazy, "Completed wallet lazy pulls");
}
lock.unlock ();
stop ();
condition.notify_all ();
}
std::size_t nano::bootstrap_attempt_wallet::wallet_size ()
{
nano::lock_guard<nano::mutex> lock{ mutex };
return wallet_accounts.size ();
}
void nano::bootstrap_attempt_wallet::get_information (boost::property_tree::ptree & tree_a)
{
nano::lock_guard<nano::mutex> lock{ mutex };
tree_a.put ("wallet_accounts", std::to_string (wallet_accounts.size ()));
}

81
nano/node/bootstrap/bootstrap_lazy.hpp
View file

@ -1,81 +0,0 @@
#pragma once
#include <nano/node/bootstrap/bootstrap_attempt.hpp>
#include <boost/multi_index_container.hpp>
#include <atomic>
#include <unordered_set>
namespace mi = boost::multi_index;
namespace nano
{
class node;
class lazy_state_backlog_item final
{
public:
nano::link link{ 0 };
nano::uint128_t balance{ 0 };
unsigned retry_limit{ 0 };
};
/**
* Lazy bootstrap session. Started with a block hash, this will "trace down" the blocks obtained to find a connection to the ledger.
* This attempts to quickly bootstrap a section of the ledger given a hash that's known to be confirmed.
*/
class bootstrap_attempt_lazy final : public bootstrap_attempt
{
public:
explicit bootstrap_attempt_lazy (std::shared_ptr<nano::node> const & node_a, uint64_t incremental_id_a, std::string const & id_a = "");
~bootstrap_attempt_lazy ();
bool process_block (std::shared_ptr<nano::block> const &, nano::account const &, uint64_t, nano::bulk_pull::count_t, bool, unsigned) override;
void run () override;
bool lazy_start (nano::hash_or_account const &);
void lazy_add (nano::hash_or_account const &, unsigned);
void lazy_add (nano::pull_info const &);
void lazy_requeue (nano::block_hash const &, nano::block_hash const &);
bool lazy_finished ();
bool lazy_has_expired () const;
uint32_t lazy_batch_size ();
void lazy_pull_flush (nano::unique_lock<nano::mutex> & lock_a);
bool process_block_lazy (std::shared_ptr<nano::block> const &, nano::account const &, uint64_t, nano::bulk_pull::count_t, unsigned);
void lazy_block_state (std::shared_ptr<nano::block> const &, unsigned);
void lazy_block_state_backlog_check (std::shared_ptr<nano::block> const &, nano::block_hash const &);
void lazy_backlog_cleanup ();
void lazy_blocks_insert (nano::block_hash const &);
void lazy_blocks_erase (nano::block_hash const &);
bool lazy_blocks_processed (nano::block_hash const &);
bool lazy_processed_or_exists (nano::block_hash const &);
void get_information (boost::property_tree::ptree &) override;
std::unordered_set<std::size_t> lazy_blocks;
std::unordered_map<nano::block_hash, nano::lazy_state_backlog_item> lazy_state_backlog;
std::unordered_set<nano::block_hash> lazy_undefined_links;
std::unordered_map<nano::block_hash, nano::uint128_t> lazy_balances;
std::unordered_set<nano::block_hash> lazy_keys;
std::deque<std::pair<nano::hash_or_account, unsigned>> lazy_pulls;
std::chrono::steady_clock::time_point lazy_start_time;
std::atomic<std::size_t> lazy_blocks_count{ 0 };
std::size_t peer_count{ 0 };
/** The maximum number of records to be read in while iterating over long lazy containers */
static uint64_t constexpr batch_read_size = 256;
};
/**
* Wallet bootstrap session. This session will trace down accounts within local wallets to try and bootstrap those blocks first.
*/
class bootstrap_attempt_wallet final : public bootstrap_attempt
{
public:
explicit bootstrap_attempt_wallet (std::shared_ptr<nano::node> const & node_a, uint64_t incremental_id_a, std::string id_a = "");
~bootstrap_attempt_wallet ();
void request_pending (nano::unique_lock<nano::mutex> &);
void requeue_pending (nano::account const &);
void run () override;
void wallet_start (std::deque<nano::account> &);
bool wallet_finished ();
std::size_t wallet_size ();
void get_information (boost::property_tree::ptree &) override;
std::deque<nano::account> wallet_accounts;
};
}

262
nano/node/bootstrap/bootstrap_legacy.cpp
View file

@ -1,262 +0,0 @@
#include <nano/node/bootstrap/bootstrap_bulk_push.hpp>
#include <nano/node/bootstrap/bootstrap_frontier.hpp>
#include <nano/node/bootstrap/bootstrap_legacy.hpp>
#include <nano/node/node.hpp>
#include <boost/format.hpp>
nano::bootstrap_attempt_legacy::bootstrap_attempt_legacy (std::shared_ptr<nano::node> const & node_a, uint64_t const incremental_id_a, std::string const & id_a, uint32_t const frontiers_age_a, nano::account const & start_account_a) :
nano::bootstrap_attempt (node_a, nano::bootstrap_mode::legacy, incremental_id_a, id_a),
frontiers_age (frontiers_age_a),
start_account (start_account_a)
{
node_a->bootstrap_initiator.notify_listeners (true);
}
bool nano::bootstrap_attempt_legacy::consume_future (std::future<bool> & future_a)
{
bool result;
try
{
result = future_a.get ();
}
catch (std::future_error &)
{
result = true;
}
return result;
}
void nano::bootstrap_attempt_legacy::stop ()
{
auto node = this->node.lock ();
if (!node)
{
return;
}
nano::unique_lock<nano::mutex> lock{ mutex };
stopped = true;
lock.unlock ();
condition.notify_all ();
lock.lock ();
if (auto i = frontiers.lock ())
{
try
{
i->promise.set_value (true);
}
catch (std::future_error &)
{
}
}
if (auto i = push.lock ())
{
try
{
i->promise.set_value (true);
}
catch (std::future_error &)
{
}
}
lock.unlock ();
node->bootstrap_initiator.connections->clear_pulls (incremental_id);
}
void nano::bootstrap_attempt_legacy::request_push (nano::unique_lock<nano::mutex> & lock_a)
{
auto node = this->node.lock ();
if (!node)
{
return;
}
bool error (false);
lock_a.unlock ();
auto connection_l (node->bootstrap_initiator.connections->find_connection (endpoint_frontier_request));
lock_a.lock ();
if (connection_l)
{
std::future<bool> future;
{
auto this_l = std::dynamic_pointer_cast<nano::bootstrap_attempt_legacy> (shared_from_this ());
auto client = std::make_shared<nano::bulk_push_client> (connection_l, this_l);
client->start ();
push = client;
future = client->promise.get_future ();
}
lock_a.unlock ();
error = consume_future (future); // This is out of scope of `client' so when the last reference via boost::asio::io_context is lost and the client is destroyed, the future throws an exception.
lock_a.lock ();
}
}
void nano::bootstrap_attempt_legacy::add_frontier (nano::pull_info const & pull_a)
{
// Prevent incorrect or malicious pulls with frontier 0 insertion
if (!pull_a.head.is_zero ())
{
nano::lock_guard<nano::mutex> lock{ mutex };
frontier_pulls.push_back (pull_a);
}
}
void nano::bootstrap_attempt_legacy::add_bulk_push_target (nano::block_hash const & head, nano::block_hash const & end)
{
nano::lock_guard<nano::mutex> lock{ mutex };
bulk_push_targets.emplace_back (head, end);
}
bool nano::bootstrap_attempt_legacy::request_bulk_push_target (std::pair<nano::block_hash, nano::block_hash> & current_target_a)
{
nano::lock_guard<nano::mutex> lock{ mutex };
auto empty (bulk_push_targets.empty ());
if (!empty)
{
current_target_a = bulk_push_targets.back ();
bulk_push_targets.pop_back ();
}
return empty;
}
void nano::bootstrap_attempt_legacy::set_start_account (nano::account const & start_account_a)
{
// Add last account from frontier request
nano::lock_guard<nano::mutex> lock{ mutex };
start_account = start_account_a;
}
bool nano::bootstrap_attempt_legacy::request_frontier (nano::unique_lock<nano::mutex> & lock_a, bool first_attempt)
{
auto node = this->node.lock ();
if (!node)
{
return true;
}
auto result (true);
lock_a.unlock ();
auto connection_l (node->bootstrap_initiator.connections->connection (shared_from_this (), first_attempt));
lock_a.lock ();
if (connection_l && !stopped)
{
endpoint_frontier_request = connection_l->channel->get_remote_endpoint ();
std::future<bool> future;
{
auto this_l = std::dynamic_pointer_cast<nano::bootstrap_attempt_legacy> (shared_from_this ());
auto client = std::make_shared<nano::frontier_req_client> (connection_l, this_l);
client->run (start_account, frontiers_age, node->config.bootstrap_frontier_request_count);
frontiers = client;
future = client->promise.get_future ();
}
lock_a.unlock ();
result = consume_future (future); // This is out of scope of `client' so when the last reference via boost::asio::io_context is lost and the client is destroyed, the future throws an exception.
lock_a.lock ();
if (result)
{
frontier_pulls.clear ();
}
else
{
account_count = nano::narrow_cast<unsigned int> (frontier_pulls.size ());
// Shuffle pulls
release_assert (std::numeric_limits<CryptoPP::word32>::max () > frontier_pulls.size ());
if (!frontier_pulls.empty ())
{
for (auto i = static_cast<CryptoPP::word32> (frontier_pulls.size () - 1); i > 0; --i)
{
auto k = nano::random_pool::generate_word32 (0, i);
std::swap (frontier_pulls[i], frontier_pulls[k]);
}
}
// Add to regular pulls
while (!frontier_pulls.empty ())
{
auto pull (frontier_pulls.front ());
lock_a.unlock ();
node->bootstrap_initiator.connections->add_pull (pull);
lock_a.lock ();
++pulling;
frontier_pulls.pop_front ();
}
}
if (!result)
{
node->logger.debug (nano::log::type::bootstrap_legacy, "Completed frontier request, {} out of sync accounts according to {}", account_count.load (), connection_l->channel->to_string ());
}
else
{
node->stats.inc (nano::stat::type::error, nano::stat::detail::frontier_req, nano::stat::dir::out);
}
}
return result;
}
void nano::bootstrap_attempt_legacy::run_start (nano::unique_lock<nano::mutex> & lock_a)
{
frontiers_received = false;
auto frontier_failure (true);
uint64_t frontier_attempts (0);
while (!stopped && frontier_failure)
{
++frontier_attempts;
frontier_failure = request_frontier (lock_a, frontier_attempts == 1);
}
frontiers_received = true;
}
void nano::bootstrap_attempt_legacy::run ()
{
auto node = this->node.lock ();
if (!node)
{
return;
}
debug_assert (started);
debug_assert (!node->flags.disable_legacy_bootstrap);
node->bootstrap_initiator.connections->populate_connections (false);
nano::unique_lock<nano::mutex> lock{ mutex };
run_start (lock);
while (still_pulling ())
{
while (still_pulling ())
{
// clang-format off
condition.wait (lock, [&stopped = stopped, &pulling = pulling] { return stopped || pulling == 0; });
}
// TODO: This check / wait is a heuristic and should be improved.
auto wait_start = std::chrono::steady_clock::now ();
while (!stopped && node->block_processor.size (nano::block_source::bootstrap_legacy) != 0 && ((std::chrono::steady_clock::now () - wait_start) < std::chrono::seconds{ 10 }))
{
condition.wait_for (lock, std::chrono::milliseconds{ 100 }, [this, node] { return stopped || node->block_processor.size (nano::block_source::bootstrap_legacy) == 0; });
}
if (start_account.number () != std::numeric_limits<nano::uint256_t>::max ())
{
node->logger.debug(nano::log::type::bootstrap_legacy, "Requesting new frontiers after: {}", start_account.to_account ());
// Requesting new frontiers
run_start (lock);
}
}
if (!stopped)
{
node->logger.debug(nano::log::type::bootstrap_legacy, "Completed legacy pulls");
if (!node->flags.disable_bootstrap_bulk_push_client)
{
request_push (lock);
}
}
lock.unlock ();
stop ();
condition.notify_all ();
}
void nano::bootstrap_attempt_legacy::get_information (boost::property_tree::ptree & tree_a)
{
nano::lock_guard<nano::mutex> lock{ mutex };
tree_a.put ("frontier_pulls", std::to_string (frontier_pulls.size ()));
tree_a.put ("frontiers_received", static_cast<bool> (frontiers_received));
tree_a.put ("frontiers_age", std::to_string (frontiers_age));
tree_a.put ("last_account", start_account.to_account ());
}

43
nano/node/bootstrap/bootstrap_legacy.hpp
View file

@ -1,43 +0,0 @@
#pragma once
#include <nano/node/bootstrap/bootstrap_attempt.hpp>
#include <boost/property_tree/ptree_fwd.hpp>
#include <atomic>
#include <deque>
#include <memory>
#include <vector>
namespace nano
{
class node;
/**
* Legacy bootstrap session. This is made up of 3 phases: frontier requests, bootstrap pulls, bootstrap pushes.
*/
class bootstrap_attempt_legacy : public bootstrap_attempt
{
public:
explicit bootstrap_attempt_legacy (std::shared_ptr<nano::node> const & node_a, uint64_t const incremental_id_a, std::string const & id_a, uint32_t const frontiers_age_a, nano::account const & start_account_a);
void run () override;
bool consume_future (std::future<bool> &);
void stop () override;
bool request_frontier (nano::unique_lock<nano::mutex> &, bool = false);
void request_push (nano::unique_lock<nano::mutex> &);
void add_frontier (nano::pull_info const &);
void add_bulk_push_target (nano::block_hash const &, nano::block_hash const &);
bool request_bulk_push_target (std::pair<nano::block_hash, nano::block_hash> &);
void set_start_account (nano::account const &);
void run_start (nano::unique_lock<nano::mutex> &);
void get_information (boost::property_tree::ptree &) override;
nano::tcp_endpoint endpoint_frontier_request;
std::weak_ptr<nano::frontier_req_client> frontiers;
std::weak_ptr<nano::bulk_push_client> push;
std::deque<nano::pull_info> frontier_pulls;
std::vector<std::pair<nano::block_hash, nano::block_hash>> bulk_push_targets;
nano::account start_account{};
std::atomic<unsigned> account_count{ 0 };
uint32_t frontiers_age;
};
}

549
nano/node/bootstrap_ascending/service.cpp → nano/node/bootstrap/bootstrap_service.cpp
View file

@ -1,9 +1,11 @@
#include <nano/lib/block_type.hpp>
#include <nano/lib/blocks.hpp>
#include <nano/lib/enum_util.hpp>
#include <nano/lib/stats_enums.hpp>
#include <nano/lib/thread_roles.hpp>
#include <nano/node/blockprocessor.hpp>
#include <nano/node/bootstrap_ascending/service.hpp>
#include <nano/node/bootstrap/bootstrap_service.hpp>
#include <nano/node/bootstrap/crawlers.hpp>
#include <nano/node/network.hpp>
#include <nano/node/nodeconfig.hpp>
#include <nano/node/transport/transport.hpp>
@ -12,15 +14,12 @@
#include <nano/secure/ledger_set_any.hpp>
#include <nano/store/account.hpp>
#include <nano/store/component.hpp>
#include <nano/store/confirmation_height.hpp>
using namespace std::chrono_literals;
/*
* bootstrap_ascending
*/
nano::bootstrap_ascending::service::service (nano::node_config const & node_config_a, nano::block_processor & block_processor_a, nano::ledger & ledger_a, nano::network & network_a, nano::stats & stat_a, nano::logger & logger_a) :
config{ node_config_a.bootstrap_ascending },
nano::bootstrap_service::bootstrap_service (nano::node_config const & node_config_a, nano::block_processor & block_processor_a, nano::ledger & ledger_a, nano::network & network_a, nano::stats & stat_a, nano::logger & logger_a) :
config{ node_config_a.bootstrap },
network_constants{ node_config_a.network_params.network },
block_processor{ block_processor_a },
ledger{ ledger_a },
@ -29,9 +28,13 @@ nano::bootstrap_ascending::service::service (nano::node_config const & node_conf
logger{ logger_a },
accounts{ config.account_sets, stats },
database_scan{ ledger },
frontiers{ config.frontier_scan, stats },
throttle{ compute_throttle_size () },
scoring{ config, node_config_a.network_params.network },
database_limiter{ config.database_rate_limit, 1.0 }
limiter{ config.rate_limit },
database_limiter{ config.database_rate_limit },
frontiers_limiter{ config.frontier_rate_limit },
workers{ 1, nano::thread_role::name::bootstrap_worker }
{
block_processor.batch_processed.add ([this] (auto const & batch) {
{
@ -50,37 +53,45 @@ nano::bootstrap_ascending::service::service (nano::node_config const & node_conf
accounts.priority_set (node_config_a.network_params.ledger.genesis->account_field ().value ());
}
nano::bootstrap_ascending::service::~service ()
nano::bootstrap_service::~bootstrap_service ()
{
// All threads must be stopped before destruction
debug_assert (!priorities_thread.joinable ());
debug_assert (!database_thread.joinable ());
debug_assert (!dependencies_thread.joinable ());
debug_assert (!frontiers_thread.joinable ());
debug_assert (!timeout_thread.joinable ());
debug_assert (!workers.alive ());
}
void nano::bootstrap_ascending::service::start ()
void nano::bootstrap_service::start ()
{
debug_assert (!priorities_thread.joinable ());
debug_assert (!database_thread.joinable ());
debug_assert (!dependencies_thread.joinable ());
debug_assert (!frontiers_thread.joinable ());
debug_assert (!timeout_thread.joinable ());
if (!config.enable)
{
logger.warn (nano::log::type::bootstrap, "Ascending bootstrap is disabled");
logger.warn (nano::log::type::bootstrap, "Bootstrap is disabled, node will not be able to synchronize with the network");
return;
}
priorities_thread = std::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::ascending_bootstrap);
run_priorities ();
});
workers.start ();
if (config.enable_scan)
{
priorities_thread = std::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::bootstrap);
run_priorities ();
});
}
if (config.enable_database_scan)
{
database_thread = std::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::ascending_bootstrap);
nano::thread_role::set (nano::thread_role::name::bootstrap_database_scan);
run_database ();
});
}
@ -88,18 +99,26 @@ void nano::bootstrap_ascending::service::start ()
if (config.enable_dependency_walker)
{
dependencies_thread = std::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::ascending_bootstrap);
nano::thread_role::set (nano::thread_role::name::bootstrap_dependendy_walker);
run_dependencies ();
});
}
if (config.enable_frontier_scan)
{
frontiers_thread = std::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::bootstrap_frontier_scan);
run_frontiers ();
});
}
timeout_thread = std::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::ascending_bootstrap);
nano::thread_role::set (nano::thread_role::name::bootstrap_cleanup);
run_timeouts ();
});
}
void nano::bootstrap_ascending::service::stop ()
void nano::bootstrap_service::stop ()
{
{
nano::lock_guard<nano::mutex> lock{ mutex };
@ -110,10 +129,13 @@ void nano::bootstrap_ascending::service::stop ()
nano::join_or_pass (priorities_thread);
nano::join_or_pass (database_thread);
nano::join_or_pass (dependencies_thread);
nano::join_or_pass (frontiers_thread);
nano::join_or_pass (timeout_thread);
workers.stop ();
}
void nano::bootstrap_ascending::service::send (std::shared_ptr<nano::transport::channel> const & channel, async_tag tag)
bool nano::bootstrap_service::send (std::shared_ptr<nano::transport::channel> const & channel, async_tag tag)
{
debug_assert (tag.type != query_type::invalid);
debug_assert (tag.source != query_source::invalid);
@ -151,44 +173,68 @@ void nano::bootstrap_ascending::service::send (std::shared_ptr<nano::transport::
request.payload = pld;
}
break;
case query_type::frontiers:
{
request.type = nano::asc_pull_type::frontiers;
nano::asc_pull_req::frontiers_payload pld;
pld.start = tag.start.as_account ();
pld.count = nano::asc_pull_ack::frontiers_payload::max_frontiers;
request.payload = pld;
}
break;
default:
debug_assert (false);
}
request.update_header ();
stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::request, nano::stat::dir::out);
stats.inc (nano::stat::type::bootstrap_ascending_request, to_stat_detail (tag.type));
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::request, nano::stat::dir::out);
stats.inc (nano::stat::type::bootstrap_request, to_stat_detail (tag.type));
// TODO: There is no feedback mechanism if bandwidth limiter starts dropping our requests
channel->send (
request, nullptr,
nano::transport::buffer_drop_policy::limiter, nano::transport::traffic_type::bootstrap);
return true; // TODO: Return channel send result
}
std::size_t nano::bootstrap_ascending::service::priority_size () const
std::size_t nano::bootstrap_service::priority_size () const
{
nano::lock_guard<nano::mutex> lock{ mutex };
return accounts.priority_size ();
}
std::size_t nano::bootstrap_ascending::service::blocked_size () const
std::size_t nano::bootstrap_service::blocked_size () const
{
nano::lock_guard<nano::mutex> lock{ mutex };
return accounts.blocked_size ();
}
std::size_t nano::bootstrap_ascending::service::score_size () const
std::size_t nano::bootstrap_service::score_size () const
{
nano::lock_guard<nano::mutex> lock{ mutex };
return scoring.size ();
}
bool nano::bootstrap_service::prioritized (nano::account const & account) const
{
nano::lock_guard<nano::mutex> lock{ mutex };
return accounts.prioritized (account);
}
bool nano::bootstrap_service::blocked (nano::account const & account) const
{
nano::lock_guard<nano::mutex> lock{ mutex };
return accounts.blocked (account);
}
/** Inspects a block that has been processed by the block processor
- Marks an account as blocked if the result code is gap source as there is no reason request additional blocks for this account until the dependency is resolved
- Marks an account as forwarded if it has been recently referenced by a block that has been inserted.
*/
void nano::bootstrap_ascending::service::inspect (secure::transaction const & tx, nano::block_status const & result, nano::block const & block, nano::block_source source)
void nano::bootstrap_service::inspect (secure::transaction const & tx, nano::block_status const & result, nano::block const & block, nano::block_source source)
{
debug_assert (!mutex.try_lock ());
@ -214,6 +260,7 @@ void nano::bootstrap_ascending::service::inspect (secure::transaction const & tx
break;
case nano::block_status::gap_source:
{
// Prevent malicious live traffic from filling up the blocked set
if (source == nano::block_source::bootstrap)
{
const auto account = block.previous ().is_zero () ? block.account_field ().value () : ledger.any.block_account (tx, block.previous ()).value_or (0);
@ -245,10 +292,9 @@ void nano::bootstrap_ascending::service::inspect (secure::transaction const & tx
}
}
void nano::bootstrap_ascending::service::wait (std::function<bool ()> const & predicate) const
void nano::bootstrap_service::wait (std::function<bool ()> const & predicate) const
{
std::unique_lock<nano::mutex> lock{ mutex };
std::chrono::milliseconds interval = 5ms;
while (!stopped && !predicate ())
{
@ -257,72 +303,68 @@ void nano::bootstrap_ascending::service::wait (std::function<bool ()> const & pr
}
}
void nano::bootstrap_ascending::service::wait_tags () const
{
wait ([this] () {
debug_assert (!mutex.try_lock ());
return tags.size () < config.max_requests;
});
}
void nano::bootstrap_ascending::service::wait_blockprocessor () const
void nano::bootstrap_service::wait_blockprocessor () const
{
wait ([this] () {
return block_processor.size (nano::block_source::bootstrap) < config.block_processor_threshold;
});
}
std::shared_ptr<nano::transport::channel> nano::bootstrap_ascending::service::wait_channel ()
std::shared_ptr<nano::transport::channel> nano::bootstrap_service::wait_channel ()
{
std::shared_ptr<nano::transport::channel> channel;
// Limit the number of in-flight requests
wait ([this] () {
return tags.size () < config.max_requests;
});
// Wait until more requests can be sent
wait ([this] () {
return limiter.should_pass (1);
});
// Wait until a channel is available
std::shared_ptr<nano::transport::channel> channel;
wait ([this, &channel] () {
debug_assert (!mutex.try_lock ());
channel = scoring.channel ();
return channel != nullptr; // Wait until a channel is available
});
return channel;
}
size_t nano::bootstrap_ascending::service::count_tags (nano::account const & account, query_source source) const
size_t nano::bootstrap_service::count_tags (nano::account const & account, query_source source) const
{
debug_assert (!mutex.try_lock ());
auto [begin, end] = tags.get<tag_account> ().equal_range (account);
return std::count_if (begin, end, [source] (auto const & tag) { return tag.source == source; });
}
size_t nano::bootstrap_ascending::service::count_tags (nano::block_hash const & hash, query_source source) const
size_t nano::bootstrap_service::count_tags (nano::block_hash const & hash, query_source source) const
{
debug_assert (!mutex.try_lock ());
auto [begin, end] = tags.get<tag_hash> ().equal_range (hash);
return std::count_if (begin, end, [source] (auto const & tag) { return tag.source == source; });
}
std::pair<nano::account, double> nano::bootstrap_ascending::service::next_priority ()
std::pair<nano::account, double> nano::bootstrap_service::next_priority ()
{
debug_assert (!mutex.try_lock ());
auto account = accounts.next_priority ([this] (nano::account const & account) {
return count_tags (account, query_source::priority) < 4;
});
if (account.is_zero ())
{
return {};
}
stats.inc (nano::stat::type::bootstrap_ascending_next, nano::stat::detail::next_priority);
stats.inc (nano::stat::type::bootstrap_next, nano::stat::detail::next_priority);
accounts.timestamp_set (account);
// TODO: Priority could be returned by the accounts.next_priority() call
return { account, accounts.priority (account) };
}
std::pair<nano::account, double> nano::bootstrap_ascending::service::wait_priority ()
std::pair<nano::account, double> nano::bootstrap_service::wait_priority ()
{
std::pair<nano::account, double> result{ 0, 0 };
wait ([this, &result] () {
debug_assert (!mutex.try_lock ());
result = next_priority ();
@ -332,11 +374,10 @@ std::pair<nano::account, double> nano::bootstrap_ascending::service::wait_priori
}
return false;
});
return result;
}
nano::account nano::bootstrap_ascending::service::next_database (bool should_throttle)
nano::account nano::bootstrap_service::next_database (bool should_throttle)
{
debug_assert (!mutex.try_lock ());
debug_assert (config.database_warmup_ratio > 0);
@ -346,24 +387,20 @@ nano::account nano::bootstrap_ascending::service::next_database (bool should_thr
{
return { 0 };
}
auto account = database_scan.next ([this] (nano::account const & account) {
return count_tags (account, query_source::database) == 0;
});
if (account.is_zero ())
{
return { 0 };
}
stats.inc (nano::stat::type::bootstrap_ascending_next, nano::stat::detail::next_database);
stats.inc (nano::stat::type::bootstrap_next, nano::stat::detail::next_database);
return account;
}
nano::account nano::bootstrap_ascending::service::wait_database (bool should_throttle)
nano::account nano::bootstrap_service::wait_database (bool should_throttle)
{
nano::account result{ 0 };
wait ([this, &result, should_throttle] () {
debug_assert (!mutex.try_lock ());
result = next_database (should_throttle);
@ -373,31 +410,27 @@ nano::account nano::bootstrap_ascending::service::wait_database (bool should_thr
}
return false;
});
return result;
}
nano::block_hash nano::bootstrap_ascending::service::next_blocking ()
nano::block_hash nano::bootstrap_service::next_blocking ()
{
debug_assert (!mutex.try_lock ());
auto blocking = accounts.next_blocking ([this] (nano::block_hash const & hash) {
return count_tags (hash, query_source::blocking) == 0;
});
if (blocking.is_zero ())
{
return { 0 };
}
stats.inc (nano::stat::type::bootstrap_ascending_next, nano::stat::detail::next_blocking);
stats.inc (nano::stat::type::bootstrap_next, nano::stat::detail::next_blocking);
return blocking;
}
nano::block_hash nano::bootstrap_ascending::service::wait_blocking ()
nano::block_hash nano::bootstrap_service::wait_blocking ()
{
nano::block_hash result{ 0 };
wait ([this, &result] () {
debug_assert (!mutex.try_lock ());
result = next_blocking ();
@ -407,11 +440,26 @@ nano::block_hash nano::bootstrap_ascending::service::wait_blocking ()
}
return false;
});
return result;
}
bool nano::bootstrap_ascending::service::request (nano::account account, size_t count, std::shared_ptr<nano::transport::channel> const & channel, query_source source)
nano::account nano::bootstrap_service::wait_frontier ()
{
nano::account result{ 0 };
wait ([this, &result] () {
debug_assert (!mutex.try_lock ());
result = frontiers.next ();
if (!result.is_zero ())
{
stats.inc (nano::stat::type::bootstrap_next, nano::stat::detail::next_frontier);
return true;
}
return false;
});
return result;
}
bool nano::bootstrap_service::request (nano::account account, size_t count, std::shared_ptr<nano::transport::channel> const & channel, query_source source)
{
debug_assert (count > 0);
debug_assert (count <= nano::bootstrap_server::max_blocks);
@ -424,45 +472,66 @@ bool nano::bootstrap_ascending::service::request (nano::account account, size_t
tag.account = account;
tag.count = count;
// Check if the account picked has blocks, if it does, start the pull from the highest block
auto info = ledger.store.account.get (ledger.store.tx_begin_read (), account);
if (info)
{
tag.type = query_type::blocks_by_hash;
tag.start = info->head;
tag.hash = info->head;
}
else
{
tag.type = query_type::blocks_by_account;
tag.start = account;
auto transaction = ledger.store.tx_begin_read ();
// Check if the account picked has blocks, if it does, start the pull from the highest block
if (auto info = ledger.store.account.get (transaction, account))
{
tag.type = query_type::blocks_by_hash;
// Probabilistically choose between requesting blocks from account frontier or confirmed frontier
// Optimistic requests start from the (possibly unconfirmed) account frontier and are vulnerable to bootstrap poisoning
// Safe requests start from the confirmed frontier and given enough time will eventually resolve forks
bool optimistic_reuest = rng.random (100) < config.optimistic_request_percentage;
if (!optimistic_reuest)
{
if (auto conf_info = ledger.store.confirmation_height.get (transaction, account))
{
stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::safe);
tag.start = conf_info->frontier;
tag.hash = conf_info->height;
}
}
if (tag.start.is_zero ())
{
stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::optimistic);
tag.start = info->head;
tag.hash = info->head;
}
}
else
{
stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::base);
tag.type = query_type::blocks_by_account;
tag.start = account;
}
}
on_request.notify (tag, channel);
send (channel, tag);
return true; // Request sent
return send (channel, tag);
}
bool nano::bootstrap_ascending::service::request_info (nano::block_hash hash, std::shared_ptr<nano::transport::channel> const & channel, query_source source)
bool nano::bootstrap_service::request_info (nano::block_hash hash, std::shared_ptr<nano::transport::channel> const & channel, query_source source)
{
async_tag tag{};
tag.type = query_type::account_info_by_hash;
tag.source = source;
tag.start = hash;
tag.hash = hash;
on_request.notify (tag, channel);
send (channel, tag);
return true; // Request sent
return send (channel, tag);
}
void nano::bootstrap_ascending::service::run_one_priority ()
bool nano::bootstrap_service::request_frontiers (nano::account start, std::shared_ptr<nano::transport::channel> const & channel, query_source source)
{
async_tag tag{};
tag.type = query_type::frontiers;
tag.source = source;
tag.start = start;
return send (channel, tag);
}
void nano::bootstrap_service::run_one_priority ()
{
wait_tags ();
wait_blockprocessor ();
auto channel = wait_channel ();
if (!channel)
@ -479,21 +548,20 @@ void nano::bootstrap_ascending::service::run_one_priority ()
request (account, count, channel, query_source::priority);
}
void nano::bootstrap_ascending::service::run_priorities ()
void nano::bootstrap_service::run_priorities ()
{
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
lock.unlock ();
stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::loop);
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::loop);
run_one_priority ();
lock.lock ();
}
}
void nano::bootstrap_ascending::service::run_one_database (bool should_throttle)
void nano::bootstrap_service::run_one_database (bool should_throttle)
{
wait_tags ();
wait_blockprocessor ();
auto channel = wait_channel ();
if (!channel)
@ -508,7 +576,7 @@ void nano::bootstrap_ascending::service::run_one_database (bool should_throttle)
request (account, 2, channel, query_source::database);
}
void nano::bootstrap_ascending::service::run_database ()
void nano::bootstrap_service::run_database ()
{
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
@ -516,16 +584,15 @@ void nano::bootstrap_ascending::service::run_database ()
// Avoid high churn rate of database requests
bool should_throttle = !database_scan.warmed_up () && throttle.throttled ();
lock.unlock ();
stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::loop_database);
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::loop_database);
run_one_database (should_throttle);
lock.lock ();
}
}
void nano::bootstrap_ascending::service::run_one_blocking ()
void nano::bootstrap_service::run_one_blocking ()
{
wait_tags ();
wait_blockprocessor ();
// No need to wait for blockprocessor, as we are not processing blocks
auto channel = wait_channel ();
if (!channel)
{
@ -539,19 +606,56 @@ void nano::bootstrap_ascending::service::run_one_blocking ()
request_info (blocking, channel, query_source::blocking);
}
void nano::bootstrap_ascending::service::run_dependencies ()
void nano::bootstrap_service::run_dependencies ()
{
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
lock.unlock ();
stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::loop_dependencies);
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::loop_dependencies);
run_one_blocking ();
lock.lock ();
}
}
void nano::bootstrap_ascending::service::cleanup_and_sync ()
void nano::bootstrap_service::run_one_frontier ()
{
// No need to wait for blockprocessor, as we are not processing blocks
wait ([this] () {
return !accounts.priority_half_full ();
});
wait ([this] () {
return frontiers_limiter.should_pass (1);
});
wait ([this] () {
return workers.queued_tasks () < config.frontier_scan.max_pending;
});
auto channel = wait_channel ();
if (!channel)
{
return;
}
auto frontier = wait_frontier ();
if (frontier.is_zero ())
{
return;
}
request_frontiers (frontier, channel, query_source::frontiers);
}
void nano::bootstrap_service::run_frontiers ()
{
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
lock.unlock ();
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::loop_frontiers);
run_one_frontier ();
lock.lock ();
}
}
void nano::bootstrap_service::cleanup_and_sync ()
{
debug_assert (!mutex.try_lock ());
@ -570,29 +674,28 @@ void nano::bootstrap_ascending::service::cleanup_and_sync ()
{
auto tag = tags_by_order.front ();
tags_by_order.pop_front ();
on_timeout.notify (tag);
stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::timeout);
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::timeout);
}
if (sync_dependencies_interval.elapsed (60s))
{
stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::sync_dependencies);
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::sync_dependencies);
accounts.sync_dependencies ();
}
}
void nano::bootstrap_ascending::service::run_timeouts ()
void nano::bootstrap_service::run_timeouts ()
{
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::loop_cleanup);
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::loop_cleanup);
cleanup_and_sync ();
condition.wait_for (lock, 5s, [this] () { return stopped; });
}
}
void nano::bootstrap_ascending::service::process (nano::asc_pull_ack const & message, std::shared_ptr<nano::transport::channel> const & channel)
void nano::bootstrap_service::process (nano::asc_pull_ack const & message, std::shared_ptr<nano::transport::channel> const & channel)
{
nano::unique_lock<nano::mutex> lock{ mutex };
@ -600,11 +703,11 @@ void nano::bootstrap_ascending::service::process (nano::asc_pull_ack const & mes
auto it = tags.get<tag_id> ().find (message.id);
if (it == tags.get<tag_id> ().end ())
{
stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::missing_tag);
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::missing_tag);
return;
}
stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::reply);
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::reply);
auto tag = *it;
tags.get<tag_id> ().erase (it); // Iterator is invalid after this point
@ -624,7 +727,7 @@ void nano::bootstrap_ascending::service::process (nano::asc_pull_ack const & mes
}
bool operator() (const nano::asc_pull_ack::frontiers_payload & response) const
{
return false; // TODO: Handle frontiers
return type == query_type::frontiers;
}
bool operator() (const nano::empty_payload & response) const
{
@ -635,39 +738,37 @@ void nano::bootstrap_ascending::service::process (nano::asc_pull_ack const & mes
bool valid = std::visit (payload_verifier{ tag.type }, message.payload);
if (!valid)
{
stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::invalid_response_type);
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::invalid_response_type);
return;
}
// Track bootstrap request response time
stats.inc (nano::stat::type::bootstrap_ascending_reply, to_stat_detail (tag.type));
stats.inc (nano::stat::type::bootstrap_reply, to_stat_detail (tag.type));
stats.sample (nano::stat::sample::bootstrap_tag_duration, nano::log::milliseconds_delta (tag.timestamp), { 0, config.request_timeout.count () });
scoring.received_message (channel);
lock.unlock ();
on_reply.notify (tag);
// Process the response payload
std::visit ([this, &tag] (auto && request) { return process (request, tag); }, message.payload);
condition.notify_all ();
}
void nano::bootstrap_ascending::service::process (const nano::asc_pull_ack::blocks_payload & response, const async_tag & tag)
void nano::bootstrap_service::process (const nano::asc_pull_ack::blocks_payload & response, const async_tag & tag)
{
debug_assert (tag.type == query_type::blocks_by_hash || tag.type == query_type::blocks_by_account);
stats.inc (nano::stat::type::bootstrap_ascending_process, nano::stat::detail::blocks);
stats.inc (nano::stat::type::bootstrap_process, nano::stat::detail::blocks);
auto result = verify (response, tag);
switch (result)
{
case verify_result::ok:
{
stats.inc (nano::stat::type::bootstrap_ascending_verify, nano::stat::detail::ok);
stats.add (nano::stat::type::bootstrap_ascending, nano::stat::detail::blocks, nano::stat::dir::in, response.blocks.size ());
stats.inc (nano::stat::type::bootstrap_verify_blocks, nano::stat::detail::ok);
stats.add (nano::stat::type::bootstrap, nano::stat::detail::blocks, nano::stat::dir::in, response.blocks.size ());
auto blocks = response.blocks;
@ -684,7 +785,7 @@ void nano::bootstrap_ascending::service::process (const nano::asc_pull_ack::bloc
{
// It's the last block submitted for this account chain, reset timestamp to allow more requests
block_processor.add (block, nano::block_source::bootstrap, nullptr, [this, account = tag.account] (auto result) {
stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::timestamp_reset);
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::timestamp_reset);
{
nano::lock_guard<nano::mutex> guard{ mutex };
accounts.timestamp_reset (account);
@ -707,7 +808,7 @@ void nano::bootstrap_ascending::service::process (const nano::asc_pull_ack::bloc
break;
case verify_result::nothing_new:
{
stats.inc (nano::stat::type::bootstrap_ascending_verify, nano::stat::detail::nothing_new);
stats.inc (nano::stat::type::bootstrap_verify_blocks, nano::stat::detail::nothing_new);
nano::lock_guard<nano::mutex> lock{ mutex };
accounts.priority_down (tag.account);
@ -719,47 +820,172 @@ void nano::bootstrap_ascending::service::process (const nano::asc_pull_ack::bloc
break;
case verify_result::invalid:
{
stats.inc (nano::stat::type::bootstrap_ascending_verify, nano::stat::detail::invalid);
stats.inc (nano::stat::type::bootstrap_verify_blocks, nano::stat::detail::invalid);
}
break;
}
}
void nano::bootstrap_ascending::service::process (const nano::asc_pull_ack::account_info_payload & response, const async_tag & tag)
void nano::bootstrap_service::process (const nano::asc_pull_ack::account_info_payload & response, const async_tag & tag)
{
debug_assert (tag.type == query_type::account_info_by_hash);
debug_assert (!tag.hash.is_zero ());
if (response.account.is_zero ())
{
stats.inc (nano::stat::type::bootstrap_ascending_process, nano::stat::detail::account_info_empty);
stats.inc (nano::stat::type::bootstrap_process, nano::stat::detail::account_info_empty);
return;
}
else
stats.inc (nano::stat::type::bootstrap_process, nano::stat::detail::account_info);
// Prioritize account containing the dependency
{
stats.inc (nano::stat::type::bootstrap_ascending_process, nano::stat::detail::account_info);
// Prioritize account containing the dependency
{
nano::lock_guard<nano::mutex> lock{ mutex };
accounts.dependency_update (tag.hash, response.account);
accounts.priority_set (response.account);
}
nano::lock_guard<nano::mutex> lock{ mutex };
accounts.dependency_update (tag.hash, response.account);
accounts.priority_set (response.account);
}
}
void nano::bootstrap_ascending::service::process (const nano::asc_pull_ack::frontiers_payload & response, const async_tag & tag)
void nano::bootstrap_service::process (const nano::asc_pull_ack::frontiers_payload & response, const async_tag & tag)
{
// TODO: Make use of frontiers info
stats.inc (nano::stat::type::bootstrap_ascending_process, nano::stat::detail::frontiers);
debug_assert (tag.type == query_type::frontiers);
debug_assert (!tag.start.is_zero ());
if (response.frontiers.empty ())
{
stats.inc (nano::stat::type::bootstrap_process, nano::stat::detail::frontiers_empty);
return;
}
stats.inc (nano::stat::type::bootstrap_process, nano::stat::detail::frontiers);
auto result = verify (response, tag);
switch (result)
{
case verify_result::ok:
{
stats.inc (nano::stat::type::bootstrap_verify_frontiers, nano::stat::detail::ok);
stats.add (nano::stat::type::bootstrap, nano::stat::detail::frontiers, nano::stat::dir::in, response.frontiers.size ());
{
nano::lock_guard<nano::mutex> lock{ mutex };
frontiers.process (tag.start.as_account (), response.frontiers);
}
// Allow some overfill to avoid unnecessarily dropping responses
if (workers.queued_tasks () < config.frontier_scan.max_pending * 4)
{
workers.post ([this, frontiers = response.frontiers] {
process_frontiers (frontiers);
});
}
else
{
stats.add (nano::stat::type::bootstrap, nano::stat::detail::frontiers_dropped, response.frontiers.size ());
}
}
break;
case verify_result::nothing_new:
{
stats.inc (nano::stat::type::bootstrap_verify_frontiers, nano::stat::detail::nothing_new);
}
break;
case verify_result::invalid:
{
stats.inc (nano::stat::type::bootstrap_verify_frontiers, nano::stat::detail::invalid);
}
break;
}
}
void nano::bootstrap_ascending::service::process (const nano::empty_payload & response, const async_tag & tag)
void nano::bootstrap_service::process (const nano::empty_payload & response, const async_tag & tag)
{
stats.inc (nano::stat::type::bootstrap_ascending_process, nano::stat::detail::empty);
stats.inc (nano::stat::type::bootstrap_process, nano::stat::detail::empty);
debug_assert (false, "empty payload"); // Should not happen
}
nano::bootstrap_ascending::service::verify_result nano::bootstrap_ascending::service::verify (const nano::asc_pull_ack::blocks_payload & response, const nano::bootstrap_ascending::service::async_tag & tag) const
void nano::bootstrap_service::process_frontiers (std::deque<std::pair<nano::account, nano::block_hash>> const & frontiers)
{
release_assert (!frontiers.empty ());
// Accounts must be passed in ascending order
debug_assert (std::adjacent_find (frontiers.begin (), frontiers.end (), [] (auto const & lhs, auto const & rhs) {
return lhs.first.number () >= rhs.first.number ();
})
== frontiers.end ());
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::processing_frontiers);
size_t outdated = 0;
size_t pending = 0;
// Accounts with outdated frontiers to sync
std::deque<nano::account> result;
{
auto transaction = ledger.tx_begin_read ();
auto const start = frontiers.front ().first;
nano::bootstrap::account_database_crawler account_crawler{ ledger.store, transaction, start };
nano::bootstrap::pending_database_crawler pending_crawler{ ledger.store, transaction, start };
auto block_exists = [&] (nano::block_hash const & hash) {
return ledger.any.block_exists_or_pruned (transaction, hash);
};
auto should_prioritize = [&] (nano::account const & account, nano::block_hash const & frontier) {
account_crawler.advance_to (account);
pending_crawler.advance_to (account);
// Check if account exists in our ledger
if (account_crawler.current && account_crawler.current->first == account)
{
// Check for frontier mismatch
if (account_crawler.current->second.head != frontier)
{
// Check if frontier block exists in our ledger
if (!block_exists (frontier))
{
outdated++;
return true; // Frontier is outdated
}
}
return false; // Account exists and frontier is up-to-date
}
// Check if account has pending blocks in our ledger
if (pending_crawler.current && pending_crawler.current->first.account == account)
{
pending++;
return true; // Account doesn't exist but has pending blocks in the ledger
}
return false; // Account doesn't exist in the ledger and has no pending blocks, can't be prioritized right now
};
for (auto const & [account, frontier] : frontiers)
{
if (should_prioritize (account, frontier))
{
result.push_back (account);
}
}
}
stats.add (nano::stat::type::bootstrap_frontiers, nano::stat::detail::processed, frontiers.size ());
stats.add (nano::stat::type::bootstrap_frontiers, nano::stat::detail::prioritized, result.size ());
stats.add (nano::stat::type::bootstrap_frontiers, nano::stat::detail::outdated, outdated);
stats.add (nano::stat::type::bootstrap_frontiers, nano::stat::detail::pending, pending);
nano::lock_guard<nano::mutex> guard{ mutex };
for (auto const & account : result)
{
accounts.priority_set (account);
}
}
auto nano::bootstrap_service::verify (const nano::asc_pull_ack::blocks_payload & response, const async_tag & tag) const -> verify_result
{
auto const & blocks = response.blocks;
@ -818,13 +1044,42 @@ nano::bootstrap_ascending::service::verify_result nano::bootstrap_ascending::ser
return verify_result::ok;
}
auto nano::bootstrap_ascending::service::info () const -> nano::bootstrap_ascending::account_sets::info_t
auto nano::bootstrap_service::verify (nano::asc_pull_ack::frontiers_payload const & response, async_tag const & tag) const -> verify_result
{
auto const & frontiers = response.frontiers;
if (frontiers.empty ())
{
return verify_result::nothing_new;
}
// Ensure frontiers accounts are in ascending order
nano::account previous{ 0 };
for (auto const & [account, _] : frontiers)
{
if (account.number () <= previous.number ())
{
return verify_result::invalid;
}
previous = account;
}
// Ensure the frontiers are larger or equal to the requested frontier
if (frontiers.front ().first.number () < tag.start.as_account ().number ())
{
return verify_result::invalid;
}
return verify_result::ok;
}
auto nano::bootstrap_service::info () const -> nano::bootstrap::account_sets::info_t
{
nano::lock_guard<nano::mutex> lock{ mutex };
return accounts.info ();
}
std::size_t nano::bootstrap_ascending::service::compute_throttle_size () const
std::size_t nano::bootstrap_service::compute_throttle_size () const
{
auto ledger_size = ledger.account_count ();
size_t target = ledger_size > 0 ? config.throttle_coefficient * static_cast<size_t> (std::log (ledger_size)) : 0;
@ -832,7 +1087,7 @@ std::size_t nano::bootstrap_ascending::service::compute_throttle_size () const
return std::max (target, min_size);
}
nano::container_info nano::bootstrap_ascending::service::container_info () const
nano::container_info nano::bootstrap_service::container_info () const
{
nano::lock_guard<nano::mutex> lock{ mutex };
@ -842,6 +1097,8 @@ nano::container_info nano::bootstrap_ascending::service::container_info () const
info.put ("throttle_successes", throttle.successes ());
info.add ("accounts", accounts.container_info ());
info.add ("database_scan", database_scan.container_info ());
info.add ("frontiers", frontiers.container_info ());
info.add ("workers", workers.container_info ());
return info;
}
@ -849,7 +1106,7 @@ nano::container_info nano::bootstrap_ascending::service::container_info () const
*
*/
nano::stat::detail nano::bootstrap_ascending::to_stat_detail (nano::bootstrap_ascending::service::query_type type)
nano::stat::detail nano::to_stat_detail (nano::bootstrap_service::query_type type)
{
return nano::enum_util::cast<nano::stat::detail> (type);
}
}

215
nano/node/bootstrap/bootstrap_service.hpp Normal file
View file

@ -0,0 +1,215 @@
#pragma once
#include <nano/lib/interval.hpp>
#include <nano/lib/locks.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/observer_set.hpp>
#include <nano/lib/random.hpp>
#include <nano/lib/rate_limiting.hpp>
#include <nano/lib/thread_pool.hpp>
#include <nano/node/bootstrap/account_sets.hpp>
#include <nano/node/bootstrap/bootstrap_config.hpp>
#include <nano/node/bootstrap/common.hpp>
#include <nano/node/bootstrap/database_scan.hpp>
#include <nano/node/bootstrap/frontier_scan.hpp>
#include <nano/node/bootstrap/peer_scoring.hpp>
#include <nano/node/bootstrap/throttle.hpp>
#include <nano/node/fwd.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/sequenced_index.hpp>
#include <boost/multi_index_container.hpp>
#include <thread>
namespace mi = boost::multi_index;
namespace nano
{
class bootstrap_service
{
public:
bootstrap_service (nano::node_config const &, nano::block_processor &, nano::ledger &, nano::network &, nano::stats &, nano::logger &);
~bootstrap_service ();
void start ();
void stop ();
/**
* Process `asc_pull_ack` message coming from network
*/
void process (nano::asc_pull_ack const & message, std::shared_ptr<nano::transport::channel> const &);
std::size_t blocked_size () const;
std::size_t priority_size () const;
std::size_t score_size () const;
bool prioritized (nano::account const &) const;
bool blocked (nano::account const &) const;
nano::container_info container_info () const;
nano::bootstrap::account_sets::info_t info () const;
private: // Dependencies
bootstrap_config const & config;
nano::network_constants const & network_constants;
nano::block_processor & block_processor;
nano::ledger & ledger;
nano::network & network;
nano::stats & stats;
nano::logger & logger;
public: // Tag
enum class query_type
{
invalid = 0, // Default initialization
blocks_by_hash,
blocks_by_account,
account_info_by_hash,
frontiers,
};
enum class query_source
{
invalid,
priority,
database,
blocking,
frontiers,
};
struct async_tag
{
using id_t = nano::bootstrap::id_t;
query_type type{ query_type::invalid };
query_source source{ query_source::invalid };
nano::hash_or_account start{ 0 };
nano::account account{ 0 };
nano::block_hash hash{ 0 };
size_t count{ 0 };
id_t id{ nano::bootstrap::generate_id () };
std::chrono::steady_clock::time_point timestamp{ std::chrono::steady_clock::now () };
};
private:
/* Inspects a block that has been processed by the block processor */
void inspect (secure::transaction const &, nano::block_status const & result, nano::block const & block, nano::block_source);
void run_priorities ();
void run_one_priority ();
void run_database ();
void run_one_database (bool should_throttle);
void run_dependencies ();
void run_one_blocking ();
void run_one_frontier ();
void run_frontiers ();
void run_timeouts ();
void cleanup_and_sync ();
/* Waits for a condition to be satisfied with incremental backoff */
void wait (std::function<bool ()> const & predicate) const;
/* Ensure there is enough space in blockprocessor for queuing new blocks */
void wait_blockprocessor () const;
/* Waits for a channel that is not full */
std::shared_ptr<nano::transport::channel> wait_channel ();
/* Waits until a suitable account outside of cool down period is available */
std::pair<nano::account, double> next_priority ();
std::pair<nano::account, double> wait_priority ();
/* Gets the next account from the database */
nano::account next_database (bool should_throttle);
nano::account wait_database (bool should_throttle);
/* Waits for next available blocking block */
nano::block_hash next_blocking ();
nano::block_hash wait_blocking ();
/* Waits for next available frontier scan range */
nano::account wait_frontier ();
bool request (nano::account, size_t count, std::shared_ptr<nano::transport::channel> const &, query_source);
bool request_info (nano::block_hash, std::shared_ptr<nano::transport::channel> const &, query_source);
bool request_frontiers (nano::account, std::shared_ptr<nano::transport::channel> const &, query_source);
bool send (std::shared_ptr<nano::transport::channel> const &, async_tag tag);
void process (nano::asc_pull_ack::blocks_payload const & response, async_tag const & tag);
void process (nano::asc_pull_ack::account_info_payload const & response, async_tag const & tag);
void process (nano::asc_pull_ack::frontiers_payload const & response, async_tag const & tag);
void process (nano::empty_payload const & response, async_tag const & tag);
void process_frontiers (std::deque<std::pair<nano::account, nano::block_hash>> const & frontiers);
enum class verify_result
{
ok,
nothing_new,
invalid,
};
/**
* Verifies whether the received response is valid. Returns:
* - invalid: when received blocks do not correspond to requested hash/account or they do not make a valid chain
* - nothing_new: when received response indicates that the account chain does not have more blocks
* - ok: otherwise, if all checks pass
*/
verify_result verify (nano::asc_pull_ack::blocks_payload const & response, async_tag const & tag) const;
verify_result verify (nano::asc_pull_ack::frontiers_payload const & response, async_tag const & tag) const;
size_t count_tags (nano::account const & account, query_source source) const;
size_t count_tags (nano::block_hash const & hash, query_source source) const;
// Calculates a lookback size based on the size of the ledger where larger ledgers have a larger sample count
std::size_t compute_throttle_size () const;
private:
nano::bootstrap::account_sets accounts;
nano::bootstrap::database_scan database_scan;
nano::bootstrap::throttle throttle;
nano::bootstrap::peer_scoring scoring;
nano::bootstrap::frontier_scan frontiers;
// clang-format off
class tag_sequenced {};
class tag_id {};
class tag_account {};
class tag_hash {};
using ordered_tags = boost::multi_index_container<async_tag,
mi::indexed_by<
mi::sequenced<mi::tag<tag_sequenced>>,
mi::hashed_unique<mi::tag<tag_id>,
mi::member<async_tag, nano::bootstrap::id_t, &async_tag::id>>,
mi::hashed_non_unique<mi::tag<tag_account>,
mi::member<async_tag, nano::account , &async_tag::account>>,
mi::hashed_non_unique<mi::tag<tag_hash>,
mi::member<async_tag, nano::block_hash, &async_tag::hash>>
>>;
// clang-format on
ordered_tags tags;
// Rate limiter for all types of requests
nano::rate_limiter limiter;
// Requests for accounts from database have much lower hitrate and could introduce strain on the network
// A separate (lower) limiter ensures that we always reserve resources for querying accounts from priority queue
nano::rate_limiter database_limiter;
nano::rate_limiter frontiers_limiter;
nano::interval sync_dependencies_interval;
bool stopped{ false };
mutable nano::mutex mutex;
mutable nano::condition_variable condition;
std::thread priorities_thread;
std::thread database_thread;
std::thread dependencies_thread;
std::thread frontiers_thread;
std::thread timeout_thread;
nano::thread_pool workers;
nano::random_generator_mt rng;
};
nano::stat::detail to_stat_detail (bootstrap_service::query_type);
}

8
nano/node/bootstrap_ascending/common.hpp → nano/node/bootstrap/common.hpp
View file

@ -2,13 +2,13 @@
#include <nano/crypto_lib/random_pool.hpp>
namespace nano::bootstrap_ascending
namespace nano::bootstrap
{
using id_t = uint64_t;
static nano::bootstrap_ascending::id_t generate_id ()
static nano::bootstrap::id_t generate_id ()
{
nano::bootstrap_ascending::id_t id;
nano::bootstrap::id_t id;
nano::random_pool::generate_block (reinterpret_cast<uint8_t *> (&id), sizeof (id));
return id;
}
} // nano::bootstrap_ascending
}

187
nano/node/bootstrap/crawlers.hpp Normal file
View file

@ -0,0 +1,187 @@
#pragma once
#include <nano/secure/account_info.hpp>
#include <nano/secure/pending_info.hpp>
#include <nano/store/account.hpp>
#include <nano/store/component.hpp>
#include <nano/store/pending.hpp>
#include <optional>
namespace nano::bootstrap
{
struct account_database_crawler
{
using value_type = std::pair<nano::account, nano::account_info>;
static constexpr size_t sequential_attempts = 10;
account_database_crawler (nano::store::component & store, nano::store::transaction const & transaction, nano::account const & start) :
store{ store },
transaction{ transaction },
it{ store.account.end (transaction) },
end{ store.account.end (transaction) }
{
seek (start);
}
void seek (nano::account const & account)
{
it = store.account.begin (transaction, account);
update_current ();
}
void advance ()
{
if (it == end)
{
debug_assert (!current);
return;
}
++it;
update_current ();
}
void advance_to (nano::account const & account)
{
if (it == end)
{
debug_assert (!current);
return;
}
// First try advancing sequentially
for (size_t count = 0; count < sequential_attempts && it != end; ++count, ++it)
{
// Break if we've reached or overshoot the target account
if (it->first.number () >= account.number ())
{
update_current ();
return;
}
}
// If that fails, perform a fresh lookup
seek (account);
}
std::optional<value_type> current{};
private:
void update_current ()
{
if (it != end)
{
current = *it;
}
else
{
current = std::nullopt;
}
}
nano::store::component & store;
nano::store::transaction const & transaction;
nano::store::account::iterator it;
nano::store::account::iterator const end;
};
struct pending_database_crawler
{
using value_type = std::pair<nano::pending_key, nano::pending_info>;
static constexpr size_t sequential_attempts = 10;
pending_database_crawler (nano::store::component & store, nano::store::transaction const & transaction, nano::account const & start) :
store{ store },
transaction{ transaction },
it{ store.pending.end (transaction) },
end{ store.pending.end (transaction) }
{
seek (start);
}
void seek (nano::account const & account)
{
it = store.pending.begin (transaction, { account, 0 });
update_current ();
}
// Advance to the next account
void advance ()
{
if (it == end)
{
debug_assert (!current);
return;
}
auto const starting_account = it->first.account;
// First try advancing sequentially
for (size_t count = 0; count < sequential_attempts && it != end; ++count, ++it)
{
// Break if we've reached the next account
if (it->first.account != starting_account)
{
update_current ();
return;
}
}
if (it != end)
{
// If that fails, perform a fresh lookup
seek (starting_account.number () + 1);
}
update_current ();
}
void advance_to (nano::account const & account)
{
if (it == end)
{
debug_assert (!current);
return;
}
// First try advancing sequentially
for (size_t count = 0; count < sequential_attempts && it != end; ++count, ++it)
{
// Break if we've reached or overshoot the target account
if (it->first.account.number () >= account.number ())
{
update_current ();
return;
}
}
// If that fails, perform a fresh lookup
seek (account);
}
std::optional<value_type> current{};
private:
void update_current ()
{
if (it != end)
{
current = *it;
}
else
{
current = std::nullopt;
}
}
nano::store::component & store;
nano::store::transaction const & transaction;
nano::store::pending::iterator it;
nano::store::pending::iterator const end;
};
}

121
nano/node/bootstrap/database_scan.cpp Normal file
View file

@ -0,0 +1,121 @@
#include <nano/lib/utility.hpp>
#include <nano/node/bootstrap/crawlers.hpp>
#include <nano/node/bootstrap/database_scan.hpp>
#include <nano/secure/common.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
#include <nano/store/account.hpp>
#include <nano/store/component.hpp>
#include <nano/store/pending.hpp>
/*
* database_scan
*/
nano::bootstrap::database_scan::database_scan (nano::ledger & ledger_a) :
ledger{ ledger_a },
account_scanner{ ledger },
pending_scanner{ ledger }
{
}
nano::account nano::bootstrap::database_scan::next (std::function<bool (nano::account const &)> const & filter)
{
if (queue.empty ())
{
fill ();
}
while (!queue.empty ())
{
auto result = queue.front ();
queue.pop_front ();
if (filter (result))
{
return result;
}
}
return { 0 };
}
void nano::bootstrap::database_scan::fill ()
{
auto transaction = ledger.store.tx_begin_read ();
auto set1 = account_scanner.next_batch (transaction, batch_size);
auto set2 = pending_scanner.next_batch (transaction, batch_size);
queue.insert (queue.end (), set1.begin (), set1.end ());
queue.insert (queue.end (), set2.begin (), set2.end ());
}
bool nano::bootstrap::database_scan::warmed_up () const
{
return account_scanner.completed > 0 && pending_scanner.completed > 0;
}
nano::container_info nano::bootstrap::database_scan::container_info () const
{
nano::container_info info;
info.put ("accounts_iterator", account_scanner.completed);
info.put ("pending_iterator", pending_scanner.completed);
return info;
}
/*
* account_database_scanner
*/
std::deque<nano::account> nano::bootstrap::account_database_scanner::next_batch (nano::store::transaction & transaction, size_t batch_size)
{
std::deque<nano::account> result;
nano::bootstrap::account_database_crawler crawler{ ledger.store, transaction, next };
for (size_t count = 0; crawler.current && count < batch_size; crawler.advance (), ++count)
{
auto const & [account, info] = crawler.current.value ();
result.push_back (account);
next = account.number () + 1; // TODO: Handle account number overflow
}
// Empty current value indicates the end of the table
if (!crawler.current)
{
// Reset for the next ledger iteration
next = { 0 };
++completed;
}
return result;
}
/*
* pending_database_scanner
*/
std::deque<nano::account> nano::bootstrap::pending_database_scanner::next_batch (nano::store::transaction & transaction, size_t batch_size)
{
std::deque<nano::account> result;
nano::bootstrap::pending_database_crawler crawler{ ledger.store, transaction, next };
for (size_t count = 0; crawler.current && count < batch_size; crawler.advance (), ++count)
{
auto const & [key, info] = crawler.current.value ();
result.push_back (key.account);
next = key.account.number () + 1; // TODO: Handle account number overflow
}
// Empty current value indicates the end of the table
if (!crawler.current)
{
// Reset for the next ledger iteration
next = { 0 };
++completed;
}
return result;
}

20
nano/node/bootstrap_ascending/database_scan.hpp → nano/node/bootstrap/database_scan.hpp
View file

@ -7,29 +7,25 @@
#include <deque>
namespace nano::bootstrap_ascending
namespace nano::bootstrap
{
struct account_database_iterator
struct account_database_scanner
{
explicit account_database_iterator (nano::ledger &);
nano::ledger & ledger;
std::deque<nano::account> next_batch (nano::store::transaction &, size_t batch_size);
bool warmed_up () const;
nano::ledger & ledger;
nano::account next{ 0 };
size_t completed{ 0 };
};
struct pending_database_iterator
struct pending_database_scanner
{
explicit pending_database_iterator (nano::ledger &);
nano::ledger & ledger;
std::deque<nano::account> next_batch (nano::store::transaction &, size_t batch_size);
bool warmed_up () const;
nano::ledger & ledger;
nano::pending_key next{ 0, 0 };
nano::account next{ 0 };
size_t completed{ 0 };
};
@ -52,8 +48,8 @@ private:
void fill ();
private:
account_database_iterator accounts_iterator;
pending_database_iterator pending_iterator;
account_database_scanner account_scanner;
pending_database_scanner pending_scanner;
std::deque<nano::account> queue;

188
nano/node/bootstrap/frontier_scan.cpp Normal file
View file

@ -0,0 +1,188 @@
#include <nano/node/bootstrap/frontier_scan.hpp>
#include <boost/multiprecision/cpp_dec_float.hpp>
#include <boost/multiprecision/cpp_int.hpp>
nano::bootstrap::frontier_scan::frontier_scan (frontier_scan_config const & config_a, nano::stats & stats_a) :
config{ config_a },
stats{ stats_a }
{
// Divide nano::account numeric range into consecutive and equal ranges
nano::uint256_t max_account = std::numeric_limits<nano::uint256_t>::max ();
nano::uint256_t range_size = max_account / config.head_parallelistm;
for (unsigned i = 0; i < config.head_parallelistm; ++i)
{
// Start at 1 to avoid the burn account
nano::uint256_t start = (i == 0) ? 1 : i * range_size;
nano::uint256_t end = (i == config.head_parallelistm - 1) ? max_account : start + range_size;
heads.emplace_back (frontier_head{ nano::account{ start }, nano::account{ end } });
}
release_assert (!heads.empty ());
}
nano::account nano::bootstrap::frontier_scan::next ()
{
auto const cutoff = std::chrono::steady_clock::now () - config.cooldown;
auto & heads_by_timestamp = heads.get<tag_timestamp> ();
for (auto it = heads_by_timestamp.begin (); it != heads_by_timestamp.end (); ++it)
{
auto const & head = *it;
if (head.requests < config.consideration_count || head.timestamp < cutoff)
{
stats.inc (nano::stat::type::bootstrap_frontier_scan, (head.requests < config.consideration_count) ? nano::stat::detail::next_by_requests : nano::stat::detail::next_by_timestamp);
debug_assert (head.next.number () >= head.start.number ());
debug_assert (head.next.number () < head.end.number ());
auto result = head.next;
heads_by_timestamp.modify (it, [this] (auto & entry) {
entry.requests += 1;
entry.timestamp = std::chrono::steady_clock::now ();
});
return result;
}
}
stats.inc (nano::stat::type::bootstrap_frontier_scan, nano::stat::detail::next_none);
return { 0 };
}
bool nano::bootstrap::frontier_scan::process (nano::account start, std::deque<std::pair<nano::account, nano::block_hash>> const & response)
{
debug_assert (std::all_of (response.begin (), response.end (), [&] (auto const & pair) { return pair.first.number () >= start.number (); }));
stats.inc (nano::stat::type::bootstrap_frontier_scan, nano::stat::detail::process);
// Find the first head with head.start <= start
auto & heads_by_start = heads.get<tag_start> ();
auto it = heads_by_start.upper_bound (start);
release_assert (it != heads_by_start.begin ());
it = std::prev (it);
bool done = false;
heads_by_start.modify (it, [this, &response, &done] (frontier_head & entry) {
entry.completed += 1;
for (auto const & [account, _] : response)
{
// Only consider candidates that actually advance the current frontier
if (account.number () > entry.next.number ())
{
entry.candidates.insert (account);
}
}
// Trim the candidates
while (entry.candidates.size () > config.candidates)
{
release_assert (!entry.candidates.empty ());
entry.candidates.erase (std::prev (entry.candidates.end ()));
}
// Special case for the last frontier head that won't receive larger than max frontier
if (entry.completed >= config.consideration_count * 2 && entry.candidates.empty ())
{
stats.inc (nano::stat::type::bootstrap_frontier_scan, nano::stat::detail::done_empty);
entry.candidates.insert (entry.end);
}
// Check if done
if (entry.completed >= config.consideration_count && !entry.candidates.empty ())
{
stats.inc (nano::stat::type::bootstrap_frontier_scan, nano::stat::detail::done);
// Take the last candidate as the next frontier
release_assert (!entry.candidates.empty ());
auto it = std::prev (entry.candidates.end ());
debug_assert (entry.next.number () < it->number ());
entry.next = *it;
entry.processed += entry.candidates.size ();
entry.candidates.clear ();
entry.requests = 0;
entry.completed = 0;
entry.timestamp = {};
// Bound the search range
if (entry.next.number () >= entry.end.number ())
{
stats.inc (nano::stat::type::bootstrap_frontier_scan, nano::stat::detail::done_range);
entry.next = entry.start;
}
done = true;
}
});
return done;
}
nano::container_info nano::bootstrap::frontier_scan::container_info () const
{
auto collect_progress = [&] () {
nano::container_info info;
for (int n = 0; n < heads.size (); ++n)
{
auto const & head = heads[n];
boost::multiprecision::cpp_dec_float_50 start{ head.start.number ().str () };
boost::multiprecision::cpp_dec_float_50 next{ head.next.number ().str () };
boost::multiprecision::cpp_dec_float_50 end{ head.end.number ().str () };
// Progress in the range [0, 1000000] since we can only represent `size_t` integers in the container_info data
boost::multiprecision::cpp_dec_float_50 progress = (next - start) * boost::multiprecision::cpp_dec_float_50 (1000000) / (end - start);
info.put (std::to_string (n), progress.convert_to<std::uint64_t> ());
}
return info;
};
auto collect_candidates = [&] () {
nano::container_info info;
for (int n = 0; n < heads.size (); ++n)
{
auto const & head = heads[n];
info.put (std::to_string (n), head.candidates.size ());
}
return info;
};
auto collect_responses = [&] () {
nano::container_info info;
for (int n = 0; n < heads.size (); ++n)
{
auto const & head = heads[n];
info.put (std::to_string (n), head.completed);
}
return info;
};
auto collect_processed = [&] () {
nano::container_info info;
for (int n = 0; n < heads.size (); ++n)
{
auto const & head = heads[n];
info.put (std::to_string (n), head.processed);
}
return info;
};
auto total_processed = std::accumulate (heads.begin (), heads.end (), std::size_t{ 0 }, [] (auto total, auto const & head) {
return total + head.processed;
});
nano::container_info info;
info.put ("total_processed", total_processed);
info.add ("progress", collect_progress ());
info.add ("candidates", collect_candidates ());
info.add ("responses", collect_responses ());
info.add ("processed", collect_processed ());
return info;
}

87
nano/node/bootstrap/frontier_scan.hpp Normal file
View file

@ -0,0 +1,87 @@
#pragma once
#include <nano/node/bootstrap/bootstrap_config.hpp>
#include <nano/node/bootstrap/common.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/mem_fun.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/random_access_index.hpp>
#include <boost/multi_index/sequenced_index.hpp>
#include <boost/multi_index_container.hpp>
#include <chrono>
#include <map>
#include <set>
namespace mi = boost::multi_index;
namespace nano::bootstrap
{
/*
* Frontier scan divides the account space into ranges and scans each range for outdated frontiers in parallel.
* This class is used to track the progress of each range.
*/
class frontier_scan
{
public:
frontier_scan (frontier_scan_config const &, nano::stats &);
nano::account next ();
bool process (nano::account start, std::deque<std::pair<nano::account, nano::block_hash>> const & response);
nano::container_info container_info () const;
private: // Dependencies
frontier_scan_config const & config;
nano::stats & stats;
private:
// Represents a range of accounts to scan, once the full range is scanned (goes past `end`) the head wraps around (to the `start`)
struct frontier_head
{
frontier_head (nano::account start_a, nano::account end_a) :
start{ start_a },
end{ end_a },
next{ start_a }
{
}
// The range of accounts to scan is [start, end)
nano::account const start;
nano::account const end;
// We scan the range by querying frontiers starting at 'next' and gathering candidates
nano::account next;
std::set<nano::account> candidates;
unsigned requests{ 0 };
unsigned completed{ 0 };
std::chrono::steady_clock::time_point timestamp{};
size_t processed{ 0 }; // Total number of accounts processed
nano::account index () const
{
return start;
}
};
// clang-format off
class tag_sequenced {};
class tag_start {};
class tag_timestamp {};
using ordered_heads = boost::multi_index_container<frontier_head,
mi::indexed_by<
mi::random_access<mi::tag<tag_sequenced>>,
mi::ordered_unique<mi::tag<tag_start>,
mi::const_mem_fun<frontier_head, nano::account, &frontier_head::index>>,
mi::ordered_non_unique<mi::tag<tag_timestamp>,
mi::member<frontier_head, std::chrono::steady_clock::time_point, &frontier_head::timestamp>>
>>;
// clang-format on
ordered_heads heads;
};
}

19
nano/node/bootstrap_ascending/peer_scoring.cpp → nano/node/bootstrap/peer_scoring.cpp
View file

@ -1,18 +1,18 @@
#include <nano/node/bootstrap/bootstrap_config.hpp>
#include <nano/node/bootstrap_ascending/peer_scoring.hpp>
#include <nano/node/bootstrap/peer_scoring.hpp>
#include <nano/node/transport/channel.hpp>
/*
* peer_scoring
*/
nano::bootstrap_ascending::peer_scoring::peer_scoring (bootstrap_ascending_config const & config_a, nano::network_constants const & network_constants_a) :
nano::bootstrap::peer_scoring::peer_scoring (bootstrap_config const & config_a, nano::network_constants const & network_constants_a) :
config{ config_a },
network_constants{ network_constants_a }
{
}
bool nano::bootstrap_ascending::peer_scoring::try_send_message (std::shared_ptr<nano::transport::channel> channel)
bool nano::bootstrap::peer_scoring::try_send_message (std::shared_ptr<nano::transport::channel> channel)
{
auto & index = scoring.get<tag_channel> ();
auto existing = index.find (channel.get ());
@ -38,7 +38,7 @@ bool nano::bootstrap_ascending::peer_scoring::try_send_message (std::shared_ptr<
return false;
}
void nano::bootstrap_ascending::peer_scoring::received_message (std::shared_ptr<nano::transport::channel> channel)
void nano::bootstrap::peer_scoring::received_message (std::shared_ptr<nano::transport::channel> channel)
{
auto & index = scoring.get<tag_channel> ();
auto existing = index.find (channel.get ());
@ -55,7 +55,7 @@ void nano::bootstrap_ascending::peer_scoring::received_message (std::shared_ptr<
}
}
std::shared_ptr<nano::transport::channel> nano::bootstrap_ascending::peer_scoring::channel ()
std::shared_ptr<nano::transport::channel> nano::bootstrap::peer_scoring::channel ()
{
auto & index = scoring.get<tag_outstanding> ();
for (auto const & score : index)
@ -74,12 +74,12 @@ std::shared_ptr<nano::transport::channel> nano::bootstrap_ascending::peer_scorin
return nullptr;
}
std::size_t nano::bootstrap_ascending::peer_scoring::size () const
std::size_t nano::bootstrap::peer_scoring::size () const
{
return scoring.size ();
}
void nano::bootstrap_ascending::peer_scoring::timeout ()
void nano::bootstrap::peer_scoring::timeout ()
{
auto & index = scoring.get<tag_channel> ();
@ -102,7 +102,7 @@ void nano::bootstrap_ascending::peer_scoring::timeout ()
}
}
void nano::bootstrap_ascending::peer_scoring::sync (std::deque<std::shared_ptr<nano::transport::channel>> const & list)
void nano::bootstrap::peer_scoring::sync (std::deque<std::shared_ptr<nano::transport::channel>> const & list)
{
auto & index = scoring.get<tag_channel> ();
for (auto const & channel : list)
@ -124,8 +124,7 @@ void nano::bootstrap_ascending::peer_scoring::sync (std::deque<std::shared_ptr<n
* peer_score
*/
nano::bootstrap_ascending::peer_scoring::peer_score::peer_score (
std::shared_ptr<nano::transport::channel> const & channel_a, uint64_t outstanding_a, uint64_t request_count_total_a, uint64_t response_count_total_a) :
nano::bootstrap::peer_scoring::peer_score::peer_score (std::shared_ptr<nano::transport::channel> const & channel_a, uint64_t outstanding_a, uint64_t request_count_total_a, uint64_t response_count_total_a) :
channel{ channel_a },
channel_ptr{ channel_a.get () },
outstanding{ outstanding_a },

14
nano/node/bootstrap_ascending/peer_scoring.hpp → nano/node/bootstrap/peer_scoring.hpp
View file

@ -1,5 +1,7 @@
#pragma once
#include <nano/node/fwd.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/ordered_index.hpp>
@ -12,19 +14,13 @@ namespace mi = boost::multi_index;
namespace nano
{
class bootstrap_ascending_config;
class network_constants;
namespace transport
{
class channel;
}
namespace bootstrap_ascending
namespace bootstrap
{
// Container for tracking and scoring peers with respect to bootstrapping
class peer_scoring
{
public:
peer_scoring (bootstrap_ascending_config const &, nano::network_constants const &);
peer_scoring (bootstrap_config const &, nano::network_constants const &);
// Returns true if channel limit has been exceeded
bool try_send_message (std::shared_ptr<nano::transport::channel> channel);
@ -37,7 +33,7 @@ namespace bootstrap_ascending
void sync (std::deque<std::shared_ptr<nano::transport::channel>> const & list);
private:
bootstrap_ascending_config const & config;
bootstrap_config const & config;
nano::network_constants const & network_constants;
private:

Some files were not shown because too many files have changed in this diff Show more