diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index 4c59a4229..6f5943f08 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -9,7 +9,6 @@ add_executable( block.cpp block_store.cpp blockprocessor.cpp - bootstrap.cpp bootstrap_ascending.cpp bootstrap_server.cpp cli.cpp diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp deleted file mode 100644 index 3d1fc18b4..000000000 --- a/nano/core_test/bootstrap.cpp +++ /dev/null @@ -1,2115 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -using namespace std::chrono_literals; - -// If the account doesn't exist, current == end so there's no iteration -TEST (bulk_pull, no_address) -{ - nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = 1; - req->end = 2; - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (request->current, request->request->end); - ASSERT_TRUE (request->current.is_zero ()); -} - -TEST (bulk_pull, genesis_to_end) -{ - nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = nano::dev::genesis_key.pub; - req->end.clear (); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (system.nodes[0]->latest (nano::dev::genesis_key.pub), request->current); - ASSERT_EQ (request->request->end, request->request->end); -} - -// If we can't find the end block, send everything -TEST (bulk_pull, no_end) -{ - nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = nano::dev::genesis_key.pub; - req->end = 1; - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (system.nodes[0]->latest (nano::dev::genesis_key.pub), request->current); - ASSERT_TRUE (request->request->end.is_zero ()); -} - -TEST (bulk_pull, end_not_owned) -{ - nano::test::system system (1); - nano::keypair key2; - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::dev::genesis_key.pub, key2.pub, 100)); - nano::block_hash latest (system.nodes[0]->latest (nano::dev::genesis_key.pub)); - nano::block_builder builder; - auto open = builder - .open () - .source (0) - .representative (1) - .account (2) - .sign (nano::keypair ().prv, 4) - .work (5) - .build (); - open->hashables.account = key2.pub; - open->hashables.representative = key2.pub; - open->hashables.source = latest; - open->refresh (); - open->signature = nano::sign_message (key2.prv, key2.pub, open->hash ()); - system.nodes[0]->work_generate_blocking (*open); - ASSERT_EQ (nano::block_status::progress, system.nodes[0]->process (open)); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = key2.pub; - req->end = nano::dev::genesis->hash (); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (request->current, request->request->end); -} - -TEST (bulk_pull, none) -{ - nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = nano::dev::genesis_key.pub; - req->end = nano::dev::genesis->hash (); - auto request (std::make_shared (connection, std::move (req))); - auto block (request->get_next ()); - ASSERT_EQ (nullptr, block); -} - -TEST (bulk_pull, get_next_on_open) -{ - nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = nano::dev::genesis_key.pub; - req->end.clear (); - auto request (std::make_shared (connection, std::move (req))); - auto block (request->get_next ()); - ASSERT_NE (nullptr, block); - ASSERT_TRUE (block->previous ().is_zero ()); - ASSERT_EQ (request->current, request->request->end); -} - -/** - Tests that the ascending flag is respected in the bulk_pull message when given a known block hash - */ -TEST (bulk_pull, ascending_one_hash) -{ - nano::test::system system{ 1 }; - auto & node = *system.nodes[0]; - nano::state_block_builder builder; - auto block1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - 100) - .link (nano::dev::genesis_key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (0) - .build (); - node.work_generate_blocking (*block1); - ASSERT_EQ (nano::block_status::progress, node.process (block1)); - auto socket = std::make_shared (node, nano::transport::socket_endpoint::server); - auto connection = std::make_shared (socket, system.nodes[0]); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = nano::dev::genesis->hash (); - req->end.clear (); - req->header.flag_set (nano::message_header::bulk_pull_ascending_flag); - auto request = std::make_shared (connection, std::move (req)); - auto block_out1 = request->get_next (); - ASSERT_NE (nullptr, block_out1); - ASSERT_EQ (block_out1->hash (), block1->hash ()); - ASSERT_EQ (nullptr, request->get_next ()); -} - -/** - Tests that the ascending flag is respected in the bulk_pull message when given an account number - */ -TEST (bulk_pull, ascending_two_account) -{ - nano::test::system system{ 1 }; - auto & node = *system.nodes[0]; - nano::state_block_builder builder; - auto block1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - 100) - .link (nano::dev::genesis_key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (0) - .build (); - node.work_generate_blocking (*block1); - ASSERT_EQ (nano::block_status::progress, node.process (block1)); - auto socket = std::make_shared (node, nano::transport::socket_endpoint::server); - auto connection = std::make_shared (socket, system.nodes[0]); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = nano::dev::genesis_key.pub; - req->end.clear (); - req->header.flag_set (nano::message_header::bulk_pull_ascending_flag); - auto request = std::make_shared (connection, std::move (req)); - auto block_out1 = request->get_next (); - ASSERT_NE (nullptr, block_out1); - ASSERT_EQ (block_out1->hash (), nano::dev::genesis->hash ()); - auto block_out2 = request->get_next (); - ASSERT_NE (nullptr, block_out2); - ASSERT_EQ (block_out2->hash (), block1->hash ()); - ASSERT_EQ (nullptr, request->get_next ()); -} - -/** - Tests that the `end' value is respected in the bulk_pull message when the ascending flag is used. - */ -TEST (bulk_pull, ascending_end) -{ - nano::test::system system{ 1 }; - auto & node = *system.nodes[0]; - nano::state_block_builder builder; - auto block1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - 100) - .link (nano::dev::genesis_key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (0) - .build (); - node.work_generate_blocking (*block1); - ASSERT_EQ (nano::block_status::progress, node.process (block1)); - auto socket = std::make_shared (node, nano::transport::socket_endpoint::server); - auto connection = std::make_shared (socket, system.nodes[0]); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = nano::dev::genesis_key.pub; - req->end = block1->hash (); - req->header.flag_set (nano::message_header::bulk_pull_ascending_flag); - auto request = std::make_shared (connection, std::move (req)); - auto block_out1 = request->get_next (); - ASSERT_NE (nullptr, block_out1); - ASSERT_EQ (block_out1->hash (), nano::dev::genesis->hash ()); - ASSERT_EQ (nullptr, request->get_next ()); -} - -TEST (bulk_pull, by_block) -{ - nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = nano::dev::genesis->hash (); - req->end.clear (); - auto request (std::make_shared (connection, std::move (req))); - auto block (request->get_next ()); - ASSERT_NE (nullptr, block); - ASSERT_EQ (block->hash (), nano::dev::genesis->hash ()); - - block = request->get_next (); - ASSERT_EQ (nullptr, block); -} - -TEST (bulk_pull, by_block_single) -{ - nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = nano::dev::genesis->hash (); - req->end = nano::dev::genesis->hash (); - auto request (std::make_shared (connection, std::move (req))); - auto block (request->get_next ()); - ASSERT_NE (nullptr, block); - ASSERT_EQ (block->hash (), nano::dev::genesis->hash ()); - - block = request->get_next (); - ASSERT_EQ (nullptr, block); -} - -TEST (bulk_pull, count_limit) -{ - nano::test::system system (1); - auto node0 (system.nodes[0]); - - nano::block_builder builder; - auto send1 = builder - .send () - .previous (node0->latest (nano::dev::genesis_key.pub)) - .destination (nano::dev::genesis_key.pub) - .balance (1) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (node0->latest (nano::dev::genesis_key.pub))) - .build (); - ASSERT_EQ (nano::block_status::progress, node0->process (send1)); - auto receive1 = builder - .receive () - .previous (send1->hash ()) - .source (send1->hash ()) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (send1->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node0->process (receive1)); - - auto connection (std::make_shared (std::make_shared (*node0, nano::transport::socket_endpoint::server), node0)); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = receive1->hash (); - req->set_count_present (true); - req->count = 2; - auto request (std::make_shared (connection, std::move (req))); - - ASSERT_EQ (request->max_count, 2); - ASSERT_EQ (request->sent_count, 0); - - auto block (request->get_next ()); - ASSERT_NE (nullptr, block); - ASSERT_EQ (receive1->hash (), block->hash ()); - - block = request->get_next (); - ASSERT_EQ (send1->hash (), block->hash ()); - - block = request->get_next (); - ASSERT_EQ (nullptr, block); -} - -TEST (bootstrap_processor, process_none) -{ - nano::test::system system (1); - auto node0 = system.nodes[0]; - auto node1 = system.make_disconnected_node (); - - std::atomic done = false; - node0->observers.socket_connected.add ([&] (nano::transport::tcp_socket & socket) { - done = true; - }); - - node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint (), false); - ASSERT_TIMELY (5s, done); -} - -// Bootstrap can pull one basic block -TEST (bootstrap_processor, process_one) -{ - nano::test::system system; - nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; - node_config.enable_voting = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - auto node0 = system.add_node (node_config, node_flags); - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - auto send (system.wallet (0)->send_action (nano::dev::genesis_key.pub, nano::dev::genesis_key.pub, 100)); - ASSERT_NE (nullptr, send); - ASSERT_TIMELY (5s, node0->latest (nano::dev::genesis_key.pub) != nano::dev::genesis->hash ()); - - node_flags.disable_rep_crawler = true; - node_config.peering_port = system.get_available_port (); - auto node1 = system.make_disconnected_node (node_config, node_flags); - ASSERT_NE (node0->latest (nano::dev::genesis_key.pub), node1->latest (nano::dev::genesis_key.pub)); - node1->bootstrap_initiator.bootstrap (node0->network.endpoint (), false); - ASSERT_TIMELY_EQ (10s, node1->latest (nano::dev::genesis_key.pub), node0->latest (nano::dev::genesis_key.pub)); -} - -TEST (bootstrap_processor, process_two) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - auto node0 (system.add_node (config, node_flags)); - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - ASSERT_TRUE (system.wallet (0)->send_action (nano::dev::genesis_key.pub, nano::dev::genesis_key.pub, 50)); - ASSERT_TRUE (system.wallet (0)->send_action (nano::dev::genesis_key.pub, nano::dev::genesis_key.pub, 50)); - ASSERT_TIMELY_EQ (5s, nano::test::account_info (*node0, nano::dev::genesis_key.pub).block_count, 3); - - // create a node manually to avoid making automatic network connections - auto node1 = system.make_disconnected_node (); - ASSERT_NE (node1->latest (nano::dev::genesis_key.pub), node0->latest (nano::dev::genesis_key.pub)); // nodes should be out of sync here - node1->bootstrap_initiator.bootstrap (node0->network.endpoint (), false); // bootstrap triggered - ASSERT_TIMELY_EQ (5s, node1->latest (nano::dev::genesis_key.pub), node0->latest (nano::dev::genesis_key.pub)); // nodes should sync up -} - -// Bootstrap can pull universal blocks -TEST (bootstrap_processor, process_state) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - auto node0 (system.add_node (config, node_flags)); - nano::state_block_builder builder; - - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - auto block1 = builder - .account (nano::dev::genesis_key.pub) - .previous (node0->latest (nano::dev::genesis_key.pub)) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - 100) - .link (nano::dev::genesis_key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (0) - .build (); - auto block2 = builder - .make_block () - .account (nano::dev::genesis_key.pub) - .previous (block1->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount) - .link (block1->hash ()) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (0) - .build (); - - node0->work_generate_blocking (*block1); - node0->work_generate_blocking (*block2); - ASSERT_EQ (nano::block_status::progress, node0->process (block1)); - ASSERT_EQ (nano::block_status::progress, node0->process (block2)); - ASSERT_TIMELY_EQ (5s, nano::test::account_info (*node0, nano::dev::genesis_key.pub).block_count, 3); - - auto node1 = system.make_disconnected_node (std::nullopt, node_flags); - ASSERT_EQ (node0->latest (nano::dev::genesis_key.pub), block2->hash ()); - ASSERT_NE (node1->latest (nano::dev::genesis_key.pub), block2->hash ()); - node1->bootstrap_initiator.bootstrap (node0->network.endpoint (), false); - ASSERT_TIMELY_EQ (5s, node1->latest (nano::dev::genesis_key.pub), block2->hash ()); -} - -TEST (bootstrap_processor, process_new) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - nano::keypair key2; - - auto node1 = system.add_node (config, node_flags); - config.peering_port = system.get_available_port (); - auto node2 = system.add_node (config, node_flags); - - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - system.wallet (1)->insert_adhoc (key2.prv); - - // send amount raw from genesis to key2, the wallet will autoreceive - auto amount = node1->config.receive_minimum.number (); - auto send = system.wallet (0)->send_action (nano::dev::genesis_key.pub, key2.pub, amount); - ASSERT_NE (nullptr, send); - ASSERT_TIMELY (5s, !node1->balance (key2.pub).is_zero ()); - - // wait for the receive block on node2 - std::shared_ptr receive; - ASSERT_TIMELY (5s, receive = node2->block (node2->latest (key2.pub))); - - // All blocks should be propagated & confirmed - ASSERT_TIMELY (5s, nano::test::confirmed (*node1, { send, receive })); - ASSERT_TIMELY (5s, nano::test::confirmed (*node2, { send, receive })); - ASSERT_TIMELY (5s, node1->active.empty ()); - ASSERT_TIMELY (5s, node2->active.empty ()); - - // create a node manually to avoid making automatic network connections - auto node3 = system.make_disconnected_node (); - node3->bootstrap_initiator.bootstrap (node1->network.endpoint (), false); - ASSERT_TIMELY_EQ (5s, node3->balance (key2.pub), amount); -} - -TEST (bootstrap_processor, pull_diamond) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - auto node0 (system.add_node (config, node_flags)); - nano::keypair key; - nano::block_builder builder; - auto send1 = builder - .send () - .previous (node0->latest (nano::dev::genesis_key.pub)) - .destination (key.pub) - .balance (0) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (node0->latest (nano::dev::genesis_key.pub))) - .build (); - ASSERT_EQ (nano::block_status::progress, node0->process (send1)); - auto open = builder - .open () - .source (send1->hash ()) - .representative (1) - .account (key.pub) - .sign (key.prv, key.pub) - .work (*system.work.generate (key.pub)) - .build (); - ASSERT_EQ (nano::block_status::progress, node0->process (open)); - auto send2 = builder - .send () - .previous (open->hash ()) - .destination (nano::dev::genesis_key.pub) - .balance (std::numeric_limits::max () - 100) - .sign (key.prv, key.pub) - .work (*system.work.generate (open->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node0->process (send2)); - auto receive = builder - .receive () - .previous (send1->hash ()) - .source (send2->hash ()) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (send1->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node0->process (receive)); - - auto node1 = system.make_disconnected_node (); - node1->bootstrap_initiator.bootstrap (node0->network.endpoint (), false); - ASSERT_TIMELY_EQ (5s, node1->balance (nano::dev::genesis_key.pub), 100); -} - -TEST (bootstrap_processor, DISABLED_pull_requeue_network_error) -{ - // Bootstrap attempt stopped before requeue & then cannot be found in attempts list - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - auto node1 (system.add_node (config, node_flags)); - config.peering_port = system.get_available_port (); - auto node2 (system.add_node (config, node_flags)); - nano::keypair key1; - - nano::state_block_builder builder; - - auto send1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (nano::dev::genesis->hash ())) - .build (); - - node1->bootstrap_initiator.bootstrap (node2->network.endpoint ()); - auto attempt (node1->bootstrap_initiator.current_attempt ()); - ASSERT_NE (nullptr, attempt); - ASSERT_TIMELY (2s, attempt->frontiers_received); - // Add non-existing pull & stop remote peer - { - nano::unique_lock lock{ node1->bootstrap_initiator.connections->mutex }; - ASSERT_FALSE (attempt->stopped); - ++attempt->pulling; - node1->bootstrap_initiator.connections->pulls.emplace_back (nano::dev::genesis_key.pub, send1->hash (), nano::dev::genesis->hash (), attempt->incremental_id); - node1->bootstrap_initiator.connections->request_pull (lock); - } - ASSERT_TIMELY (5s, attempt == nullptr || attempt->requeued_pulls == 1); - ASSERT_EQ (0, node1->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in)); // Requeue is not increasing failed attempts -} - -TEST (bootstrap_processor, push_diamond) -{ - nano::test::system system; - nano::keypair key; - - auto node1 = system.make_disconnected_node (); - auto wallet1 (node1->wallets.create (100)); - wallet1->insert_adhoc (nano::dev::genesis_key.prv); - wallet1->insert_adhoc (key.prv); - - nano::block_builder builder; - - // send all balance from genesis to key - auto send1 = builder - .send () - .previous (nano::dev::genesis->hash ()) - .destination (key.pub) - .balance (0) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (nano::dev::genesis->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (send1)); - - // open key account receiving all balance of genesis - auto open = builder - .open () - .source (send1->hash ()) - .representative (1) - .account (key.pub) - .sign (key.prv, key.pub) - .work (*system.work.generate (key.pub)) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (open)); - - // send from key to genesis 100 raw - auto send2 = builder - .send () - .previous (open->hash ()) - .destination (nano::dev::genesis_key.pub) - .balance (std::numeric_limits::max () - 100) - .sign (key.prv, key.pub) - .work (*system.work.generate (open->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (send2)); - - // receive the 100 raw on genesis - auto receive = builder - .receive () - .previous (send1->hash ()) - .source (send2->hash ()) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (send1->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (receive)); - - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags flags; - flags.disable_ongoing_bootstrap = true; - flags.disable_ascending_bootstrap = true; - auto node2 = system.add_node (config, flags); - node1->bootstrap_initiator.bootstrap (node2->network.endpoint (), false); - ASSERT_TIMELY_EQ (5s, node2->balance (nano::dev::genesis_key.pub), 100); -} - -TEST (bootstrap_processor, push_diamond_pruning) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags0; - node_flags0.disable_ascending_bootstrap = true; - node_flags0.disable_ongoing_bootstrap = true; - auto node0 (system.add_node (config, node_flags0)); - nano::keypair key; - - config.enable_voting = false; // Remove after allowing pruned voting - nano::node_flags node_flags; - node_flags.enable_pruning = true; - config.peering_port = system.get_available_port (); - auto node1 = system.make_disconnected_node (config, node_flags); - - nano::block_builder builder; - - // send all balance from genesis to key - auto send1 = builder - .send () - .previous (nano::dev::genesis->hash ()) - .destination (key.pub) - .balance (0) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (nano::dev::genesis->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (send1)); - - // receive all balance on key - auto open = builder - .open () - .source (send1->hash ()) - .representative (1) - .account (key.pub) - .sign (key.prv, key.pub) - .work (*system.work.generate (key.pub)) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (open)); - - // 1st bootstrap - node1->bootstrap_initiator.bootstrap (node0->network.endpoint (), false); - ASSERT_TIMELY_EQ (5s, node0->balance (key.pub), nano::dev::constants.genesis_amount); - ASSERT_TIMELY_EQ (5s, node1->balance (key.pub), nano::dev::constants.genesis_amount); - - // Process more blocks & prune old - - // send 100 raw from key to genesis - auto send2 = builder - .send () - .previous (open->hash ()) - .destination (nano::dev::genesis_key.pub) - .balance (std::numeric_limits::max () - 100) - .sign (key.prv, key.pub) - .work (*system.work.generate (open->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (send2)); - - // receive the 100 raw from key on genesis - auto receive = builder - .receive () - .previous (send1->hash ()) - .source (send2->hash ()) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (send1->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (receive)); - - { - auto transaction = node1->ledger.tx_begin_write (); - node1->ledger.confirm (transaction, open->hash ()); - ASSERT_EQ (1, node1->ledger.pruning_action (transaction, send1->hash (), 2)); - ASSERT_EQ (1, node1->ledger.pruning_action (transaction, open->hash (), 1)); - ASSERT_TRUE (node1->ledger.any.block_exists (transaction, nano::dev::genesis->hash ())); - ASSERT_FALSE (node1->ledger.any.block_exists (transaction, send1->hash ())); - ASSERT_TRUE (node1->store.pruned.exists (transaction, send1->hash ())); - ASSERT_FALSE (node1->ledger.any.block_exists (transaction, open->hash ())); - ASSERT_TRUE (node1->store.pruned.exists (transaction, open->hash ())); - ASSERT_TRUE (node1->ledger.any.block_exists (transaction, send2->hash ())); - ASSERT_TRUE (node1->ledger.any.block_exists (transaction, receive->hash ())); - ASSERT_EQ (2, node1->ledger.pruned_count ()); - ASSERT_EQ (5, node1->ledger.block_count ()); - } - - // 2nd bootstrap - node1->bootstrap_initiator.bootstrap (node0->network.endpoint (), false); - ASSERT_TIMELY_EQ (5s, node0->balance (nano::dev::genesis_key.pub), 100); - ASSERT_TIMELY_EQ (5s, node1->balance (nano::dev::genesis_key.pub), 100); -} - -TEST (bootstrap_processor, push_one) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - auto node0 (system.add_node (config)); - nano::keypair key1; - auto node1 = system.make_disconnected_node (); - auto wallet = node1->wallets.create (nano::random_wallet_id ()); - ASSERT_NE (nullptr, wallet); - wallet->insert_adhoc (nano::dev::genesis_key.prv); - - // send 100 raw from genesis to key1 - nano::uint128_t genesis_balance = node1->balance (nano::dev::genesis_key.pub); - auto send = wallet->send_action (nano::dev::genesis_key.pub, key1.pub, 100); - ASSERT_NE (nullptr, send); - ASSERT_TIMELY_EQ (5s, genesis_balance - 100, node1->balance (nano::dev::genesis_key.pub)); - - node1->bootstrap_initiator.bootstrap (node0->network.endpoint (), false); - ASSERT_TIMELY_EQ (5s, node0->balance (nano::dev::genesis_key.pub), genesis_balance - 100); -} - -TEST (bootstrap_processor, lazy_hash) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - auto node0 (system.add_node (config, node_flags)); - nano::keypair key1; - nano::keypair key2; - // Generating test chain - - nano::state_block_builder builder; - - auto send1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (nano::dev::genesis->hash ())) - .build (); - auto receive1 = builder - .make_block () - .account (key1.pub) - .previous (0) - .representative (key1.pub) - .balance (nano::Knano_ratio) - .link (send1->hash ()) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (key1.pub)) - .build (); - auto send2 = builder - .make_block () - .account (key1.pub) - .previous (receive1->hash ()) - .representative (key1.pub) - .balance (0) - .link (key2.pub) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (receive1->hash ())) - .build (); - auto receive2 = builder - .make_block () - .account (key2.pub) - .previous (0) - .representative (key2.pub) - .balance (nano::Knano_ratio) - .link (send2->hash ()) - .sign (key2.prv, key2.pub) - .work (*node0->work_generate_blocking (key2.pub)) - .build (); - - // Processing test chain - node0->block_processor.add (send1); - node0->block_processor.add (receive1); - node0->block_processor.add (send2); - node0->block_processor.add (receive2); - ASSERT_TIMELY (5s, nano::test::exists (*node0, { send1, receive1, send2, receive2 })); - - // Start lazy bootstrap with last block in chain known - auto node1 = system.make_disconnected_node (); - nano::test::establish_tcp (system, *node1, node0->network.endpoint ()); - node1->bootstrap_initiator.bootstrap_lazy (receive2->hash (), true); - { - auto lazy_attempt (node1->bootstrap_initiator.current_lazy_attempt ()); - ASSERT_NE (nullptr, lazy_attempt); - ASSERT_EQ (receive2->hash ().to_string (), lazy_attempt->id); - } - // Check processed blocks - ASSERT_TIMELY (10s, node1->balance (key2.pub) != 0); -} - -TEST (bootstrap_processor, lazy_hash_bootstrap_id) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - auto node0 (system.add_node (config, node_flags)); - nano::keypair key1; - nano::keypair key2; - // Generating test chain - - nano::state_block_builder builder; - - auto send1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (nano::dev::genesis->hash ())) - .build (); - auto receive1 = builder - .make_block () - .account (key1.pub) - .previous (0) - .representative (key1.pub) - .balance (nano::Knano_ratio) - .link (send1->hash ()) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (key1.pub)) - .build (); - auto send2 = builder - .make_block () - .account (key1.pub) - .previous (receive1->hash ()) - .representative (key1.pub) - .balance (0) - .link (key2.pub) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (receive1->hash ())) - .build (); - auto receive2 = builder - .make_block () - .account (key2.pub) - .previous (0) - .representative (key2.pub) - .balance (nano::Knano_ratio) - .link (send2->hash ()) - .sign (key2.prv, key2.pub) - .work (*node0->work_generate_blocking (key2.pub)) - .build (); - - // Processing test chain - node0->block_processor.add (send1); - node0->block_processor.add (receive1); - node0->block_processor.add (send2); - node0->block_processor.add (receive2); - ASSERT_TIMELY (5s, nano::test::exists (*node0, { send1, receive1, send2, receive2 })); - - // Start lazy bootstrap with last block in chain known - auto node1 = system.make_disconnected_node (); - nano::test::establish_tcp (system, *node1, node0->network.endpoint ()); - node1->bootstrap_initiator.bootstrap_lazy (receive2->hash (), true, "123456"); - { - auto lazy_attempt (node1->bootstrap_initiator.current_lazy_attempt ()); - ASSERT_NE (nullptr, lazy_attempt); - ASSERT_EQ ("123456", lazy_attempt->id); - } - // Check processed blocks - ASSERT_TIMELY (10s, node1->balance (key2.pub) != 0); -} - -TEST (bootstrap_processor, lazy_hash_pruning) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - config.enable_voting = false; // Remove after allowing pruned voting - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - node_flags.enable_pruning = true; - auto node0 = system.add_node (config, node_flags); - - nano::state_block_builder builder; - - // send Knano_ratio raw from genesis to genesis - auto send1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (nano::dev::genesis_key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (nano::dev::genesis->hash ())) - .build (); - - // receive send1 - auto receive1 = builder - .make_block () - .account (nano::dev::genesis_key.pub) - .previous (send1->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount) - .link (send1->hash ()) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (send1->hash ())) - .build (); - - // change rep of genesis account to be key1 - nano::keypair key1; - auto change1 = builder - .make_block () - .account (nano::dev::genesis_key.pub) - .previous (receive1->hash ()) - .representative (key1.pub) - .balance (nano::dev::constants.genesis_amount) - .link (0) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (receive1->hash ())) - .build (); - - // change rep of genesis account to be rep2 - nano::keypair key2; - auto change2 = builder - .make_block () - .account (nano::dev::genesis_key.pub) - .previous (change1->hash ()) - .representative (key2.pub) - .balance (nano::dev::constants.genesis_amount) - .link (0) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (change1->hash ())) - .build (); - - // send Knano_ratio from genesis to key1 and genesis rep back to genesis account - auto send2 = builder - .make_block () - .account (nano::dev::genesis_key.pub) - .previous (change2->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (change2->hash ())) - .build (); - - // receive send2 and rep of key1 to be itself - auto receive2 = builder - .make_block () - .account (key1.pub) - .previous (0) - .representative (key1.pub) - .balance (nano::Knano_ratio) - .link (send2->hash ()) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (key1.pub)) - .build (); - - // send all available balance from key1 to key2 - auto send3 = builder - .make_block () - .account (key1.pub) - .previous (receive2->hash ()) - .representative (key1.pub) - .balance (0) - .link (key2.pub) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (receive2->hash ())) - .build (); - - // receive send3 on key2, set rep of key2 to be itself - auto receive3 = builder - .make_block () - .account (key2.pub) - .previous (0) - .representative (key2.pub) - .balance (nano::Knano_ratio) - .link (send3->hash ()) - .sign (key2.prv, key2.pub) - .work (*node0->work_generate_blocking (key2.pub)) - .build (); - - std::vector> blocks = { send1, receive1, change1, change2, send2, receive2, send3, receive3 }; - ASSERT_TRUE (nano::test::process (*node0, blocks)); - nano::test::confirm (node0->ledger, blocks); - - config.peering_port = system.get_available_port (); - auto node1 = system.make_disconnected_node (config, node_flags); - - // Processing chain to prune for node1 - node1->process_active (send1); - node1->process_active (receive1); - node1->process_active (change1); - node1->process_active (change2); - ASSERT_TIMELY (5s, nano::test::exists (*node1, { send1, receive1, change1, change2 })); - - // Confirm last block to prune previous - nano::test::confirm (node1->ledger, { send1, receive1, change1, change2 }); - ASSERT_EQ (5, node1->ledger.block_count ()); - ASSERT_EQ (5, node1->ledger.cemented_count ()); - - // Pruning action - node1->ledger_pruning (2, false); - ASSERT_EQ (9, node0->ledger.block_count ()); - ASSERT_EQ (0, node0->ledger.pruned_count ()); - ASSERT_EQ (5, node1->ledger.block_count ()); - ASSERT_EQ (3, node1->ledger.pruned_count ()); - - // Start lazy bootstrap with last block in chain known - nano::test::establish_tcp (system, *node1, node0->network.endpoint ()); - node1->bootstrap_initiator.bootstrap_lazy (receive3->hash (), true); - - // Check processed blocks - ASSERT_TIMELY_EQ (5s, node1->ledger.block_count (), 9); - ASSERT_TIMELY (5s, node1->balance (key2.pub) != 0); - ASSERT_TIMELY (5s, !node1->bootstrap_initiator.in_progress ()); -} - -TEST (bootstrap_processor, lazy_max_pull_count) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - auto node0 (system.add_node (config, node_flags)); - nano::keypair key1; - nano::keypair key2; - // Generating test chain - - nano::state_block_builder builder; - - auto send1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (nano::dev::genesis->hash ())) - .build (); - auto receive1 = builder - .make_block () - .account (key1.pub) - .previous (0) - .representative (key1.pub) - .balance (nano::Knano_ratio) - .link (send1->hash ()) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (key1.pub)) - .build (); - auto send2 = builder - .make_block () - .account (key1.pub) - .previous (receive1->hash ()) - .representative (key1.pub) - .balance (0) - .link (key2.pub) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (receive1->hash ())) - .build (); - auto receive2 = builder - .make_block () - .account (key2.pub) - .previous (0) - .representative (key2.pub) - .balance (nano::Knano_ratio) - .link (send2->hash ()) - .sign (key2.prv, key2.pub) - .work (*node0->work_generate_blocking (key2.pub)) - .build (); - auto change1 = builder - .make_block () - .account (key2.pub) - .previous (receive2->hash ()) - .representative (key1.pub) - .balance (nano::Knano_ratio) - .link (0) - .sign (key2.prv, key2.pub) - .work (*node0->work_generate_blocking (receive2->hash ())) - .build (); - auto change2 = builder - .make_block () - .account (key2.pub) - .previous (change1->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::Knano_ratio) - .link (0) - .sign (key2.prv, key2.pub) - .work (*node0->work_generate_blocking (change1->hash ())) - .build (); - auto change3 = builder - .make_block () - .account (key2.pub) - .previous (change2->hash ()) - .representative (key2.pub) - .balance (nano::Knano_ratio) - .link (0) - .sign (key2.prv, key2.pub) - .work (*node0->work_generate_blocking (change2->hash ())) - .build (); - // Processing test chain - node0->block_processor.add (send1); - node0->block_processor.add (receive1); - node0->block_processor.add (send2); - node0->block_processor.add (receive2); - node0->block_processor.add (change1); - node0->block_processor.add (change2); - node0->block_processor.add (change3); - ASSERT_TIMELY (5s, nano::test::exists (*node0, { send1, receive1, send2, receive2, change1, change2, change3 })); - - // Start lazy bootstrap with last block in chain known - auto node1 = system.make_disconnected_node (); - nano::test::establish_tcp (system, *node1, node0->network.endpoint ()); - node1->bootstrap_initiator.bootstrap_lazy (change3->hash ()); - // Check processed blocks - ASSERT_TIMELY (10s, node1->block (change3->hash ())); -} - -TEST (bootstrap_processor, lazy_unclear_state_link) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - node_flags.disable_legacy_bootstrap = true; - node_flags.disable_ascending_bootstrap = true; - node_flags.disable_ongoing_bootstrap = true; - auto node1 = system.add_node (config, node_flags); - nano::keypair key; - - nano::block_builder builder; - - auto send1 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (nano::dev::genesis->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (send1)); - auto send2 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (send1->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - 2 * nano::Knano_ratio) - .link (key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (send1->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (send2)); - auto open = builder - .open () - .source (send1->hash ()) - .representative (key.pub) - .account (key.pub) - .sign (key.prv, key.pub) - .work (*system.work.generate (key.pub)) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (open)); - auto receive = builder - .state () - .account (key.pub) - .previous (open->hash ()) - .representative (key.pub) - .balance (2 * nano::Knano_ratio) - .link (send2->hash ()) - .sign (key.prv, key.pub) - .work (*system.work.generate (open->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (receive)); - - ASSERT_TIMELY (5s, nano::test::exists (*node1, { send1, send2, open, receive })); - - // Start lazy bootstrap with last block in chain known - auto node2 = system.make_disconnected_node (std::nullopt, node_flags); - nano::test::establish_tcp (system, *node2, node1->network.endpoint ()); - node2->bootstrap_initiator.bootstrap_lazy (receive->hash ()); - ASSERT_TIMELY (5s, nano::test::exists (*node2, { send1, send2, open, receive })); - ASSERT_EQ (0, node2->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in)); -} - -TEST (bootstrap_processor, lazy_unclear_state_link_not_existing) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - node_flags.disable_legacy_bootstrap = true; - node_flags.disable_ascending_bootstrap = true; - node_flags.disable_ongoing_bootstrap = true; - auto node1 = system.add_node (config, node_flags); - nano::keypair key, key2; - // Generating test chain - - nano::block_builder builder; - - auto send1 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (nano::dev::genesis->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (send1)); - auto open = builder - .open () - .source (send1->hash ()) - .representative (key.pub) - .account (key.pub) - .sign (key.prv, key.pub) - .work (*system.work.generate (key.pub)) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (open)); - auto send2 = builder - .state () - .account (key.pub) - .previous (open->hash ()) - .representative (key.pub) - .balance (0) - .link (key2.pub) - .sign (key.prv, key.pub) - .work (*system.work.generate (open->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (send2)); - - // Start lazy bootstrap with last block in chain known - auto node2 = system.make_disconnected_node (std::nullopt, node_flags); - nano::test::establish_tcp (system, *node2, node1->network.endpoint ()); - node2->bootstrap_initiator.bootstrap_lazy (send2->hash ()); - // Check processed blocks - ASSERT_TIMELY (15s, !node2->bootstrap_initiator.in_progress ()); - ASSERT_TIMELY (15s, nano::test::block_or_pruned_all_exists (*node2, { send1, open, send2 })); - ASSERT_EQ (1, node2->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in)); -} - -TEST (bootstrap_processor, lazy_destinations) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - node_flags.disable_legacy_bootstrap = true; - node_flags.disable_ascending_bootstrap = true; - node_flags.disable_ongoing_bootstrap = true; - auto node1 = system.add_node (config, node_flags); - nano::keypair key1, key2; - - nano::block_builder builder; - - // send Knano_ratio raw from genesis to key1 - auto send1 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (nano::dev::genesis->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (send1)); - - // send Knano_ratio raw from genesis to key2 - auto send2 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (send1->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - 2 * nano::Knano_ratio) - .link (key2.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (send1->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (send2)); - - // receive send1 on key1 - auto open = builder - .open () - .source (send1->hash ()) - .representative (key1.pub) - .account (key1.pub) - .sign (key1.prv, key1.pub) - .work (*system.work.generate (key1.pub)) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (open)); - - // receive send2 on key2 - auto state_open = builder - .state () - .account (key2.pub) - .previous (0) - .representative (key2.pub) - .balance (nano::Knano_ratio) - .link (send2->hash ()) - .sign (key2.prv, key2.pub) - .work (*system.work.generate (key2.pub)) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (state_open)); - - // Start lazy bootstrap with last block in sender chain - auto node2 = system.make_disconnected_node (std::nullopt, node_flags); - nano::test::establish_tcp (system, *node2, node1->network.endpoint ()); - node2->bootstrap_initiator.bootstrap_lazy (send2->hash ()); - - // Check processed blocks - ASSERT_TIMELY (5s, !node2->bootstrap_initiator.in_progress ()); - ASSERT_TIMELY (5s, node2->block_or_pruned_exists (send1->hash ())); - ASSERT_TIMELY (5s, node2->block_or_pruned_exists (send2->hash ())); - ASSERT_FALSE (node2->block_or_pruned_exists (open->hash ())); - ASSERT_FALSE (node2->block_or_pruned_exists (state_open->hash ())); -} - -TEST (bootstrap_processor, lazy_pruning_missing_block) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - config.enable_voting = false; // Remove after allowing pruned voting - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - node_flags.disable_legacy_bootstrap = true; - node_flags.disable_ascending_bootstrap = true; - node_flags.disable_ongoing_bootstrap = true; - node_flags.enable_pruning = true; - auto node1 = system.add_node (config, node_flags); - nano::keypair key1, key2; - - nano::block_builder builder; - - // send from genesis to key1 - auto send1 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (nano::dev::genesis->hash ())) - .build (); - - // send from genesis to key2 - auto send2 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (send1->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - 2 * nano::Knano_ratio) - .link (key2.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (send1->hash ())) - .build (); - - // open account key1 - auto open = builder - .open () - .source (send1->hash ()) - .representative (key1.pub) - .account (key1.pub) - .sign (key1.prv, key1.pub) - .work (*system.work.generate (key1.pub)) - .build (); - - // open account key2 - auto state_open = builder - .state () - .account (key2.pub) - .previous (0) - .representative (key2.pub) - .balance (nano::Knano_ratio) - .link (send2->hash ()) - .sign (key2.prv, key2.pub) - .work (*system.work.generate (key2.pub)) - .build (); - - // add the blocks without starting elections because elections publish blocks - // and the publishing would interefere with the testing - std::vector> const blocks{ send1, send2, open, state_open }; - ASSERT_TRUE (nano::test::process (*node1, blocks)); - ASSERT_TIMELY (5s, nano::test::exists (*node1, blocks)); - nano::test::confirm (node1->ledger, blocks); - ASSERT_TIMELY (5s, nano::test::confirmed (*node1, blocks)); - ASSERT_EQ (5, node1->ledger.block_count ()); - ASSERT_EQ (5, node1->ledger.cemented_count ()); - - // Pruning action, send1 should get pruned - ASSERT_EQ (0, node1->ledger.pruned_count ()); - node1->ledger_pruning (2, false); - ASSERT_EQ (1, node1->ledger.pruned_count ()); - ASSERT_EQ (5, node1->ledger.block_count ()); - ASSERT_TRUE (node1->ledger.store.pruned.exists (node1->ledger.store.tx_begin_read (), send1->hash ())); - ASSERT_TRUE (nano::test::exists (*node1, { send2, open, state_open })); - - // Start lazy bootstrap with last block in sender chain - config.peering_port = system.get_available_port (); - auto node2 = system.make_disconnected_node (config, node_flags); - nano::test::establish_tcp (system, *node2, node1->network.endpoint ()); - node2->bootstrap_initiator.bootstrap_lazy (send2->hash ()); - - // Check processed blocks - auto lazy_attempt (node2->bootstrap_initiator.current_lazy_attempt ()); - ASSERT_NE (nullptr, lazy_attempt); - ASSERT_TIMELY (5s, lazy_attempt->stopped || lazy_attempt->requeued_pulls >= 4); - - // Some blocks cannot be retrieved from pruned node - ASSERT_EQ (1, node2->ledger.block_count ()); - ASSERT_TRUE (nano::test::block_or_pruned_none_exists (*node2, { send1, send2, open, state_open })); - { - auto transaction (node2->store.tx_begin_read ()); - ASSERT_TRUE (node2->unchecked.exists (nano::unchecked_key (send2->root ().as_block_hash (), send2->hash ()))); - } - - // Insert missing block - node2->process_active (send1); - ASSERT_TIMELY_EQ (5s, 3, node2->ledger.block_count ()); - ASSERT_TIMELY (5s, nano::test::exists (*node2, { send1, send2 })); - ASSERT_TRUE (nano::test::block_or_pruned_none_exists (*node2, { open, state_open })); -} - -TEST (bootstrap_processor, lazy_cancel) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - auto node0 (system.add_node (config, node_flags)); - nano::keypair key1; - // Generating test chain - - auto send1 = nano::state_block_builder () - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (nano::dev::genesis->hash ())) - .build (); - - // Start lazy bootstrap with last block in chain known - auto node1 = system.make_disconnected_node (); - nano::test::establish_tcp (system, *node1, node0->network.endpoint ()); - node1->bootstrap_initiator.bootstrap_lazy (send1->hash (), true); // Start "confirmed" block bootstrap - { - auto lazy_attempt (node1->bootstrap_initiator.current_lazy_attempt ()); - ASSERT_NE (nullptr, lazy_attempt); - ASSERT_EQ (send1->hash ().to_string (), lazy_attempt->id); - } - // Cancel failing lazy bootstrap - ASSERT_TIMELY (10s, !node1->bootstrap_initiator.in_progress ()); -} - -TEST (bootstrap_processor, wallet_lazy_frontier) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - node_flags.disable_legacy_bootstrap = true; - node_flags.disable_ascending_bootstrap = true; - node_flags.disable_ongoing_bootstrap = true; - auto node0 = system.add_node (config, node_flags); - nano::keypair key1; - nano::keypair key2; - // Generating test chain - - nano::state_block_builder builder; - - auto send1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (nano::dev::genesis->hash ())) - .build (); - auto receive1 = builder - .make_block () - .account (key1.pub) - .previous (0) - .representative (key1.pub) - .balance (nano::Knano_ratio) - .link (send1->hash ()) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (key1.pub)) - .build (); - auto send2 = builder - .make_block () - .account (key1.pub) - .previous (receive1->hash ()) - .representative (key1.pub) - .balance (0) - .link (key2.pub) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (receive1->hash ())) - .build (); - auto receive2 = builder - .make_block () - .account (key2.pub) - .previous (0) - .representative (key2.pub) - .balance (nano::Knano_ratio) - .link (send2->hash ()) - .sign (key2.prv, key2.pub) - .work (*node0->work_generate_blocking (key2.pub)) - .build (); - - // Processing test chain - node0->block_processor.add (send1); - node0->block_processor.add (receive1); - node0->block_processor.add (send2); - node0->block_processor.add (receive2); - ASSERT_TIMELY (5s, nano::test::exists (*node0, { send1, receive1, send2, receive2 })); - - // Start wallet lazy bootstrap - auto node1 = system.make_disconnected_node (); - nano::test::establish_tcp (system, *node1, node0->network.endpoint ()); - auto wallet (node1->wallets.create (nano::random_wallet_id ())); - ASSERT_NE (nullptr, wallet); - wallet->insert_adhoc (key2.prv); - node1->bootstrap_wallet (); - { - auto wallet_attempt (node1->bootstrap_initiator.current_wallet_attempt ()); - ASSERT_NE (nullptr, wallet_attempt); - ASSERT_EQ (key2.pub.to_account (), wallet_attempt->id); - } - // Check processed blocks - ASSERT_TIMELY (10s, node1->block_or_pruned_exists (receive2->hash ())); -} - -TEST (bootstrap_processor, wallet_lazy_pending) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - node_flags.disable_legacy_bootstrap = true; - node_flags.disable_ascending_bootstrap = true; - node_flags.disable_ongoing_bootstrap = true; - auto node0 = system.add_node (config, node_flags); - nano::keypair key1; - nano::keypair key2; - // Generating test chain - - nano::state_block_builder builder; - - auto send1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (nano::dev::genesis->hash ())) - .build (); - auto receive1 = builder - .make_block () - .account (key1.pub) - .previous (0) - .representative (key1.pub) - .balance (nano::Knano_ratio) - .link (send1->hash ()) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (key1.pub)) - .build (); - auto send2 = builder - .make_block () - .account (key1.pub) - .previous (receive1->hash ()) - .representative (key1.pub) - .balance (0) - .link (key2.pub) - .sign (key1.prv, key1.pub) - .work (*node0->work_generate_blocking (receive1->hash ())) - .build (); - - // Processing test chain - node0->block_processor.add (send1); - node0->block_processor.add (receive1); - node0->block_processor.add (send2); - nano::test::exists (*node0, { send1, receive1, send2 }); - - // Start wallet lazy bootstrap - auto node1 = system.add_node (); - nano::test::establish_tcp (system, *node1, node0->network.endpoint ()); - auto wallet (node1->wallets.create (nano::random_wallet_id ())); - ASSERT_NE (nullptr, wallet); - wallet->insert_adhoc (key2.prv); - node1->bootstrap_wallet (); - // Check processed blocks - ASSERT_TIMELY (10s, node1->block_or_pruned_exists (send2->hash ())); -} - -TEST (bootstrap_processor, multiple_attempts) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - auto node1 = system.add_node (config, node_flags); - nano::keypair key1; - nano::keypair key2; - // Generating test chain - - nano::state_block_builder builder; - - auto send1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node1->work_generate_blocking (nano::dev::genesis->hash ())) - .build (); - auto receive1 = builder - .make_block () - .account (key1.pub) - .previous (0) - .representative (key1.pub) - .balance (nano::Knano_ratio) - .link (send1->hash ()) - .sign (key1.prv, key1.pub) - .work (*node1->work_generate_blocking (key1.pub)) - .build (); - auto send2 = builder - .make_block () - .account (key1.pub) - .previous (receive1->hash ()) - .representative (key1.pub) - .balance (0) - .link (key2.pub) - .sign (key1.prv, key1.pub) - .work (*node1->work_generate_blocking (receive1->hash ())) - .build (); - auto receive2 = builder - .make_block () - .account (key2.pub) - .previous (0) - .representative (key2.pub) - .balance (nano::Knano_ratio) - .link (send2->hash ()) - .sign (key2.prv, key2.pub) - .work (*node1->work_generate_blocking (key2.pub)) - .build (); - - // Processing test chain - node1->block_processor.add (send1); - node1->block_processor.add (receive1); - node1->block_processor.add (send2); - node1->block_processor.add (receive2); - nano::test::exists (*node1, { send1, receive1, send2, receive2 }); - - // Start 2 concurrent bootstrap attempts - nano::node_config node_config = system.default_config (); - node_config.bootstrap_initiator_threads = 3; - - auto node2 = system.make_disconnected_node (node_config); - nano::test::establish_tcp (system, *node2, node1->network.endpoint ()); - node2->bootstrap_initiator.bootstrap_lazy (receive2->hash (), true); - node2->bootstrap_initiator.bootstrap (); - auto lazy_attempt (node2->bootstrap_initiator.current_lazy_attempt ()); - auto legacy_attempt (node2->bootstrap_initiator.current_attempt ()); - ASSERT_TIMELY (5s, lazy_attempt->started && legacy_attempt->started); - // Check processed blocks - ASSERT_TIMELY (10s, node2->balance (key2.pub) != 0); - // Check attempts finish - ASSERT_TIMELY_EQ (5s, node2->bootstrap_initiator.attempts.size (), 0); -} - -TEST (frontier_req_response, DISABLED_destruction) -{ - { - std::shared_ptr hold; // Destructing tcp acceptor on non-existent io_context - { - nano::test::system system (1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start.clear (); - req->age = std::numeric_limitsage)>::max (); - req->count = std::numeric_limitscount)>::max (); - hold = std::make_shared (connection, std::move (req)); - } - } - ASSERT_TRUE (true); -} - -TEST (frontier_req, begin) -{ - nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start.clear (); - req->age = std::numeric_limitsage)>::max (); - req->count = std::numeric_limitscount)>::max (); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (nano::dev::genesis_key.pub, request->current); - ASSERT_EQ (nano::dev::genesis->hash (), request->frontier); -} - -TEST (frontier_req, end) -{ - nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start = nano::dev::genesis_key.pub.number () + 1; - req->age = std::numeric_limitsage)>::max (); - req->count = std::numeric_limitscount)>::max (); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_TRUE (request->current.is_zero ()); -} - -TEST (frontier_req, count) -{ - nano::test::system system (1); - auto node1 = system.nodes[0]; - // Public key FB93... after genesis in accounts table - nano::keypair key1 ("ED5AE0A6505B14B67435C29FD9FEEBC26F597D147BC92F6D795FFAD7AFD3D967"); - nano::state_block_builder builder; - - auto send1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (0) - .build (); - node1->work_generate_blocking (*send1); - ASSERT_EQ (nano::block_status::progress, node1->process (send1)); - auto receive1 = builder - .make_block () - .account (key1.pub) - .previous (0) - .representative (nano::dev::genesis_key.pub) - .balance (nano::Knano_ratio) - .link (send1->hash ()) - .sign (key1.prv, key1.pub) - .work (0) - .build (); - node1->work_generate_blocking (*receive1); - ASSERT_EQ (nano::block_status::progress, node1->process (receive1)); - - auto connection (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); - auto req = std::make_unique (nano::dev::network_params.network); - req->start.clear (); - req->age = std::numeric_limitsage)>::max (); - req->count = 1; - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (nano::dev::genesis_key.pub, request->current); - ASSERT_EQ (send1->hash (), request->frontier); -} - -TEST (frontier_req, time_bound) -{ - nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start.clear (); - req->age = 1; - req->count = std::numeric_limitscount)>::max (); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (nano::dev::genesis_key.pub, request->current); - // Wait 2 seconds until age of account will be > 1 seconds - std::this_thread::sleep_for (std::chrono::milliseconds (2100)); - auto req2 (std::make_unique (nano::dev::network_params.network)); - req2->start.clear (); - req2->age = 1; - req2->count = std::numeric_limitscount)>::max (); - auto connection2 (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto request2 (std::make_shared (connection, std::move (req2))); - ASSERT_TRUE (request2->current.is_zero ()); -} - -TEST (frontier_req, time_cutoff) -{ - nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto req = std::make_unique (nano::dev::network_params.network); - req->start.clear (); - req->age = 3; - req->count = std::numeric_limitscount)>::max (); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (nano::dev::genesis_key.pub, request->current); - ASSERT_EQ (nano::dev::genesis->hash (), request->frontier); - // Wait 4 seconds until age of account will be > 3 seconds - std::this_thread::sleep_for (std::chrono::milliseconds (4100)); - auto req2 (std::make_unique (nano::dev::network_params.network)); - req2->start.clear (); - req2->age = 3; - req2->count = std::numeric_limitscount)>::max (); - auto connection2 (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - auto request2 (std::make_shared (connection, std::move (req2))); - ASSERT_TRUE (request2->frontier.is_zero ()); -} - -TEST (frontier_req, confirmed_frontier) -{ - nano::test::system system (1); - auto node1 = system.nodes[0]; - nano::keypair key_before_genesis; - // Public key before genesis in accounts table - while (key_before_genesis.pub.number () >= nano::dev::genesis_key.pub.number ()) - { - key_before_genesis = nano::keypair (); - } - nano::keypair key_after_genesis; - // Public key after genesis in accounts table - while (key_after_genesis.pub.number () <= nano::dev::genesis_key.pub.number ()) - { - key_after_genesis = nano::keypair (); - } - nano::state_block_builder builder; - - auto send1 = builder - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key_before_genesis.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (0) - .build (); - node1->work_generate_blocking (*send1); - ASSERT_EQ (nano::block_status::progress, node1->process (send1)); - auto send2 = builder - .make_block () - .account (nano::dev::genesis_key.pub) - .previous (send1->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - 2 * nano::Knano_ratio) - .link (key_after_genesis.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (0) - .build (); - node1->work_generate_blocking (*send2); - ASSERT_EQ (nano::block_status::progress, node1->process (send2)); - auto receive1 = builder - .make_block () - .account (key_before_genesis.pub) - .previous (0) - .representative (nano::dev::genesis_key.pub) - .balance (nano::Knano_ratio) - .link (send1->hash ()) - .sign (key_before_genesis.prv, key_before_genesis.pub) - .work (0) - .build (); - node1->work_generate_blocking (*receive1); - ASSERT_EQ (nano::block_status::progress, node1->process (receive1)); - auto receive2 = builder - .make_block () - .account (key_after_genesis.pub) - .previous (0) - .representative (nano::dev::genesis_key.pub) - .balance (nano::Knano_ratio) - .link (send2->hash ()) - .sign (key_after_genesis.prv, key_after_genesis.pub) - .work (0) - .build (); - node1->work_generate_blocking (*receive2); - ASSERT_EQ (nano::block_status::progress, node1->process (receive2)); - - // Request for all accounts (confirmed only) - auto connection (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); - auto req = std::make_unique (nano::dev::network_params.network); - req->start.clear (); - req->age = std::numeric_limitsage)>::max (); - req->count = std::numeric_limitscount)>::max (); - ASSERT_FALSE (req->header.frontier_req_is_only_confirmed_present ()); - req->header.flag_set (nano::message_header::frontier_req_only_confirmed); - ASSERT_TRUE (req->header.frontier_req_is_only_confirmed_present ()); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (nano::dev::genesis_key.pub, request->current); - ASSERT_EQ (nano::dev::genesis->hash (), request->frontier); - - // Request starting with account before genesis (confirmed only) - auto connection2 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); - auto req2 = std::make_unique (nano::dev::network_params.network); - req2->start = key_before_genesis.pub; - req2->age = std::numeric_limitsage)>::max (); - req2->count = std::numeric_limitscount)>::max (); - ASSERT_FALSE (req2->header.frontier_req_is_only_confirmed_present ()); - req2->header.flag_set (nano::message_header::frontier_req_only_confirmed); - ASSERT_TRUE (req2->header.frontier_req_is_only_confirmed_present ()); - auto request2 (std::make_shared (connection2, std::move (req2))); - ASSERT_EQ (nano::dev::genesis_key.pub, request2->current); - ASSERT_EQ (nano::dev::genesis->hash (), request2->frontier); - - // Request starting with account after genesis (confirmed only) - auto connection3 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); - auto req3 = std::make_unique (nano::dev::network_params.network); - req3->start = key_after_genesis.pub; - req3->age = std::numeric_limitsage)>::max (); - req3->count = std::numeric_limitscount)>::max (); - ASSERT_FALSE (req3->header.frontier_req_is_only_confirmed_present ()); - req3->header.flag_set (nano::message_header::frontier_req_only_confirmed); - ASSERT_TRUE (req3->header.frontier_req_is_only_confirmed_present ()); - auto request3 (std::make_shared (connection3, std::move (req3))); - ASSERT_TRUE (request3->current.is_zero ()); - ASSERT_TRUE (request3->frontier.is_zero ()); - - // Request for all accounts (unconfirmed blocks) - auto connection4 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); - auto req4 = std::make_unique (nano::dev::network_params.network); - req4->start.clear (); - req4->age = std::numeric_limitsage)>::max (); - req4->count = std::numeric_limitscount)>::max (); - ASSERT_FALSE (req4->header.frontier_req_is_only_confirmed_present ()); - auto request4 (std::make_shared (connection4, std::move (req4))); - ASSERT_EQ (key_before_genesis.pub, request4->current); - ASSERT_EQ (receive1->hash (), request4->frontier); - - // Request starting with account after genesis (unconfirmed blocks) - auto connection5 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); - auto req5 = std::make_unique (nano::dev::network_params.network); - req5->start = key_after_genesis.pub; - req5->age = std::numeric_limitsage)>::max (); - req5->count = std::numeric_limitscount)>::max (); - ASSERT_FALSE (req5->header.frontier_req_is_only_confirmed_present ()); - auto request5 (std::make_shared (connection5, std::move (req5))); - ASSERT_EQ (key_after_genesis.pub, request5->current); - ASSERT_EQ (receive2->hash (), request5->frontier); - - // Confirm account before genesis (confirmed only) - nano::test::confirm (node1->ledger, receive1); - auto connection6 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); - auto req6 = std::make_unique (nano::dev::network_params.network); - req6->start = key_before_genesis.pub; - req6->age = std::numeric_limitsage)>::max (); - req6->count = std::numeric_limitscount)>::max (); - ASSERT_FALSE (req6->header.frontier_req_is_only_confirmed_present ()); - req6->header.flag_set (nano::message_header::frontier_req_only_confirmed); - ASSERT_TRUE (req6->header.frontier_req_is_only_confirmed_present ()); - auto request6 (std::make_shared (connection6, std::move (req6))); - ASSERT_EQ (key_before_genesis.pub, request6->current); - ASSERT_EQ (receive1->hash (), request6->frontier); - - // Confirm account after genesis (confirmed only) - nano::test::confirm (node1->ledger, receive2); - auto connection7 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); - auto req7 = std::make_unique (nano::dev::network_params.network); - req7->start = key_after_genesis.pub; - req7->age = std::numeric_limitsage)>::max (); - req7->count = std::numeric_limitscount)>::max (); - ASSERT_FALSE (req7->header.frontier_req_is_only_confirmed_present ()); - req7->header.flag_set (nano::message_header::frontier_req_only_confirmed); - ASSERT_TRUE (req7->header.frontier_req_is_only_confirmed_present ()); - auto request7 (std::make_shared (connection7, std::move (req7))); - ASSERT_EQ (key_after_genesis.pub, request7->current); - ASSERT_EQ (receive2->hash (), request7->frontier); -} - -TEST (bulk, genesis) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - node_flags.disable_lazy_bootstrap = true; - auto node1 = system.add_node (config, node_flags); - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - - auto node2 = system.make_disconnected_node (); - nano::block_hash latest1 (node1->latest (nano::dev::genesis_key.pub)); - nano::block_hash latest2 (node2->latest (nano::dev::genesis_key.pub)); - ASSERT_EQ (latest1, latest2); - nano::keypair key2; - auto send (system.wallet (0)->send_action (nano::dev::genesis_key.pub, key2.pub, 100)); - ASSERT_NE (nullptr, send); - nano::block_hash latest3 (node1->latest (nano::dev::genesis_key.pub)); - ASSERT_NE (latest1, latest3); - - node2->bootstrap_initiator.bootstrap (node1->network.endpoint (), false); - ASSERT_TIMELY_EQ (10s, node2->latest (nano::dev::genesis_key.pub), node1->latest (nano::dev::genesis_key.pub)); - ASSERT_EQ (node2->latest (nano::dev::genesis_key.pub), node1->latest (nano::dev::genesis_key.pub)); -} - -TEST (bulk, offline_send) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - node_flags.disable_lazy_bootstrap = true; - - auto node1 = system.add_node (config, node_flags); - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - const auto amount = node1->config.receive_minimum.number (); - auto node2 = system.make_disconnected_node (); - nano::keypair key2; - auto wallet (node2->wallets.create (nano::random_wallet_id ())); - wallet->insert_adhoc (key2.prv); - - // send amount from genesis to key2, it will be autoreceived - auto send1 = system.wallet (0)->send_action (nano::dev::genesis_key.pub, key2.pub, amount); - ASSERT_NE (nullptr, send1); - - // Wait to finish election background tasks - ASSERT_TIMELY (5s, node1->active.empty ()); - ASSERT_TIMELY (5s, node1->block_confirmed (send1->hash ())); - ASSERT_EQ (std::numeric_limits::max () - amount, node1->balance (nano::dev::genesis_key.pub)); - - // Initiate bootstrap - node2->bootstrap_initiator.bootstrap (node1->network.endpoint ()); - - // Nodes should find each other after bootstrap initiation - ASSERT_TIMELY (5s, !node1->network.empty ()); - ASSERT_TIMELY (5s, !node2->network.empty ()); - - // Send block arrival via bootstrap - ASSERT_TIMELY_EQ (5s, node2->balance (nano::dev::genesis_key.pub), std::numeric_limits::max () - amount); - // Receiving send block - ASSERT_TIMELY_EQ (5s, node2->balance (key2.pub), amount); -} - -TEST (bulk, genesis_pruning) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.backlog_population.enable = false; - config.enable_voting = false; // Remove after allowing pruned voting - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - node_flags.disable_lazy_bootstrap = true; - node_flags.disable_ongoing_bootstrap = true; - node_flags.disable_ascending_bootstrap = true; - node_flags.enable_pruning = true; - - auto node1 = system.add_node (config, node_flags); - auto blocks = nano::test::setup_chain (system, *node1, 3); - auto send1 = blocks[0]; - auto send2 = blocks[1]; - auto send3 = blocks[2]; - - ASSERT_EQ (4, node1->ledger.block_count ()); - node1->ledger_pruning (2, false); - ASSERT_EQ (2, node1->ledger.pruned_count ()); - ASSERT_EQ (4, node1->ledger.block_count ()); - ASSERT_TRUE (node1->ledger.store.pruned.exists (node1->ledger.store.tx_begin_read (), send1->hash ())); - ASSERT_FALSE (nano::test::exists (*node1, { send1 })); - ASSERT_TRUE (node1->ledger.store.pruned.exists (node1->ledger.store.tx_begin_read (), send2->hash ())); - ASSERT_FALSE (nano::test::exists (*node1, { send2 })); - ASSERT_TRUE (nano::test::exists (*node1, { send3 })); - - // Bootstrap with missing blocks for node2 - node_flags.enable_pruning = false; - auto node2 = system.make_disconnected_node (std::nullopt, node_flags); - node2->bootstrap_initiator.bootstrap (node1->network.endpoint (), false); - node2->network.merge_peer (node1->network.endpoint ()); - ASSERT_TIMELY (5s, node2->stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate, nano::stat::dir::out) >= 1); - ASSERT_TIMELY (5s, !node2->bootstrap_initiator.in_progress ()); - - // node2 still missing blocks - ASSERT_EQ (1, node2->ledger.block_count ()); - { - auto transaction (node2->store.tx_begin_write ()); - node2->unchecked.clear (); - } - - // Insert pruned blocks - node2->process_active (send1); - node2->process_active (send2); - ASSERT_TIMELY_EQ (5s, 3, node2->ledger.block_count ()); - - // New bootstrap to sync up everything - ASSERT_TIMELY_EQ (5s, node2->bootstrap_initiator.connections->connections_count, 0); - node2->bootstrap_initiator.bootstrap (node1->network.endpoint (), false); - ASSERT_TIMELY_EQ (5s, node2->latest (nano::dev::genesis_key.pub), node1->latest (nano::dev::genesis_key.pub)); -} - -TEST (bulk_pull_account, basics) -{ - nano::test::system system (1); - system.nodes[0]->config.receive_minimum = 20; - nano::keypair key1; - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - system.wallet (0)->insert_adhoc (key1.prv); - auto send1 (system.wallet (0)->send_action (nano::dev::genesis_key.pub, key1.pub, 25)); - auto send2 (system.wallet (0)->send_action (nano::dev::genesis_key.pub, key1.pub, 10)); - auto send3 (system.wallet (0)->send_action (nano::dev::genesis_key.pub, key1.pub, 2)); - ASSERT_TIMELY_EQ (5s, system.nodes[0]->balance (key1.pub), 25); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); - - { - auto req = std::make_unique (nano::dev::network_params.network); - req->account = key1.pub; - req->minimum_amount = 5; - req->flags = nano::bulk_pull_account_flags (); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_FALSE (request->invalid_request); - ASSERT_FALSE (request->pending_include_address); - ASSERT_FALSE (request->pending_address_only); - ASSERT_EQ (request->current_key.account, key1.pub); - ASSERT_EQ (request->current_key.hash, 0); - auto block_data (request->get_next ()); - ASSERT_EQ (send2->hash (), block_data.first.get ()->hash); - ASSERT_EQ (nano::uint128_union (10), block_data.second.get ()->amount); - ASSERT_EQ (nano::dev::genesis_key.pub, block_data.second.get ()->source); - ASSERT_EQ (nullptr, request->get_next ().first.get ()); - } - - { - auto req = std::make_unique (nano::dev::network_params.network); - req->account = key1.pub; - req->minimum_amount = 0; - req->flags = nano::bulk_pull_account_flags::pending_address_only; - auto request (std::make_shared (connection, std::move (req))); - ASSERT_TRUE (request->pending_address_only); - auto block_data (request->get_next ()); - ASSERT_NE (nullptr, block_data.first.get ()); - ASSERT_NE (nullptr, block_data.second.get ()); - ASSERT_EQ (nano::dev::genesis_key.pub, block_data.second.get ()->source); - block_data = request->get_next (); - ASSERT_EQ (nullptr, block_data.first.get ()); - ASSERT_EQ (nullptr, block_data.second.get ()); - } -} - -TEST (block_deserializer, construction) -{ - auto deserializer = std::make_shared (); -} diff --git a/nano/core_test/network.cpp b/nano/core_test/network.cpp index 3ae42f764..7d95fcf35 100644 --- a/nano/core_test/network.cpp +++ b/nano/core_test/network.cpp @@ -265,8 +265,6 @@ TEST (network, send_valid_publish) nano::test::system system (2, type, node_flags); auto & node1 (*system.nodes[0]); auto & node2 (*system.nodes[1]); - node1.bootstrap_initiator.stop (); - node2.bootstrap_initiator.stop (); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); nano::keypair key2; system.wallet (1)->insert_adhoc (key2.prv); diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index 5380ddde9..b38dbaa47 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -278,7 +278,6 @@ TEST (node, auto_bootstrap) system.nodes.push_back (node1); ASSERT_NE (nullptr, nano::test::establish_tcp (system, *node1, node0->network.endpoint ())); ASSERT_TIMELY_EQ (10s, node1->balance (key2.pub), node0->config.receive_minimum.number ()); - ASSERT_TIMELY (10s, !node1->bootstrap_initiator.in_progress ()); ASSERT_TRUE (node1->block_or_pruned_exists (send1->hash ())); // Wait block receive ASSERT_TIMELY_EQ (5s, node1->ledger.block_count (), 3); @@ -307,27 +306,6 @@ TEST (node, auto_bootstrap_reverse) ASSERT_TIMELY_EQ (10s, node1->balance (key2.pub), node0->config.receive_minimum.number ()); } -TEST (node, auto_bootstrap_age) -{ - nano::test::system system; - nano::node_config config (system.get_available_port ()); - config.backlog_population.enable = false; - nano::node_flags node_flags; - node_flags.disable_bootstrap_bulk_push_client = true; - node_flags.disable_lazy_bootstrap = true; - node_flags.bootstrap_interval = 1; - auto node0 = system.add_node (config, node_flags); - auto node1 (std::make_shared (system.io_ctx, system.get_available_port (), nano::unique_path (), system.work, node_flags)); - ASSERT_FALSE (node1->init_error ()); - node1->start (); - system.nodes.push_back (node1); - ASSERT_NE (nullptr, nano::test::establish_tcp (system, *node1, node0->network.endpoint ())); - // 4 bootstraps with frontiers age - ASSERT_TIMELY (10s, node0->stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate_legacy_age, nano::stat::dir::out) >= 3); - // More attempts with frontiers age - ASSERT_GE (node0->stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate_legacy_age, nano::stat::dir::out), node0->stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate, nano::stat::dir::out)); -} - TEST (node, merge_peers) { nano::test::system system (1); @@ -1107,7 +1085,6 @@ TEST (node, DISABLED_fork_stale) nano::test::system system2 (1); auto & node1 (*system1.nodes[0]); auto & node2 (*system2.nodes[0]); - node2.bootstrap_initiator.bootstrap (node1.network.endpoint (), false); auto channel = nano::test::establish_tcp (system1, node2, node1.network.endpoint ()); auto vote = std::make_shared (nano::dev::genesis_key.pub, nano::dev::genesis_key.prv, 0, 0, std::vector ()); @@ -1162,7 +1139,6 @@ TEST (node, DISABLED_fork_stale) node1.process_active (send2); node2.process_active (send1); node2.process_active (send2); - node2.bootstrap_initiator.bootstrap (node1.network.endpoint (), false); while (node2.block (send1->hash ()) == nullptr) { system1.poll (); @@ -1376,8 +1352,6 @@ TEST (node, DISABLED_bootstrap_no_publish) auto transaction = node0->ledger.tx_begin_write (); ASSERT_EQ (nano::block_status::progress, node0->ledger.process (transaction, send0)); } - ASSERT_FALSE (node1->bootstrap_initiator.in_progress ()); - node1->bootstrap_initiator.bootstrap (node0->network.endpoint (), false); ASSERT_TRUE (node1->active.empty ()); system1.deadline_set (10s); while (node1->block (send0->hash ()) == nullptr) @@ -1391,51 +1365,6 @@ TEST (node, DISABLED_bootstrap_no_publish) } } -// Check that an outgoing bootstrap request can push blocks -// Test disabled because it's failing intermittently. -// PR in which it got disabled: https://github.com/nanocurrency/nano-node/pull/3512 -// Issue for investigating it: https://github.com/nanocurrency/nano-node/issues/3515 -TEST (node, DISABLED_bootstrap_bulk_push) -{ - nano::test::system system; - nano::test::system system0; - nano::test::system system1; - nano::node_config config0 (system.get_available_port ()); - config0.backlog_population.enable = false; - auto node0 (system0.add_node (config0)); - nano::node_config config1 (system.get_available_port ()); - config1.backlog_population.enable = false; - auto node1 (system1.add_node (config1)); - nano::keypair key0; - // node0 knows about send0 but node1 doesn't. - auto send0 = nano::send_block_builder () - .previous (nano::dev::genesis->hash ()) - .destination (key0.pub) - .balance (500) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*node0->work_generate_blocking (nano::dev::genesis->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node0->process (send0)); - - ASSERT_FALSE (node0->bootstrap_initiator.in_progress ()); - ASSERT_FALSE (node1->bootstrap_initiator.in_progress ()); - ASSERT_TRUE (node1->active.empty ()); - node0->bootstrap_initiator.bootstrap (node1->network.endpoint (), false); - system1.deadline_set (10s); - while (node1->block (send0->hash ()) == nullptr) - { - ASSERT_NO_ERROR (system0.poll ()); - ASSERT_NO_ERROR (system1.poll ()); - } - // since this uses bulk_push, the new block should be republished - system1.deadline_set (10s); - while (node1->active.empty ()) - { - ASSERT_NO_ERROR (system0.poll ()); - ASSERT_NO_ERROR (system1.poll ()); - } -} - // Bootstrapping a forked open block should succeed. TEST (node, bootstrap_fork_open) { @@ -1486,8 +1415,6 @@ TEST (node, bootstrap_fork_open) ASSERT_EQ (nano::block_status::progress, node1->process (open1)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); ASSERT_FALSE (node1->block_or_pruned_exists (open0->hash ())); - ASSERT_FALSE (node1->bootstrap_initiator.in_progress ()); - node1->bootstrap_initiator.bootstrap (node0->network.endpoint (), false); ASSERT_TIMELY (1s, node1->active.empty ()); ASSERT_TIMELY (10s, !node1->block_or_pruned_exists (open1->hash ()) && node1->block_or_pruned_exists (open0->hash ())); } @@ -1495,12 +1422,10 @@ TEST (node, bootstrap_fork_open) // Unconfirmed blocks from bootstrap should be confirmed TEST (node, bootstrap_confirm_frontiers) { - // create 2 separate systems, the 2 system do not interact with each other automatically - nano::test::system system0 (1); - nano::test::system system1 (1); - auto node0 = system0.nodes[0]; - auto node1 = system1.nodes[0]; - system0.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); + nano::test::system system; + auto node0 = system.add_node (); + auto node1 = system.add_node (); + system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); nano::keypair key0; // create block to send 500 raw from genesis to key0 and save into node0 ledger without immediately triggering an election @@ -1512,25 +1437,7 @@ TEST (node, bootstrap_confirm_frontiers) .work (*node0->work_generate_blocking (nano::dev::genesis->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, node0->process (send0)); - - // each system only has one node, so there should be no bootstrapping going on - ASSERT_FALSE (node0->bootstrap_initiator.in_progress ()); - ASSERT_FALSE (node1->bootstrap_initiator.in_progress ()); - ASSERT_TRUE (node1->active.empty ()); - - // create a bootstrap connection from node1 to node0 - // this also has the side effect of adding node0 to node1's list of peers, which will trigger realtime connections too - node1->bootstrap_initiator.bootstrap (node0->network.endpoint ()); - - // Wait until the block is confirmed on node1. Poll more than usual because we are polling - // on 2 different systems at once and in sequence and there might be strange timing effects. - system0.deadline_set (10s); - system1.deadline_set (10s); - while (!node1->ledger.confirmed.block_exists_or_pruned (node1->ledger.tx_begin_read (), send0->hash ())) - { - ASSERT_NO_ERROR (system0.poll (std::chrono::milliseconds (1))); - ASSERT_NO_ERROR (system1.poll (std::chrono::milliseconds (1))); - } + ASSERT_TIMELY (10s, node1->block_confirmed (send0->hash ())); } // Test that if we create a block that isn't confirmed, the bootstrapping processes sync the missing block. @@ -1644,37 +1551,6 @@ TEST (node, balance_observer) } } -TEST (node, bootstrap_connection_scaling) -{ - nano::test::system system (1); - auto & node1 (*system.nodes[0]); - ASSERT_EQ (34, node1.bootstrap_initiator.connections->target_connections (5000, 1)); - ASSERT_EQ (4, node1.bootstrap_initiator.connections->target_connections (0, 1)); - ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (50000, 1)); - ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (10000000000, 1)); - ASSERT_EQ (32, node1.bootstrap_initiator.connections->target_connections (5000, 0)); - ASSERT_EQ (1, node1.bootstrap_initiator.connections->target_connections (0, 0)); - ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (50000, 0)); - ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (10000000000, 0)); - ASSERT_EQ (36, node1.bootstrap_initiator.connections->target_connections (5000, 2)); - ASSERT_EQ (8, node1.bootstrap_initiator.connections->target_connections (0, 2)); - ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (50000, 2)); - ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (10000000000, 2)); - node1.config.bootstrap_connections = 128; - ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (0, 1)); - ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (50000, 1)); - ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (0, 2)); - ASSERT_EQ (64, node1.bootstrap_initiator.connections->target_connections (50000, 2)); - node1.config.bootstrap_connections_max = 256; - ASSERT_EQ (128, node1.bootstrap_initiator.connections->target_connections (0, 1)); - ASSERT_EQ (256, node1.bootstrap_initiator.connections->target_connections (50000, 1)); - ASSERT_EQ (256, node1.bootstrap_initiator.connections->target_connections (0, 2)); - ASSERT_EQ (256, node1.bootstrap_initiator.connections->target_connections (50000, 2)); - node1.config.bootstrap_connections_max = 0; - ASSERT_EQ (1, node1.bootstrap_initiator.connections->target_connections (0, 1)); - ASSERT_EQ (1, node1.bootstrap_initiator.connections->target_connections (50000, 1)); -} - TEST (node, online_reps) { nano::test::system system (1); diff --git a/nano/core_test/websocket.cpp b/nano/core_test/websocket.cpp index 90991493d..da16b4160 100644 --- a/nano/core_test/websocket.cpp +++ b/nano/core_test/websocket.cpp @@ -840,121 +840,6 @@ TEST (websocket, work) ASSERT_EQ (contents.get ("reason"), ""); } -// Test client subscribing to notifications for bootstrap -TEST (websocket, bootstrap) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.websocket_config.enabled = true; - config.websocket_config.port = system.get_available_port (); - auto node1 (system.add_node (config)); - - ASSERT_EQ (0, node1->websocket.server->subscriber_count (nano::websocket::topic::bootstrap)); - - // Subscribe to bootstrap and wait for response asynchronously - std::atomic ack_ready{ false }; - auto task = ([&ack_ready, config, &node1] () { - fake_websocket_client client (node1->websocket.server->listening_port ()); - client.send_message (R"json({"action": "subscribe", "topic": "bootstrap", "ack": true})json"); - client.await_ack (); - ack_ready = true; - EXPECT_EQ (1, node1->websocket.server->subscriber_count (nano::websocket::topic::bootstrap)); - return client.get_response (); - }); - auto future = std::async (std::launch::async, task); - - // Wait for acknowledge - ASSERT_TIMELY (5s, ack_ready); - - // Start bootstrap attempt - node1->bootstrap_initiator.bootstrap (true, "123abc"); - ASSERT_TIMELY_EQ (5s, nullptr, node1->bootstrap_initiator.current_attempt ()); - - // Wait for the bootstrap notification - ASSERT_TIMELY_EQ (5s, future.wait_for (0s), std::future_status::ready); - - // Check the bootstrap notification message - auto response = future.get (); - ASSERT_TRUE (response); - std::stringstream stream; - stream << response; - boost::property_tree::ptree event; - boost::property_tree::read_json (stream, event); - ASSERT_EQ (event.get ("topic"), "bootstrap"); - - auto & contents = event.get_child ("message"); - ASSERT_EQ (contents.get ("reason"), "started"); - ASSERT_EQ (contents.get ("id"), "123abc"); - ASSERT_EQ (contents.get ("mode"), "legacy"); - - // Wait for bootstrap finish - ASSERT_TIMELY (5s, !node1->bootstrap_initiator.in_progress ()); -} - -TEST (websocket, bootstrap_exited) -{ - nano::test::system system; - nano::node_config config = system.default_config (); - config.websocket_config.enabled = true; - config.websocket_config.port = system.get_available_port (); - auto node1 (system.add_node (config)); - - // Start bootstrap, exit after subscription - std::atomic bootstrap_started{ false }; - nano::test::counted_completion subscribed_completion (1); - std::thread bootstrap_thread ([node1, &system, &bootstrap_started, &subscribed_completion] () { - std::shared_ptr attempt; - while (attempt == nullptr) - { - std::this_thread::sleep_for (50ms); - node1->bootstrap_initiator.bootstrap (true, "123abc"); - attempt = node1->bootstrap_initiator.current_attempt (); - } - ASSERT_NE (nullptr, attempt); - bootstrap_started = true; - EXPECT_FALSE (subscribed_completion.await_count_for (5s)); - }); - - // Wait for bootstrap start - ASSERT_TIMELY (5s, bootstrap_started); - - // Subscribe to bootstrap and wait for response asynchronously - std::atomic ack_ready{ false }; - auto task = ([&ack_ready, config, &node1] () { - fake_websocket_client client (node1->websocket.server->listening_port ()); - client.send_message (R"json({"action": "subscribe", "topic": "bootstrap", "ack": true})json"); - client.await_ack (); - ack_ready = true; - EXPECT_EQ (1, node1->websocket.server->subscriber_count (nano::websocket::topic::bootstrap)); - return client.get_response (); - }); - auto future = std::async (std::launch::async, task); - - // Wait for acknowledge - ASSERT_TIMELY (5s, ack_ready); - - // Wait for the bootstrap notification - subscribed_completion.increment (); - bootstrap_thread.join (); - ASSERT_TIMELY_EQ (5s, future.wait_for (0s), std::future_status::ready); - - // Check the bootstrap notification message - auto response = future.get (); - ASSERT_TRUE (response); - std::stringstream stream; - stream << response; - boost::property_tree::ptree event; - boost::property_tree::read_json (stream, event); - ASSERT_EQ (event.get ("topic"), "bootstrap"); - - auto & contents = event.get_child ("message"); - ASSERT_EQ (contents.get ("reason"), "exited"); - ASSERT_EQ (contents.get ("id"), "123abc"); - ASSERT_EQ (contents.get ("mode"), "legacy"); - ASSERT_EQ (contents.get ("total_blocks"), 0U); - ASSERT_LT (contents.get ("duration"), 15000U); -} - // Tests sending keepalive TEST (websocket, ws_keepalive) { diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index f7d9a962e..60c0e516b 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -26,24 +26,8 @@ add_library( bootstrap_weights_live.hpp bootstrap/block_deserializer.hpp bootstrap/block_deserializer.cpp - bootstrap/bootstrap_attempt.hpp - bootstrap/bootstrap_attempt.cpp - bootstrap/bootstrap_bulk_pull.hpp - bootstrap/bootstrap_bulk_pull.cpp - bootstrap/bootstrap_bulk_push.hpp - bootstrap/bootstrap_bulk_push.cpp bootstrap/bootstrap_config.hpp bootstrap/bootstrap_config.cpp - bootstrap/bootstrap_connections.hpp - bootstrap/bootstrap_connections.cpp - bootstrap/bootstrap_frontier.hpp - bootstrap/bootstrap_frontier.cpp - bootstrap/bootstrap_lazy.hpp - bootstrap/bootstrap_lazy.cpp - bootstrap/bootstrap_legacy.hpp - bootstrap/bootstrap_legacy.cpp - bootstrap/bootstrap.hpp - bootstrap/bootstrap.cpp bootstrap/bootstrap_server.hpp bootstrap/bootstrap_server.cpp bootstrap_ascending/account_sets.hpp diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp deleted file mode 100644 index 8779400ee..000000000 --- a/nano/node/bootstrap/bootstrap.cpp +++ /dev/null @@ -1,390 +0,0 @@ -#include -#include -#include -#include -#include - -#include -#include - -nano::bootstrap_initiator::bootstrap_initiator (nano::node & node_a) : - node (node_a) -{ - connections = std::make_shared (node); - bootstrap_initiator_threads.push_back (boost::thread ([this] () { - nano::thread_role::set (nano::thread_role::name::bootstrap_connections); - connections->run (); - })); - for (std::size_t i = 0; i < node.config.bootstrap_initiator_threads; ++i) - { - bootstrap_initiator_threads.push_back (boost::thread ([this] () { - nano::thread_role::set (nano::thread_role::name::bootstrap_initiator); - run_bootstrap (); - })); - } -} - -nano::bootstrap_initiator::~bootstrap_initiator () -{ - stop (); -} - -void nano::bootstrap_initiator::bootstrap (bool force, std::string id_a, uint32_t const frontiers_age_a, nano::account const & start_account_a) -{ - if (force) - { - stop_attempts (); - } - nano::unique_lock lock{ mutex }; - if (!stopped && find_attempt (nano::bootstrap_mode::legacy) == nullptr) - { - node.stats.inc (nano::stat::type::bootstrap, frontiers_age_a == std::numeric_limits::max () ? nano::stat::detail::initiate : nano::stat::detail::initiate_legacy_age, nano::stat::dir::out); - auto legacy_attempt (std::make_shared (node.shared (), attempts.incremental++, id_a, frontiers_age_a, start_account_a)); - attempts_list.push_back (legacy_attempt); - attempts.add (legacy_attempt); - lock.unlock (); - condition.notify_all (); - } -} - -void nano::bootstrap_initiator::bootstrap (nano::endpoint const & endpoint_a, bool add_to_peers, std::string id_a) -{ - if (add_to_peers) - { - if (!node.flags.disable_tcp_realtime) - { - node.network.merge_peer (nano::transport::map_endpoint_to_v6 (endpoint_a)); - } - } - if (!stopped) - { - stop_attempts (); - node.stats.inc (nano::stat::type::bootstrap, nano::stat::detail::initiate, nano::stat::dir::out); - nano::lock_guard lock{ mutex }; - auto legacy_attempt (std::make_shared (node.shared (), attempts.incremental++, id_a, std::numeric_limits::max (), 0)); - attempts_list.push_back (legacy_attempt); - attempts.add (legacy_attempt); - if (!node.network.excluded_peers.check (nano::transport::map_endpoint_to_tcp (endpoint_a))) - { - connections->add_connection (endpoint_a); - } - } - condition.notify_all (); -} - -bool nano::bootstrap_initiator::bootstrap_lazy (nano::hash_or_account const & hash_or_account_a, bool force, std::string id_a) -{ - bool key_inserted (false); - auto lazy_attempt (current_lazy_attempt ()); - if (lazy_attempt == nullptr || force) - { - if (force) - { - stop_attempts (); - } - node.stats.inc (nano::stat::type::bootstrap, nano::stat::detail::initiate_lazy, nano::stat::dir::out); - nano::lock_guard lock{ mutex }; - if (!stopped && find_attempt (nano::bootstrap_mode::lazy) == nullptr) - { - lazy_attempt = std::make_shared (node.shared (), attempts.incremental++, id_a.empty () ? hash_or_account_a.to_string () : id_a); - attempts_list.push_back (lazy_attempt); - attempts.add (lazy_attempt); - key_inserted = lazy_attempt->lazy_start (hash_or_account_a); - } - } - else - { - key_inserted = lazy_attempt->lazy_start (hash_or_account_a); - } - condition.notify_all (); - return key_inserted; -} - -void nano::bootstrap_initiator::bootstrap_wallet (std::deque & accounts_a) -{ - debug_assert (!accounts_a.empty ()); - auto wallet_attempt (current_wallet_attempt ()); - node.stats.inc (nano::stat::type::bootstrap, nano::stat::detail::initiate_wallet_lazy, nano::stat::dir::out); - if (wallet_attempt == nullptr) - { - nano::lock_guard lock{ mutex }; - std::string id (!accounts_a.empty () ? accounts_a[0].to_account () : ""); - wallet_attempt = std::make_shared (node.shared (), attempts.incremental++, id); - attempts_list.push_back (wallet_attempt); - attempts.add (wallet_attempt); - wallet_attempt->wallet_start (accounts_a); - } - else - { - wallet_attempt->wallet_start (accounts_a); - } - condition.notify_all (); -} - -void nano::bootstrap_initiator::run_bootstrap () -{ - nano::unique_lock lock{ mutex }; - while (!stopped) - { - if (has_new_attempts ()) - { - auto attempt (new_attempt ()); - lock.unlock (); - if (attempt != nullptr) - { - attempt->run (); - remove_attempt (attempt); - } - lock.lock (); - } - else - { - condition.wait (lock); - } - } -} - -void nano::bootstrap_initiator::lazy_requeue (nano::block_hash const & hash_a, nano::block_hash const & previous_a) -{ - auto lazy_attempt (current_lazy_attempt ()); - if (lazy_attempt != nullptr) - { - lazy_attempt->lazy_requeue (hash_a, previous_a); - } -} - -void nano::bootstrap_initiator::add_observer (std::function const & observer_a) -{ - nano::lock_guard lock{ observers_mutex }; - observers.push_back (observer_a); -} - -bool nano::bootstrap_initiator::in_progress () -{ - nano::lock_guard lock{ mutex }; - return !attempts_list.empty (); -} - -void nano::bootstrap_initiator::block_processed (store::transaction const & tx, nano::block_status const & result, nano::block const & block) -{ - nano::lock_guard lock{ mutex }; - for (auto & i : attempts_list) - { - i->block_processed (tx, result, block); - } -} - -std::shared_ptr nano::bootstrap_initiator::find_attempt (nano::bootstrap_mode mode_a) -{ - for (auto & i : attempts_list) - { - if (i->mode == mode_a) - { - return i; - } - } - return nullptr; -} - -void nano::bootstrap_initiator::remove_attempt (std::shared_ptr attempt_a) -{ - nano::unique_lock lock{ mutex }; - auto attempt (std::find (attempts_list.begin (), attempts_list.end (), attempt_a)); - if (attempt != attempts_list.end ()) - { - auto attempt_ptr (*attempt); - attempts.remove (attempt_ptr->incremental_id); - attempts_list.erase (attempt); - debug_assert (attempts.size () == attempts_list.size ()); - lock.unlock (); - attempt_ptr->stop (); - } - else - { - lock.unlock (); - } - condition.notify_all (); -} - -std::shared_ptr nano::bootstrap_initiator::new_attempt () -{ - for (auto & i : attempts_list) - { - if (!i->started.exchange (true)) - { - return i; - } - } - return nullptr; -} - -bool nano::bootstrap_initiator::has_new_attempts () -{ - for (auto & i : attempts_list) - { - if (!i->started) - { - return true; - } - } - return false; -} - -std::shared_ptr nano::bootstrap_initiator::current_attempt () -{ - nano::lock_guard lock{ mutex }; - return find_attempt (nano::bootstrap_mode::legacy); -} - -std::shared_ptr nano::bootstrap_initiator::current_lazy_attempt () -{ - nano::lock_guard lock{ mutex }; - return std::dynamic_pointer_cast (find_attempt (nano::bootstrap_mode::lazy)); -} - -std::shared_ptr nano::bootstrap_initiator::current_wallet_attempt () -{ - nano::lock_guard lock{ mutex }; - return std::dynamic_pointer_cast (find_attempt (nano::bootstrap_mode::wallet_lazy)); -} - -void nano::bootstrap_initiator::stop_attempts () -{ - nano::unique_lock lock{ mutex }; - std::vector> copy_attempts; - copy_attempts.swap (attempts_list); - attempts.clear (); - lock.unlock (); - for (auto & i : copy_attempts) - { - i->stop (); - } -} - -void nano::bootstrap_initiator::stop () -{ - if (!stopped.exchange (true)) - { - stop_attempts (); - connections->stop (); - condition.notify_all (); - - for (auto & thread : bootstrap_initiator_threads) - { - if (thread.joinable ()) - { - thread.join (); - } - } - } -} - -void nano::bootstrap_initiator::notify_listeners (bool in_progress_a) -{ - nano::lock_guard lock{ observers_mutex }; - for (auto & i : observers) - { - i (in_progress_a); - } -} - -nano::container_info nano::bootstrap_initiator::container_info () const -{ - nano::container_info info; - { - nano::lock_guard guard{ observers_mutex }; - info.put ("observers", observers.size ()); - } - { - nano::lock_guard guard{ cache.pulls_cache_mutex }; - info.put ("pulls_cache", cache.cache.size ()); - } - return info; -} - -void nano::pulls_cache::add (nano::pull_info const & pull_a) -{ - if (pull_a.processed > 500) - { - nano::lock_guard guard{ pulls_cache_mutex }; - // Clean old pull - if (cache.size () > cache_size_max) - { - cache.erase (cache.begin ()); - } - debug_assert (cache.size () <= cache_size_max); - nano::uint512_union head_512 (pull_a.account_or_head.as_union (), pull_a.head_original); - auto existing (cache.get ().find (head_512)); - if (existing == cache.get ().end ()) - { - // Insert new pull - auto inserted (cache.emplace (nano::cached_pulls{ std::chrono::steady_clock::now (), head_512, pull_a.head })); - (void)inserted; - debug_assert (inserted.second); - } - else - { - // Update existing pull - cache.get ().modify (existing, [pull_a] (nano::cached_pulls & cache_a) { - cache_a.time = std::chrono::steady_clock::now (); - cache_a.new_head = pull_a.head; - }); - } - } -} - -void nano::pulls_cache::update_pull (nano::pull_info & pull_a) -{ - nano::lock_guard guard{ pulls_cache_mutex }; - nano::uint512_union head_512 (pull_a.account_or_head.as_union (), pull_a.head_original); - auto existing (cache.get ().find (head_512)); - if (existing != cache.get ().end ()) - { - pull_a.head = existing->new_head; - } -} - -void nano::pulls_cache::remove (nano::pull_info const & pull_a) -{ - nano::lock_guard guard{ pulls_cache_mutex }; - nano::uint512_union head_512 (pull_a.account_or_head.as_union (), pull_a.head_original); - cache.get ().erase (head_512); -} - -void nano::bootstrap_attempts::add (std::shared_ptr attempt_a) -{ - nano::lock_guard lock{ bootstrap_attempts_mutex }; - attempts.emplace (attempt_a->incremental_id, attempt_a); -} - -void nano::bootstrap_attempts::remove (uint64_t incremental_id_a) -{ - nano::lock_guard lock{ bootstrap_attempts_mutex }; - attempts.erase (incremental_id_a); -} - -void nano::bootstrap_attempts::clear () -{ - nano::lock_guard lock{ bootstrap_attempts_mutex }; - attempts.clear (); -} - -std::shared_ptr nano::bootstrap_attempts::find (uint64_t incremental_id_a) -{ - nano::lock_guard lock{ bootstrap_attempts_mutex }; - auto find_attempt (attempts.find (incremental_id_a)); - if (find_attempt != attempts.end ()) - { - return find_attempt->second; - } - else - { - return nullptr; - } -} - -std::size_t nano::bootstrap_attempts::size () -{ - nano::lock_guard lock{ bootstrap_attempts_mutex }; - return attempts.size (); -} diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp deleted file mode 100644 index f06f83196..000000000 --- a/nano/node/bootstrap/bootstrap.hpp +++ /dev/null @@ -1,157 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include -#include -#include - -#include - -namespace mi = boost::multi_index; - -namespace nano::store -{ -class transaction; -} - -namespace nano -{ -class node; - -class bootstrap_connections; -namespace transport -{ - class tcp_channel; -} -enum class bootstrap_mode -{ - legacy, - lazy, - wallet_lazy, - ascending -}; -enum class sync_result -{ - success, - error, - fork -}; -class cached_pulls final -{ -public: - std::chrono::steady_clock::time_point time; - nano::uint512_union account_head; - nano::block_hash new_head; -}; -class pulls_cache final -{ -public: - void add (nano::pull_info const &); - void update_pull (nano::pull_info &); - void remove (nano::pull_info const &); - mutable nano::mutex pulls_cache_mutex; - class account_head_tag - { - }; - // clang-format off - boost::multi_index_container>, - mi::hashed_unique, - mi::member>>> - cache; - // clang-format on - constexpr static std::size_t cache_size_max = 10000; -}; - -/** - * Container for bootstrap sessions that are active. Owned by bootstrap_initiator. - */ -class bootstrap_attempts final -{ -public: - void add (std::shared_ptr); - void remove (uint64_t); - void clear (); - std::shared_ptr find (uint64_t); - std::size_t size (); - std::atomic incremental{ 0 }; - nano::mutex bootstrap_attempts_mutex; - std::map> attempts; -}; - -class bootstrap_attempt_lazy; -class bootstrap_attempt_wallet; -/** - * Client side portion to initiate bootstrap sessions. Prevents multiple legacy-type bootstrap sessions from being started at the same time. Does permit - * lazy/wallet bootstrap sessions to overlap with legacy sessions. - */ -class bootstrap_initiator final -{ -public: - explicit bootstrap_initiator (nano::node &); - ~bootstrap_initiator (); - void bootstrap (nano::endpoint const &, bool add_to_peers = true, std::string id_a = ""); - void bootstrap (bool force = false, std::string id_a = "", uint32_t const frontiers_age_a = std::numeric_limits::max (), nano::account const & start_account_a = nano::account{}); - bool bootstrap_lazy (nano::hash_or_account const &, bool force = false, std::string id_a = ""); - void bootstrap_wallet (std::deque &); - void run_bootstrap (); - void lazy_requeue (nano::block_hash const &, nano::block_hash const &); - void notify_listeners (bool); - void add_observer (std::function const &); - bool in_progress (); - void block_processed (store::transaction const & tx, nano::block_status const & result, nano::block const & block); - std::shared_ptr connections; - std::shared_ptr new_attempt (); - bool has_new_attempts (); - void remove_attempt (std::shared_ptr); - std::shared_ptr current_attempt (); - std::shared_ptr current_lazy_attempt (); - std::shared_ptr current_wallet_attempt (); - nano::pulls_cache cache; - nano::bootstrap_attempts attempts; - void stop (); - nano::container_info container_info () const; - -private: - nano::node & node; - std::shared_ptr find_attempt (nano::bootstrap_mode); - void stop_attempts (); - std::vector> attempts_list; - std::atomic stopped{ false }; - mutable nano::mutex mutex; - nano::condition_variable condition; - mutable nano::mutex observers_mutex; - std::vector> observers; - std::vector bootstrap_initiator_threads; -}; - -/** - * Defines the numeric values for the bootstrap feature. - */ -class bootstrap_limits final -{ -public: - static constexpr double bootstrap_connection_scale_target_blocks = 10000.0; - static constexpr double bootstrap_connection_warmup_time_sec = 5.0; - static constexpr double bootstrap_minimum_blocks_per_sec = 10.0; - static constexpr double bootstrap_minimum_elapsed_seconds_blockrate = 0.02; - static constexpr double bootstrap_minimum_frontier_blocks_per_sec = 1000.0; - static constexpr double bootstrap_minimum_termination_time_sec = 30.0; - static constexpr unsigned bootstrap_max_new_connections = 32; - static constexpr unsigned requeued_pulls_limit = 256; - static constexpr unsigned requeued_pulls_limit_dev = 1; - static constexpr unsigned requeued_pulls_processed_blocks_factor = 4096; - static constexpr uint64_t pull_count_per_check = 8 * 1024; - static constexpr unsigned bulk_push_cost_limit = 200; - static constexpr std::chrono::seconds lazy_flush_delay_sec = std::chrono::seconds (5); - static constexpr uint64_t lazy_batch_pull_count_resize_blocks_limit = 4 * 1024 * 1024; - static constexpr double lazy_batch_pull_count_resize_ratio = 2.0; - static constexpr std::size_t lazy_blocks_restart_limit = 1024 * 1024; -}; -} diff --git a/nano/node/bootstrap/bootstrap_attempt.cpp b/nano/node/bootstrap/bootstrap_attempt.cpp deleted file mode 100644 index e40a9e0f8..000000000 --- a/nano/node/bootstrap/bootstrap_attempt.cpp +++ /dev/null @@ -1,145 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include - -constexpr unsigned nano::bootstrap_limits::requeued_pulls_limit; -constexpr unsigned nano::bootstrap_limits::requeued_pulls_limit_dev; - -nano::bootstrap_attempt::bootstrap_attempt (std::shared_ptr const & node_a, nano::bootstrap_mode mode_a, uint64_t incremental_id_a, std::string id_a) : - node (node_a), - incremental_id (incremental_id_a), - id (id_a), - mode (mode_a) -{ - if (id.empty ()) - { - id = nano::hardened_constants::get ().random_128.to_string (); - } - - node_a->logger.debug (nano::log::type::bootstrap, "Starting bootstrap attempt with ID: {} (mode: {})", mode_text (), id); - - node_a->bootstrap_initiator.notify_listeners (true); - if (node_a->websocket.server) - { - nano::websocket::message_builder builder; - node_a->websocket.server->broadcast (builder.bootstrap_started (id, mode_text ())); - } -} - -nano::bootstrap_attempt::~bootstrap_attempt () -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - - node->logger.debug (nano::log::type::bootstrap, "Exiting bootstrap attempt with ID: {} (mode: {})", mode_text (), id); - - node->bootstrap_initiator.notify_listeners (false); - if (node->websocket.server) - { - nano::websocket::message_builder builder; - node->websocket.server->broadcast (builder.bootstrap_exited (id, mode_text (), attempt_start, total_blocks)); - } -} - -bool nano::bootstrap_attempt::should_log () -{ - nano::lock_guard guard{ next_log_mutex }; - auto result (false); - auto now (std::chrono::steady_clock::now ()); - if (next_log < now) - { - result = true; - next_log = now + std::chrono::seconds (15); - } - return result; -} - -bool nano::bootstrap_attempt::still_pulling () -{ - debug_assert (!mutex.try_lock ()); - auto running (!stopped); - auto still_pulling (pulling > 0); - return running && still_pulling; -} - -void nano::bootstrap_attempt::pull_started () -{ - { - nano::lock_guard guard{ mutex }; - ++pulling; - } - condition.notify_all (); -} - -void nano::bootstrap_attempt::pull_finished () -{ - { - nano::lock_guard guard{ mutex }; - --pulling; - } - condition.notify_all (); -} - -void nano::bootstrap_attempt::stop () -{ - { - nano::lock_guard lock{ mutex }; - stopped = true; - } - condition.notify_all (); - auto node_l = node.lock (); - if (!node_l) - { - return; - } - node_l->bootstrap_initiator.connections->clear_pulls (incremental_id); -} - -char const * nano::bootstrap_attempt::mode_text () -{ - switch (mode) - { - case nano::bootstrap_mode::legacy: - return "legacy"; - case nano::bootstrap_mode::lazy: - return "lazy"; - case nano::bootstrap_mode::wallet_lazy: - return "wallet_lazy"; - case nano::bootstrap_mode::ascending: - return "ascending"; - } - return "unknown"; -} - -bool nano::bootstrap_attempt::process_block (std::shared_ptr const & block_a, nano::account const & known_account_a, uint64_t pull_blocks_processed, nano::bulk_pull::count_t max_blocks, bool block_expected, unsigned retry_limit) -{ - auto node_l = node.lock (); - if (!node_l) - { - return true; - } - bool stop_pull (false); - // If block already exists in the ledger, then we can avoid next part of long account chain - if (pull_blocks_processed % nano::bootstrap_limits::pull_count_per_check == 0 && node_l->block_or_pruned_exists (block_a->hash ())) - { - stop_pull = true; - } - else - { - node_l->block_processor.add (block_a, nano::block_source::bootstrap_legacy); - } - return stop_pull; -} - -void nano::bootstrap_attempt::block_processed (store::transaction const & tx, nano::block_status const & result, nano::block const & block) -{ -} diff --git a/nano/node/bootstrap/bootstrap_attempt.hpp b/nano/node/bootstrap/bootstrap_attempt.hpp deleted file mode 100644 index ea64fe9fd..000000000 --- a/nano/node/bootstrap/bootstrap_attempt.hpp +++ /dev/null @@ -1,54 +0,0 @@ -#pragma once - -#include - -#include -#include - -namespace nano::store -{ -class transaction; -} - -namespace nano -{ -class node; - -class frontier_req_client; -class bulk_push_client; - -/** - * Polymorphic base class for bootstrap sessions. - */ -class bootstrap_attempt : public std::enable_shared_from_this -{ -public: - explicit bootstrap_attempt (std::shared_ptr const & node_a, nano::bootstrap_mode mode_a, uint64_t incremental_id_a, std::string id_a); - virtual ~bootstrap_attempt (); - virtual void run () = 0; - virtual void stop (); - bool still_pulling (); - void pull_started (); - void pull_finished (); - bool should_log (); - char const * mode_text (); - virtual bool process_block (std::shared_ptr const &, nano::account const &, uint64_t, nano::bulk_pull::count_t, bool, unsigned); - virtual void get_information (boost::property_tree::ptree &) = 0; - virtual void block_processed (store::transaction const & tx, nano::block_status const & result, nano::block const & block); - nano::mutex next_log_mutex; - std::chrono::steady_clock::time_point next_log{ std::chrono::steady_clock::now () }; - std::atomic pulling{ 0 }; - std::weak_ptr node; - std::atomic total_blocks{ 0 }; - std::atomic requeued_pulls{ 0 }; - std::atomic started{ false }; - std::atomic stopped{ false }; - uint64_t incremental_id{ 0 }; - std::string id; - std::chrono::steady_clock::time_point attempt_start{ std::chrono::steady_clock::now () }; - std::atomic frontiers_received{ false }; - nano::bootstrap_mode mode; - nano::mutex mutex; - nano::condition_variable condition; -}; -} diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.cpp b/nano/node/bootstrap/bootstrap_bulk_pull.cpp deleted file mode 100644 index f17b08f93..000000000 --- a/nano/node/bootstrap/bootstrap_bulk_pull.cpp +++ /dev/null @@ -1,912 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -nano::pull_info::pull_info (nano::hash_or_account const & account_or_head_a, nano::block_hash const & head_a, nano::block_hash const & end_a, uint64_t bootstrap_id_a, count_t count_a, unsigned retry_limit_a) : - account_or_head (account_or_head_a), - head (head_a), - head_original (head_a), - end (end_a), - count (count_a), - retry_limit (retry_limit_a), - bootstrap_id (bootstrap_id_a) -{ -} - -nano::bulk_pull_client::bulk_pull_client (std::shared_ptr const & connection_a, std::shared_ptr const & attempt_a, nano::pull_info const & pull_a) : - connection{ connection_a }, - attempt{ attempt_a }, - pull{ pull_a }, - block_deserializer{ std::make_shared () } -{ - attempt->condition.notify_all (); -} - -nano::bulk_pull_client::~bulk_pull_client () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - /* If received end block is not expected end block - Or if given start and end blocks are from different chains (i.e. forked node or malicious node) */ - if (expected != pull.end && !expected.is_zero ()) - { - pull.head = expected; - if (attempt->mode != nano::bootstrap_mode::legacy) - { - pull.account_or_head = expected; - } - pull.processed += pull_blocks - unexpected_count; - node->bootstrap_initiator.connections->requeue_pull (pull, network_error); - - node->logger.debug (nano::log::type::bulk_pull_client, "Bulk pull end block is not expected {} for account {} or head block {}", pull.end.to_string (), pull.account_or_head.to_account (), pull.account_or_head.to_string ()); - } - else - { - node->bootstrap_initiator.cache.remove (pull); - } - attempt->pull_finished (); -} - -void nano::bulk_pull_client::request () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - debug_assert (!pull.head.is_zero () || pull.retry_limit <= node->network_params.bootstrap.lazy_retry_limit); - expected = pull.head; - nano::bulk_pull req{ node->network_params.network }; - if (pull.head == pull.head_original && pull.attempts % 4 < 3) - { - // Account for new pulls - req.start = pull.account_or_head; - } - else - { - // Head for cached pulls or accounts with public key equal to existing block hash (25% of attempts) - req.start = pull.head; - } - req.end = pull.end; - req.count = pull.count; - req.set_count_present (pull.count != 0); - - node->logger.trace (nano::log::type::bulk_pull_client, nano::log::detail::requesting_account_or_head, - nano::log::arg{ "account_or_head", pull.account_or_head }, - nano::log::arg{ "channel", connection->channel }); - - if (attempt->should_log ()) - { - node->logger.debug (nano::log::type::bulk_pull_client, "Accounts in pull queue: {}", attempt->pulling.load ()); - } - - auto this_l (shared_from_this ()); - connection->channel->send ( - req, [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - auto node = this_l->connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - this_l->throttled_receive_block (); - } - else - { - node->logger.debug (nano::log::type::bulk_pull_client, "Error sending bulk pull request to: {} ({})", this_l->connection->channel->to_string (), ec.message ()); - node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_request_failure, nano::stat::dir::in); - } - }, - nano::transport::buffer_drop_policy::no_limiter_drop); -} - -void nano::bulk_pull_client::throttled_receive_block () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - debug_assert (!network_error); - if (node->block_processor.size (nano::block_source::bootstrap_legacy) < 1024 && !node->block_processor.flushing) - { - receive_block (); - } - else - { - auto this_l (shared_from_this ()); - node->workers.post_delayed (std::chrono::seconds (1), [this_l] () { - if (!this_l->connection->pending_stop && !this_l->attempt->stopped) - { - this_l->throttled_receive_block (); - } - }); - } -} - -void nano::bulk_pull_client::receive_block () -{ - block_deserializer->read (*connection->socket, [this_l = shared_from_this ()] (boost::system::error_code ec, std::shared_ptr block) { - this_l->received_block (ec, block); - }); -} - -void nano::bulk_pull_client::received_block (boost::system::error_code ec, std::shared_ptr block) -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (ec) - { - network_error = true; - return; - } - if (block == nullptr) - { - // Avoid re-using slow peers, or peers that sent the wrong blocks. - if (!connection->pending_stop && (expected == pull.end || (pull.count != 0 && pull.count == pull_blocks))) - { - node->bootstrap_initiator.connections->pool_connection (connection); - } - return; - } - if (node->network_params.work.validate_entry (*block)) - { - node->logger.debug (nano::log::type::bulk_pull_client, "Insufficient work for bulk pull block: {}", block->hash ().to_string ()); - node->stats.inc (nano::stat::type::error, nano::stat::detail::insufficient_work); - return; - } - auto hash = block->hash (); - - node->logger.trace (nano::log::type::bulk_pull_client, nano::log::detail::pulled_block, nano::log::arg{ "block", block }); - - // Is block expected? - bool block_expected (false); - // Unconfirmed head is used only for lazy destinations if legacy bootstrap is not available, see nano::bootstrap_attempt::lazy_destinations_increment (...) - bool unconfirmed_account_head = node->flags.disable_legacy_bootstrap && pull_blocks == 0 && pull.retry_limit <= node->network_params.bootstrap.lazy_retry_limit && (expected == pull.account_or_head.as_block_hash ()) && (block->account_field ().value_or (0) == pull.account_or_head.as_account ()); - if (hash == expected || unconfirmed_account_head) - { - expected = block->previous (); - block_expected = true; - } - else - { - unexpected_count++; - } - if (pull_blocks == 0 && block_expected) - { - known_account = block->account_field ().value_or (0); - } - if (connection->block_count++ == 0) - { - connection->set_start_time (std::chrono::steady_clock::now ()); - } - attempt->total_blocks++; - pull_blocks++; - bool stop_pull (attempt->process_block (block, known_account, pull_blocks, pull.count, block_expected, pull.retry_limit)); - if (!stop_pull && !connection->hard_stop.load ()) - { - /* Process block in lazy pull if not stopped - Stop usual pull request with unexpected block & more than 16k blocks processed - to prevent spam */ - if (attempt->mode != nano::bootstrap_mode::legacy || unexpected_count < 16384) - { - throttled_receive_block (); - } - } - else if (!stop_pull && block_expected) - { - node->bootstrap_initiator.connections->pool_connection (connection); - } -} - -nano::bulk_pull_account_client::bulk_pull_account_client (std::shared_ptr const & connection_a, std::shared_ptr const & attempt_a, nano::account const & account_a) : - connection (connection_a), - attempt (attempt_a), - account (account_a), - pull_blocks (0) -{ - attempt->condition.notify_all (); -} - -nano::bulk_pull_account_client::~bulk_pull_account_client () -{ - attempt->pull_finished (); -} - -void nano::bulk_pull_account_client::request () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - nano::bulk_pull_account req{ node->network_params.network }; - req.account = account; - req.minimum_amount = node->config.receive_minimum; - req.flags = nano::bulk_pull_account_flags::pending_hash_and_amount; - - node->logger.trace (nano::log::type::bulk_pull_account_client, nano::log::detail::requesting_pending, - nano::log::arg{ "account", req.account.to_account () }, // TODO: Convert to lazy eval - nano::log::arg{ "connection", connection->channel }); - - if (attempt->should_log ()) - { - node->logger.debug (nano::log::type::bulk_pull_account_client, "Accounts in pull queue: {}", attempt->wallet_size ()); - } - - auto this_l (shared_from_this ()); - connection->channel->send ( - req, [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - auto node = this_l->connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - this_l->receive_pending (); - } - else - { - node->logger.debug (nano::log::type::bulk_pull_account_client, "Error starting bulk pull request to: {} ({})", this_l->connection->channel->to_string (), ec.message ()); - node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_error_starting_request, nano::stat::dir::in); - - this_l->attempt->requeue_pending (this_l->account); - } - }, - nano::transport::buffer_drop_policy::no_limiter_drop); -} - -void nano::bulk_pull_account_client::receive_pending () -{ - auto this_l (shared_from_this ()); - std::size_t size_l (sizeof (nano::uint256_union) + sizeof (nano::uint128_union)); - connection->socket->async_read (connection->receive_buffer, size_l, [this_l, size_l] (boost::system::error_code const & ec, std::size_t size_a) { - auto node = this_l->connection->node.lock (); - if (!node) - { - return; - } - // An issue with asio is that sometimes, instead of reporting a bad file descriptor during disconnect, - // we simply get a size of 0. - if (size_a == size_l) - { - if (!ec) - { - nano::block_hash pending; - nano::bufferstream frontier_stream (this_l->connection->receive_buffer->data (), sizeof (nano::uint256_union)); - auto error1 (nano::try_read (frontier_stream, pending)); - (void)error1; - debug_assert (!error1); - nano::amount balance; - nano::bufferstream balance_stream (this_l->connection->receive_buffer->data () + sizeof (nano::uint256_union), sizeof (nano::uint128_union)); - auto error2 (nano::try_read (balance_stream, balance)); - (void)error2; - debug_assert (!error2); - if (this_l->pull_blocks == 0 || !pending.is_zero ()) - { - if (this_l->pull_blocks == 0 || balance.number () >= node->config.receive_minimum.number ()) - { - this_l->pull_blocks++; - { - if (!pending.is_zero ()) - { - if (!node->block_or_pruned_exists (pending)) - { - node->bootstrap_initiator.bootstrap_lazy (pending, false); - } - } - } - this_l->receive_pending (); - } - else - { - this_l->attempt->requeue_pending (this_l->account); - } - } - else - { - node->bootstrap_initiator.connections->pool_connection (this_l->connection); - } - } - else - { - node->logger.debug (nano::log::type::bulk_pull_account_client, "Error while receiving bulk pull account frontier: {}", ec.message ()); - - this_l->attempt->requeue_pending (this_l->account); - } - } - else - { - node->logger.debug (nano::log::type::bulk_pull_account_client, "Invalid size: Expected {}, got: {}", size_l, size_a); - - this_l->attempt->requeue_pending (this_l->account); - } - }); -} - -/** - * Handle a request for the pull of all blocks associated with an account - * The account is supplied as the "start" member, and the final block to - * send is the "end" member. The "start" member may also be a block - * hash, in which case the that hash is used as the start of a chain - * to send. To determine if "start" is interpreted as an account or - * hash, the ledger is checked to see if the block specified exists, - * if not then it is interpreted as an account. - * - * Additionally, if "start" is specified as a block hash the range - * is inclusive of that block hash, that is the range will be: - * [start, end); In the case that a block hash is not specified the - * range will be exclusive of the frontier for that account with - * a range of (frontier, end) - */ -void nano::bulk_pull_server::set_current_end () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - include_start = false; - debug_assert (request != nullptr); - auto transaction = node->ledger.tx_begin_read (); - if (!node->ledger.any.block_exists (transaction, request->end)) - { - node->logger.debug (nano::log::type::bulk_pull_server, "Bulk pull end block doesn't exist: {}, sending everything", request->end.to_string ()); - - request->end.clear (); - } - - if (node->ledger.any.block_exists (transaction, request->start.as_block_hash ())) - { - node->logger.debug (nano::log::type::bulk_pull_server, "Bulk pull request for block hash: {}", request->start.to_string ()); - - current = ascending () ? node->ledger.any.block_successor (transaction, request->start.as_block_hash ()).value_or (0) : request->start.as_block_hash (); - include_start = true; - } - else - { - auto info = node->ledger.any.account_get (transaction, request->start.as_account ()); - if (!info) - { - node->logger.debug (nano::log::type::bulk_pull_server, "Request for unknown account: {}", request->start.to_account ()); - - current = request->end; - } - else - { - current = ascending () ? info->open_block : info->head; - if (!request->end.is_zero ()) - { - auto account (node->ledger.any.block_account (transaction, request->end)); - if (account.value_or (0) != request->start.as_account ()) - { - node->logger.debug (nano::log::type::bulk_pull_server, "Request for block that is not on account chain: {} not on {}", request->end.to_string (), request->start.to_account ()); - - current = request->end; - } - } - } - } - - sent_count = 0; - if (request->is_count_present ()) - { - max_count = request->count; - } - else - { - max_count = 0; - } -} - -void nano::bulk_pull_server::send_next () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - auto block = get_next (); - if (block != nullptr) - { - node->logger.trace (nano::log::type::bulk_pull_server, nano::log::detail::sending_block, - nano::log::arg{ "block", block }, - nano::log::arg{ "socket", connection->socket }); - - std::vector send_buffer; - { - nano::vectorstream stream (send_buffer); - nano::serialize_block (stream, *block); - } - - connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l = shared_from_this ()] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->sent_action (ec, size_a); - }); - } - else - { - send_finished (); - } -} - -std::shared_ptr nano::bulk_pull_server::get_next () -{ - auto node = connection->node.lock (); - if (!node) - { - return nullptr; - } - std::shared_ptr result; - bool send_current = false, set_current_to_end = false; - - /* - * Determine if we should reply with a block - * - * If our cursor is on the final block, we should signal that we - * are done by returning a null result. - * - * Unless we are including the "start" member and this is the - * start member, then include it anyway. - */ - if (current != request->end) - { - send_current = true; - } - else if (current == request->end && include_start == true) - { - send_current = true; - - /* - * We also need to ensure that the next time - * are invoked that we return a null result - */ - set_current_to_end = true; - } - - /* - * Account for how many blocks we have provided. If this - * exceeds the requested maximum, return an empty object - * to signal the end of results - */ - if (max_count != 0 && sent_count >= max_count) - { - send_current = false; - } - - if (send_current) - { - result = node->block (current); - if (result != nullptr && set_current_to_end == false) - { - auto next = ascending () ? result->sideband ().successor : result->previous (); - if (!next.is_zero ()) - { - current = next; - } - else - { - current = request->end; - } - } - else - { - current = request->end; - } - - sent_count++; - } - - /* - * Once we have processed "get_next()" once our cursor is no longer on - * the "start" member, so this flag is not relevant is always false. - */ - include_start = false; - - return result; -} - -void nano::bulk_pull_server::sent_action (boost::system::error_code const & ec, std::size_t size_a) -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - node->bootstrap_workers.post ([this_l = shared_from_this ()] () { - this_l->send_next (); - }); - } - else - { - node->logger.debug (nano::log::type::bulk_pull_server, "Unable to bulk send block: {}", ec.message ()); - } -} - -void nano::bulk_pull_server::send_finished () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - nano::shared_const_buffer send_buffer (static_cast (nano::block_type::not_a_block)); - auto this_l (shared_from_this ()); - - node->logger.debug (nano::log::type::bulk_pull_server, "Bulk sending finished"); - - connection->socket->async_write (send_buffer, [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->no_block_sent (ec, size_a); - }); -} - -void nano::bulk_pull_server::no_block_sent (boost::system::error_code const & ec, std::size_t size_a) -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - debug_assert (size_a == 1); - connection->start (); - } - else - { - node->logger.debug (nano::log::type::bulk_pull_server, "Unable to bulk send not-a-block: {}", ec.message ()); - } -} - -bool nano::bulk_pull_server::ascending () const -{ - return request->header.bulk_pull_ascending (); -} - -nano::bulk_pull_server::bulk_pull_server (std::shared_ptr const & connection_a, std::unique_ptr request_a) : - connection (connection_a), - request (std::move (request_a)) -{ - set_current_end (); -} - -/** - * Bulk pull blocks related to an account - */ -void nano::bulk_pull_account_server::set_params () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - debug_assert (request != nullptr); - - /* - * Parse the flags - */ - invalid_request = false; - pending_include_address = false; - pending_address_only = false; - if (request->flags == nano::bulk_pull_account_flags::pending_address_only) - { - pending_address_only = true; - } - else if (request->flags == nano::bulk_pull_account_flags::pending_hash_amount_and_address) - { - /** - ** This is the same as "pending_hash_and_amount" but with the - ** sending address appended, for UI purposes mainly. - **/ - pending_include_address = true; - } - else if (request->flags == nano::bulk_pull_account_flags::pending_hash_and_amount) - { - /** The defaults are set above **/ - } - else - { - node->logger.debug (nano::log::type::bulk_pull_account_server, "Invalid bulk_pull_account flags supplied: {}", static_cast (request->flags)); - - invalid_request = true; - return; - } - - /* - * Initialize the current item from the requested account - */ - current_key.account = request->account; - current_key.hash = 0; -} - -void nano::bulk_pull_account_server::send_frontier () -{ - /* - * This function is really the entry point into this class, - * so handle the invalid_request case by terminating the - * request without any response - */ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (!invalid_request) - { - auto stream_transaction = node->ledger.tx_begin_read (); - - // Get account balance and frontier block hash - auto account_frontier_hash (node->ledger.any.account_head (stream_transaction, request->account)); - auto account_frontier_balance_int (node->ledger.any.account_balance (stream_transaction, request->account).value_or (0)); - nano::uint128_union account_frontier_balance (account_frontier_balance_int); - - // Write the frontier block hash and balance into a buffer - std::vector send_buffer; - { - nano::vectorstream output_stream (send_buffer); - write (output_stream, account_frontier_hash.bytes); - write (output_stream, account_frontier_balance.bytes); - } - - // Send the buffer to the requestor - auto this_l (shared_from_this ()); - connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->sent_action (ec, size_a); - }); - } -} - -void nano::bulk_pull_account_server::send_next_block () -{ - /* - * Get the next item from the queue, it is a tuple with the key (which - * contains the account and hash) and data (which contains the amount) - */ - auto node = connection->node.lock (); - if (!node) - { - return; - } - auto block_data (get_next ()); - auto block_info_key (block_data.first.get ()); - auto block_info (block_data.second.get ()); - - if (block_info_key != nullptr) - { - /* - * If we have a new item, emit it to the socket - */ - - std::vector send_buffer; - if (pending_address_only) - { - node->logger.trace (nano::log::type::bulk_pull_account_server, nano::log::detail::sending_pending, - nano::log::arg{ "pending", block_info->source }); - - nano::vectorstream output_stream (send_buffer); - write (output_stream, block_info->source.bytes); - } - else - { - node->logger.trace (nano::log::type::bulk_pull_account_server, nano::log::detail::sending_block, - nano::log::arg{ "block", block_info_key->hash }); - - nano::vectorstream output_stream (send_buffer); - write (output_stream, block_info_key->hash.bytes); - write (output_stream, block_info->amount.bytes); - - if (pending_include_address) - { - /** - ** Write the source address as well, if requested - **/ - write (output_stream, block_info->source.bytes); - } - } - - auto this_l (shared_from_this ()); - connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->sent_action (ec, size_a); - }); - } - else - { - /* - * Otherwise, finalize the connection - */ - node->logger.debug (nano::log::type::bulk_pull_account_server, "Done sending blocks"); - - send_finished (); - } -} - -std::pair, std::unique_ptr> nano::bulk_pull_account_server::get_next () -{ - auto node = connection->node.lock (); - if (!node) - { - return { nullptr, nullptr }; - } - std::pair, std::unique_ptr> result; - - while (true) - { - /* - * For each iteration of this loop, establish and then - * destroy a database transaction, to avoid locking the - * database for a prolonged period. - */ - auto tx = node->ledger.tx_begin_read (); - auto & ledger = node->ledger; - auto stream = ledger.any.receivable_upper_bound (tx, current_key.account, current_key.hash); - - if (stream == ledger.any.receivable_end ()) - { - break; - } - auto const & [key, info] = *stream; - current_key = key; - - /* - * Skip entries where the amount is less than the requested - * minimum - */ - if (info.amount < request->minimum_amount) - { - continue; - } - - /* - * If the pending_address_only flag is set, de-duplicate the - * responses. The responses are the address of the sender, - * so they are part of the pending table's information - * and not key, so we have to de-duplicate them manually. - */ - if (pending_address_only) - { - if (!deduplication.insert (info.source).second) - { - /* - * If the deduplication map gets too - * large, clear it out. This may - * result in some duplicates getting - * sent to the client, but we do not - * want to commit too much memory - */ - if (deduplication.size () > 4096) - { - deduplication.clear (); - } - continue; - } - } - - result.first = std::make_unique (key); - result.second = std::make_unique (info); - - break; - } - - return result; -} - -void nano::bulk_pull_account_server::sent_action (boost::system::error_code const & ec, std::size_t size_a) -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - node->bootstrap_workers.post ([this_l = shared_from_this ()] () { - this_l->send_next_block (); - }); - } - else - { - node->logger.debug (nano::log::type::bulk_pull_account_server, "Unable to bulk send block: {}", ec.message ()); - } -} - -void nano::bulk_pull_account_server::send_finished () -{ - /* - * The "bulk_pull_account" final sequence is a final block of all - * zeros. If we are sending only account public keys (with the - * "pending_address_only" flag) then it will be 256-bits of zeros, - * otherwise it will be either 384-bits of zeros (if the - * "pending_include_address" flag is not set) or 640-bits of zeros - * (if that flag is set). - */ - auto node = connection->node.lock (); - if (!node) - { - return; - } - std::vector send_buffer; - { - nano::vectorstream output_stream (send_buffer); - nano::uint256_union account_zero (0); - nano::uint128_union balance_zero (0); - - write (output_stream, account_zero.bytes); - - if (!pending_address_only) - { - write (output_stream, balance_zero.bytes); - if (pending_include_address) - { - write (output_stream, account_zero.bytes); - } - } - } - - node->logger.debug (nano::log::type::bulk_pull_account_server, "Bulk sending for an account finished"); - - auto this_l (shared_from_this ()); - connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->complete (ec, size_a); - }); -} - -void nano::bulk_pull_account_server::complete (boost::system::error_code const & ec, std::size_t size_a) -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - if (pending_address_only) - { - debug_assert (size_a == 32); - } - else - { - if (pending_include_address) - { - debug_assert (size_a == 80); - } - else - { - debug_assert (size_a == 48); - } - } - - connection->start (); - } - else - { - node->logger.debug (nano::log::type::bulk_pull_account_server, "Unable to pending-as-zero: {}", ec.message ()); - } -} - -nano::bulk_pull_account_server::bulk_pull_account_server (std::shared_ptr const & connection_a, std::unique_ptr request_a) : - connection (connection_a), - request (std::move (request_a)), - current_key (0, 0) -{ - /* - * Setup the streaming response for the first call to "send_frontier" and "send_next_block" - */ - set_params (); -} diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.hpp b/nano/node/bootstrap/bootstrap_bulk_pull.hpp deleted file mode 100644 index a2e410db4..000000000 --- a/nano/node/bootstrap/bootstrap_bulk_pull.hpp +++ /dev/null @@ -1,140 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -namespace nano -{ -class bootstrap_attempt; -namespace transport -{ - class tcp_server; -} -namespace bootstrap -{ - class block_deserializer; -}; - -class pull_info -{ -public: - using count_t = nano::bulk_pull::count_t; - pull_info () = default; - pull_info (nano::hash_or_account const &, nano::block_hash const &, nano::block_hash const &, uint64_t, count_t = 0, unsigned = 16); - nano::hash_or_account account_or_head{ 0 }; - nano::block_hash head{ 0 }; - nano::block_hash head_original{ 0 }; - nano::block_hash end{ 0 }; - count_t count{ 0 }; - unsigned attempts{ 0 }; - uint64_t processed{ 0 }; - unsigned retry_limit{ 0 }; - uint64_t bootstrap_id{ 0 }; -}; -class bootstrap_client; - -/** - * Client side of a bulk_pull request. Created when the bootstrap_attempt wants to make a bulk_pull request to the remote side. - */ -class bulk_pull_client final : public std::enable_shared_from_this -{ -public: - bulk_pull_client (std::shared_ptr const &, std::shared_ptr const &, nano::pull_info const &); - ~bulk_pull_client (); - void request (); - void receive_block (); - void throttled_receive_block (); - void received_block (boost::system::error_code ec, std::shared_ptr block); - nano::block_hash first (); - std::shared_ptr connection; - std::shared_ptr attempt; - bool network_error{ false }; - -private: - /** - * Tracks the next block expected to be received starting with the block hash that was expected and followed by previous blocks for this account chain - */ - nano::block_hash expected{ 0 }; - /** - * Tracks the account number for this account chain - * Used when an account chain has a mix between state blocks and legacy blocks which do not encode the account number in the block - * 0 if the account is unknown - */ - nano::account known_account{ 0 }; - /** - * Original pull request - */ - nano::pull_info pull; - /** - * Tracks the number of blocks successfully deserialized - */ - uint64_t pull_blocks{ 0 }; - /** - * Tracks the number of times an unexpected block was received - */ - uint64_t unexpected_count{ 0 }; - std::shared_ptr block_deserializer; -}; -class bootstrap_attempt_wallet; -class bulk_pull_account_client final : public std::enable_shared_from_this -{ -public: - bulk_pull_account_client (std::shared_ptr const &, std::shared_ptr const &, nano::account const &); - ~bulk_pull_account_client (); - void request (); - void receive_pending (); - std::shared_ptr connection; - std::shared_ptr attempt; - nano::account account; - uint64_t pull_blocks; -}; - -class bulk_pull; - -/** - * Server side of a bulk_pull request. Created when tcp_server receives a bulk_pull message and is exited after the contents - * have been sent. If the 'start' in the bulk_pull message is an account, send blocks for that account down to 'end'. If the 'start' - * is a block hash, send blocks for that chain down to 'end'. If end doesn't exist, send all accounts in the chain. - */ -class bulk_pull_server final : public std::enable_shared_from_this -{ -public: - bulk_pull_server (std::shared_ptr const &, std::unique_ptr); - void set_current_end (); - std::shared_ptr get_next (); - void send_next (); - void sent_action (boost::system::error_code const &, std::size_t); - void send_finished (); - void no_block_sent (boost::system::error_code const &, std::size_t); - bool ascending () const; - std::shared_ptr connection; - std::unique_ptr request; - nano::block_hash current; - bool include_start; - nano::bulk_pull::count_t max_count; - nano::bulk_pull::count_t sent_count; -}; -class bulk_pull_account; -class bulk_pull_account_server final : public std::enable_shared_from_this -{ -public: - bulk_pull_account_server (std::shared_ptr const &, std::unique_ptr); - void set_params (); - std::pair, std::unique_ptr> get_next (); - void send_frontier (); - void send_next_block (); - void sent_action (boost::system::error_code const &, std::size_t); - void send_finished (); - void complete (boost::system::error_code const &, std::size_t); - std::shared_ptr connection; - std::unique_ptr request; - std::unordered_set deduplication; - nano::pending_key current_key; - bool pending_address_only; - bool pending_include_address; - bool invalid_request; -}; -} diff --git a/nano/node/bootstrap/bootstrap_bulk_push.cpp b/nano/node/bootstrap/bootstrap_bulk_push.cpp deleted file mode 100644 index c023ec1fe..000000000 --- a/nano/node/bootstrap/bootstrap_bulk_push.cpp +++ /dev/null @@ -1,279 +0,0 @@ -#include -#include -#include -#include -#include - -#include - -nano::bulk_push_client::bulk_push_client (std::shared_ptr const & connection_a, std::shared_ptr const & attempt_a) : - connection (connection_a), - attempt (attempt_a) -{ -} - -nano::bulk_push_client::~bulk_push_client () -{ -} - -void nano::bulk_push_client::start () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - nano::bulk_push message{ node->network_params.network }; - auto this_l (shared_from_this ()); - connection->channel->send ( - message, [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - auto node = this_l->connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - this_l->push (); - } - else - { - node->logger.debug (nano::log::type::bulk_push_client, "Unable to send bulk push request: {}", ec.message ()); - } - }, - nano::transport::buffer_drop_policy::no_limiter_drop); -} - -void nano::bulk_push_client::push () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - std::shared_ptr block; - bool finished (false); - while (block == nullptr && !finished) - { - if (current_target.first.is_zero () || current_target.first == current_target.second) - { - finished = attempt->request_bulk_push_target (current_target); - } - if (!finished) - { - block = node->block (current_target.first); - if (block == nullptr) - { - current_target.first = nano::block_hash (0); - } - else - { - node->logger.debug (nano::log::type::bulk_push_client, "Bulk pushing range: [{}:{}]", current_target.first.to_string (), current_target.second.to_string ()); - } - } - } - if (finished) - { - send_finished (); - } - else - { - current_target.first = block->previous (); - push_block (*block); - } -} - -void nano::bulk_push_client::send_finished () -{ - nano::shared_const_buffer buffer (static_cast (nano::block_type::not_a_block)); - auto this_l (shared_from_this ()); - connection->channel->send_buffer (buffer, [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - try - { - this_l->promise.set_value (false); - } - catch (std::future_error &) - { - } - }); -} - -void nano::bulk_push_client::push_block (nano::block const & block_a) -{ - std::vector buffer; - { - nano::vectorstream stream (buffer); - nano::serialize_block (stream, block_a); - } - auto this_l (shared_from_this ()); - connection->channel->send_buffer (nano::shared_const_buffer (std::move (buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - auto node = this_l->connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - this_l->push (); - } - else - { - node->logger.debug (nano::log::type::bulk_push_client, "Error sending block during bulk push: {}", ec.message ()); - } - }); -} - -nano::bulk_push_server::bulk_push_server (std::shared_ptr const & connection_a) : - receive_buffer (std::make_shared> ()), - connection (connection_a) -{ - receive_buffer->resize (256); -} - -void nano::bulk_push_server::throttled_receive () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (node->block_processor.size (nano::block_source::bootstrap_legacy) < 1024) - { - receive (); - } - else - { - auto this_l (shared_from_this ()); - node->workers.post_delayed (std::chrono::seconds (1), [this_l] () { - if (!this_l->connection->stopped) - { - this_l->throttled_receive (); - } - }); - } -} - -void nano::bulk_push_server::receive () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (node->bootstrap_initiator.in_progress ()) - { - node->logger.debug (nano::log::type::bulk_push_server, "Aborting bulk push because a bootstrap attempt is in progress"); - } - else - { - auto this_l (shared_from_this ()); - connection->socket->async_read (receive_buffer, 1, [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - auto node = this_l->connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - this_l->received_type (); - } - else - { - node->logger.debug (nano::log::type::bulk_push_server, "Error receiving block type: {}", ec.message ()); - } - }); - } -} - -void nano::bulk_push_server::received_type () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - auto this_l (shared_from_this ()); - nano::block_type type (static_cast (receive_buffer->data ()[0])); - switch (type) - { - case nano::block_type::send: - { - node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::send, nano::stat::dir::in); - connection->socket->async_read (receive_buffer, nano::send_block::size, [this_l, type] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->received_block (ec, size_a, type); - }); - break; - } - case nano::block_type::receive: - { - node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::receive, nano::stat::dir::in); - connection->socket->async_read (receive_buffer, nano::receive_block::size, [this_l, type] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->received_block (ec, size_a, type); - }); - break; - } - case nano::block_type::open: - { - node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::open, nano::stat::dir::in); - connection->socket->async_read (receive_buffer, nano::open_block::size, [this_l, type] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->received_block (ec, size_a, type); - }); - break; - } - case nano::block_type::change: - { - node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::change, nano::stat::dir::in); - connection->socket->async_read (receive_buffer, nano::change_block::size, [this_l, type] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->received_block (ec, size_a, type); - }); - break; - } - case nano::block_type::state: - { - node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::state_block, nano::stat::dir::in); - connection->socket->async_read (receive_buffer, nano::state_block::size, [this_l, type] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->received_block (ec, size_a, type); - }); - break; - } - case nano::block_type::not_a_block: - { - connection->start (); - break; - } - default: - { - node->logger.debug (nano::log::type::bulk_push_server, "Unknown type received as block type"); - break; - } - } -} - -void nano::bulk_push_server::received_block (boost::system::error_code const & ec, std::size_t size_a, nano::block_type type_a) -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - nano::bufferstream stream (receive_buffer->data (), size_a); - auto block (nano::deserialize_block (stream, type_a)); - if (block != nullptr) - { - if (node->network_params.work.validate_entry (*block)) - { - node->logger.debug (nano::log::type::bulk_push_server, "Insufficient work for bulk push block: {}", block->hash ().to_string ()); - node->stats.inc (nano::stat::type::error, nano::stat::detail::insufficient_work); - return; - } - node->process_active (std::move (block)); - throttled_receive (); - } - else - { - node->logger.debug (nano::log::type::bulk_push_server, "Error deserializing block received from pull request"); - } - } -} diff --git a/nano/node/bootstrap/bootstrap_bulk_push.hpp b/nano/node/bootstrap/bootstrap_bulk_push.hpp deleted file mode 100644 index d40e0fe5a..000000000 --- a/nano/node/bootstrap/bootstrap_bulk_push.hpp +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include - -#include - -namespace nano -{ -class bootstrap_attempt_legacy; -class bootstrap_client; -namespace transport -{ - class tcp_server; -} - -/** - * Client side of a bulk_push request. Sends a sequence of blocks the other side did not report in their frontier_req response. - */ -class bulk_push_client final : public std::enable_shared_from_this -{ -public: - explicit bulk_push_client (std::shared_ptr const &, std::shared_ptr const &); - ~bulk_push_client (); - void start (); - void push (); - void push_block (nano::block const &); - void send_finished (); - std::shared_ptr connection; - std::shared_ptr attempt; - std::promise promise; - std::pair current_target; -}; - -/** - * Server side of a bulk_push request. Receives blocks and puts them in the block processor to be processed. - */ -class bulk_push_server final : public std::enable_shared_from_this -{ -public: - explicit bulk_push_server (std::shared_ptr const &); - void throttled_receive (); - void receive (); - void received_type (); - void received_block (boost::system::error_code const &, std::size_t, nano::block_type); - std::shared_ptr> receive_buffer; - std::shared_ptr connection; -}; -} diff --git a/nano/node/bootstrap/bootstrap_connections.cpp b/nano/node/bootstrap/bootstrap_connections.cpp deleted file mode 100644 index b3f1334b8..000000000 --- a/nano/node/bootstrap/bootstrap_connections.cpp +++ /dev/null @@ -1,489 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include - -constexpr double nano::bootstrap_limits::bootstrap_connection_scale_target_blocks; -constexpr double nano::bootstrap_limits::bootstrap_minimum_blocks_per_sec; -constexpr double nano::bootstrap_limits::bootstrap_minimum_termination_time_sec; -constexpr unsigned nano::bootstrap_limits::bootstrap_max_new_connections; -constexpr unsigned nano::bootstrap_limits::requeued_pulls_processed_blocks_factor; - -nano::bootstrap_client::bootstrap_client (std::shared_ptr const & node_a, std::shared_ptr const & channel_a, std::shared_ptr const & socket_a) : - node (node_a), - channel (channel_a), - socket (socket_a), - receive_buffer (std::make_shared> ()), - start_time_m (std::chrono::steady_clock::now ()) -{ - ++node_a->bootstrap_initiator.connections->connections_count; - receive_buffer->resize (256); - channel->update_endpoints (); -} - -nano::bootstrap_client::~bootstrap_client () -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - --node->bootstrap_initiator.connections->connections_count; -} - -double nano::bootstrap_client::sample_block_rate () -{ - auto elapsed = std::max (elapsed_seconds (), nano::bootstrap_limits::bootstrap_minimum_elapsed_seconds_blockrate); - block_rate = static_cast (block_count.load ()) / elapsed; - return block_rate; -} - -void nano::bootstrap_client::set_start_time (std::chrono::steady_clock::time_point start_time_a) -{ - nano::lock_guard guard{ start_time_mutex }; - start_time_m = start_time_a; -} - -double nano::bootstrap_client::elapsed_seconds () const -{ - nano::lock_guard guard{ start_time_mutex }; - return std::chrono::duration_cast> (std::chrono::steady_clock::now () - start_time_m).count (); -} - -void nano::bootstrap_client::stop (bool force) -{ - pending_stop = true; - if (force) - { - hard_stop = true; - } -} - -nano::bootstrap_connections::bootstrap_connections (nano::node & node_a) : - node (node_a) -{ -} - -std::shared_ptr nano::bootstrap_connections::connection (std::shared_ptr const & attempt_a, bool use_front_connection) -{ - nano::unique_lock lock{ mutex }; - condition.wait (lock, [&stopped = stopped, &idle = idle, &new_connections_empty = new_connections_empty] { return stopped || !idle.empty () || new_connections_empty; }); - std::shared_ptr result; - if (!stopped && !idle.empty ()) - { - if (!use_front_connection) - { - result = idle.back (); - idle.pop_back (); - } - else - { - result = idle.front (); - idle.pop_front (); - } - } - if (result == nullptr && connections_count == 0 && new_connections_empty && attempt_a != nullptr) - { - node.logger.debug (nano::log::type::bootstrap, "Bootstrap attempt stopped because there are no peers"); - - lock.unlock (); - attempt_a->stop (); - } - return result; -} - -void nano::bootstrap_connections::pool_connection (std::shared_ptr const & client_a, bool new_client, bool push_front) -{ - nano::unique_lock lock{ mutex }; - auto const & socket_l = client_a->socket; - if (!stopped && !client_a->pending_stop && !node.network.excluded_peers.check (client_a->channel->get_remote_endpoint ())) - { - socket_l->set_timeout (node.network_params.network.idle_timeout); - // Push into idle deque - if (!push_front) - { - idle.push_back (client_a); - } - else - { - idle.push_front (client_a); - } - if (new_client) - { - clients.push_back (client_a); - } - } - else - { - socket_l->close (); - } - lock.unlock (); - condition.notify_all (); -} - -void nano::bootstrap_connections::add_connection (nano::endpoint const & endpoint_a) -{ - connect_client (nano::tcp_endpoint (endpoint_a.address (), endpoint_a.port ()), true); -} - -std::shared_ptr nano::bootstrap_connections::find_connection (nano::tcp_endpoint const & endpoint_a) -{ - nano::lock_guard lock{ mutex }; - std::shared_ptr result; - for (auto i (idle.begin ()), end (idle.end ()); i != end && !stopped; ++i) - { - if ((*i)->channel->get_remote_endpoint () == endpoint_a) - { - result = *i; - idle.erase (i); - break; - } - } - return result; -} - -void nano::bootstrap_connections::connect_client (nano::tcp_endpoint const & endpoint_a, bool push_front) -{ - ++connections_count; - auto socket (std::make_shared (node)); - auto this_l (shared_from_this ()); - socket->async_connect (endpoint_a, - [this_l, socket, endpoint_a, push_front] (boost::system::error_code const & ec) { - if (!ec) - { - this_l->node.logger.debug (nano::log::type::bootstrap, "Connection established to: {}", nano::util::to_str (endpoint_a)); - - auto client (std::make_shared (this_l->node.shared (), std::make_shared (*this_l->node.shared (), socket), socket)); - this_l->pool_connection (client, true, push_front); - } - else - { - switch (ec.value ()) - { - default: - this_l->node.logger.debug (nano::log::type::bootstrap, "Error initiating bootstrap connection to: {} ({})", nano::util::to_str (endpoint_a), ec.message ()); - break; - case boost::system::errc::connection_refused: - case boost::system::errc::operation_canceled: - case boost::system::errc::timed_out: - case 995: // Windows The I/O operation has been aborted because of either a thread exit or an application request - case 10061: // Windows No connection could be made because the target machine actively refused it - break; - } - } - --this_l->connections_count; - }); -} - -unsigned nano::bootstrap_connections::target_connections (std::size_t pulls_remaining, std::size_t attempts_count) const -{ - auto const attempts_factor = nano::narrow_cast (node.config.bootstrap_connections * attempts_count); - if (attempts_factor >= node.config.bootstrap_connections_max) - { - return std::max (1U, node.config.bootstrap_connections_max); - } - - // Only scale up to bootstrap_connections_max for large pulls. - double step_scale = std::min (1.0, std::max (0.0, (double)pulls_remaining / nano::bootstrap_limits::bootstrap_connection_scale_target_blocks)); - double target = (double)attempts_factor + (double)(node.config.bootstrap_connections_max - attempts_factor) * step_scale; - return std::max (1U, (unsigned)(target + 0.5f)); -} - -struct block_rate_cmp -{ - bool operator() (std::shared_ptr const & lhs, std::shared_ptr const & rhs) const - { - return lhs->block_rate > rhs->block_rate; - } -}; - -void nano::bootstrap_connections::populate_connections (bool repeat) -{ - double rate_sum = 0.0; - std::size_t num_pulls = 0; - std::size_t attempts_count = node.bootstrap_initiator.attempts.size (); - std::priority_queue, std::vector>, block_rate_cmp> sorted_connections; - std::unordered_set endpoints; - { - nano::unique_lock lock{ mutex }; - num_pulls = pulls.size (); - std::deque> new_clients; - for (auto & c : clients) - { - if (auto client = c.lock ()) - { - new_clients.push_back (client); - endpoints.insert (client->socket->remote_endpoint ()); - double elapsed_sec = client->elapsed_seconds (); - auto blocks_per_sec = client->sample_block_rate (); - rate_sum += blocks_per_sec; - if (client->elapsed_seconds () > nano::bootstrap_limits::bootstrap_connection_warmup_time_sec && client->block_count > 0) - { - sorted_connections.push (client); - } - // Force-stop the slowest peers, since they can take the whole bootstrap hostage by dribbling out blocks on the last remaining pull. - // This is ~1.5kilobits/sec. - if (elapsed_sec > nano::bootstrap_limits::bootstrap_minimum_termination_time_sec && blocks_per_sec < nano::bootstrap_limits::bootstrap_minimum_blocks_per_sec) - { - node.logger.debug (nano::log::type::bootstrap, "Stopping slow peer {} (elapsed sec {} > {} and {} blocks per second < {})", - client->channel->to_string (), - elapsed_sec, - nano::bootstrap_limits::bootstrap_minimum_termination_time_sec, - blocks_per_sec, - nano::bootstrap_limits::bootstrap_minimum_blocks_per_sec); - - client->stop (true); - new_clients.pop_back (); - } - } - } - // Cleanup expired clients - clients.swap (new_clients); - } - - auto target = target_connections (num_pulls, attempts_count); - - // We only want to drop slow peers when more than 2/3 are active. 2/3 because 1/2 is too aggressive, and 100% rarely happens. - // Probably needs more tuning. - if (sorted_connections.size () >= (target * 2) / 3 && target >= 4) - { - // 4 -> 1, 8 -> 2, 16 -> 4, arbitrary, but seems to work well. - auto drop = (int)roundf (sqrtf ((float)target - 2.0f)); - - node.logger.debug (nano::log::type::bootstrap, "Dropping {} bulk pull peers, target connections {}", drop, target); - - for (int i = 0; i < drop; i++) - { - auto client = sorted_connections.top (); - - node.logger.debug (nano::log::type::bootstrap, "Dropping peer with block rate {} and block count {} ({})", - client->block_rate.load (), - client->block_count.load (), - client->channel->to_string ()); - - client->stop (false); - sorted_connections.pop (); - } - } - - node.logger.debug (nano::log::type::bootstrap, "Bulk pull connections: {}, rate: {} blocks/sec, bootstrap attempts {}, remaining pulls: {}", - connections_count.load (), - (int)rate_sum, - attempts_count, - num_pulls); - - if (connections_count < target && (attempts_count != 0 || new_connections_empty) && !stopped) - { - auto delta = std::min ((target - connections_count) * 2, nano::bootstrap_limits::bootstrap_max_new_connections); - // TODO - tune this better - // Not many peers respond, need to try to make more connections than we need. - for (auto i = 0u; i < delta; i++) - { - auto endpoint (node.network.bootstrap_peer ()); // Legacy bootstrap is compatible with older version of protocol - if (endpoint != nano::tcp_endpoint (boost::asio::ip::address_v6::any (), 0) && (node.flags.allow_bootstrap_peers_duplicates || endpoints.find (endpoint) == endpoints.end ()) && !node.network.excluded_peers.check (endpoint)) - { - connect_client (endpoint); - endpoints.insert (endpoint); - nano::lock_guard lock{ mutex }; - new_connections_empty = false; - } - else if (connections_count == 0) - { - { - nano::lock_guard lock{ mutex }; - new_connections_empty = true; - } - condition.notify_all (); - } - } - } - if (!stopped && repeat) - { - std::weak_ptr this_w (shared_from_this ()); - node.workers.post_delayed (std::chrono::seconds (1), [this_w] () { - if (auto this_l = this_w.lock ()) - { - this_l->populate_connections (); - } - }); - } -} - -void nano::bootstrap_connections::start_populate_connections () -{ - if (!populate_connections_started.exchange (true)) - { - populate_connections (); - } -} - -void nano::bootstrap_connections::add_pull (nano::pull_info const & pull_a) -{ - nano::pull_info pull (pull_a); - node.bootstrap_initiator.cache.update_pull (pull); - { - nano::lock_guard lock{ mutex }; - pulls.push_back (pull); - } - condition.notify_all (); -} - -void nano::bootstrap_connections::request_pull (nano::unique_lock & lock_a) -{ - lock_a.unlock (); - auto connection_l (connection ()); - lock_a.lock (); - if (connection_l != nullptr && !pulls.empty ()) - { - std::shared_ptr attempt_l; - nano::pull_info pull; - // Search pulls with existing attempts - while (attempt_l == nullptr && !pulls.empty ()) - { - pull = pulls.front (); - pulls.pop_front (); - attempt_l = node.bootstrap_initiator.attempts.find (pull.bootstrap_id); - // Check if lazy pull is obsolete (head was processed or head is 0 for destinations requests) - if (auto lazy = std::dynamic_pointer_cast (attempt_l)) - { - if (!pull.head.is_zero () && lazy->lazy_processed_or_exists (pull.head)) - { - attempt_l->pull_finished (); - attempt_l = nullptr; - } - } - } - if (attempt_l != nullptr) - { - // The bulk_pull_client destructor attempt to requeue_pull which can cause a deadlock if this is the last reference - // Dispatch request in an external thread in case it needs to be destroyed - node.background ([connection_l, attempt_l, pull] () { - auto client (std::make_shared (connection_l, attempt_l, pull)); - client->request (); - }); - } - } - else if (connection_l != nullptr) - { - // Reuse connection if pulls deque become empty - lock_a.unlock (); - pool_connection (connection_l); - lock_a.lock (); - } -} - -void nano::bootstrap_connections::requeue_pull (nano::pull_info const & pull_a, bool network_error) -{ - auto pull (pull_a); - if (!network_error) - { - ++pull.attempts; - } - auto attempt_l (node.bootstrap_initiator.attempts.find (pull.bootstrap_id)); - if (attempt_l != nullptr) - { - auto lazy = std::dynamic_pointer_cast (attempt_l); - ++attempt_l->requeued_pulls; - if (lazy) - { - pull.count = lazy->lazy_batch_size (); - } - if (attempt_l->mode == nano::bootstrap_mode::legacy && (pull.attempts < pull.retry_limit + (pull.processed / nano::bootstrap_limits::requeued_pulls_processed_blocks_factor))) - { - { - nano::lock_guard lock{ mutex }; - pulls.push_front (pull); - } - attempt_l->pull_started (); - condition.notify_all (); - } - else if (lazy && (pull.attempts <= pull.retry_limit + (pull.processed / node.network_params.bootstrap.lazy_max_pull_blocks))) - { - debug_assert (pull.account_or_head.as_block_hash () == pull.head); - if (!lazy->lazy_processed_or_exists (pull.account_or_head.as_block_hash ())) - { - { - nano::lock_guard lock{ mutex }; - pulls.push_back (pull); - } - attempt_l->pull_started (); - condition.notify_all (); - } - } - else - { - node.stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in); - node.logger.debug (nano::log::type::bootstrap, "Failed to pull account {} or head block {} down to {} after {} attempts and {} blocks processed", - pull.account_or_head.to_account (), - pull.account_or_head.to_string (), - pull.end.to_string (), - pull.attempts, - pull.processed); - - if (lazy && pull.processed > 0) - { - lazy->lazy_add (pull); - } - else if (attempt_l->mode == nano::bootstrap_mode::legacy) - { - node.bootstrap_initiator.cache.add (pull); - } - } - } -} - -void nano::bootstrap_connections::clear_pulls (uint64_t bootstrap_id_a) -{ - { - nano::lock_guard lock{ mutex }; - - erase_if (pulls, [bootstrap_id_a] (auto const & pull) { - return pull.bootstrap_id == bootstrap_id_a; - }); - } - condition.notify_all (); -} - -void nano::bootstrap_connections::run () -{ - start_populate_connections (); - nano::unique_lock lock{ mutex }; - while (!stopped) - { - if (!pulls.empty ()) - { - request_pull (lock); - } - else - { - condition.wait (lock); - } - } - stopped = true; - lock.unlock (); - condition.notify_all (); -} - -void nano::bootstrap_connections::stop () -{ - nano::unique_lock lock{ mutex }; - stopped = true; - lock.unlock (); - condition.notify_all (); - lock.lock (); - for (auto const & i : clients) - { - if (auto client = i.lock ()) - { - client->socket->close (); - } - } - clients.clear (); - idle.clear (); -} diff --git a/nano/node/bootstrap/bootstrap_connections.hpp b/nano/node/bootstrap/bootstrap_connections.hpp deleted file mode 100644 index d246c23bb..000000000 --- a/nano/node/bootstrap/bootstrap_connections.hpp +++ /dev/null @@ -1,81 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -namespace nano -{ -class node; -namespace transport -{ - class tcp_channel; -} - -class bootstrap_attempt; -class bootstrap_connections; -class frontier_req_client; -class pull_info; - -/** - * Owns the client side of the bootstrap connection. - */ -class bootstrap_client final : public std::enable_shared_from_this -{ -public: - bootstrap_client (std::shared_ptr const & node_a, std::shared_ptr const & channel_a, std::shared_ptr const & socket_a); - ~bootstrap_client (); - void stop (bool force); - double sample_block_rate (); - double elapsed_seconds () const; - void set_start_time (std::chrono::steady_clock::time_point start_time_a); - std::weak_ptr node; - std::shared_ptr channel; - std::shared_ptr socket; - std::shared_ptr> receive_buffer; - std::atomic block_count{ 0 }; - std::atomic block_rate{ 0 }; - std::atomic pending_stop{ false }; - std::atomic hard_stop{ false }; - -private: - mutable nano::mutex start_time_mutex; - std::chrono::steady_clock::time_point start_time_m; -}; - -/** - * Container for bootstrap_client objects. Owned by bootstrap_initiator which pools open connections and makes them available - * for use by different bootstrap sessions. - */ -class bootstrap_connections final : public std::enable_shared_from_this -{ -public: - explicit bootstrap_connections (nano::node & node_a); - std::shared_ptr connection (std::shared_ptr const & attempt_a = nullptr, bool use_front_connection = false); - void pool_connection (std::shared_ptr const & client_a, bool new_client = false, bool push_front = false); - void add_connection (nano::endpoint const & endpoint_a); - std::shared_ptr find_connection (nano::tcp_endpoint const & endpoint_a); - void connect_client (nano::tcp_endpoint const & endpoint_a, bool push_front = false); - unsigned target_connections (std::size_t pulls_remaining, std::size_t attempts_count) const; - void populate_connections (bool repeat = true); - void start_populate_connections (); - void add_pull (nano::pull_info const & pull_a); - void request_pull (nano::unique_lock & lock_a); - void requeue_pull (nano::pull_info const & pull_a, bool network_error = false); - void clear_pulls (uint64_t); - void run (); - void stop (); - std::deque> clients; - std::atomic connections_count{ 0 }; - nano::node & node; - std::deque> idle; - std::deque pulls; - std::atomic populate_connections_started{ false }; - std::atomic new_connections_empty{ false }; - std::atomic stopped{ false }; - nano::mutex mutex; - nano::condition_variable condition; -}; -} diff --git a/nano/node/bootstrap/bootstrap_frontier.cpp b/nano/node/bootstrap/bootstrap_frontier.cpp deleted file mode 100644 index 8a6067ef2..000000000 --- a/nano/node/bootstrap/bootstrap_frontier.cpp +++ /dev/null @@ -1,424 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -constexpr double nano::bootstrap_limits::bootstrap_connection_warmup_time_sec; -constexpr double nano::bootstrap_limits::bootstrap_minimum_elapsed_seconds_blockrate; -constexpr double nano::bootstrap_limits::bootstrap_minimum_frontier_blocks_per_sec; -constexpr unsigned nano::bootstrap_limits::bulk_push_cost_limit; - -constexpr std::size_t nano::frontier_req_client::size_frontier; - -void nano::frontier_req_client::run (nano::account const & start_account_a, uint32_t const frontiers_age_a, uint32_t const count_a) -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - nano::frontier_req request{ node->network_params.network }; - request.start = (start_account_a.is_zero () || start_account_a.number () == std::numeric_limits::max ()) ? start_account_a.number () : start_account_a.number () + 1; - request.age = frontiers_age_a; - request.count = count_a; - current = start_account_a; - frontiers_age = frontiers_age_a; - count_limit = count_a; - next (); // Load accounts from disk - auto this_l (shared_from_this ()); - connection->channel->send ( - request, [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - auto node = this_l->connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - this_l->receive_frontier (); - } - else - { - node->logger.debug (nano::log::type::frontier_req_client, "Error while sending bootstrap request: {}", ec.message ()); - } - }, - nano::transport::buffer_drop_policy::no_limiter_drop); -} - -nano::frontier_req_client::frontier_req_client (std::shared_ptr const & connection_a, std::shared_ptr const & attempt_a) : - connection (connection_a), - attempt (attempt_a), - count (0), - bulk_push_cost (0) -{ -} - -void nano::frontier_req_client::receive_frontier () -{ - auto this_l (shared_from_this ()); - connection->socket->async_read (connection->receive_buffer, nano::frontier_req_client::size_frontier, [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - auto node = this_l->connection->node.lock (); - if (!node) - { - return; - } - // An issue with asio is that sometimes, instead of reporting a bad file descriptor during disconnect, - // we simply get a size of 0. - if (size_a == nano::frontier_req_client::size_frontier) - { - node->bootstrap_workers.post ([this_l, ec, size_a] () { - this_l->received_frontier (ec, size_a); - }); - } - else - { - node->logger.debug (nano::log::type::frontier_req_client, "Invalid size: expected {}, got {}", nano::frontier_req_client::size_frontier, size_a); - } - }); -} - -bool nano::frontier_req_client::bulk_push_available () -{ - return bulk_push_cost < nano::bootstrap_limits::bulk_push_cost_limit && frontiers_age == std::numeric_limits::max (); -} - -void nano::frontier_req_client::unsynced (nano::block_hash const & head, nano::block_hash const & end) -{ - if (bulk_push_available ()) - { - attempt->add_bulk_push_target (head, end); - if (end.is_zero ()) - { - bulk_push_cost += 2; - } - else - { - bulk_push_cost += 1; - } - } -} - -void nano::frontier_req_client::received_frontier (boost::system::error_code const & ec, std::size_t size_a) -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - debug_assert (size_a == nano::frontier_req_client::size_frontier); - nano::account account; - nano::bufferstream account_stream (connection->receive_buffer->data (), sizeof (account)); - auto error1 (nano::try_read (account_stream, account)); - (void)error1; - debug_assert (!error1); - nano::block_hash latest; - nano::bufferstream latest_stream (connection->receive_buffer->data () + sizeof (account), sizeof (latest)); - auto error2 (nano::try_read (latest_stream, latest)); - (void)error2; - debug_assert (!error2); - if (count == 0) - { - start_time = std::chrono::steady_clock::now (); - } - ++count; - std::chrono::duration time_span = std::chrono::duration_cast> (std::chrono::steady_clock::now () - start_time); - - double elapsed_sec = std::max (time_span.count (), nano::bootstrap_limits::bootstrap_minimum_elapsed_seconds_blockrate); - double blocks_per_sec = static_cast (count) / elapsed_sec; - double age_factor = (frontiers_age == std::numeric_limits::max ()) ? 1.0 : 1.5; // Allow slower frontiers receive for requests with age - if (elapsed_sec > nano::bootstrap_limits::bootstrap_connection_warmup_time_sec && blocks_per_sec * age_factor < nano::bootstrap_limits::bootstrap_minimum_frontier_blocks_per_sec) - { - node->logger.debug (nano::log::type::frontier_req_client, "Aborting frontier req because it was too slow: {} frontiers per second, last {}", blocks_per_sec, account.to_account ()); - - promise.set_value (true); - return; - } - - if (attempt->should_log ()) - { - node->logger.debug (nano::log::type::frontier_req_client, "Received {} frontiers from {}", count, connection->channel->to_string ()); - } - - if (!account.is_zero () && count <= count_limit) - { - last_account = account; - while (!current.is_zero () && current < account) - { - // We know about an account they don't. - unsynced (frontier, 0); - next (); - } - if (!current.is_zero ()) - { - if (account == current) - { - if (latest == frontier) - { - // In sync - } - else - { - if (node->block_or_pruned_exists (latest)) - { - // We know about a block they don't. - unsynced (frontier, latest); - } - else - { - attempt->add_frontier (nano::pull_info (account, latest, frontier, attempt->incremental_id, 0, node->network_params.bootstrap.frontier_retry_limit)); - // Either we're behind or there's a fork we differ on - // Either way, bulk pushing will probably not be effective - bulk_push_cost += 5; - } - } - next (); - } - else - { - debug_assert (account < current); - attempt->add_frontier (nano::pull_info (account, latest, nano::block_hash (0), attempt->incremental_id, 0, node->network_params.bootstrap.frontier_retry_limit)); - } - } - else - { - attempt->add_frontier (nano::pull_info (account, latest, nano::block_hash (0), attempt->incremental_id, 0, node->network_params.bootstrap.frontier_retry_limit)); - } - receive_frontier (); - } - else - { - if (count <= count_limit) - { - while (!current.is_zero () && bulk_push_available ()) - { - // We know about an account they don't. - unsynced (frontier, 0); - next (); - } - // Prevent new frontier_req requests - attempt->set_start_account (std::numeric_limits::max ()); - - node->logger.debug (nano::log::type::frontier_req_client, "Bulk push cost: {}", bulk_push_cost); - } - else - { - // Set last processed account as new start target - attempt->set_start_account (last_account); - } - node->bootstrap_initiator.connections->pool_connection (connection); - try - { - promise.set_value (false); - } - catch (std::future_error &) - { - } - } - } - else - { - node->logger.debug (nano::log::type::frontier_req_client, "Error while receiving frontier: {}", ec.message ()); - } -} - -void nano::frontier_req_client::next () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - // Filling accounts deque to prevent often read transactions - if (accounts.empty ()) - { - std::size_t max_size (128); - auto transaction (node->store.tx_begin_read ()); - for (auto i (node->store.account.begin (transaction, current.number () + 1)), n (node->store.account.end (transaction)); i != n && accounts.size () != max_size; ++i) - { - nano::account_info const & info (i->second); - nano::account const & account (i->first); - accounts.emplace_back (account, info.head); - } - - /* If loop breaks before max_size, then accounts_end () is reached. Add empty record */ - if (accounts.size () != max_size) - { - accounts.emplace_back (nano::account{}, nano::block_hash (0)); - } - } - // Retrieving accounts from deque - auto const & account_pair (accounts.front ()); - current = account_pair.first; - frontier = account_pair.second; - accounts.pop_front (); -} - -nano::frontier_req_server::frontier_req_server (std::shared_ptr const & connection_a, std::unique_ptr request_a) : - connection (connection_a), - current (request_a->start.number () - 1), - frontier (0), - request (std::move (request_a)), - count (0) -{ - next (); -} - -void nano::frontier_req_server::send_next () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (!current.is_zero () && count < request->count) - { - node->logger.trace (nano::log::type::frontier_req_server, nano::log::detail::sending_frontier, - nano::log::arg{ "account", current.to_account () }, // TODO: Convert to lazy eval - nano::log::arg{ "frontier", frontier }, - nano::log::arg{ "socket", connection->socket }); - - std::vector send_buffer; - { - nano::vectorstream stream (send_buffer); - write (stream, current.bytes); - write (stream, frontier.bytes); - debug_assert (!current.is_zero ()); - debug_assert (!frontier.is_zero ()); - } - - auto this_l (shared_from_this ()); - next (); - connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->sent_action (ec, size_a); - }); - } - else - { - send_finished (); - } -} - -void nano::frontier_req_server::send_finished () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - std::vector send_buffer; - { - nano::vectorstream stream (send_buffer); - nano::uint256_union zero (0); - write (stream, zero.bytes); - write (stream, zero.bytes); - } - - node->logger.debug (nano::log::type::frontier_req_server, "Frontier sending finished"); - - auto this_l (shared_from_this ()); - connection->socket->async_write (nano::shared_const_buffer (std::move (send_buffer)), [this_l] (boost::system::error_code const & ec, std::size_t size_a) { - this_l->no_block_sent (ec, size_a); - }); -} - -void nano::frontier_req_server::no_block_sent (boost::system::error_code const & ec, std::size_t size_a) -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - connection->start (); - } - else - { - node->logger.debug (nano::log::type::frontier_req_server, "Error sending frontier finish: {}", ec.message ()); - } -} - -void nano::frontier_req_server::sent_action (boost::system::error_code const & ec, std::size_t size_a) -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - if (!ec) - { - count++; - - node->bootstrap_workers.post ([this_l = shared_from_this ()] () { - this_l->send_next (); - }); - } - else - { - node->logger.debug (nano::log::type::frontier_req_server, "Error sending frontier pair: {}", ec.message ()); - } -} - -void nano::frontier_req_server::next () -{ - auto node = connection->node.lock (); - if (!node) - { - return; - } - // Filling accounts deque to prevent often read transactions - if (accounts.empty ()) - { - auto now (nano::seconds_since_epoch ()); - bool disable_age_filter (request->age == std::numeric_limitsage)>::max ()); - std::size_t max_size (128); - auto transaction (node->store.tx_begin_read ()); - if (!send_confirmed ()) - { - for (auto i (node->store.account.begin (transaction, current.number () + 1)), n (node->store.account.end (transaction)); i != n && accounts.size () != max_size; ++i) - { - nano::account_info const & info (i->second); - if (disable_age_filter || (now - info.modified) <= request->age) - { - nano::account const & account (i->first); - accounts.emplace_back (account, info.head); - } - } - } - else - { - for (auto i (node->store.confirmation_height.begin (transaction, current.number () + 1)), n (node->store.confirmation_height.end (transaction)); i != n && accounts.size () != max_size; ++i) - { - nano::confirmation_height_info const & info (i->second); - nano::block_hash const & confirmed_frontier (info.frontier); - if (!confirmed_frontier.is_zero ()) - { - nano::account const & account (i->first); - accounts.emplace_back (account, confirmed_frontier); - } - } - } - - /* If loop breaks before max_size, then accounts_end () is reached. Add empty record to finish frontier_req_server */ - if (accounts.size () != max_size) - { - accounts.emplace_back (nano::account{}, nano::block_hash (0)); - } - } - // Retrieving accounts from deque - auto const & account_pair (accounts.front ()); - current = account_pair.first; - frontier = account_pair.second; - accounts.pop_front (); -} - -bool nano::frontier_req_server::send_confirmed () -{ - return request->header.frontier_req_is_only_confirmed_present (); -} diff --git a/nano/node/bootstrap/bootstrap_frontier.hpp b/nano/node/bootstrap/bootstrap_frontier.hpp deleted file mode 100644 index 75ee9c1ab..000000000 --- a/nano/node/bootstrap/bootstrap_frontier.hpp +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include - -#include -#include -#include - -namespace nano -{ -class bootstrap_attempt_legacy; -class bootstrap_client; -namespace transport -{ - class tcp_server; -} - -/** - * Client side of a frontier request. Created to send and listen for frontier sequences from the server. - */ -class frontier_req_client final : public std::enable_shared_from_this -{ -public: - explicit frontier_req_client (std::shared_ptr const &, std::shared_ptr const &); - void run (nano::account const & start_account_a, uint32_t const frontiers_age_a, uint32_t const count_a); - void receive_frontier (); - void received_frontier (boost::system::error_code const &, std::size_t); - bool bulk_push_available (); - void unsynced (nano::block_hash const &, nano::block_hash const &); - void next (); - std::shared_ptr connection; - std::shared_ptr attempt; - nano::account current; - nano::block_hash frontier; - unsigned count; - nano::account last_account{ std::numeric_limits::max () }; // Using last possible account stop further frontier requests - std::chrono::steady_clock::time_point start_time; - std::promise promise; - /** A very rough estimate of the cost of `bulk_push`ing missing blocks */ - uint64_t bulk_push_cost; - std::deque> accounts; - uint32_t frontiers_age{ std::numeric_limits::max () }; - uint32_t count_limit{ std::numeric_limits::max () }; - static std::size_t constexpr size_frontier = sizeof (nano::account) + sizeof (nano::block_hash); -}; - -class frontier_req; - -/** - * Server side of a frontier request. Created when a tcp_server receives a frontier_req message and exited when end-of-list is reached. - */ -class frontier_req_server final : public std::enable_shared_from_this -{ -public: - frontier_req_server (std::shared_ptr const &, std::unique_ptr); - void send_next (); - void sent_action (boost::system::error_code const &, std::size_t); - void send_finished (); - void no_block_sent (boost::system::error_code const &, std::size_t); - void next (); - bool send_confirmed (); - std::shared_ptr connection; - nano::account current; - nano::block_hash frontier; - std::unique_ptr request; - std::size_t count; - std::deque> accounts; -}; -} diff --git a/nano/node/bootstrap/bootstrap_lazy.cpp b/nano/node/bootstrap/bootstrap_lazy.cpp deleted file mode 100644 index 439aa0ced..000000000 --- a/nano/node/bootstrap/bootstrap_lazy.cpp +++ /dev/null @@ -1,634 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -constexpr std::chrono::seconds nano::bootstrap_limits::lazy_flush_delay_sec; -constexpr uint64_t nano::bootstrap_limits::lazy_batch_pull_count_resize_blocks_limit; -constexpr double nano::bootstrap_limits::lazy_batch_pull_count_resize_ratio; -constexpr std::size_t nano::bootstrap_limits::lazy_blocks_restart_limit; - -nano::bootstrap_attempt_lazy::bootstrap_attempt_lazy (std::shared_ptr const & node_a, uint64_t incremental_id_a, std::string const & id_a) : - nano::bootstrap_attempt (node_a, nano::bootstrap_mode::lazy, incremental_id_a, id_a) -{ - node_a->bootstrap_initiator.notify_listeners (true); -} - -nano::bootstrap_attempt_lazy::~bootstrap_attempt_lazy () -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - debug_assert (lazy_blocks.size () == lazy_blocks_count); - node->bootstrap_initiator.notify_listeners (false); -} - -bool nano::bootstrap_attempt_lazy::lazy_start (nano::hash_or_account const & hash_or_account_a) -{ - auto node = this->node.lock (); - if (!node) - { - return false; - } - nano::unique_lock lock{ mutex }; - bool inserted (false); - // Add start blocks, limit 1024 (4k with disabled legacy bootstrap) - std::size_t max_keys (node->flags.disable_legacy_bootstrap ? 4 * 1024 : 1024); - if (lazy_keys.size () < max_keys && lazy_keys.find (hash_or_account_a.as_block_hash ()) == lazy_keys.end () && !lazy_blocks_processed (hash_or_account_a.as_block_hash ())) - { - lazy_keys.insert (hash_or_account_a.as_block_hash ()); - lazy_pulls.emplace_back (hash_or_account_a, node->network_params.bootstrap.lazy_retry_limit); - lock.unlock (); - condition.notify_all (); - inserted = true; - } - return inserted; -} - -void nano::bootstrap_attempt_lazy::lazy_add (nano::hash_or_account const & hash_or_account_a, unsigned retry_limit) -{ - // Add only unknown blocks - debug_assert (!mutex.try_lock ()); - if (!lazy_blocks_processed (hash_or_account_a.as_block_hash ())) - { - lazy_pulls.emplace_back (hash_or_account_a, retry_limit); - } -} - -void nano::bootstrap_attempt_lazy::lazy_add (nano::pull_info const & pull_a) -{ - debug_assert (pull_a.account_or_head.as_block_hash () == pull_a.head); - nano::lock_guard lock{ mutex }; - lazy_add (pull_a.account_or_head, pull_a.retry_limit); -} - -void nano::bootstrap_attempt_lazy::lazy_requeue (nano::block_hash const & hash_a, nano::block_hash const & previous_a) -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - nano::unique_lock lock{ mutex }; - // Add only known blocks - if (lazy_blocks_processed (hash_a)) - { - lazy_blocks_erase (hash_a); - lock.unlock (); - node->bootstrap_initiator.connections->requeue_pull (nano::pull_info (hash_a, hash_a, previous_a, incremental_id, static_cast (1), node->network_params.bootstrap.lazy_destinations_retry_limit)); - } -} - -uint32_t nano::bootstrap_attempt_lazy::lazy_batch_size () -{ - auto node = this->node.lock (); - if (!node) - { - return 0; - } - auto result (node->network_params.bootstrap.lazy_max_pull_blocks); - if (total_blocks > nano::bootstrap_limits::lazy_batch_pull_count_resize_blocks_limit && lazy_blocks_count != 0) - { - auto lazy_blocks_ratio (static_cast (total_blocks / lazy_blocks_count)); - if (lazy_blocks_ratio > nano::bootstrap_limits::lazy_batch_pull_count_resize_ratio) - { - // Increasing blocks ratio weight as more important (^3). Small batch count should lower blocks ratio below target - double lazy_blocks_factor (std::pow (lazy_blocks_ratio / nano::bootstrap_limits::lazy_batch_pull_count_resize_ratio, 3.0)); - // Decreasing total block count weight as less important (sqrt) - double total_blocks_factor (std::sqrt (total_blocks / nano::bootstrap_limits::lazy_batch_pull_count_resize_blocks_limit)); - uint32_t batch_count_min (node->network_params.bootstrap.lazy_max_pull_blocks / static_cast (lazy_blocks_factor * total_blocks_factor)); - result = std::max (node->network_params.bootstrap.lazy_min_pull_blocks, batch_count_min); - } - } - return result; -} - -void nano::bootstrap_attempt_lazy::lazy_pull_flush (nano::unique_lock & lock_a) -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - static std::size_t const max_pulls (static_cast (nano::bootstrap_limits::bootstrap_connection_scale_target_blocks) * 3); - if (pulling < max_pulls) - { - debug_assert (node->network_params.bootstrap.lazy_max_pull_blocks <= std::numeric_limits::max ()); - nano::pull_info::count_t batch_count (lazy_batch_size ()); - uint64_t read_count (0); - std::size_t count (0); - auto transaction = node->ledger.tx_begin_read (); - while (!lazy_pulls.empty () && count < max_pulls) - { - auto pull_start (lazy_pulls.front ()); - lazy_pulls.pop_front (); - // Recheck if block was already processed - if (!lazy_blocks_processed (pull_start.first.as_block_hash ()) && !node->ledger.any.block_exists_or_pruned (transaction, pull_start.first.as_block_hash ())) - { - lock_a.unlock (); - node->bootstrap_initiator.connections->add_pull (nano::pull_info (pull_start.first, pull_start.first.as_block_hash (), nano::block_hash (0), incremental_id, batch_count, pull_start.second)); - ++pulling; - ++count; - lock_a.lock (); - } - // We don't want to open read transactions for too long - ++read_count; - if (read_count % batch_read_size == 0) - { - lock_a.unlock (); - transaction.refresh (); - lock_a.lock (); - } - } - } -} - -bool nano::bootstrap_attempt_lazy::lazy_finished () -{ - auto node = this->node.lock (); - if (!node) - { - return true; - } - debug_assert (!mutex.try_lock ()); - if (stopped) - { - return true; - } - bool result (true); - uint64_t read_count (0); - auto transaction = node->ledger.tx_begin_read (); - for (auto it (lazy_keys.begin ()), end (lazy_keys.end ()); it != end && !stopped;) - { - if (node->ledger.any.block_exists_or_pruned (transaction, *it)) - { - it = lazy_keys.erase (it); - } - else - { - result = false; - break; - // No need to increment `it` as we break above. - } - // We don't want to open read transactions for too long - ++read_count; - if (read_count % batch_read_size == 0) - { - transaction.refresh (); - } - } - // Finish lazy bootstrap without lazy pulls (in combination with still_pulling ()) - if (!result && lazy_pulls.empty () && lazy_state_backlog.empty ()) - { - result = true; - } - return result; -} - -bool nano::bootstrap_attempt_lazy::lazy_has_expired () const -{ - auto node = this->node.lock (); - if (!node) - { - return true; - } - bool result (false); - // Max 30 minutes run with enabled legacy bootstrap - static std::chrono::minutes const max_lazy_time (node->flags.disable_legacy_bootstrap ? 7 * 24 * 60 : 30); - if (std::chrono::steady_clock::now () - lazy_start_time >= max_lazy_time) - { - result = true; - } - else if (!node->flags.disable_legacy_bootstrap && lazy_blocks_count > nano::bootstrap_limits::lazy_blocks_restart_limit) - { - result = true; - } - return result; -} - -void nano::bootstrap_attempt_lazy::run () -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - debug_assert (started); - debug_assert (!node->flags.disable_lazy_bootstrap); - node->bootstrap_initiator.connections->populate_connections (false); - lazy_start_time = std::chrono::steady_clock::now (); - nano::unique_lock lock{ mutex }; - while ((still_pulling () || !lazy_finished ()) && !lazy_has_expired ()) - { - unsigned iterations (0); - while (still_pulling () && !lazy_has_expired ()) - { - condition.wait (lock, [this, &stopped = stopped, &pulling = pulling, &lazy_pulls = lazy_pulls] { return stopped || pulling == 0 || (pulling < nano::bootstrap_limits::bootstrap_connection_scale_target_blocks && !lazy_pulls.empty ()) || lazy_has_expired (); }); - ++iterations; - // Flushing lazy pulls - lazy_pull_flush (lock); - // Start backlog cleanup - if (iterations % 100 == 0) - { - lazy_backlog_cleanup (); - } - } - // Flushing lazy pulls - lazy_pull_flush (lock); - // Check if some blocks required for backlog were processed. Start destinations check - if (pulling == 0) - { - lazy_backlog_cleanup (); - lazy_pull_flush (lock); - } - } - if (!stopped) - { - node->logger.debug (nano::log::type::bootstrap_lazy, "Completed lazy pulls"); - } - if (lazy_has_expired ()) - { - node->logger.debug (nano::log::type::bootstrap_lazy, "Lazy bootstrap attempt ID {} expired", id); - } - lock.unlock (); - stop (); - condition.notify_all (); -} - -bool nano::bootstrap_attempt_lazy::process_block (std::shared_ptr const & block_a, nano::account const & known_account_a, uint64_t pull_blocks_processed, nano::bulk_pull::count_t max_blocks, bool block_expected, unsigned retry_limit) -{ - bool stop_pull (false); - if (block_expected) - { - stop_pull = process_block_lazy (block_a, known_account_a, pull_blocks_processed, max_blocks, retry_limit); - } - else - { - // Drop connection with unexpected block for lazy bootstrap - stop_pull = true; - } - return stop_pull; -} - -bool nano::bootstrap_attempt_lazy::process_block_lazy (std::shared_ptr const & block_a, nano::account const & known_account_a, uint64_t pull_blocks_processed, nano::bulk_pull::count_t max_blocks, unsigned retry_limit) -{ - auto node = this->node.lock (); - if (!node) - { - return true; - } - bool stop_pull (false); - auto hash (block_a->hash ()); - nano::unique_lock lock{ mutex }; - // Processing new blocks - if (!lazy_blocks_processed (hash)) - { - // Search for new dependencies - if (block_a->source_field () && !node->block_or_pruned_exists (block_a->source_field ().value ()) && block_a->source_field ().value () != node->network_params.ledger.genesis->account ().as_union ()) - { - lazy_add (block_a->source_field ().value (), retry_limit); - } - else if (block_a->type () == nano::block_type::state) - { - lazy_block_state (block_a, retry_limit); - } - lazy_blocks_insert (hash); - // Adding lazy balances for first processed block in pull - if (pull_blocks_processed == 1 && (block_a->type () == nano::block_type::state || block_a->type () == nano::block_type::send)) - { - lazy_balances.emplace (hash, block_a->balance_field ().value ().number ()); - } - // Clearing lazy balances for previous block - if (!block_a->previous ().is_zero () && lazy_balances.find (block_a->previous ()) != lazy_balances.end ()) - { - lazy_balances.erase (block_a->previous ()); - } - lazy_block_state_backlog_check (block_a, hash); - lock.unlock (); - node->block_processor.add (block_a, nano::block_source::bootstrap_legacy); - } - // Force drop lazy bootstrap connection for long bulk_pull - if (pull_blocks_processed > max_blocks) - { - stop_pull = true; - } - return stop_pull; -} - -void nano::bootstrap_attempt_lazy::lazy_block_state (std::shared_ptr const & block_a, unsigned retry_limit) -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - std::shared_ptr block_l (std::static_pointer_cast (block_a)); - if (block_l != nullptr) - { - auto transaction = node->ledger.tx_begin_read (); - nano::uint128_t balance (block_l->hashables.balance.number ()); - auto const & link (block_l->hashables.link); - // If link is not epoch link or 0. And if block from link is unknown - if (!link.is_zero () && !node->ledger.is_epoch_link (link) && !lazy_blocks_processed (link.as_block_hash ()) && !node->ledger.any.block_exists_or_pruned (transaction, link.as_block_hash ())) - { - auto const & previous (block_l->hashables.previous); - // If state block previous is 0 then source block required - if (previous.is_zero ()) - { - lazy_add (link, retry_limit); - } - // In other cases previous block balance required to find out subtype of state block - else if (node->ledger.any.block_exists_or_pruned (transaction, previous)) - { - auto previous_balance = node->ledger.any.block_balance (transaction, previous); - if (previous_balance) - { - if (previous_balance.value ().number () <= balance) - { - lazy_add (link, retry_limit); - } - } - // Else ignore pruned blocks - } - // Search balance of already processed previous blocks - else if (lazy_blocks_processed (previous)) - { - auto previous_balance (lazy_balances.find (previous)); - if (previous_balance != lazy_balances.end ()) - { - if (previous_balance->second <= balance) - { - lazy_add (link, retry_limit); - } - lazy_balances.erase (previous_balance); - } - } - // Insert in backlog state blocks if previous wasn't already processed - else - { - lazy_state_backlog.emplace (previous, nano::lazy_state_backlog_item{ link, balance, retry_limit }); - } - } - } -} - -void nano::bootstrap_attempt_lazy::lazy_block_state_backlog_check (std::shared_ptr const & block_a, nano::block_hash const & hash_a) -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - // Search unknown state blocks balances - auto find_state (lazy_state_backlog.find (hash_a)); - if (find_state != lazy_state_backlog.end ()) - { - auto next_block (find_state->second); - // Retrieve balance for previous state & send blocks - if (block_a->type () == nano::block_type::state || block_a->type () == nano::block_type::send) - { - if (block_a->balance_field ().value ().number () <= next_block.balance) // balance - { - lazy_add (next_block.link, next_block.retry_limit); // link - } - } - // Assumption for other legacy block types - else if (lazy_undefined_links.find (next_block.link.as_block_hash ()) == lazy_undefined_links.end ()) - { - lazy_add (next_block.link, node->network_params.bootstrap.lazy_retry_limit); // Head is not confirmed. It can be account or hash or non-existing - lazy_undefined_links.insert (next_block.link.as_block_hash ()); - } - lazy_state_backlog.erase (find_state); - } -} - -void nano::bootstrap_attempt_lazy::lazy_backlog_cleanup () -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - uint64_t read_count (0); - auto transaction = node->ledger.tx_begin_read (); - for (auto it (lazy_state_backlog.begin ()), end (lazy_state_backlog.end ()); it != end && !stopped;) - { - if (node->ledger.any.block_exists_or_pruned (transaction, it->first)) - { - auto next_block (it->second); - auto balance = node->ledger.any.block_balance (transaction, it->first); - if (balance) - { - if (balance.value ().number () <= next_block.balance) // balance - { - lazy_add (next_block.link, next_block.retry_limit); // link - } - } - else - { - lazy_add (next_block.link, node->network_params.bootstrap.lazy_retry_limit); // Not confirmed - } - it = lazy_state_backlog.erase (it); - } - else - { - lazy_add (it->first, it->second.retry_limit); - ++it; - } - // We don't want to open read transactions for too long - ++read_count; - if (read_count % batch_read_size == 0) - { - transaction.refresh (); - } - } -} - -void nano::bootstrap_attempt_lazy::lazy_blocks_insert (nano::block_hash const & hash_a) -{ - debug_assert (!mutex.try_lock ()); - auto inserted (lazy_blocks.insert (std::hash<::nano::block_hash> () (hash_a))); - if (inserted.second) - { - ++lazy_blocks_count; - debug_assert (lazy_blocks_count > 0); - } -} - -void nano::bootstrap_attempt_lazy::lazy_blocks_erase (nano::block_hash const & hash_a) -{ - debug_assert (!mutex.try_lock ()); - auto erased (lazy_blocks.erase (std::hash<::nano::block_hash> () (hash_a))); - if (erased) - { - --lazy_blocks_count; - debug_assert (lazy_blocks_count != std::numeric_limits::max ()); - } -} - -bool nano::bootstrap_attempt_lazy::lazy_blocks_processed (nano::block_hash const & hash_a) -{ - return lazy_blocks.find (std::hash<::nano::block_hash> () (hash_a)) != lazy_blocks.end (); -} - -bool nano::bootstrap_attempt_lazy::lazy_processed_or_exists (nano::block_hash const & hash_a) -{ - auto node = this->node.lock (); - if (!node) - { - return true; - } - bool result (false); - nano::unique_lock lock{ mutex }; - if (lazy_blocks_processed (hash_a)) - { - result = true; - } - else - { - lock.unlock (); - if (node->block_or_pruned_exists (hash_a)) - { - result = true; - } - } - return result; -} - -void nano::bootstrap_attempt_lazy::get_information (boost::property_tree::ptree & tree_a) -{ - nano::lock_guard lock{ mutex }; - tree_a.put ("lazy_blocks", std::to_string (lazy_blocks.size ())); - tree_a.put ("lazy_state_backlog", std::to_string (lazy_state_backlog.size ())); - tree_a.put ("lazy_balances", std::to_string (lazy_balances.size ())); - tree_a.put ("lazy_undefined_links", std::to_string (lazy_undefined_links.size ())); - tree_a.put ("lazy_pulls", std::to_string (lazy_pulls.size ())); - tree_a.put ("lazy_keys", std::to_string (lazy_keys.size ())); - if (!lazy_keys.empty ()) - { - tree_a.put ("lazy_key_1", (*(lazy_keys.begin ())).to_string ()); - } -} - -nano::bootstrap_attempt_wallet::bootstrap_attempt_wallet (std::shared_ptr const & node_a, uint64_t incremental_id_a, std::string id_a) : - nano::bootstrap_attempt (node_a, nano::bootstrap_mode::wallet_lazy, incremental_id_a, id_a) -{ - node_a->bootstrap_initiator.notify_listeners (true); -} - -nano::bootstrap_attempt_wallet::~bootstrap_attempt_wallet () -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - node->bootstrap_initiator.notify_listeners (false); -} - -void nano::bootstrap_attempt_wallet::request_pending (nano::unique_lock & lock_a) -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - lock_a.unlock (); - auto connection_l (node->bootstrap_initiator.connections->connection (shared_from_this ())); - lock_a.lock (); - if (connection_l && !stopped) - { - auto account (wallet_accounts.front ()); - wallet_accounts.pop_front (); - ++pulling; - auto this_l = std::dynamic_pointer_cast (shared_from_this ()); - // The bulk_pull_account_client destructor attempt to requeue_pull which can cause a deadlock if this is the last reference - // Dispatch request in an external thread in case it needs to be destroyed - node->background ([connection_l, this_l, account] () { - auto client (std::make_shared (connection_l, this_l, account)); - client->request (); - }); - } -} - -void nano::bootstrap_attempt_wallet::requeue_pending (nano::account const & account_a) -{ - auto account (account_a); - { - nano::lock_guard lock{ mutex }; - wallet_accounts.push_front (account); - } - condition.notify_all (); -} - -void nano::bootstrap_attempt_wallet::wallet_start (std::deque & accounts_a) -{ - { - nano::lock_guard lock{ mutex }; - wallet_accounts.swap (accounts_a); - } - condition.notify_all (); -} - -bool nano::bootstrap_attempt_wallet::wallet_finished () -{ - debug_assert (!mutex.try_lock ()); - auto running (!stopped); - auto more_accounts (!wallet_accounts.empty ()); - auto still_pulling (pulling > 0); - return running && (more_accounts || still_pulling); -} - -void nano::bootstrap_attempt_wallet::run () -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - debug_assert (started); - debug_assert (!node->flags.disable_wallet_bootstrap); - node->bootstrap_initiator.connections->populate_connections (false); - auto start_time (std::chrono::steady_clock::now ()); - auto max_time (std::chrono::minutes (10)); - nano::unique_lock lock{ mutex }; - while (wallet_finished () && std::chrono::steady_clock::now () - start_time < max_time) - { - if (!wallet_accounts.empty ()) - { - request_pending (lock); - } - else - { - condition.wait_for (lock, std::chrono::seconds (1)); - } - } - if (!stopped) - { - node->logger.info (nano::log::type::bootstrap_lazy, "Completed wallet lazy pulls"); - } - lock.unlock (); - stop (); - condition.notify_all (); -} - -std::size_t nano::bootstrap_attempt_wallet::wallet_size () -{ - nano::lock_guard lock{ mutex }; - return wallet_accounts.size (); -} - -void nano::bootstrap_attempt_wallet::get_information (boost::property_tree::ptree & tree_a) -{ - nano::lock_guard lock{ mutex }; - tree_a.put ("wallet_accounts", std::to_string (wallet_accounts.size ())); -} diff --git a/nano/node/bootstrap/bootstrap_lazy.hpp b/nano/node/bootstrap/bootstrap_lazy.hpp deleted file mode 100644 index d355eb796..000000000 --- a/nano/node/bootstrap/bootstrap_lazy.hpp +++ /dev/null @@ -1,81 +0,0 @@ -#pragma once - -#include - -#include - -#include -#include - -namespace mi = boost::multi_index; - -namespace nano -{ -class node; -class lazy_state_backlog_item final -{ -public: - nano::link link{ 0 }; - nano::uint128_t balance{ 0 }; - unsigned retry_limit{ 0 }; -}; - -/** - * Lazy bootstrap session. Started with a block hash, this will "trace down" the blocks obtained to find a connection to the ledger. - * This attempts to quickly bootstrap a section of the ledger given a hash that's known to be confirmed. - */ -class bootstrap_attempt_lazy final : public bootstrap_attempt -{ -public: - explicit bootstrap_attempt_lazy (std::shared_ptr const & node_a, uint64_t incremental_id_a, std::string const & id_a = ""); - ~bootstrap_attempt_lazy (); - bool process_block (std::shared_ptr const &, nano::account const &, uint64_t, nano::bulk_pull::count_t, bool, unsigned) override; - void run () override; - bool lazy_start (nano::hash_or_account const &); - void lazy_add (nano::hash_or_account const &, unsigned); - void lazy_add (nano::pull_info const &); - void lazy_requeue (nano::block_hash const &, nano::block_hash const &); - bool lazy_finished (); - bool lazy_has_expired () const; - uint32_t lazy_batch_size (); - void lazy_pull_flush (nano::unique_lock & lock_a); - bool process_block_lazy (std::shared_ptr const &, nano::account const &, uint64_t, nano::bulk_pull::count_t, unsigned); - void lazy_block_state (std::shared_ptr const &, unsigned); - void lazy_block_state_backlog_check (std::shared_ptr const &, nano::block_hash const &); - void lazy_backlog_cleanup (); - void lazy_blocks_insert (nano::block_hash const &); - void lazy_blocks_erase (nano::block_hash const &); - bool lazy_blocks_processed (nano::block_hash const &); - bool lazy_processed_or_exists (nano::block_hash const &); - void get_information (boost::property_tree::ptree &) override; - std::unordered_set lazy_blocks; - std::unordered_map lazy_state_backlog; - std::unordered_set lazy_undefined_links; - std::unordered_map lazy_balances; - std::unordered_set lazy_keys; - std::deque> lazy_pulls; - std::chrono::steady_clock::time_point lazy_start_time; - std::atomic lazy_blocks_count{ 0 }; - std::size_t peer_count{ 0 }; - /** The maximum number of records to be read in while iterating over long lazy containers */ - static uint64_t constexpr batch_read_size = 256; -}; - -/** - * Wallet bootstrap session. This session will trace down accounts within local wallets to try and bootstrap those blocks first. - */ -class bootstrap_attempt_wallet final : public bootstrap_attempt -{ -public: - explicit bootstrap_attempt_wallet (std::shared_ptr const & node_a, uint64_t incremental_id_a, std::string id_a = ""); - ~bootstrap_attempt_wallet (); - void request_pending (nano::unique_lock &); - void requeue_pending (nano::account const &); - void run () override; - void wallet_start (std::deque &); - bool wallet_finished (); - std::size_t wallet_size (); - void get_information (boost::property_tree::ptree &) override; - std::deque wallet_accounts; -}; -} diff --git a/nano/node/bootstrap/bootstrap_legacy.cpp b/nano/node/bootstrap/bootstrap_legacy.cpp deleted file mode 100644 index 47368eccf..000000000 --- a/nano/node/bootstrap/bootstrap_legacy.cpp +++ /dev/null @@ -1,262 +0,0 @@ -#include -#include -#include -#include - -#include - -nano::bootstrap_attempt_legacy::bootstrap_attempt_legacy (std::shared_ptr const & node_a, uint64_t const incremental_id_a, std::string const & id_a, uint32_t const frontiers_age_a, nano::account const & start_account_a) : - nano::bootstrap_attempt (node_a, nano::bootstrap_mode::legacy, incremental_id_a, id_a), - frontiers_age (frontiers_age_a), - start_account (start_account_a) -{ - node_a->bootstrap_initiator.notify_listeners (true); -} - -bool nano::bootstrap_attempt_legacy::consume_future (std::future & future_a) -{ - bool result; - try - { - result = future_a.get (); - } - catch (std::future_error &) - { - result = true; - } - return result; -} - -void nano::bootstrap_attempt_legacy::stop () -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - nano::unique_lock lock{ mutex }; - stopped = true; - lock.unlock (); - condition.notify_all (); - lock.lock (); - if (auto i = frontiers.lock ()) - { - try - { - i->promise.set_value (true); - } - catch (std::future_error &) - { - } - } - if (auto i = push.lock ()) - { - try - { - i->promise.set_value (true); - } - catch (std::future_error &) - { - } - } - lock.unlock (); - node->bootstrap_initiator.connections->clear_pulls (incremental_id); -} - -void nano::bootstrap_attempt_legacy::request_push (nano::unique_lock & lock_a) -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - bool error (false); - lock_a.unlock (); - auto connection_l (node->bootstrap_initiator.connections->find_connection (endpoint_frontier_request)); - lock_a.lock (); - if (connection_l) - { - std::future future; - { - auto this_l = std::dynamic_pointer_cast (shared_from_this ()); - auto client = std::make_shared (connection_l, this_l); - client->start (); - push = client; - future = client->promise.get_future (); - } - lock_a.unlock (); - error = consume_future (future); // This is out of scope of `client' so when the last reference via boost::asio::io_context is lost and the client is destroyed, the future throws an exception. - lock_a.lock (); - } -} - -void nano::bootstrap_attempt_legacy::add_frontier (nano::pull_info const & pull_a) -{ - // Prevent incorrect or malicious pulls with frontier 0 insertion - if (!pull_a.head.is_zero ()) - { - nano::lock_guard lock{ mutex }; - frontier_pulls.push_back (pull_a); - } -} - -void nano::bootstrap_attempt_legacy::add_bulk_push_target (nano::block_hash const & head, nano::block_hash const & end) -{ - nano::lock_guard lock{ mutex }; - bulk_push_targets.emplace_back (head, end); -} - -bool nano::bootstrap_attempt_legacy::request_bulk_push_target (std::pair & current_target_a) -{ - nano::lock_guard lock{ mutex }; - auto empty (bulk_push_targets.empty ()); - if (!empty) - { - current_target_a = bulk_push_targets.back (); - bulk_push_targets.pop_back (); - } - return empty; -} - -void nano::bootstrap_attempt_legacy::set_start_account (nano::account const & start_account_a) -{ - // Add last account from frontier request - nano::lock_guard lock{ mutex }; - start_account = start_account_a; -} - -bool nano::bootstrap_attempt_legacy::request_frontier (nano::unique_lock & lock_a, bool first_attempt) -{ - auto node = this->node.lock (); - if (!node) - { - return true; - } - auto result (true); - lock_a.unlock (); - auto connection_l (node->bootstrap_initiator.connections->connection (shared_from_this (), first_attempt)); - lock_a.lock (); - if (connection_l && !stopped) - { - endpoint_frontier_request = connection_l->channel->get_remote_endpoint (); - std::future future; - { - auto this_l = std::dynamic_pointer_cast (shared_from_this ()); - auto client = std::make_shared (connection_l, this_l); - client->run (start_account, frontiers_age, node->config.bootstrap_frontier_request_count); - frontiers = client; - future = client->promise.get_future (); - } - lock_a.unlock (); - result = consume_future (future); // This is out of scope of `client' so when the last reference via boost::asio::io_context is lost and the client is destroyed, the future throws an exception. - lock_a.lock (); - if (result) - { - frontier_pulls.clear (); - } - else - { - account_count = nano::narrow_cast (frontier_pulls.size ()); - // Shuffle pulls - release_assert (std::numeric_limits::max () > frontier_pulls.size ()); - if (!frontier_pulls.empty ()) - { - for (auto i = static_cast (frontier_pulls.size () - 1); i > 0; --i) - { - auto k = nano::random_pool::generate_word32 (0, i); - std::swap (frontier_pulls[i], frontier_pulls[k]); - } - } - // Add to regular pulls - while (!frontier_pulls.empty ()) - { - auto pull (frontier_pulls.front ()); - lock_a.unlock (); - node->bootstrap_initiator.connections->add_pull (pull); - lock_a.lock (); - ++pulling; - frontier_pulls.pop_front (); - } - } - if (!result) - { - node->logger.debug (nano::log::type::bootstrap_legacy, "Completed frontier request, {} out of sync accounts according to {}", account_count.load (), connection_l->channel->to_string ()); - } - else - { - node->stats.inc (nano::stat::type::error, nano::stat::detail::frontier_req, nano::stat::dir::out); - } - } - return result; -} - -void nano::bootstrap_attempt_legacy::run_start (nano::unique_lock & lock_a) -{ - frontiers_received = false; - auto frontier_failure (true); - uint64_t frontier_attempts (0); - while (!stopped && frontier_failure) - { - ++frontier_attempts; - frontier_failure = request_frontier (lock_a, frontier_attempts == 1); - } - frontiers_received = true; -} - -void nano::bootstrap_attempt_legacy::run () -{ - auto node = this->node.lock (); - if (!node) - { - return; - } - debug_assert (started); - debug_assert (!node->flags.disable_legacy_bootstrap); - node->bootstrap_initiator.connections->populate_connections (false); - nano::unique_lock lock{ mutex }; - run_start (lock); - while (still_pulling ()) - { - while (still_pulling ()) - { - // clang-format off - condition.wait (lock, [&stopped = stopped, &pulling = pulling] { return stopped || pulling == 0; }); - } - - // TODO: This check / wait is a heuristic and should be improved. - auto wait_start = std::chrono::steady_clock::now (); - while (!stopped && node->block_processor.size (nano::block_source::bootstrap_legacy) != 0 && ((std::chrono::steady_clock::now () - wait_start) < std::chrono::seconds{ 10 })) - { - condition.wait_for (lock, std::chrono::milliseconds{ 100 }, [this, node] { return stopped || node->block_processor.size (nano::block_source::bootstrap_legacy) == 0; }); - } - - if (start_account.number () != std::numeric_limits::max ()) - { - node->logger.debug(nano::log::type::bootstrap_legacy, "Requesting new frontiers after: {}", start_account.to_account ()); - - // Requesting new frontiers - run_start (lock); - } - } - if (!stopped) - { - node->logger.debug(nano::log::type::bootstrap_legacy, "Completed legacy pulls"); - - if (!node->flags.disable_bootstrap_bulk_push_client) - { - request_push (lock); - } - } - lock.unlock (); - stop (); - condition.notify_all (); -} - -void nano::bootstrap_attempt_legacy::get_information (boost::property_tree::ptree & tree_a) -{ - nano::lock_guard lock{ mutex }; - tree_a.put ("frontier_pulls", std::to_string (frontier_pulls.size ())); - tree_a.put ("frontiers_received", static_cast (frontiers_received)); - tree_a.put ("frontiers_age", std::to_string (frontiers_age)); - tree_a.put ("last_account", start_account.to_account ()); -} diff --git a/nano/node/bootstrap/bootstrap_legacy.hpp b/nano/node/bootstrap/bootstrap_legacy.hpp deleted file mode 100644 index 51423d778..000000000 --- a/nano/node/bootstrap/bootstrap_legacy.hpp +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once - -#include - -#include - -#include -#include -#include -#include - -namespace nano -{ -class node; - -/** - * Legacy bootstrap session. This is made up of 3 phases: frontier requests, bootstrap pulls, bootstrap pushes. - */ -class bootstrap_attempt_legacy : public bootstrap_attempt -{ -public: - explicit bootstrap_attempt_legacy (std::shared_ptr const & node_a, uint64_t const incremental_id_a, std::string const & id_a, uint32_t const frontiers_age_a, nano::account const & start_account_a); - void run () override; - bool consume_future (std::future &); - void stop () override; - bool request_frontier (nano::unique_lock &, bool = false); - void request_push (nano::unique_lock &); - void add_frontier (nano::pull_info const &); - void add_bulk_push_target (nano::block_hash const &, nano::block_hash const &); - bool request_bulk_push_target (std::pair &); - void set_start_account (nano::account const &); - void run_start (nano::unique_lock &); - void get_information (boost::property_tree::ptree &) override; - nano::tcp_endpoint endpoint_frontier_request; - std::weak_ptr frontiers; - std::weak_ptr push; - std::deque frontier_pulls; - std::vector> bulk_push_targets; - nano::account start_account{}; - std::atomic account_count{ 0 }; - uint32_t frontiers_age; -}; -} diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 288296d05..0ecacfe53 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -250,7 +249,6 @@ nano::account_info nano::json_handler::account_info_impl (secure::transaction co if (!info) { ec = nano::error_common::account_not_found; - node.bootstrap_initiator.bootstrap_lazy (account_a, false, account_a.to_account ()); } else { @@ -1804,16 +1802,7 @@ void nano::json_handler::bootstrap () uint16_t port; if (!nano::parse_port (port_text, port)) { - if (!node.flags.disable_legacy_bootstrap) - { - std::string bootstrap_id (request.get ("id", "")); - node.bootstrap_initiator.bootstrap (nano::endpoint (address, port), true, bootstrap_id); - response_l.put ("success", ""); - } - else - { - ec = nano::error_rpc::disabled_bootstrap_legacy; - } + ec = nano::error_rpc::disabled_bootstrap_legacy; } else { @@ -1830,22 +1819,7 @@ void nano::json_handler::bootstrap () void nano::json_handler::bootstrap_any () { bool const force = request.get ("force", false); - if (!node.flags.disable_legacy_bootstrap) - { - nano::account start_account{}; - boost::optional account_text (request.get_optional ("account")); - if (account_text.is_initialized ()) - { - start_account = account_impl (account_text.get ()); - } - std::string bootstrap_id (request.get ("id", "")); - node.bootstrap_initiator.bootstrap (force, bootstrap_id, std::numeric_limits::max (), start_account); - response_l.put ("success", ""); - } - else - { - ec = nano::error_rpc::disabled_bootstrap_legacy; - } + ec = nano::error_rpc::disabled_bootstrap_legacy; response_errors (); } @@ -1855,19 +1829,7 @@ void nano::json_handler::bootstrap_lazy () bool const force = request.get ("force", false); if (!ec) { - if (!node.flags.disable_lazy_bootstrap) - { - auto existed (node.bootstrap_initiator.current_lazy_attempt () != nullptr); - std::string bootstrap_id (request.get ("id", "")); - auto key_inserted (node.bootstrap_initiator.bootstrap_lazy (hash, force, bootstrap_id)); - bool started = !existed && key_inserted; - response_l.put ("started", started ? "1" : "0"); - response_l.put ("key_inserted", key_inserted ? "1" : "0"); - } - else - { - ec = nano::error_rpc::disabled_bootstrap_lazy; - } + ec = nano::error_rpc::disabled_bootstrap_lazy; } response_errors (); } @@ -1877,39 +1839,7 @@ void nano::json_handler::bootstrap_lazy () */ void nano::json_handler::bootstrap_status () { - auto attempts_count (node.bootstrap_initiator.attempts.size ()); - response_l.put ("bootstrap_threads", std::to_string (node.config.bootstrap_initiator_threads)); - response_l.put ("running_attempts_count", std::to_string (attempts_count)); - response_l.put ("total_attempts_count", std::to_string (node.bootstrap_initiator.attempts.incremental)); - boost::property_tree::ptree connections; - { - nano::lock_guard connections_lock (node.bootstrap_initiator.connections->mutex); - connections.put ("clients", std::to_string (node.bootstrap_initiator.connections->clients.size ())); - connections.put ("connections", std::to_string (node.bootstrap_initiator.connections->connections_count)); - connections.put ("idle", std::to_string (node.bootstrap_initiator.connections->idle.size ())); - connections.put ("target_connections", std::to_string (node.bootstrap_initiator.connections->target_connections (node.bootstrap_initiator.connections->pulls.size (), attempts_count))); - connections.put ("pulls", std::to_string (node.bootstrap_initiator.connections->pulls.size ())); - } - response_l.add_child ("connections", connections); - boost::property_tree::ptree attempts; - { - nano::lock_guard attempts_lock (node.bootstrap_initiator.attempts.bootstrap_attempts_mutex); - for (auto i : node.bootstrap_initiator.attempts.attempts) - { - boost::property_tree::ptree entry; - auto & attempt (i.second); - entry.put ("id", attempt->id); - entry.put ("mode", attempt->mode_text ()); - entry.put ("started", static_cast (attempt->started)); - entry.put ("pulling", std::to_string (attempt->pulling)); - entry.put ("total_blocks", std::to_string (attempt->total_blocks)); - entry.put ("requeued_pulls", std::to_string (attempt->requeued_pulls)); - attempt->get_information (entry); - entry.put ("duration", std::chrono::duration_cast (std::chrono::steady_clock::now () - attempt->attempt_start).count ()); - attempts.push_back (std::make_pair ("", entry)); - } - } - response_l.add_child ("attempts", attempts); + // TODO: Bootstrap status for ascending bootstrap response_errors (); } diff --git a/nano/node/node.cpp b/nano/node/node.cpp index e556fda03..dc5919a18 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -108,7 +108,6 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy network (*this, config.peering_port.has_value () ? *config.peering_port : 0), telemetry_impl{ std::make_unique (flags, *this, network, observers, network_params, stats) }, telemetry{ *telemetry_impl }, - bootstrap_initiator (*this), bootstrap_server{ config.bootstrap_server, store, ledger, network_params.network, stats }, // BEWARE: `bootstrap` takes `network.port` instead of `config.peering_port` because when the user doesn't specify // a peering port and wants the OS to pick one, the picking happens when `network` gets initialized @@ -564,10 +563,6 @@ void nano::node::start () network.start (); message_processor.start (); - if (!flags.disable_legacy_bootstrap && !flags.disable_ongoing_bootstrap) - { - ongoing_bootstrap (); - } if (flags.enable_pruning) { auto this_l (shared ()); @@ -608,14 +603,6 @@ void nano::node::start () { search_receivable_all (); } - if (!flags.disable_wallet_bootstrap) - { - // Delay to start wallet lazy bootstrap - auto this_l (shared ()); - workers.post_delayed (std::chrono::minutes (1), [this_l] () { - this_l->bootstrap_wallet (); - }); - } // Start port mapping if external address is not defined and TCP ports are enabled if (config.external_address == boost::asio::ip::address_v6::any ().to_string () && tcp_enabled) { @@ -684,7 +671,6 @@ void nano::node::stop () telemetry.stop (); websocket.stop (); bootstrap_server.stop (); - bootstrap_initiator.stop (); port_mapping.stop (); wallets.stop (); stats.stop (); @@ -777,69 +763,6 @@ void nano::node::long_inactivity_cleanup () } } -void nano::node::ongoing_bootstrap () -{ - auto next_wakeup = network_params.network.bootstrap_interval; - if (warmed_up < 3) - { - // Re-attempt bootstrapping more aggressively on startup - next_wakeup = std::chrono::seconds (5); - if (!bootstrap_initiator.in_progress () && !network.empty ()) - { - ++warmed_up; - } - } - if (network_params.network.is_dev_network () && flags.bootstrap_interval != 0) - { - // For test purposes allow faster automatic bootstraps - next_wakeup = std::chrono::seconds (flags.bootstrap_interval); - ++warmed_up; - } - // Differential bootstrap with max age (75% of all legacy attempts) - uint32_t frontiers_age (std::numeric_limits::max ()); - auto bootstrap_weight_reached (ledger.block_count () >= ledger.bootstrap_weight_max_blocks); - auto previous_bootstrap_count (stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate, nano::stat::dir::out) + stats.count (nano::stat::type::bootstrap, nano::stat::detail::initiate_legacy_age, nano::stat::dir::out)); - /* - - Maximum value for 25% of attempts or if block count is below preconfigured value (initial bootstrap not finished) - - Node shutdown time minus 1 hour for start attempts (warm up) - - Default age value otherwise (1 day for live network, 1 hour for beta) - */ - if (bootstrap_weight_reached) - { - if (warmed_up < 3) - { - // Find last online weight sample (last active time for node) - uint64_t last_sample_time (0); - { - auto transaction = store.tx_begin_read (); - auto last_record = store.online_weight.rbegin (transaction); - if (last_record != store.online_weight.end (transaction)) - { - last_sample_time = last_record->first; - } - } - uint64_t time_since_last_sample = std::chrono::duration_cast (std::chrono::system_clock::now ().time_since_epoch ()).count () - static_cast (last_sample_time / std::pow (10, 9)); // Nanoseconds to seconds - if (time_since_last_sample + 60 * 60 < std::numeric_limits::max ()) - { - frontiers_age = std::max (static_cast (time_since_last_sample + 60 * 60), network_params.bootstrap.default_frontiers_age_seconds); - } - } - else if (previous_bootstrap_count % 4 != 0) - { - frontiers_age = network_params.bootstrap.default_frontiers_age_seconds; - } - } - // Bootstrap and schedule for next attempt - bootstrap_initiator.bootstrap (false, boost::str (boost::format ("auto_bootstrap_%1%") % previous_bootstrap_count), frontiers_age); - std::weak_ptr node_w (shared_from_this ()); - workers.post_delayed (next_wakeup, [node_w] () { - if (auto node_l = node_w.lock ()) - { - node_l->ongoing_bootstrap (); - } - }); -} - void nano::node::backup_wallet () { auto transaction (wallets.tx_begin_read ()); @@ -870,29 +793,6 @@ void nano::node::search_receivable_all () }); } -void nano::node::bootstrap_wallet () -{ - std::deque accounts; - { - nano::lock_guard lock{ wallets.mutex }; - auto const transaction (wallets.tx_begin_read ()); - for (auto i (wallets.items.begin ()), n (wallets.items.end ()); i != n && accounts.size () < 128; ++i) - { - auto & wallet (*i->second); - nano::lock_guard wallet_lock{ wallet.store.mutex }; - for (auto j (wallet.store.begin (transaction)), m (wallet.store.end (transaction)); j != m && accounts.size () < 128; ++j) - { - nano::account account (j->first); - accounts.push_back (account); - } - } - } - if (!accounts.empty ()) - { - bootstrap_initiator.bootstrap_wallet (accounts); - } -} - bool nano::node::collect_ledger_pruning_targets (std::deque & pruning_targets_a, nano::account & last_account_a, uint64_t const batch_read_size_a, uint64_t const max_depth_a, uint64_t const cutoff_time_a) { uint64_t read_operations (0); @@ -1285,7 +1185,6 @@ nano::container_info nano::node::container_info () const info.add ("work", work.container_info ()); info.add ("ledger", ledger.container_info ()); info.add ("active", active.container_info ()); - info.add ("bootstrap_initiator", bootstrap_initiator.container_info ()); info.add ("tcp_listener", tcp_listener.container_info ()); info.add ("network", network.container_info ()); info.add ("telemetry", telemetry.container_info ()); diff --git a/nano/node/node.hpp b/nano/node/node.hpp index ac9a4bb25..d774adb21 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -6,8 +6,6 @@ #include #include #include -#include -#include #include #include #include @@ -104,10 +102,8 @@ public: std::pair balance_pending (nano::account const &, bool only_confirmed); nano::uint128_t weight (nano::account const &); nano::uint128_t minimum_principal_weight (); - void ongoing_bootstrap (); void backup_wallet (); void search_receivable_all (); - void bootstrap_wallet (); bool collect_ledger_pruning_targets (std::deque &, nano::account &, uint64_t const, uint64_t const, uint64_t const); void ledger_pruning (uint64_t const, bool); void ongoing_ledger_pruning (); @@ -180,7 +176,6 @@ public: nano::network network; std::unique_ptr telemetry_impl; nano::telemetry & telemetry; - nano::bootstrap_initiator bootstrap_initiator; nano::bootstrap_server bootstrap_server; std::unique_ptr tcp_listener_impl; nano::transport::tcp_listener & tcp_listener; diff --git a/nano/node/transport/tcp_server.cpp b/nano/node/transport/tcp_server.cpp index 881da00c5..1097301cb 100644 --- a/nano/node/transport/tcp_server.cpp +++ b/nano/node/transport/tcp_server.cpp @@ -1,5 +1,3 @@ -#include -#include #include #include #include @@ -516,79 +514,26 @@ nano::transport::tcp_server::bootstrap_message_visitor::bootstrap_message_visito void nano::transport::tcp_server::bootstrap_message_visitor::bulk_pull (const nano::bulk_pull & message) { - auto node = server->node.lock (); - if (!node) - { - return; - } - if (node->flags.disable_bootstrap_bulk_pull_server) - { - return; - } - - node->bootstrap_workers.post ([server = server, message = message] () { - // TODO: Add completion callback to bulk pull server - // TODO: There should be no need to re-copy message as unique pointer, refactor those bulk/frontier pull/push servers - auto bulk_pull_server = std::make_shared (server, std::make_unique (message)); - bulk_pull_server->send_next (); - }); - - processed = true; + // Ignored since V28 + // TODO: Abort connection? } void nano::transport::tcp_server::bootstrap_message_visitor::bulk_pull_account (const nano::bulk_pull_account & message) { - auto node = server->node.lock (); - if (!node) - { - return; - } - if (node->flags.disable_bootstrap_bulk_pull_server) - { - return; - } - - node->bootstrap_workers.post ([server = server, message = message] () { - // TODO: Add completion callback to bulk pull server - // TODO: There should be no need to re-copy message as unique pointer, refactor those bulk/frontier pull/push servers - auto bulk_pull_account_server = std::make_shared (server, std::make_unique (message)); - bulk_pull_account_server->send_frontier (); - }); - - processed = true; + // Ignored since V28 + // TODO: Abort connection? } void nano::transport::tcp_server::bootstrap_message_visitor::bulk_push (const nano::bulk_push &) { - auto node = server->node.lock (); - if (!node) - { - return; - } - node->bootstrap_workers.post ([server = server] () { - // TODO: Add completion callback to bulk pull server - auto bulk_push_server = std::make_shared (server); - bulk_push_server->throttled_receive (); - }); - - processed = true; + // Ignored since V28 + // TODO: Abort connection? } void nano::transport::tcp_server::bootstrap_message_visitor::frontier_req (const nano::frontier_req & message) { - auto node = server->node.lock (); - if (!node) - { - return; - } - - node->bootstrap_workers.post ([server = server, message = message] () { - // TODO: There should be no need to re-copy message as unique pointer, refactor those bulk/frontier pull/push servers - auto response = std::make_shared (server, std::make_unique (message)); - response->send_next (); - }); - - processed = true; + // Ignored since V28 + // TODO: Abort connection? } /* diff --git a/nano/qt/qt.cpp b/nano/qt/qt.cpp index 8f24aeb70..2782e1018 100644 --- a/nano/qt/qt.cpp +++ b/nano/qt/qt.cpp @@ -399,11 +399,6 @@ nano_qt::import::import (nano_qt::wallet & wallet_a) : { this->wallet.account = this->wallet.wallet_m->change_seed (transaction, seed_l); successful = true; - // Pending check for accounts to restore if bootstrap is in progress - if (this->wallet.node.bootstrap_initiator.in_progress ()) - { - this->wallet.needs_deterministic_restore = true; - } } else { @@ -1379,31 +1374,6 @@ void nano_qt::wallet::start () })); } }); - node.bootstrap_initiator.add_observer ([this_w] (bool active_a) { - if (auto this_l = this_w.lock ()) - { - this_l->application.postEvent (&this_l->processor, new eventloop_event ([this_w, active_a] () { - if (auto this_l = this_w.lock ()) - { - if (active_a) - { - this_l->active_status.insert (nano_qt::status_types::synchronizing); - } - else - { - this_l->active_status.erase (nano_qt::status_types::synchronizing); - // Check for accounts to restore - if (this_l->needs_deterministic_restore) - { - this_l->needs_deterministic_restore = false; - auto transaction (this_l->wallet_m->wallets.tx_begin_write ()); - this_l->wallet_m->deterministic_restore (transaction); - } - } - } - })); - } - }); node.work.work_observers.add ([this_w] (bool working) { if (auto this_l = this_w.lock ()) { @@ -1785,7 +1755,6 @@ nano_qt::advanced_actions::advanced_actions (nano_qt::wallet & wallet_a) : bootstrap_label (new QLabel ("IPV6:port \"::ffff:192.168.0.1:7075\"")), peer_count_label (new QLabel ("")), bootstrap_line (new QLineEdit), - peers_bootstrap (new QPushButton ("Initiate Bootstrap")), peers_refresh (new QPushButton ("Refresh")), peers_back (new QPushButton ("Back")), wallet (wallet_a) @@ -1827,7 +1796,6 @@ nano_qt::advanced_actions::advanced_actions (nano_qt::wallet & wallet_a) : peer_summary_layout->addWidget (peer_count_label); peers_layout->addLayout (peer_summary_layout); peers_layout->addWidget (bootstrap_line); - peers_layout->addWidget (peers_bootstrap); peers_layout->addWidget (peers_refresh); peers_layout->addWidget (peers_back); peers_layout->setContentsMargins (0, 0, 0, 0); @@ -1887,20 +1855,6 @@ nano_qt::advanced_actions::advanced_actions (nano_qt::wallet & wallet_a) : QObject::connect (peers_back, &QPushButton::released, [this] () { this->wallet.pop_main_stack (); }); - QObject::connect (peers_bootstrap, &QPushButton::released, [this] () { - nano::endpoint endpoint; - auto error (nano::parse_endpoint (bootstrap_line->text ().toStdString (), endpoint)); - if (!error) - { - show_line_ok (*bootstrap_line); - bootstrap_line->clear (); - this->wallet.node.bootstrap_initiator.bootstrap (endpoint); - } - else - { - show_line_error (*bootstrap_line); - } - }); QObject::connect (peers_refresh, &QPushButton::released, [this] () { refresh_peers (); }); @@ -1914,7 +1868,6 @@ nano_qt::advanced_actions::advanced_actions (nano_qt::wallet & wallet_a) : std::thread ([this] { this->wallet.wallet_m->search_receivable (this->wallet.wallet_m->wallets.tx_begin_read ()); }).detach (); }); QObject::connect (bootstrap, &QPushButton::released, [this] () { - std::thread ([this] { this->wallet.node.bootstrap_initiator.bootstrap (); }).detach (); }); QObject::connect (create_block, &QPushButton::released, [this] () { this->wallet.push_main_stack (this->wallet.block_creation.window); diff --git a/nano/qt/qt.hpp b/nano/qt/qt.hpp index fd94052bb..3b1c870d6 100644 --- a/nano/qt/qt.hpp +++ b/nano/qt/qt.hpp @@ -85,7 +85,6 @@ public: QLabel * bootstrap_label; QLabel * peer_count_label; QLineEdit * bootstrap_line; - QPushButton * peers_bootstrap; QPushButton * peers_refresh; QPushButton * peers_back; diff --git a/nano/qt_test/qt.cpp b/nano/qt_test/qt.cpp index b4070ca80..bb55c37dc 100644 --- a/nano/qt_test/qt.cpp +++ b/nano/qt_test/qt.cpp @@ -905,38 +905,6 @@ TEST (wallet, import_locked) system.wallet (0)->store.seed (seed3, transaction); ASSERT_EQ (seed1, seed3); } -// DISABLED: this always fails -TEST (wallet, DISABLED_synchronizing) -{ - nano_qt::eventloop_processor processor; - nano::test::system system0 (1); - nano::test::system system1 (1); - auto key1 (system0.wallet (0)->deterministic_insert ()); - auto wallet (std::make_shared (*test_application, processor, *system0.nodes[0], system0.wallet (0), key1)); - wallet->start (); - { - auto transaction = system1.nodes[0]->ledger.tx_begin_write (); - auto latest (system1.nodes[0]->ledger.any.account_head (transaction, nano::dev::genesis_key.pub)); - auto send = std::make_shared (latest, key1, 0, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, *system1.work.generate (latest)); - system1.nodes[0]->ledger.process (transaction, send); - } - ASSERT_EQ (0, wallet->active_status.active.count (nano_qt::status_types::synchronizing)); - system0.nodes[0]->bootstrap_initiator.bootstrap (system1.nodes[0]->network.endpoint ()); - system1.deadline_set (10s); - while (wallet->active_status.active.count (nano_qt::status_types::synchronizing) == 0) - { - ASSERT_NO_ERROR (system0.poll ()); - ASSERT_NO_ERROR (system1.poll ()); - test_application->processEvents (); - } - system1.deadline_set (25s); - while (wallet->active_status.active.count (nano_qt::status_types::synchronizing) == 1) - { - ASSERT_NO_ERROR (system0.poll ()); - ASSERT_NO_ERROR (system1.poll ()); - test_application->processEvents (); - } -} TEST (wallet, epoch_2_validation) { diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index ce050df02..a9757dd76 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -6471,59 +6471,6 @@ TEST (rpc, epoch_upgrade_multithreaded) } } -// FIXME: This test is testing legacy bootstrap, the current behavior is different -TEST (rpc, DISABLED_account_lazy_start) -{ - nano::test::system system{}; - nano::node_flags node_flags{}; - node_flags.disable_legacy_bootstrap = true; - auto node1 = system.add_node (node_flags); - nano::keypair key{}; - nano::block_builder builder; - // Generating test chain - auto send1 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Knano_ratio) - .link (key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*system.work.generate (nano::dev::genesis->hash ())) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (send1)); - auto open = builder - .open () - .source (send1->hash ()) - .representative (key.pub) - .account (key.pub) - .sign (key.prv, key.pub) - .work (*system.work.generate (key.pub)) - .build (); - ASSERT_EQ (nano::block_status::progress, node1->process (open)); - - // Start lazy bootstrap with account - nano::node_config node_config = system.default_config (); - node_config.ipc_config.transport_tcp.enabled = true; - node_config.ipc_config.transport_tcp.port = system.get_available_port (); - auto node2 = system.add_node (node_config, node_flags); - nano::test::establish_tcp (system, *node2, node1->network.endpoint ()); - auto const rpc_ctx = add_rpc (system, node2); - boost::property_tree::ptree request; - request.put ("action", "account_info"); - request.put ("account", key.pub.to_account ()); - auto response = wait_response (system, rpc_ctx, request); - boost::optional account_error{ response.get_optional ("error") }; - ASSERT_TRUE (account_error.is_initialized ()); - - // Check processed blocks - ASSERT_TIMELY (10s, !node2->bootstrap_initiator.in_progress ()); - - // needs timed assert because the writing (put) operation is done by a different - // thread, it might not get done before DB get operation. - ASSERT_TIMELY (15s, nano::test::block_or_pruned_all_exists (*node2, { send1, open })); -} - TEST (rpc, receive) { nano::test::system system;