diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp index f0f160dd..2af247b6 100644 --- a/nano/core_test/bootstrap.cpp +++ b/nano/core_test/bootstrap.cpp @@ -303,6 +303,39 @@ TEST (bootstrap_processor, pull_diamond) node1->stop (); } +TEST (bootstrap_processor, pull_requeue_network_error) +{ + nano::system system (24000, 2); + auto node1 = system.nodes[0]; + auto node2 = system.nodes[1]; + nano::genesis genesis; + nano::keypair key1; + auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (genesis.hash ()))); + + node1->bootstrap_initiator.bootstrap (node2->network.endpoint ()); + auto attempt (node1->bootstrap_initiator.current_attempt ()); + ASSERT_NE (nullptr, attempt); + system.deadline_set (2s); + while (!attempt->frontiers_received) + { + ASSERT_NO_ERROR (system.poll ()); + } + // Add non-existing pull & stop remote peer + { + nano::unique_lock lock (attempt->mutex); + ASSERT_FALSE (attempt->stopped); + attempt->pulls.push_back (nano::pull_info (nano::test_genesis_key.pub, send1->hash (), genesis.hash ())); + attempt->request_pull (lock); + node2->stop (); + } + system.deadline_set (5s); + while (attempt != nullptr && attempt->requeued_pulls < 1) + { + ASSERT_NO_ERROR (system.poll ()); + } + ASSERT_EQ (0, node1->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in)); // Requeue is not increasing failed attempts +} + TEST (bootstrap_processor, frontiers_unconfirmed) { nano::system system; @@ -337,40 +370,29 @@ TEST (bootstrap_processor, frontiers_unconfirmed) auto open3 (std::make_shared (key1.pub, 0, key1.pub, nano::xrb_ratio, send3->hash (), key1.prv, key1.pub, *system.work.generate (key1.pub))); ASSERT_EQ (nano::process_result::progress, node2->process (*open3).code); system.wallet (1)->insert_adhoc (nano::test_genesis_key.prv); - node_config.peering_port = 24002; - auto node3 = system.add_node (node_config, node_flags); - ASSERT_EQ (nano::process_result::progress, node3->process (*send3).code); - ASSERT_EQ (nano::process_result::progress, node3->process (*open3).code); - node_config.peering_port = 24003; - auto node4 = system.add_node (node_config, node_flags); - ASSERT_EQ (nano::process_result::progress, node4->process (*send3).code); - ASSERT_EQ (nano::process_result::progress, node4->process (*open3).code); // Test node to restart bootstrap - node_config.peering_port = 24004; + node_config.peering_port = 24002; node_flags.disable_legacy_bootstrap = false; - auto node5 = system.add_node (node_config, node_flags); + auto node3 = system.add_node (node_config, node_flags); system.deadline_set (5s); - while (node5->rep_crawler.representative_count () == 0) + while (node3->rep_crawler.representative_count () == 0) { ASSERT_NO_ERROR (system.poll ()); } //Add single excluded peers record (2 records are required to drop peer) - node5->bootstrap_initiator.excluded_peers.add (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ()), 0); - ASSERT_FALSE (node5->bootstrap_initiator.excluded_peers.check (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ()))); - node5->bootstrap_initiator.bootstrap (node1->network.endpoint ()); + node3->bootstrap_initiator.excluded_peers.add (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ()), 0); + ASSERT_FALSE (node3->bootstrap_initiator.excluded_peers.check (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ()))); + node3->bootstrap_initiator.bootstrap (node1->network.endpoint ()); system.deadline_set (15s); - while (node5->bootstrap_initiator.in_progress ()) + while (node3->bootstrap_initiator.in_progress ()) { ASSERT_NO_ERROR (system.poll ()); } - system.deadline_set (5s); - while (node5->balance (key1.pub) != nano::xrb_ratio) - { - ASSERT_NO_ERROR (system.poll ()); - } - ASSERT_EQ (1, node5->stats.count (nano::stat::type::bootstrap, nano::stat::detail::frontier_confirmation_failed, nano::stat::dir::in)); // failed request from node1 - ASSERT_TRUE (node5->bootstrap_initiator.excluded_peers.check (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ()))); + ASSERT_FALSE (node3->ledger.block_exists (send1->hash ())); + ASSERT_FALSE (node3->ledger.block_exists (open1->hash ())); + ASSERT_EQ (1, node3->stats.count (nano::stat::type::bootstrap, nano::stat::detail::frontier_confirmation_failed, nano::stat::dir::in)); // failed request from node1 + ASSERT_TRUE (node3->bootstrap_initiator.excluded_peers.check (nano::transport::map_endpoint_to_tcp (node1->network.endpoint ()))); } TEST (bootstrap_processor, frontiers_confirmed) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index d13da8c7..a99ec77e 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -207,6 +207,7 @@ bool nano::bootstrap_attempt::still_pulling () void nano::bootstrap_attempt::run_start (nano::unique_lock & lock_a) { + frontiers_received = false; frontiers_confirmed = false; total_blocks = 0; requeued_pulls = 0; @@ -216,6 +217,7 @@ void nano::bootstrap_attempt::run_start (nano::unique_lock & lock_a) { frontier_failure = request_frontier (lock_a); } + frontiers_received = true; // Shuffle pulls. release_assert (std::numeric_limits::max () > pulls.size ()); if (!pulls.empty ()) @@ -549,10 +551,13 @@ void nano::bootstrap_attempt::add_pull (nano::pull_info const & pull_a) condition.notify_all (); } -void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) +void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a, bool network_error) { auto pull (pull_a); - ++pull.attempts; + if (!network_error) + { + ++pull.attempts; + } ++requeued_pulls; if (pull.attempts < (!node->network_params.network.is_test_network () ? nano::bootstrap_limits::bootstrap_frontier_retry_limit : 1 + (pull.processed / 10000))) { diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp index f7850fbd..04b8ddaa 100644 --- a/nano/node/bootstrap/bootstrap.hpp +++ b/nano/node/bootstrap/bootstrap.hpp @@ -72,7 +72,7 @@ public: void connect_client (nano::tcp_endpoint const &); void pool_connection (std::shared_ptr); void stop (); - void requeue_pull (nano::pull_info const &); + void requeue_pull (nano::pull_info const &, bool = false); void add_pull (nano::pull_info const &); bool still_pulling (); void run_start (nano::unique_lock &); @@ -123,6 +123,7 @@ public: std::atomic runs_count{ 0 }; std::atomic requeued_pulls{ 0 }; std::vector> bulk_push_targets; + std::atomic frontiers_received{ false }; std::atomic frontiers_confirmed{ false }; std::atomic stopped{ false }; std::chrono::steady_clock::time_point attempt_start{ std::chrono::steady_clock::now () }; diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.cpp b/nano/node/bootstrap/bootstrap_bulk_pull.cpp index 63233167..66295ee9 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.cpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.cpp @@ -34,7 +34,7 @@ nano::bulk_pull_client::~bulk_pull_client () pull.account_or_head = expected; } pull.processed += pull_blocks - unexpected_count; - connection->attempt->requeue_pull (pull); + connection->attempt->requeue_pull (pull, network_error); if (connection->node->config.logging.bulk_pull_logging ()) { connection->node->logger.try_log (boost::str (boost::format ("Bulk pull end block is not expected %1% for account %2%") % pull.end.to_string () % pull.account_or_head.to_account ())); @@ -101,6 +101,7 @@ void nano::bulk_pull_client::request () void nano::bulk_pull_client::throttled_receive_block () { + assert (!network_error); if (!connection->node->block_processor.half_full ()) { receive_block (); @@ -132,6 +133,7 @@ void nano::bulk_pull_client::receive_block () this_l->connection->node->logger.try_log (boost::str (boost::format ("Error receiving block type: %1%") % ec.message ())); } this_l->connection->node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_receive_block_failure, nano::stat::dir::in); + this_l->network_error = true; } }); } @@ -266,6 +268,7 @@ void nano::bulk_pull_client::received_block (boost::system::error_code const & e connection->node->logger.try_log (boost::str (boost::format ("Error bulk receiving block: %1%") % ec.message ())); } connection->node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_receive_block_failure, nano::stat::dir::in); + network_error = true; } } diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.hpp b/nano/node/bootstrap/bootstrap_bulk_pull.hpp index daf849cb..060fe13a 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.hpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.hpp @@ -40,6 +40,7 @@ public: nano::pull_info pull; uint64_t pull_blocks; uint64_t unexpected_count; + bool network_error{ false }; }; class bulk_pull_account_client final : public std::enable_shared_from_this { diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index f78629c3..26948b03 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -1681,6 +1681,7 @@ void nano::json_handler::bootstrap_status () response_l.put ("total_blocks", std::to_string (attempt->total_blocks)); response_l.put ("runs_count", std::to_string (attempt->runs_count)); response_l.put ("requeued_pulls", std::to_string (attempt->requeued_pulls)); + response_l.put ("frontiers_received", static_cast (attempt->frontiers_received)); response_l.put ("frontiers_confirmed", static_cast (attempt->frontiers_confirmed)); std::string mode_text; if (attempt->mode == nano::bootstrap_mode::legacy)