diff --git a/nano/node/bootstrap_ascending/frontier_scan.cpp b/nano/node/bootstrap_ascending/frontier_scan.cpp index 28fab99de..e955fd3fc 100644 --- a/nano/node/bootstrap_ascending/frontier_scan.cpp +++ b/nano/node/bootstrap_ascending/frontier_scan.cpp @@ -136,6 +136,7 @@ std::unique_ptr nano::bootstrap_ascending::front boost::multiprecision::cpp_dec_float_50 next{ head.next.number ().str () }; boost::multiprecision::cpp_dec_float_50 end{ head.end.number ().str () }; + // Progress in the range [0, 1000000] since we can only represent `size_t` integers in the container_info data boost::multiprecision::cpp_dec_float_50 progress = (next - start) * boost::multiprecision::cpp_dec_float_50 (1000000) / (end - start); composite->add_component (std::make_unique (container_info{ std::to_string (n), progress.convert_to (), 6 })); diff --git a/nano/node/bootstrap_ascending/frontier_scan.hpp b/nano/node/bootstrap_ascending/frontier_scan.hpp index 382c59b47..bee3db24f 100644 --- a/nano/node/bootstrap_ascending/frontier_scan.hpp +++ b/nano/node/bootstrap_ascending/frontier_scan.hpp @@ -19,6 +19,10 @@ namespace mi = boost::multi_index; namespace nano::bootstrap_ascending { +/* + * Frontier scan divides the account space into ranges and scans each range for outdated frontiers in parallel. + * This class is used to track the progress of each range. + */ class frontier_scan { public: @@ -34,6 +38,7 @@ private: // Dependencies nano::stats & stats; private: + // Represents a range of accounts to scan, once the full range is scanned (goes past `end`) the head wraps around (to the `start`) struct frontier_head { frontier_head (nano::account start_a, nano::account end_a) : @@ -44,9 +49,10 @@ private: } // The range of accounts to scan is [start, end) - nano::account start; - nano::account end; + nano::account const start; + nano::account const end; + // We scan the range by querying frontiers starting at 'next' and gathering candidates nano::account next; std::set candidates; @@ -54,6 +60,11 @@ private: unsigned completed{ 0 }; std::chrono::steady_clock::time_point timestamp{}; size_t processed{ 0 }; // Total number of accounts processed + + nano::account index () const + { + return start; + } }; // clang-format off @@ -65,7 +76,7 @@ private: mi::indexed_by< mi::random_access>, mi::ordered_unique, - mi::member>, + mi::const_mem_fun>, mi::ordered_non_unique, mi::member> >>; diff --git a/nano/node/bootstrap_ascending/service.cpp b/nano/node/bootstrap_ascending/service.cpp index 0739040ca..67ba813c9 100644 --- a/nano/node/bootstrap_ascending/service.cpp +++ b/nano/node/bootstrap_ascending/service.cpp @@ -635,86 +635,6 @@ void nano::bootstrap_ascending::service::run_frontiers () } } -void nano::bootstrap_ascending::service::process_frontiers (std::deque> const & frontiers) -{ - release_assert (!frontiers.empty ()); - - // Accounts must be passed in ascending order - debug_assert (std::adjacent_find (frontiers.begin (), frontiers.end (), [] (auto const & lhs, auto const & rhs) { - return lhs.first.number () >= rhs.first.number (); - }) - == frontiers.end ()); - - stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::process_frontiers); - - size_t outdated = 0; - size_t pending = 0; - - // Accounts with outdated frontiers to sync - std::deque result; - { - auto transaction = ledger.tx_begin_read (); - - auto const start = frontiers.front ().first; - account_database_crawler account_crawler{ ledger.store, transaction, start }; - pending_database_crawler pending_crawler{ ledger.store, transaction, start }; - - auto block_exists = [&] (nano::block_hash const & hash) { - return ledger.any.block_exists_or_pruned (transaction, hash); - }; - - auto should_prioritize = [&] (nano::account const & account, nano::block_hash const & frontier) { - account_crawler.advance_to (account); - pending_crawler.advance_to (account); - - // Check if account exists in our ledger - if (account_crawler.current && account_crawler.current->first == account) - { - // Check for frontier mismatch - if (account_crawler.current->second.head != frontier) - { - // Check if frontier block exists in our ledger - if (!block_exists (frontier)) - { - outdated++; - return true; // Frontier is outdated - } - } - return false; // Account exists and frontier is up-to-date - } - - // Check if account has pending blocks in our ledger - if (pending_crawler.current && pending_crawler.current->first.account == account) - { - pending++; - return true; // Account doesn't exist but has pending blocks in the ledger - } - - return false; // Account doesn't exist in the ledger and has no pending blocks, can't be prioritized right now - }; - - for (auto const & [account, frontier] : frontiers) - { - if (should_prioritize (account, frontier)) - { - result.push_back (account); - } - } - } - - stats.add (nano::stat::type::bootstrap_ascending, nano::stat::detail::frontiers_processed, frontiers.size ()); - stats.add (nano::stat::type::bootstrap_ascending, nano::stat::detail::frontiers_prioritized, result.size ()); - stats.add (nano::stat::type::bootstrap_ascending, nano::stat::detail::frontiers_outdated, outdated); - stats.add (nano::stat::type::bootstrap_ascending, nano::stat::detail::frontiers_pending, pending); - - nano::lock_guard guard{ mutex }; - - for (auto const & account : result) - { - accounts.priority_set (account); - } -} - void nano::bootstrap_ascending::service::cleanup_and_sync () { debug_assert (!mutex.try_lock ()); @@ -968,6 +888,86 @@ void nano::bootstrap_ascending::service::process (const nano::empty_payload & re debug_assert (false, "empty payload"); // Should not happen } +void nano::bootstrap_ascending::service::process_frontiers (std::deque> const & frontiers) +{ + release_assert (!frontiers.empty ()); + + // Accounts must be passed in ascending order + debug_assert (std::adjacent_find (frontiers.begin (), frontiers.end (), [] (auto const & lhs, auto const & rhs) { + return lhs.first.number () >= rhs.first.number (); + }) + == frontiers.end ()); + + stats.inc (nano::stat::type::bootstrap_ascending, nano::stat::detail::process_frontiers); + + size_t outdated = 0; + size_t pending = 0; + + // Accounts with outdated frontiers to sync + std::deque result; + { + auto transaction = ledger.tx_begin_read (); + + auto const start = frontiers.front ().first; + account_database_crawler account_crawler{ ledger.store, transaction, start }; + pending_database_crawler pending_crawler{ ledger.store, transaction, start }; + + auto block_exists = [&] (nano::block_hash const & hash) { + return ledger.any.block_exists_or_pruned (transaction, hash); + }; + + auto should_prioritize = [&] (nano::account const & account, nano::block_hash const & frontier) { + account_crawler.advance_to (account); + pending_crawler.advance_to (account); + + // Check if account exists in our ledger + if (account_crawler.current && account_crawler.current->first == account) + { + // Check for frontier mismatch + if (account_crawler.current->second.head != frontier) + { + // Check if frontier block exists in our ledger + if (!block_exists (frontier)) + { + outdated++; + return true; // Frontier is outdated + } + } + return false; // Account exists and frontier is up-to-date + } + + // Check if account has pending blocks in our ledger + if (pending_crawler.current && pending_crawler.current->first.account == account) + { + pending++; + return true; // Account doesn't exist but has pending blocks in the ledger + } + + return false; // Account doesn't exist in the ledger and has no pending blocks, can't be prioritized right now + }; + + for (auto const & [account, frontier] : frontiers) + { + if (should_prioritize (account, frontier)) + { + result.push_back (account); + } + } + } + + stats.add (nano::stat::type::bootstrap_ascending, nano::stat::detail::frontiers_processed, frontiers.size ()); + stats.add (nano::stat::type::bootstrap_ascending, nano::stat::detail::frontiers_prioritized, result.size ()); + stats.add (nano::stat::type::bootstrap_ascending, nano::stat::detail::frontiers_outdated, outdated); + stats.add (nano::stat::type::bootstrap_ascending, nano::stat::detail::frontiers_pending, pending); + + nano::lock_guard guard{ mutex }; + + for (auto const & account : result) + { + accounts.priority_set (account); + } +} + nano::bootstrap_ascending::service::verify_result nano::bootstrap_ascending::service::verify (const nano::asc_pull_ack::blocks_payload & response, const nano::bootstrap_ascending::service::async_tag & tag) const { auto const & blocks = response.blocks; diff --git a/nano/node/bootstrap_ascending/service.hpp b/nano/node/bootstrap_ascending/service.hpp index 652c7b862..ba49ea6c4 100644 --- a/nano/node/bootstrap_ascending/service.hpp +++ b/nano/node/bootstrap_ascending/service.hpp @@ -109,7 +109,6 @@ namespace bootstrap_ascending void run_frontiers (); void run_timeouts (); void cleanup_and_sync (); - void process_frontiers (std::deque> const & frontiers); /* Waits for a condition to be satisfied with incremental backoff */ void wait (std::function const & predicate) const; @@ -142,6 +141,8 @@ namespace bootstrap_ascending void process (nano::asc_pull_ack::frontiers_payload const & response, async_tag const & tag); void process (nano::empty_payload const & response, async_tag const & tag); + void process_frontiers (std::deque> const & frontiers); + enum class verify_result { ok,