dn724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 1

Merge branch 'develop' into reset-bootstrap-rpc

Browse source 
# Conflicts:
#	nano/lib/stats_enums.hpp
#	nano/node/bootstrap/account_sets.hpp
#	nano/node/node.hpp
This commit is contained in:
Piotr Wójcik 2025-03-04 22:18:22 +01:00
commit 702ced1e43
159 changed files with 4191 additions and 1527 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/unit_tests.yml vendored
View file

@ -12,7 +12,7 @@ jobs:
RELEASE:
- ${{ startsWith(github.ref, 'refs/tags/') }}
env:
BACKEND: ${{ matrix.BACKEND }}
NANO_BACKEND: ${{ matrix.BACKEND }}
BUILD_TYPE: ${{ matrix.RELEASE && 'RelWithDebInfo' || 'Debug' }}
TEST_USE_ROCKSDB: ${{ matrix.BACKEND == 'rocksdb' && '1' || '0' }}
DEADLINE_SCALE_FACTOR: ${{ matrix.BACKEND == 'rocksdb' && '2' || '1' }}

49
CMakeLists.txt
View file

@ -122,18 +122,6 @@ set(NANO_ROCKSDB_TOOLS
OFF
CACHE BOOL "")
option(NANO_STACKTRACE_BACKTRACE
"Use BOOST_STACKTRACE_USE_BACKTRACE in stacktraces, for POSIX" OFF)
if(NANO_STACKTRACE_BACKTRACE)
add_definitions(-DNANO_STACKTRACE_BACKTRACE)
add_definitions(-DBOOST_STACKTRACE_USE_BACKTRACE)
if(NANO_BACKTRACE_INCLUDE)
add_definitions(
-DBOOST_STACKTRACE_BACKTRACE_INCLUDE_FILE=${NANO_BACKTRACE_INCLUDE})
endif()
endif()
# Enable NANO_TRACING by default in Debug builds
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
option(NANO_TRACING "Enable trace logging" ON)
@ -400,6 +388,22 @@ endif()
include_directories(${CMAKE_SOURCE_DIR})
list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules")
# Find libbacktrace first (Unix/Linux only)
if(NOT WIN32)
find_path(
BACKTRACE_INCLUDE_DIR
NAMES backtrace.h
PATHS /usr/local/include /usr/include)
if(BACKTRACE_INCLUDE_DIR)
set(BACKTRACE_FOUND TRUE)
# Most systems just need -lbacktrace
set(BACKTRACE_LIBRARIES backtrace)
message(STATUS "Found backtrace.h at ${BACKTRACE_INCLUDE_DIR}")
endif()
endif()
# Then set up boost modules
set(Boost_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/submodules/boost/libs/config/include)
set(BOOST_MODULE_LIBS
algorithm
@ -495,8 +499,25 @@ foreach(lib IN LISTS BOOST_MODULE_LIBS)
add_subdirectory(submodules/boost/libs/${lib} EXCLUDE_FROM_ALL)
endforeach()
include_directories(${BOOST_LIBRARY_INCLUDES})
add_library(Boost::stacktrace ALIAS boost_stacktrace_basic)
add_definitions(-DBOOST_STACKTRACE_GNU_SOURCE_NOT_REQUIRED)
# Configure stacktrace with appropriate backend
if(WIN32)
message(STATUS "Windows platform - Using WinDbg stacktrace")
add_definitions(-DBOOST_STACKTRACE_USE_WINDBG)
add_library(Boost::stacktrace ALIAS boost_stacktrace_windbg)
target_link_libraries(boost_stacktrace_windbg PRIVATE dbghelp)
elseif(BOOST_STACKTRACE_HAS_BACKTRACE)
message(
STATUS "Found libbacktrace - enabling Boost stacktrace backtrace support")
add_definitions(-DBOOST_STACKTRACE_USE_BACKTRACE)
add_library(Boost::stacktrace ALIAS boost_stacktrace_backtrace)
target_link_libraries(boost_stacktrace_backtrace
PRIVATE ${BACKTRACE_LIBRARIES})
else()
message(STATUS "Using basic stacktrace backend")
add_definitions(-DBOOST_STACKTRACE_GNU_SOURCE_NOT_REQUIRED)
add_library(Boost::stacktrace ALIAS boost_stacktrace_basic)
endif()
# Workaround for missing reference errata in the boost property_tree module
target_link_libraries(boost_property_tree INTERFACE Boost::any)

11
ci/build-ci.sh
View file

@ -28,17 +28,6 @@ fi
ulimit -S -n 8192
if [[ "$OS" == 'Linux' ]]; then
if clang --version && [ ${LCOV:-0} == 0 ]; then
BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON \
-DNANO_BACKTRACE_INCLUDE=</tmp/backtrace.h>"
else
BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON"
fi
else
BACKTRACE=""
fi
cmake \
-G'Unix Makefiles' \
-DACTIVE_NETWORK=nano_dev_network \

9
ci/build.sh
View file

@ -10,15 +10,6 @@ fi
SRC=${SRC:-${PWD}}
OS=$(uname)
CMAKE_BACKTRACE=""
if [[ ${OS} == 'Linux' ]]; then
CMAKE_BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON"
if [[ ${COMPILER:-} == 'clang' ]]; then
CMAKE_BACKTRACE="${CMAKE_BACKTRACE} -DNANO_BACKTRACE_INCLUDE=</tmp/backtrace.h>"
fi
fi
CMAKE_QT_DIR=""
if [[ ${QT_DIR:-} ]]; then
CMAKE_QT_DIR="-DQt5_DIR=${QT_DIR}"

2
ci/prepare/linux/prepare-clang.sh
View file

@ -15,4 +15,4 @@ update-alternatives --install /usr/bin/lldb lldb /usr/bin/lldb-$CLANG_VERSION 10
# Workaround to get a path that can be easily passed into cmake for BOOST_STACKTRACE_BACKTRACE_INCLUDE_FILE
# See https://www.boost.org/doc/libs/1_70_0/doc/html/stacktrace/configuration_and_build.html#stacktrace.configuration_and_build.f3
backtrace_file=$(find /usr/lib/gcc/ -name 'backtrace.h' | head -n 1) && test -f $backtrace_file && ln -s $backtrace_file /tmp/backtrace.h
backtrace_file=$(find /usr/lib/gcc/ -name 'backtrace.h' | head -n 1) && test -f $backtrace_file && ln -s $backtrace_file /usr/local/include/backtrace.h

2
nano/core_test/CMakeLists.txt
View file

@ -11,6 +11,7 @@ add_executable(
block_store.cpp
block_processor.cpp
bootstrap.cpp
bootstrap_frontier_scan.cpp
bootstrap_server.cpp
bucketing.cpp
cli.cpp
@ -57,6 +58,7 @@ add_executable(
request_aggregator.cpp
signal_manager.cpp
socket.cpp
stacktrace.cpp
system.cpp
tcp_listener.cpp
telemetry.cpp

4
nano/core_test/active_elections.cpp
View file

@ -971,7 +971,7 @@ TEST (active_elections, fork_replacement_tally)
node_config.peering_port = system.get_available_port ();
auto & node2 (*system.add_node (node_config));
node1.network.filter.clear ();
node2.network.flood_block (send_last, nano::transport::traffic_type::test);
ASSERT_TRUE (node2.network.flood_block (send_last, nano::transport::traffic_type::test));
ASSERT_TIMELY (3s, node1.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::in) > 0);
// Correct block without votes is ignored
@ -985,7 +985,7 @@ TEST (active_elections, fork_replacement_tally)
// ensure vote arrives before the block
ASSERT_TIMELY_EQ (5s, 1, node1.vote_cache.find (send_last->hash ()).size ());
node1.network.filter.clear ();
node2.network.flood_block (send_last, nano::transport::traffic_type::test);
ASSERT_TRUE (node2.network.flood_block (send_last, nano::transport::traffic_type::test));
ASSERT_TIMELY (5s, node1.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::in) > 1);
// the send_last block should replace one of the existing block of the election because it has higher vote weight

67
nano/core_test/block.cpp
View file

@ -186,73 +186,6 @@ TEST (block, change_serialize_json)
ASSERT_EQ (*block1, block2);
}
TEST (uint512_union, parse_zero)
{
nano::uint512_union input (nano::uint512_t (0));
std::string text;
input.encode_hex (text);
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_FALSE (error);
ASSERT_EQ (input, output);
ASSERT_TRUE (output.number ().is_zero ());
}
TEST (uint512_union, parse_zero_short)
{
std::string text ("0");
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_FALSE (error);
ASSERT_TRUE (output.number ().is_zero ());
}
TEST (uint512_union, parse_one)
{
nano::uint512_union input (nano::uint512_t (1));
std::string text;
input.encode_hex (text);
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_FALSE (error);
ASSERT_EQ (input, output);
ASSERT_EQ (1, output.number ());
}
TEST (uint512_union, parse_error_symbol)
{
nano::uint512_union input (nano::uint512_t (1000));
std::string text;
input.encode_hex (text);
text[5] = '!';
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_TRUE (error);
}
TEST (uint512_union, max)
{
nano::uint512_union input (std::numeric_limits<nano::uint512_t>::max ());
std::string text;
input.encode_hex (text);
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_FALSE (error);
ASSERT_EQ (input, output);
ASSERT_EQ (nano::uint512_t ("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), output.number ());
}
TEST (uint512_union, parse_error_overflow)
{
nano::uint512_union input (std::numeric_limits<nano::uint512_t>::max ());
std::string text;
input.encode_hex (text);
text.push_back (0);
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_TRUE (error);
}
TEST (send_block, deserialize)
{
nano::block_builder builder;

8
nano/core_test/block_store.cpp
View file

@ -633,7 +633,7 @@ TEST (mdb_block_store, supported_version_upgrades)
{
nano::store::lmdb::component store (logger, path, nano::dev::constants);
nano::stats stats{ logger };
nano::ledger ledger (store, stats, nano::dev::constants);
nano::ledger ledger (store, nano::dev::constants, stats, logger);
auto transaction (store.tx_begin_write ());
store.initialize (transaction, ledger.cache, nano::dev::constants);
// Lower the database to the max version unsupported for upgrades
@ -651,7 +651,7 @@ TEST (mdb_block_store, supported_version_upgrades)
{
nano::store::lmdb::component store (logger, path1, nano::dev::constants);
nano::stats stats{ logger };
nano::ledger ledger (store, stats, nano::dev::constants);
nano::ledger ledger (store, nano::dev::constants, stats, logger);
auto transaction (store.tx_begin_write ());
store.initialize (transaction, ledger.cache, nano::dev::constants);
// Lower the database version to the minimum version supported for upgrade.
@ -899,7 +899,7 @@ TEST (block_store, cemented_count_cache)
ASSERT_TRUE (!store->init_error ());
auto transaction (store->tx_begin_write ());
nano::stats stats{ logger };
nano::ledger ledger (*store, stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
store->initialize (transaction, ledger.cache, nano::dev::constants);
ASSERT_EQ (1, ledger.cemented_count ());
}
@ -987,7 +987,7 @@ TEST (mdb_block_store, sideband_height)
nano::store::lmdb::component store (logger, nano::unique_path () / "data.ldb", nano::dev::constants);
ASSERT_FALSE (store.init_error ());
nano::stats stats{ logger };
nano::ledger ledger (store, stats, nano::dev::constants);
nano::ledger ledger (store, nano::dev::constants, stats, logger);
nano::block_builder builder;
auto transaction = ledger.tx_begin_write ();
store.initialize (transaction, ledger.cache, nano::dev::constants);

77
nano/core_test/bootstrap.cpp
View file

@ -178,6 +178,83 @@ TEST (account_sets, saturate_priority)
ASSERT_EQ (sets.priority (account), nano::bootstrap::account_sets::priority_max);
}
TEST (account_sets, decay_blocking)
{
using namespace std::chrono_literals;
nano::test::system system;
nano::account_sets_config config;
config.blocking_decay = 1s;
nano::bootstrap::account_sets sets{ config, system.stats };
// Test empty set
ASSERT_EQ (0, sets.decay_blocking ());
// Create test accounts and timestamps
nano::account account1{ 1 };
nano::account account2{ 2 };
nano::account account3{ 3 };
auto now = std::chrono::steady_clock::now ();
// Add first account
sets.priority_up (account1);
sets.block (account1, random_hash (), now);
ASSERT_TRUE (sets.blocked (account1));
ASSERT_EQ (1, sets.blocked_size ());
// Decay before timeout should not remove entry
ASSERT_EQ (0, sets.decay_blocking (now));
ASSERT_TRUE (sets.blocked (account1));
ASSERT_EQ (1, sets.blocked_size ());
// Add second account after 500ms
now += 500ms;
sets.priority_up (account2);
sets.block (account2, random_hash (), now);
ASSERT_TRUE (sets.blocked (account2));
ASSERT_EQ (2, sets.blocked_size ());
// Add third account after another 500ms
now += 500ms;
sets.priority_up (account3);
sets.block (account3, random_hash (), now);
ASSERT_TRUE (sets.blocked (account3));
ASSERT_EQ (3, sets.blocked_size ());
// Decay at 1.5s - should remove first two accounts
now += 500ms;
ASSERT_EQ (2, sets.decay_blocking (now));
ASSERT_FALSE (sets.blocked (account1));
ASSERT_FALSE (sets.blocked (account2));
ASSERT_TRUE (sets.blocked (account3));
ASSERT_EQ (1, sets.blocked_size ());
// Reinsert second account
auto hash2 = random_hash ();
sets.priority_up (account2);
sets.block (account2, hash2, now);
ASSERT_TRUE (sets.blocked (account2));
ASSERT_EQ (2, sets.blocked_size ());
// Immediate decay should not affect reinserted account
ASSERT_EQ (0, sets.decay_blocking (now));
ASSERT_TRUE (sets.blocked (account2));
// Decay at 2s - should remove account3 but keep reinserted account2
now += 500ms;
ASSERT_EQ (1, sets.decay_blocking (now));
ASSERT_FALSE (sets.blocked (account3));
ASSERT_TRUE (sets.blocked (account2));
ASSERT_EQ (1, sets.blocked_size ());
// Final decay after another second - should remove remaining account
now += 1s;
ASSERT_EQ (1, sets.decay_blocking (now));
ASSERT_FALSE (sets.blocked (account2));
ASSERT_EQ (0, sets.blocked_size ());
}
/*
* bootstrap
*/

242
nano/core_test/bootstrap_frontier_scan.cpp Normal file
View file

@ -0,0 +1,242 @@
#include <nano/lib/blocks.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/stats.hpp>
#include <nano/lib/tomlconfig.hpp>
#include <nano/node/bootstrap/bootstrap_service.hpp>
#include <nano/node/bootstrap/frontier_scan.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>
#include <gtest/gtest.h>
#include <sstream>
using namespace std::chrono_literals;
namespace
{
struct test_context
{
nano::stats stats;
nano::frontier_scan_config config;
nano::bootstrap::frontier_scan frontier_scan;
explicit test_context (nano::frontier_scan_config config_a = {}) :
stats{ nano::default_logger () },
config{ config_a },
frontier_scan{ config, stats }
{
}
};
}
TEST (bootstrap_frontier_scan, construction)
{
test_context ctx{};
auto & frontier_scan = ctx.frontier_scan;
}
TEST (bootstrap_frontier_scan, next_basic)
{
nano::frontier_scan_config config;
config.head_parallelism = 2; // Two heads for simpler testing
config.consideration_count = 3;
test_context ctx{ config };
auto & frontier_scan = ctx.frontier_scan;
// First call should return first head, account number 1 (avoiding burn account 0)
auto first = frontier_scan.next ();
ASSERT_EQ (first.number (), 1);
// Second call should return second head, account number 0x7FF... (half the range)
auto second = frontier_scan.next ();
ASSERT_EQ (second.number (), nano::account{ "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" });
// Third call should return first head again, sequentially iterating through heads
auto third = frontier_scan.next ();
ASSERT_EQ (third.number (), 1);
}
TEST (bootstrap_frontier_scan, process_basic)
{
nano::frontier_scan_config config;
config.head_parallelism = 1; // Single head for simpler testing
config.consideration_count = 3;
config.candidates = 5;
test_context ctx{ config };
auto & frontier_scan = ctx.frontier_scan;
// Get initial account to scan
auto start = frontier_scan.next ();
ASSERT_EQ (start.number (), 1);
// Create response with some frontiers
std::deque<std::pair<nano::account, nano::block_hash>> response;
response.push_back ({ nano::account{ 2 }, nano::block_hash{ 1 } });
response.push_back ({ nano::account{ 3 }, nano::block_hash{ 2 } });
// Process should not be done until consideration_count is reached
ASSERT_FALSE (frontier_scan.process (start, response));
ASSERT_FALSE (frontier_scan.process (start, response));
// Head should not advance before reaching `consideration_count` responses
ASSERT_EQ (frontier_scan.next (), 1);
// After consideration_count responses, should be done
ASSERT_TRUE (frontier_scan.process (start, response));
// Head should advance to next account and start subsequent scan from there
ASSERT_EQ (frontier_scan.next (), 3);
}
TEST (bootstrap_frontier_scan, range_wrap_around)
{
nano::frontier_scan_config config;
config.head_parallelism = 1;
config.consideration_count = 1;
config.candidates = 1;
test_context ctx{ config };
auto & frontier_scan = ctx.frontier_scan;
auto start = frontier_scan.next ();
// Create response that would push next beyond the range end
std::deque<std::pair<nano::account, nano::block_hash>> response;
response.push_back ({ nano::account{ std::numeric_limits<nano::uint256_t>::max () }, nano::block_hash{ 1 } });
// Process should succeed and wrap around
ASSERT_TRUE (frontier_scan.process (start, response));
// Next account should be back at start of range
auto next = frontier_scan.next ();
ASSERT_EQ (next.number (), 1);
}
TEST (bootstrap_frontier_scan, cooldown)
{
nano::frontier_scan_config config;
config.head_parallelism = 1;
config.consideration_count = 1;
config.cooldown = 250ms;
test_context ctx{ config };
auto & frontier_scan = ctx.frontier_scan;
// First call should succeed
auto first = frontier_scan.next ();
ASSERT_NE (first.number (), 0);
// Immediate second call should fail (return 0)
auto second = frontier_scan.next ();
ASSERT_EQ (second.number (), 0);
// After cooldown, should succeed again
std::this_thread::sleep_for (500ms);
auto third = frontier_scan.next ();
ASSERT_NE (third.number (), 0);
}
TEST (bootstrap_frontier_scan, candidate_trimming)
{
nano::frontier_scan_config config;
config.head_parallelism = 1;
config.consideration_count = 2;
config.candidates = 3; // Only keep the lowest candidates
test_context ctx{ config };
auto & frontier_scan = ctx.frontier_scan;
auto start = frontier_scan.next ();
ASSERT_EQ (start.number (), 1);
// Create response with more candidates than limit
// Response contains: 1, 4, 7, 10
std::deque<std::pair<nano::account, nano::block_hash>> response1;
for (int i = 0; i <= 9; i += 3)
{
response1.push_back ({ nano::account{ start.number () + i }, nano::block_hash{ static_cast<uint64_t> (i) } });
}
ASSERT_FALSE (frontier_scan.process (start, response1));
// Response contains: 1, 3, 5, 7, 9
std::deque<std::pair<nano::account, nano::block_hash>> response2;
for (int i = 0; i <= 8; i += 2)
{
response2.push_back ({ nano::account{ start.number () + i }, nano::block_hash{ static_cast<uint64_t> (i) } });
}
ASSERT_TRUE (frontier_scan.process (start, response2));
// After processing replies candidates should be ordered and trimmed
auto next = frontier_scan.next ();
ASSERT_EQ (next.number (), 5);
}
TEST (bootstrap_frontier_scan, heads_distribution)
{
nano::frontier_scan_config config;
config.head_parallelism = 4;
test_context ctx{ config };
auto & frontier_scan = ctx.frontier_scan;
// Collect initial accounts from each head
std::vector<nano::account> initial_accounts;
for (int i = 0; i < 4; i++)
{
initial_accounts.push_back (frontier_scan.next ());
}
// Verify accounts are properly distributed across the range
for (size_t i = 1; i < initial_accounts.size (); i++)
{
ASSERT_GT (initial_accounts[i].number (), initial_accounts[i - 1].number ());
}
}
TEST (bootstrap_frontier_scan, invalid_response_ordering)
{
nano::frontier_scan_config config;
config.head_parallelism = 1;
config.consideration_count = 1;
test_context ctx{ config };
auto & frontier_scan = ctx.frontier_scan;
auto start = frontier_scan.next ();
// Create response with out-of-order accounts
std::deque<std::pair<nano::account, nano::block_hash>> response;
response.push_back ({ nano::account{ start.number () + 2 }, nano::block_hash{ 1 } });
response.push_back ({ nano::account{ start.number () + 1 }, nano::block_hash{ 2 } }); // Out of order
// Should still process successfully
ASSERT_TRUE (frontier_scan.process (start, response));
ASSERT_EQ (frontier_scan.next (), start.number () + 2);
}
TEST (bootstrap_frontier_scan, empty_responses)
{
nano::frontier_scan_config config;
config.head_parallelism = 1;
config.consideration_count = 2;
test_context ctx{ config };
auto & frontier_scan = ctx.frontier_scan;
auto start = frontier_scan.next ();
// Empty response should not advance head even after receiving `consideration_count` responses
std::deque<std::pair<nano::account, nano::block_hash>> empty_response;
ASSERT_FALSE (frontier_scan.process (start, empty_response));
ASSERT_FALSE (frontier_scan.process (start, empty_response));
ASSERT_EQ (frontier_scan.next (), start);
// Let the head advance
std::deque<std::pair<nano::account, nano::block_hash>> response;
response.push_back ({ nano::account{ start.number () + 1 }, nano::block_hash{ 1 } });
ASSERT_TRUE (frontier_scan.process (start, response));
ASSERT_EQ (frontier_scan.next (), start.number () + 1);
// However, after receiving enough empty responses, head should wrap around to the start
ASSERT_FALSE (frontier_scan.process (start, empty_response));
ASSERT_FALSE (frontier_scan.process (start, empty_response));
ASSERT_FALSE (frontier_scan.process (start, empty_response));
ASSERT_EQ (frontier_scan.next (), start.number () + 1);
ASSERT_TRUE (frontier_scan.process (start, empty_response));
ASSERT_EQ (frontier_scan.next (), start); // Wraps around
}

24
nano/core_test/confirming_set.cpp
View file

@ -4,6 +4,7 @@
#include <nano/node/block_processor.hpp>
#include <nano/node/confirming_set.hpp>
#include <nano/node/election.hpp>
#include <nano/node/ledger_notifications.hpp>
#include <nano/node/make_store.hpp>
#include <nano/node/unchecked_map.hpp>
#include <nano/secure/ledger.hpp>
@ -26,17 +27,15 @@ struct confirming_set_context
nano::stats & stats;
nano::ledger & ledger;
nano::unchecked_map unchecked;
nano::block_processor block_processor;
nano::ledger_notifications ledger_notifications;
nano::confirming_set confirming_set;
explicit confirming_set_context (nano::test::ledger_context & ledger_context, nano::node_config node_config = {}) :
logger{ ledger_context.logger () },
stats{ ledger_context.stats () },
ledger{ ledger_context.ledger () },
unchecked{ 0, stats, false },
block_processor{ node_config, ledger, unchecked, stats, logger },
confirming_set{ node_config.confirming_set, ledger, block_processor, stats, logger }
ledger_notifications{ node_config, stats, logger },
confirming_set{ node_config.confirming_set, ledger, ledger_notifications, stats, logger }
{
}
};
@ -78,21 +77,20 @@ TEST (confirming_set, process_one)
TEST (confirming_set, process_multiple)
{
nano::test::system system;
auto & node = *system.add_node ();
auto ctx = nano::test::ledger_send_receive ();
nano::confirming_set_config config{};
nano::confirming_set confirming_set{ config, ctx.ledger (), node.block_processor, ctx.stats (), ctx.logger () };
auto ledger_ctx = nano::test::ledger_send_receive ();
confirming_set_context ctx{ ledger_ctx };
nano::confirming_set & confirming_set = ctx.confirming_set;
std::atomic<int> count = 0;
std::mutex mutex;
std::condition_variable condition;
confirming_set.cemented_observers.add ([&] (auto const &) { ++count; condition.notify_all (); });
confirming_set.add (ctx.blocks ()[0]->hash ());
confirming_set.add (ctx.blocks ()[1]->hash ());
confirming_set.add (ledger_ctx.blocks ()[0]->hash ());
confirming_set.add (ledger_ctx.blocks ()[1]->hash ());
nano::test::start_stop_guard guard{ confirming_set };
std::unique_lock lock{ mutex };
ASSERT_TRUE (condition.wait_for (lock, 5s, [&] () { return count == 2; }));
ASSERT_EQ (2, ctx.stats ().count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
ASSERT_EQ (3, ctx.ledger ().cemented_count ());
ASSERT_EQ (2, ctx.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
ASSERT_EQ (3, ctx.ledger.cemented_count ());
}
TEST (confirmation_callback, observer_callbacks)

3
nano/core_test/election.cpp
View file

@ -198,8 +198,6 @@ TEST (election, quorum_minimum_confirm_fail)
ASSERT_FALSE (election->confirmed ());
}
namespace nano
{
// FIXME: this test fails on rare occasions. It needs a review.
TEST (election, quorum_minimum_update_weight_before_quorum_checks)
{
@ -267,7 +265,6 @@ TEST (election, quorum_minimum_update_weight_before_quorum_checks)
ASSERT_TIMELY (5s, election->confirmed ());
ASSERT_NE (nullptr, node1.block (send1->hash ()));
}
}
TEST (election, continuous_voting)
{

101
nano/core_test/ledger.cpp
View file

@ -871,7 +871,7 @@ TEST (ledger, double_open)
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::stats stats{ logger };
nano::ledger ledger (*store, stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
@ -2251,51 +2251,87 @@ TEST (ledger, send_open_receive_rollback)
ASSERT_EQ (nano::dev::constants.genesis_amount - 0, ledger.weight (nano::dev::genesis_key.pub));
}
TEST (ledger, bootstrap_rep_weight)
TEST (ledger, bootstrap_weights)
{
auto ctx = nano::test::ledger_empty ();
auto & ledger = ctx.ledger ();
auto & store = ctx.store ();
nano::keypair key2;
auto & pool = ctx.pool ();
nano::keypair rep_key1, rep_key2, throwaway;
// Genesis key has all the voting weight
ASSERT_EQ (std::numeric_limits<nano::uint128_t>::max (), ledger.weight (nano::dev::genesis_key.pub));
// Throwaway transaction to reduce genesis weight
{
auto transaction = ledger.tx_begin_write ();
auto info1 = ledger.any.account_get (transaction, nano::dev::genesis_key.pub);
ASSERT_TRUE (info1);
auto genesis_info = ledger.any.account_get (transaction, nano::dev::genesis_key.pub);
ASSERT_TRUE (genesis_info);
nano::block_builder builder;
auto send = builder
.send ()
.previous (info1->head)
.destination (key2.pub)
.previous (genesis_info->head)
.destination (throwaway.pub)
.balance (std::numeric_limits<nano::uint128_t>::max () - 50)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*pool.generate (info1->head))
.work (*pool.generate (genesis_info->head))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send));
}
// Override bootstrap weights
ledger.bootstrap_weight_max_blocks = 4;
ledger.bootstrap_weights[rep_key1.pub] = 1000;
// Should use bootstrap weights
ASSERT_EQ (2, ledger.block_count ());
ASSERT_FALSE (ledger.bootstrap_height_reached ());
ASSERT_EQ (0, ledger.weight (nano::dev::genesis_key.pub));
ASSERT_EQ (1000, ledger.weight (rep_key1.pub));
{
ledger.bootstrap_weight_max_blocks = 3;
ledger.bootstrap_weights[key2.pub] = 1000;
ASSERT_EQ (1000, ledger.weight (key2.pub));
auto snapshot = ledger.rep_weights_snapshot ();
ASSERT_EQ (1, snapshot.size ());
ASSERT_EQ (1000, snapshot[rep_key1.pub]);
}
// Open normal representative account, should not use bootstrap weights anymore
{
auto transaction = ledger.tx_begin_write ();
auto info1 = ledger.any.account_get (transaction, nano::dev::genesis_key.pub);
ASSERT_TRUE (info1);
auto genesis_info = ledger.any.account_get (transaction, nano::dev::genesis_key.pub);
ASSERT_TRUE (genesis_info);
nano::block_builder builder;
auto send = builder
.send ()
.previous (info1->head)
.destination (key2.pub)
.previous (genesis_info->head)
.destination (rep_key2.pub)
.balance (std::numeric_limits<nano::uint128_t>::max () - 100)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*pool.generate (info1->head))
.work (*pool.generate (genesis_info->head))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send));
auto open = builder
.open ()
.source (send->hash ())
.representative (rep_key2.pub)
.account (rep_key2.pub)
.sign (rep_key2.prv, rep_key2.pub)
.work (*pool.generate (rep_key2.pub))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, open));
}
ASSERT_EQ (4, ledger.block_count ());
ASSERT_TRUE (ledger.bootstrap_height_reached ());
ASSERT_EQ (std::numeric_limits<nano::uint128_t>::max () - 100, ledger.weight (nano::dev::genesis_key.pub));
ASSERT_EQ (0, ledger.weight (rep_key1.pub));
ASSERT_EQ (50, ledger.weight (rep_key2.pub));
{
auto snapshot = ledger.rep_weights_snapshot ();
ASSERT_EQ (2, snapshot.size ());
ASSERT_EQ (std::numeric_limits<nano::uint128_t>::max () - 100, snapshot[nano::dev::genesis_key.pub]);
ASSERT_EQ (50, snapshot[rep_key2.pub]);
}
ASSERT_EQ (3, ledger.block_count ());
ASSERT_EQ (0, ledger.weight (key2.pub));
}
TEST (ledger, block_destination_source)
@ -4700,7 +4736,7 @@ TEST (ledger, dependents_confirmed_pruning)
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::stats stats{ logger };
nano::ledger ledger (*store, stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
@ -4775,6 +4811,7 @@ TEST (ledger, cache)
auto & ledger = ctx.ledger ();
auto & store = ctx.store ();
auto & stats = ctx.stats ();
auto & logger = ctx.logger ();
auto & pool = ctx.pool ();
nano::block_builder builder;
@ -4825,7 +4862,7 @@ TEST (ledger, cache)
++block_count;
--genesis_weight;
cache_check (ledger);
cache_check (nano::ledger (store, stats, nano::dev::constants));
cache_check (nano::ledger (store, nano::dev::constants, stats, logger));
{
auto transaction = ledger.tx_begin_write ();
@ -4835,7 +4872,7 @@ TEST (ledger, cache)
++block_count;
++account_count;
cache_check (ledger);
cache_check (nano::ledger (store, stats, nano::dev::constants));
cache_check (nano::ledger (store, nano::dev::constants, stats, logger));
{
auto transaction = ledger.tx_begin_write ();
@ -4845,7 +4882,7 @@ TEST (ledger, cache)
++cemented_count;
cache_check (ledger);
cache_check (nano::ledger (store, stats, nano::dev::constants));
cache_check (nano::ledger (store, nano::dev::constants, stats, logger));
{
auto transaction = ledger.tx_begin_write ();
@ -4855,7 +4892,7 @@ TEST (ledger, cache)
++cemented_count;
cache_check (ledger);
cache_check (nano::ledger (store, stats, nano::dev::constants));
cache_check (nano::ledger (store, nano::dev::constants, stats, logger));
{
auto transaction = ledger.tx_begin_write ();
@ -4863,7 +4900,7 @@ TEST (ledger, cache)
}
++pruned_count;
cache_check (ledger);
cache_check (nano::ledger (store, stats, nano::dev::constants));
cache_check (nano::ledger (store, nano::dev::constants, stats, logger));
}
}
@ -4873,7 +4910,7 @@ TEST (ledger, pruning_action)
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::stats stats{ logger };
nano::ledger ledger (*store, stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
@ -4959,7 +4996,7 @@ TEST (ledger, pruning_large_chain)
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::stats stats{ logger };
nano::ledger ledger (*store, stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
@ -5015,7 +5052,7 @@ TEST (ledger, pruning_source_rollback)
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::stats stats{ logger };
nano::ledger ledger (*store, stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
@ -5104,7 +5141,7 @@ TEST (ledger, pruning_source_rollback_legacy)
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::stats stats{ logger };
nano::ledger ledger (*store, stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
@ -5218,7 +5255,7 @@ TEST (ledger, pruning_legacy_blocks)
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::stats stats{ logger };
nano::ledger ledger (*store, stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
nano::keypair key1;
auto transaction = ledger.tx_begin_write ();
@ -5305,7 +5342,7 @@ TEST (ledger, pruning_safe_functions)
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::stats stats{ logger };
nano::ledger ledger (*store, stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
@ -5357,7 +5394,7 @@ TEST (ledger, random_blocks)
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::stats stats{ logger };
nano::ledger ledger (*store, stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
@ -5445,7 +5482,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
boost::asio::ip::address_v6 address (boost::asio::ip::make_address_v6 ("::ffff:127.0.0.1"));
uint16_t port = 100;
nano::store::lmdb::component store{ logger, path / "data.ldb", nano::dev::constants };
nano::ledger ledger{ store, system.stats, nano::dev::constants };
nano::ledger ledger{ store, nano::dev::constants, system.stats, system.logger };
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
std::shared_ptr<nano::block> send = nano::state_block_builder ()

4
nano/core_test/ledger_confirm.cpp
View file

@ -760,7 +760,7 @@ TEST (ledger_confirm, pruned_source)
auto path (nano::unique_path ());
auto store = nano::make_store (system.logger, path, nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::ledger ledger (*store, system.stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, system.stats, system.logger);
ledger.pruning = true;
nano::store::write_queue write_queue;
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
@ -845,7 +845,7 @@ TEST (ledger_confirmDeathTest, rollback_added_block)
auto path (nano::unique_path ());
auto store = nano::make_store (system.logger, path, nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::ledger ledger (*store, system.stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, system.stats, system.logger);
nano::store::write_queue write_queue;
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
nano::keypair key1;

22
nano/core_test/network.cpp
View file

@ -195,7 +195,8 @@ TEST (network, send_discarded_publish)
.build ();
{
auto transaction = node1.ledger.tx_begin_read ();
node1.network.flood_block (block, nano::transport::traffic_type::test);
auto sent = node1.network.flood_block (block, nano::transport::traffic_type::test);
ASSERT_EQ (1, sent);
ASSERT_EQ (nano::dev::genesis->hash (), node1.ledger.any.account_head (transaction, nano::dev::genesis_key.pub));
ASSERT_EQ (nano::dev::genesis->hash (), node2.latest (nano::dev::genesis_key.pub));
}
@ -221,7 +222,8 @@ TEST (network, send_invalid_publish)
.build ();
{
auto transaction = node1.ledger.tx_begin_read ();
node1.network.flood_block (block, nano::transport::traffic_type::test);
auto sent = node1.network.flood_block (block, nano::transport::traffic_type::test);
ASSERT_EQ (1, sent);
ASSERT_EQ (nano::dev::genesis->hash (), node1.ledger.any.account_head (transaction, nano::dev::genesis_key.pub));
ASSERT_EQ (nano::dev::genesis->hash (), node2.latest (nano::dev::genesis_key.pub));
}
@ -1149,3 +1151,19 @@ TEST (network, purge_dead_channel_remote)
};
ASSERT_TIMELY (5s, !channel_exists (node2, channel));
}
TEST (network, flood_vote)
{
nano::test::system system{ 4 };
auto & node = *system.nodes[0];
// Make one of the nodes a representative
system.wallet (1)->insert_adhoc (nano::dev::genesis_key.prv);
ASSERT_TIMELY_EQ (5s, node.rep_crawler.representative_count (), 1);
auto vote = nano::test::make_vote (nano::dev::genesis_key, { nano::dev::genesis->hash () });
ASSERT_EQ (3, node.network.flood_vote_rebroadcasted (vote, 999.0f));
ASSERT_EQ (2, node.network.flood_vote_non_pr (vote, 999.0f));
ASSERT_EQ (1, node.network.flood_vote_pr (vote));
}

54
nano/core_test/node.cpp
View file

@ -10,6 +10,7 @@
#include <nano/node/make_store.hpp>
#include <nano/node/online_reps.hpp>
#include <nano/node/portmapping.hpp>
#include <nano/node/pruning.hpp>
#include <nano/node/scheduler/component.hpp>
#include <nano/node/scheduler/manual.hpp>
#include <nano/node/scheduler/priority.hpp>
@ -1385,6 +1386,14 @@ TEST (node, bootstrap_fork_open)
{
nano::test::system system;
nano::node_config node_config (system.get_available_port ());
node_config.bootstrap.account_sets.cooldown = 100ms; // Reduce cooldown to speed up fork resolution
node_config.bootstrap.frontier_scan.head_parallelism = 3; // Make sure we can process the full account number range
node_config.bootstrap.frontier_rate_limit = 0; // Disable rate limiting to speed up the scan
// Disable automatic election activation
node_config.backlog_scan.enable = false;
node_config.priority_scheduler.enable = false;
node_config.hinted_scheduler.enable = false;
node_config.optimistic_scheduler.enable = false;
auto node0 = system.add_node (node_config);
node_config.peering_port = system.get_available_port ();
auto node1 = system.add_node (node_config);
@ -1411,26 +1420,30 @@ TEST (node, bootstrap_fork_open)
.sign (key0.prv, key0.pub)
.work (*system.work.generate (key0.pub))
.build ();
// Both know about send0
ASSERT_EQ (nano::block_status::progress, node0->process (send0));
ASSERT_EQ (nano::block_status::progress, node1->process (send0));
// Confirm send0 to allow starting and voting on the following blocks
for (auto node : system.nodes)
{
node->start_election (node->block (node->latest (nano::dev::genesis_key.pub)));
ASSERT_TIMELY (1s, node->active.election (send0->qualified_root ()));
auto election = node->active.election (send0->qualified_root ());
ASSERT_NE (nullptr, election);
election->force_confirm ();
ASSERT_TIMELY (2s, node->active.empty ());
}
ASSERT_TIMELY (3s, node0->block_confirmed (send0->hash ()));
nano::test::confirm (*node0, { send0 });
nano::test::confirm (*node1, { send0 });
ASSERT_TIMELY (5s, node0->block_confirmed (send0->hash ()));
ASSERT_TIMELY (5s, node1->block_confirmed (send0->hash ()));
// They disagree about open0/open1
ASSERT_EQ (nano::block_status::progress, node0->process (open0));
node0->confirming_set.add (open0->hash ());
ASSERT_TIMELY (5s, node0->block_confirmed (open0->hash ()));
ASSERT_EQ (nano::block_status::progress, node1->process (open1));
ASSERT_TRUE (node1->block_or_pruned_exists (open1->hash ()));
node1->start_election (open1); // Start election for open block which is necessary to resolve the fork
ASSERT_TIMELY (5s, node1->active.active (*open1));
// Allow node0 to vote on its fork
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
ASSERT_FALSE (node1->block_or_pruned_exists (open0->hash ()));
ASSERT_TIMELY (1s, node1->active.empty ());
ASSERT_TIMELY (10s, !node1->block_or_pruned_exists (open1->hash ()) && node1->block_or_pruned_exists (open0->hash ()));
}
@ -2225,11 +2238,11 @@ TEST (node, DISABLED_fork_invalid_block_signature)
node1.process_active (send1);
ASSERT_TIMELY (5s, node1.block (send1->hash ()));
// Send the vote with the corrupt block signature
node2.network.flood_vote (vote_corrupt, 1.0f);
ASSERT_TRUE (node2.network.flood_vote_rebroadcasted (vote_corrupt, 1.0f));
// Wait for the rollback
ASSERT_TIMELY (5s, node1.stats.count (nano::stat::type::rollback));
// Send the vote with the correct block
node2.network.flood_vote (vote, 1.0f);
ASSERT_TRUE (node2.network.flood_vote_rebroadcasted (vote, 1.0f));
ASSERT_TIMELY (10s, !node1.block (send1->hash ()));
ASSERT_TIMELY (10s, node1.block (send2->hash ()));
ASSERT_EQ (node1.block (send2->hash ())->block_signature (), send2->block_signature ());
@ -2770,9 +2783,8 @@ TEST (node, rollback_vote_self)
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
// Without the rollback being finished, the aggregator should not reply with any vote
auto channel = std::make_shared<nano::transport::fake::channel> (node);
node.aggregator.request ({ { send2->hash (), send2->root () } }, channel);
ASSERT_ALWAYS_EQ (1s, node.stats.count (nano::stat::type::request_aggregator_replies), 0);
node.aggregator.request ({ { send2->hash (), send2->root () } }, node.loopback_channel);
ASSERT_ALWAYS (1s, !election->votes ().contains (nano::dev::genesis_key.pub));
// Going out of the scope allows the rollback to complete
}
@ -3446,13 +3458,13 @@ TEST (node, DISABLED_pruning_age)
ASSERT_EQ (3, node1.ledger.block_count ());
// Pruning with default age 1 day
node1.ledger_pruning (1, true);
node1.pruning.ledger_pruning (1, true);
ASSERT_EQ (0, node1.ledger.pruned_count ());
ASSERT_EQ (3, node1.ledger.block_count ());
// Pruning with max age 0
node1.config.max_pruning_age = std::chrono::seconds{ 0 };
node1.ledger_pruning (1, true);
node1.pruning.ledger_pruning (1, true);
ASSERT_EQ (1, node1.ledger.pruned_count ());
ASSERT_EQ (3, node1.ledger.block_count ());
@ -3507,13 +3519,13 @@ TEST (node, DISABLED_pruning_depth)
ASSERT_EQ (3, node1.ledger.block_count ());
// Pruning with default depth (unlimited)
node1.ledger_pruning (1, true);
node1.pruning.ledger_pruning (1, true);
ASSERT_EQ (0, node1.ledger.pruned_count ());
ASSERT_EQ (3, node1.ledger.block_count ());
// Pruning with max depth 1
node1.config.max_pruning_depth = 1;
node1.ledger_pruning (1, true);
node1.pruning.ledger_pruning (1, true);
ASSERT_EQ (1, node1.ledger.pruned_count ());
ASSERT_EQ (3, node1.ledger.block_count ());

143
nano/core_test/numbers.cpp
View file

@ -345,8 +345,7 @@ TEST (uint256_union, decode_empty)
TEST (uint256_union, parse_zero)
{
nano::uint256_union input (nano::uint256_t (0));
std::string text;
input.encode_hex (text);
std::string text = input.to_string ();
nano::uint256_union output;
auto error (output.decode_hex (text));
ASSERT_FALSE (error);
@ -366,8 +365,7 @@ TEST (uint256_union, parse_zero_short)
TEST (uint256_union, parse_one)
{
nano::uint256_union input (nano::uint256_t (1));
std::string text;
input.encode_hex (text);
std::string text = input.to_string ();
nano::uint256_union output;
auto error (output.decode_hex (text));
ASSERT_FALSE (error);
@ -378,8 +376,7 @@ TEST (uint256_union, parse_one)
TEST (uint256_union, parse_error_symbol)
{
nano::uint256_union input (nano::uint256_t (1000));
std::string text;
input.encode_hex (text);
std::string text = input.to_string ();
text[5] = '!';
nano::uint256_union output;
auto error (output.decode_hex (text));
@ -389,8 +386,7 @@ TEST (uint256_union, parse_error_symbol)
TEST (uint256_union, max_hex)
{
nano::uint256_union input (std::numeric_limits<nano::uint256_t>::max ());
std::string text;
input.encode_hex (text);
std::string text = input.to_string ();
nano::uint256_union output;
auto error (output.decode_hex (text));
ASSERT_FALSE (error);
@ -409,8 +405,7 @@ TEST (uint256_union, decode_dec)
TEST (uint256_union, max_dec)
{
nano::uint256_union input (std::numeric_limits<nano::uint256_t>::max ());
std::string text;
input.encode_dec (text);
std::string text = input.to_string_dec ();
nano::uint256_union output;
auto error (output.decode_dec (text));
ASSERT_FALSE (error);
@ -445,8 +440,7 @@ TEST (uint256_union, decode_dec_leading_zero)
TEST (uint256_union, parse_error_overflow)
{
nano::uint256_union input (std::numeric_limits<nano::uint256_t>::max ());
std::string text;
input.encode_hex (text);
std::string text = input.to_string ();
text.push_back (0);
nano::uint256_union output;
auto error (output.decode_hex (text));
@ -650,6 +644,68 @@ TEST (uint512_union, hash)
}
}
TEST (uint512_union, parse_zero)
{
nano::uint512_union input (nano::uint512_t (0));
std::string text = input.to_string ();
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_FALSE (error);
ASSERT_EQ (input, output);
ASSERT_TRUE (output.number ().is_zero ());
}
TEST (uint512_union, parse_zero_short)
{
std::string text ("0");
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_FALSE (error);
ASSERT_TRUE (output.number ().is_zero ());
}
TEST (uint512_union, parse_one)
{
nano::uint512_union input (nano::uint512_t (1));
std::string text = input.to_string ();
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_FALSE (error);
ASSERT_EQ (input, output);
ASSERT_EQ (1, output.number ());
}
TEST (uint512_union, parse_error_symbol)
{
nano::uint512_union input (nano::uint512_t (1000));
std::string text = input.to_string ();
text[5] = '!';
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_TRUE (error);
}
TEST (uint512_union, max)
{
nano::uint512_union input (std::numeric_limits<nano::uint512_t>::max ());
std::string text = input.to_string ();
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_FALSE (error);
ASSERT_EQ (input, output);
ASSERT_EQ (nano::uint512_t ("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), output.number ());
}
TEST (uint512_union, parse_error_overflow)
{
nano::uint512_union input (std::numeric_limits<nano::uint512_t>::max ());
std::string text = input.to_string ();
text.push_back (0);
nano::uint512_union output;
auto error (output.decode_hex (text));
ASSERT_TRUE (error);
}
TEST (sat_math, add_sat)
{
// Test uint128_t
@ -743,4 +799,67 @@ TEST (sat_math, sub_sat)
ASSERT_EQ (nano::sub_sat (hundred, nano::uint512_t (200)), min);
ASSERT_EQ (nano::sub_sat (min, max), min);
}
}
TEST (account, encode_zero)
{
nano::account number0{};
std::stringstream stream;
number0.encode_account (stream);
auto str0 = stream.str ();
/*
* Handle different lengths for "xrb_" prefixed and "nano_" prefixed accounts
*/
ASSERT_EQ ((str0.front () == 'x') ? 64 : 65, str0.size ());
ASSERT_EQ (65, str0.size ());
nano::account number1;
ASSERT_FALSE (number1.decode_account (str0));
ASSERT_EQ (number0, number1);
}
TEST (account, encode_all)
{
nano::account number0;
number0.decode_hex ("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff");
std::stringstream stream;
number0.encode_account (stream);
auto str0 = stream.str ();
/*
* Handle different lengths for "xrb_" prefixed and "nano_" prefixed accounts
*/
ASSERT_EQ ((str0.front () == 'x') ? 64 : 65, str0.size ());
nano::account number1;
ASSERT_FALSE (number1.decode_account (str0));
ASSERT_EQ (number0, number1);
}
TEST (account, encode_fail)
{
nano::account number0{};
std::stringstream stream;
number0.encode_account (stream);
auto str0 = stream.str ();
str0[16] ^= 1;
nano::account number1;
ASSERT_TRUE (number1.decode_account (str0));
}
TEST (account, known_addresses)
{
nano::account account1{ "0000000000000000000000000000000000000000000000000000000000000000" };
ASSERT_EQ (account1.to_account (), "nano_1111111111111111111111111111111111111111111111111111hifc8npp");
nano::account account2{ "B0311EA55708D6A53C75CDBF88300259C6D018522FE3D4D0A242E431F9E8B6D0" };
ASSERT_EQ (account2.to_account (), "nano_3e3j5tkog48pnny9dmfzj1r16pg8t1e76dz5tmac6iq689wyjfpiij4txtdo");
nano::account account3{ "45C6FF9D1706D61F0821327752671BDA9F9ED2DA40326B01935AB566FB9E08ED" };
ASSERT_EQ (account3.to_account (), "nano_1jg8zygjg3pp5w644emqcbmjqpnzmubfni3kfe1s8pooeuxsw49fdq1mco9j");
nano::account account4{ "E89208DD038FBB269987689621D52292AE9C35941A7484756ECCED92A65093BA" };
ASSERT_EQ (account4.to_account (), "nano_3t6k35gi95xu6tergt6p69ck76ogmitsa8mnijtpxm9fkcm736xtoncuohr3");
nano::account account5{ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" };
ASSERT_EQ (account5.to_account (), "nano_3zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzc3yoon41");
}

206
nano/core_test/online_reps.cpp
View file

@ -1,3 +1,5 @@
#include "nano/node/election.hpp"
#include <nano/node/online_reps.hpp>
#include <nano/node/transport/fake.hpp>
#include <nano/secure/vote.hpp>
@ -8,8 +10,8 @@
TEST (online_reps, basic)
{
nano::test::system system (1);
auto & node1 (*system.nodes[0]);
nano::test::system system;
auto & node1 = *system.add_node ();
// 1 sample of minimum weight
ASSERT_EQ (node1.config.online_weight_minimum, node1.online_reps.trended ());
auto vote (std::make_shared<nano::vote> ());
@ -69,4 +71,204 @@ TEST (online_reps, election)
ASSERT_EQ (0, node1.online_reps.online ());
node1.vote_processor.vote_blocking (vote, std::make_shared<nano::transport::fake::channel> (node1));
ASSERT_EQ (nano::dev::constants.genesis_amount - nano::Knano_ratio, node1.online_reps.online ());
}
// Online reps should be able to observe remote representative
TEST (online_reps, observe)
{
nano::test::system system;
auto & node = *system.add_node ();
ASSERT_EQ (0, node.online_reps.online ());
// Addd genesis representative
auto & node_rep = *system.add_node ();
system.wallet (1)->insert_adhoc (nano::dev::genesis_key.prv);
// The node should see that weight as online
ASSERT_TIMELY_EQ (10s, node.online_reps.online (), nano::dev::constants.genesis_amount);
ASSERT_ALWAYS_EQ (1s, node.online_reps.online (), nano::dev::constants.genesis_amount);
}
TEST (online_reps, observe_multiple)
{
nano::test::system system;
auto & node = *system.add_node ();
ASSERT_EQ (0, node.online_reps.online ());
auto & node_rep1 = *system.add_node (); // key1
auto & node_rep2 = *system.add_node (); // key2 & key3
auto const weight_1 = nano::nano_ratio * 1000;
auto const weight_2 = nano::nano_ratio * 1000000;
auto const weight_3 = nano::nano_ratio * 10000000;
nano::keypair key1, key2, key3;
// Distribute genesis voting weight
{
nano::block_builder builder;
auto send1 = builder.state ()
.account (nano::dev::genesis_key.pub)
.previous (nano::dev::genesis->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - weight_1)
.link (key1.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (nano::dev::genesis->hash ()))
.build ();
auto send2 = builder.state ()
.account (nano::dev::genesis_key.pub)
.previous (send1->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - weight_1 - weight_2)
.link (key2.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (send1->hash ()))
.build ();
auto send3 = builder.state ()
.account (nano::dev::genesis_key.pub)
.previous (send2->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - weight_1 - weight_2 - weight_3)
.link (key3.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (send2->hash ()))
.build ();
auto open1 = builder.state ()
.account (key1.pub)
.previous (0)
.representative (key1.pub)
.balance (weight_1)
.link (send1->hash ())
.sign (key1.prv, key1.pub)
.work (*system.work.generate (key1.pub))
.build ();
auto open2 = builder.state ()
.account (key2.pub)
.previous (0)
.representative (key2.pub)
.balance (weight_2)
.link (send2->hash ())
.sign (key2.prv, key2.pub)
.work (*system.work.generate (key2.pub))
.build ();
auto open3 = builder.state ()
.account (key3.pub)
.previous (0)
.representative (key3.pub)
.balance (weight_3)
.link (send3->hash ())
.sign (key3.prv, key3.pub)
.work (*system.work.generate (key3.pub))
.build ();
ASSERT_TRUE (nano::test::process (node_rep1, { send1, send2, send3, open1, open2, open3 }));
ASSERT_TRUE (nano::test::process (node_rep2, { send1, send2, send3, open1, open2, open3 }));
}
// Add rep keys to nodes
system.wallet (1)->insert_adhoc (key1.prv);
system.wallet (2)->insert_adhoc (key2.prv);
system.wallet (2)->insert_adhoc (key3.prv);
ASSERT_TIMELY_EQ (10s, node.online_reps.online (), weight_1 + weight_2 + weight_3);
ASSERT_ALWAYS_EQ (1s, node.online_reps.online (), weight_1 + weight_2 + weight_3);
}
// Online weight calculation should include local representative
TEST (online_reps, observe_local)
{
nano::test::system system;
auto & node = *system.add_node ();
ASSERT_EQ (0, node.online_reps.online ());
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
ASSERT_TIMELY_EQ (10s, node.online_reps.online (), nano::dev::constants.genesis_amount);
ASSERT_ALWAYS_EQ (1s, node.online_reps.online (), nano::dev::constants.genesis_amount);
}
// Online weight calculation should include slower but active representatives
TEST (online_reps, observe_slow)
{
nano::test::system system;
auto & node = *system.add_node ();
ASSERT_EQ (0, node.online_reps.online ());
// Enough to reach quorum by a single vote
auto const weight = nano::nano_ratio * 80000000;
nano::keypair key1, key2; // Fast and slow reps
// Distribute genesis voting weight
nano::block_builder builder;
auto send1 = builder.state ()
.account (nano::dev::genesis_key.pub)
.previous (nano::dev::genesis->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - weight)
.link (key1.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (nano::dev::genesis->hash ()))
.build ();
auto send2 = builder.state ()
.account (nano::dev::genesis_key.pub)
.previous (send1->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - weight * 2)
.link (key2.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (send1->hash ()))
.build ();
auto open1 = builder.state ()
.account (key1.pub)
.previous (0)
.representative (key1.pub)
.balance (weight)
.link (send1->hash ())
.sign (key1.prv, key1.pub)
.work (*system.work.generate (key1.pub))
.build ();
auto open2 = builder.state ()
.account (key2.pub)
.previous (0)
.representative (key2.pub)
.balance (weight)
.link (send2->hash ())
.sign (key2.prv, key2.pub)
.work (*system.work.generate (key2.pub))
.build ();
ASSERT_TRUE (nano::test::process (node, { send1, send2, open1, open2 }));
nano::test::confirm (node, { send1, send2, open1, open2 });
ASSERT_EQ (node.active.size (), 0);
// Add a block that we can vote on
auto send_dummy = builder.state ()
.account (nano::dev::genesis_key.pub)
.previous (send2->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - weight * 2 - nano::nano_ratio)
.link (nano::keypair{}.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (send2->hash ()))
.build ();
ASSERT_TRUE (nano::test::process (node, { send_dummy }));
// Wait for election for the block to be activated
std::shared_ptr<nano::election> election;
ASSERT_TIMELY (5s, election = node.active.election (send_dummy->qualified_root ()));
ASSERT_TRUE (election->contains (send_dummy->hash ()));
// Issue vote from a fast rep
auto vote_fast = nano::test::make_final_vote (key1, { send_dummy });
node.vote_processor.vote_blocking (vote_fast, nano::test::fake_channel (node));
ASSERT_TIMELY (5s, election->confirmed ());
ASSERT_TIMELY (5s, !node.active.active (send_dummy->qualified_root ())); // No longer present in AEC
ASSERT_TIMELY_EQ (5s, node.online_reps.online (), weight);
// Issue vote from a slow rep
auto vote_slow = nano::test::make_final_vote (key2, { send_dummy });
node.vote_processor.vote_blocking (vote_slow, nano::test::fake_channel (node));
// The slow rep weight should still be counted as online, even though it arrived slightly after the election already reached quorum
ASSERT_TIMELY_EQ (5s, node.online_reps.online (), weight * 2);
}

13
nano/core_test/optimistic_scheduler.cpp
View file

@ -31,8 +31,9 @@ TEST (optimistic_scheduler, activate_one)
// Ensure unconfirmed account head block gets activated
auto const & block = blocks.back ();
ASSERT_TIMELY (5s, node.vote_router.active (block->hash ()));
ASSERT_EQ (node.active.election (block->qualified_root ())->behavior (), nano::election_behavior::optimistic);
std::shared_ptr<nano::election> election;
ASSERT_TIMELY (5s, election = node.active.election (block->qualified_root ()));
ASSERT_EQ (election->behavior (), nano::election_behavior::optimistic);
}
/*
@ -52,8 +53,9 @@ TEST (optimistic_scheduler, activate_one_zero_conf)
// Ensure unconfirmed account head block gets activated
auto const & block = blocks.back ();
ASSERT_TIMELY (5s, node.vote_router.active (block->hash ()));
ASSERT_EQ (node.active.election (block->qualified_root ())->behavior (), nano::election_behavior::optimistic);
std::shared_ptr<nano::election> election;
ASSERT_TIMELY (5s, election = node.active.election (block->qualified_root ()));
ASSERT_EQ (election->behavior (), nano::election_behavior::optimistic);
}
/*
@ -74,7 +76,8 @@ TEST (optimistic_scheduler, activate_many)
ASSERT_TIMELY (5s, std::all_of (chains.begin (), chains.end (), [&] (auto const & entry) {
auto const & [account, blocks] = entry;
auto const & block = blocks.back ();
return node.vote_router.active (block->hash ()) && node.active.election (block->qualified_root ())->behavior () == nano::election_behavior::optimistic;
auto election = node.active.election (block->qualified_root ());
return election && election->behavior () == nano::election_behavior::optimistic;
}));
}

21
nano/core_test/peer_container.cpp
View file

@ -169,13 +169,13 @@ TEST (channels, fill_random_part)
}
// TODO: remove node instantiation requirement for testing with bigger network size
TEST (peer_container, list_fanout)
TEST (peer_container, DISABLED_list_fanout)
{
nano::test::system system{ 1 };
auto node = system.nodes[0];
ASSERT_EQ (0, node->network.size ());
ASSERT_EQ (0.0, node->network.size_sqrt ());
ASSERT_EQ (0, node->network.fanout ());
ASSERT_EQ (0.0f, node->network.size_log ());
ASSERT_EQ (1, node->network.fanout ());
ASSERT_TRUE (node->network.list (node->network.fanout ()).empty ());
auto add_peer = [&node, &system] () {
@ -185,26 +185,27 @@ TEST (peer_container, list_fanout)
add_peer ();
ASSERT_TIMELY_EQ (5s, 1, node->network.size ());
ASSERT_EQ (1.f, node->network.size_sqrt ());
ASSERT_EQ (0.0f, node->network.size_log ());
ASSERT_EQ (1, node->network.fanout ());
ASSERT_EQ (1, node->network.list (node->network.fanout ()).size ());
add_peer ();
ASSERT_TIMELY_EQ (5s, 2, node->network.size ());
ASSERT_EQ (std::sqrt (2.f), node->network.size_sqrt ());
add_peer ();
ASSERT_TIMELY_EQ (5s, 3, node->network.size ());
ASSERT_EQ (std::log (3.0f), node->network.size_log ());
ASSERT_EQ (2, node->network.fanout ());
ASSERT_EQ (2, node->network.list (node->network.fanout ()).size ());
unsigned number_of_peers = 10;
for (unsigned i = 2; i < number_of_peers; ++i)
for (unsigned i = 3; i < number_of_peers; ++i)
{
add_peer ();
}
ASSERT_TIMELY_EQ (5s, number_of_peers, node->network.size ());
ASSERT_EQ (std::sqrt (float (number_of_peers)), node->network.size_sqrt ());
ASSERT_EQ (4, node->network.fanout ());
ASSERT_EQ (4, node->network.list (node->network.fanout ()).size ());
ASSERT_EQ (std::log (float (number_of_peers)), node->network.size_log ());
ASSERT_EQ (3, node->network.fanout ());
ASSERT_EQ (3, node->network.list (node->network.fanout ()).size ());
}
// Test to make sure we don't repeatedly send keepalive messages to nodes that aren't responding

4
nano/core_test/processor_service.cpp
View file

@ -18,7 +18,7 @@ TEST (processor_service, bad_send_signature)
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::ledger ledger (*store, system.stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, system.stats, system.logger);
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
@ -44,7 +44,7 @@ TEST (processor_service, bad_receive_signature)
auto store = nano::make_store (system.logger, nano::unique_path (), nano::dev::constants);
ASSERT_FALSE (store->init_error ());
nano::ledger ledger (*store, system.stats, nano::dev::constants);
nano::ledger ledger (*store, nano::dev::constants, system.stats, system.logger);
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };

3
nano/core_test/rep_crawler.cpp
View file

@ -254,7 +254,8 @@ TEST (rep_crawler, rep_connection_close)
// The behaviour of this test previously was the opposite, that the repcrawler eventually send out such a block and deleted the block
// from the recently confirmed list to try to make ammends for sending it, which is bad behaviour.
// In the long term, we should have a better way to check for reps and this test should become redundant
TEST (rep_crawler, recently_confirmed)
// DISABLED as behaviour changed, and we now only query confirmed blocks
TEST (rep_crawler, DISABLED_recently_confirmed)
{
nano::test::system system (1);
auto & node1 (*system.nodes[0]);

309
nano/core_test/request_aggregator.cpp
View file

@ -1,4 +1,5 @@
#include <nano/lib/blocks.hpp>
#include <nano/lib/function.hpp>
#include <nano/lib/jsonconfig.hpp>
#include <nano/node/active_elections.hpp>
#include <nano/node/confirming_set.hpp>
@ -9,12 +10,17 @@
#include <nano/node/transport/inproc.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_confirmed.hpp>
#include <nano/secure/vote.hpp>
#include <nano/test_common/network.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>
#include <gtest/gtest.h>
#include <atomic>
#include <chrono>
#include <future>
using namespace std::chrono_literals;
TEST (request_aggregator, one)
@ -22,7 +28,9 @@ TEST (request_aggregator, one)
nano::test::system system;
nano::node_config node_config = system.default_config ();
node_config.backlog_scan.enable = false;
auto & node (*system.add_node (node_config));
nano::node_flags node_flags;
node_flags.disable_rep_crawler = true;
auto & node (*system.add_node (node_config, node_flags));
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
nano::block_builder builder;
auto send1 = builder
@ -71,7 +79,9 @@ TEST (request_aggregator, one_update)
nano::test::system system;
nano::node_config node_config = system.default_config ();
node_config.backlog_scan.enable = false;
auto & node (*system.add_node (node_config));
nano::node_flags node_flags;
node_flags.disable_rep_crawler = true;
auto & node (*system.add_node (node_config, node_flags));
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
nano::keypair key1;
auto send1 = nano::state_block_builder ()
@ -137,7 +147,9 @@ TEST (request_aggregator, two)
nano::test::system system;
nano::node_config node_config = system.default_config ();
node_config.backlog_scan.enable = false;
auto & node (*system.add_node (node_config));
nano::node_flags node_flags;
node_flags.disable_rep_crawler = true;
auto & node (*system.add_node (node_config, node_flags));
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
nano::keypair key1;
nano::state_block_builder builder;
@ -267,7 +279,9 @@ TEST (request_aggregator, split)
nano::test::system system;
nano::node_config node_config = system.default_config ();
node_config.backlog_scan.enable = false;
auto & node (*system.add_node (node_config));
nano::node_flags node_flags;
node_flags.disable_rep_crawler = true;
auto & node (*system.add_node (node_config, node_flags));
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
std::vector<std::pair<nano::block_hash, nano::root>> request;
std::vector<std::shared_ptr<nano::block>> blocks;
@ -322,7 +336,9 @@ TEST (request_aggregator, channel_max_queue)
nano::node_config node_config = system.default_config ();
node_config.backlog_scan.enable = false;
node_config.request_aggregator.max_queue = 0;
auto & node (*system.add_node (node_config));
nano::node_flags node_flags;
node_flags.disable_rep_crawler = true;
auto & node (*system.add_node (node_config, node_flags));
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
nano::block_builder builder;
auto send1 = builder
@ -384,6 +400,7 @@ TEST (request_aggregator, cannot_vote)
nano::test::system system;
nano::node_flags flags;
flags.disable_request_loop = true;
flags.disable_rep_crawler = true;
auto & node (*system.add_node (flags));
nano::state_block_builder builder;
auto send1 = builder.make_block ()
@ -450,3 +467,285 @@ TEST (request_aggregator, cannot_vote)
ASSERT_EQ (0, node.stats.count (nano::stat::type::requests, nano::stat::detail::requests_unknown));
ASSERT_TIMELY (3s, 1 <= node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out));
}
namespace
{
std::future<nano::confirm_ack> observe_confirm_ack (std::shared_ptr<nano::transport::test_channel> const & channel)
{
std::promise<nano::confirm_ack> promise;
auto future = promise.get_future ();
struct confirm_ack_visitor : public nano::message_visitor
{
std::optional<nano::confirm_ack> result;
void confirm_ack (nano::confirm_ack const & msg) override
{
result = msg;
}
};
channel->observers.clear ();
channel->observers.add (nano::wrap_move_only ([&, promise = std::move (promise)] (nano::message const & message, nano::transport::traffic_type const & type) mutable {
confirm_ack_visitor visitor{};
message.visit (visitor);
if (visitor.result)
{
promise.set_value (visitor.result.value ());
}
}));
return future;
}
}
/*
* Request for a forked open block should return vote for the correct fork alternative
*/
TEST (request_aggregator, forked_open)
{
nano::test::system system;
auto & node = *system.add_node ();
// Voting needs a rep key set up on the node
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
// Setup two forks of the open block
nano::keypair key;
nano::block_builder builder;
auto send0 = builder.send ()
.previous (nano::dev::genesis->hash ())
.destination (key.pub)
.balance (nano::dev::constants.genesis_amount - 500)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (nano::dev::genesis->hash ()))
.build ();
auto open0 = builder.open ()
.source (send0->hash ())
.representative (1)
.account (key.pub)
.sign (key.prv, key.pub)
.work (*system.work.generate (key.pub))
.build ();
auto open1 = builder.open ()
.source (send0->hash ())
.representative (2)
.account (key.pub)
.sign (key.prv, key.pub)
.work (*system.work.generate (key.pub))
.build ();
nano::test::process (node, { send0, open0 });
nano::test::confirm (node, { open0 });
auto channel = nano::test::test_channel (node);
auto future = observe_confirm_ack (channel);
// Request vote for the wrong fork
std::vector<std::pair<nano::block_hash, nano::root>> request{ { open1->hash (), open1->root () } };
ASSERT_TRUE (node.aggregator.request (request, channel));
ASSERT_EQ (future.wait_for (5s), std::future_status::ready);
auto ack = future.get ();
ASSERT_EQ (ack.vote->hashes.size (), 1);
ASSERT_EQ (ack.vote->hashes[0], open0->hash ()); // Vote for the correct fork alternative
ASSERT_EQ (ack.vote->account, nano::dev::genesis_key.pub);
}
/*
* Request for a conflicting epoch block should return vote for the correct alternative
*/
TEST (request_aggregator, epoch_conflict)
{
nano::test::system system;
nano::node_flags node_flags;
// Workaround for vote spacing dropping requests with the same root
// FIXME: Vote spacing should use full qualified root
node_flags.disable_rep_crawler = true;
auto & node = *system.add_node (node_flags);
// Voting needs a rep key set up on the node
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
// Setup the initial chain and the conflicting blocks
nano::keypair key;
nano::keypair epoch_signer (nano::dev::genesis_key);
nano::state_block_builder builder;
// Create initial chain: send -> open -> change
auto send = builder.make_block ()
.account (nano::dev::genesis_key.pub)
.previous (nano::dev::genesis->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - 1)
.link (key.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (nano::dev::genesis->hash ()))
.build ();
auto open = builder.make_block ()
.account (key.pub)
.previous (0)
.representative (key.pub)
.balance (1)
.link (send->hash ())
.sign (key.prv, key.pub)
.work (*system.work.generate (key.pub))
.build ();
// Change block root is the open block hash, qualified root: {open, open}
auto change = builder.make_block ()
.account (key.pub)
.previous (open->hash ())
.representative (key.pub)
.balance (1)
.link (0)
.sign (key.prv, key.pub)
.work (*system.work.generate (open->hash ()))
.build ();
// Pending entry is needed first to process the epoch open block
auto pending = builder.make_block ()
.account (nano::dev::genesis_key.pub)
.previous (send->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - 2)
.link (change->root ().as_account ())
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (send->hash ()))
.build ();
// Create conflicting epoch block with the same root as the change block, qualified root: {open, 0}
// This block is intentionally not processed immediately so the node doesn't know about it
auto epoch_open = builder.make_block ()
.account (change->root ().as_account ())
.previous (0)
.representative (0)
.balance (0)
.link (node.ledger.epoch_link (nano::epoch::epoch_1))
.sign (epoch_signer.prv, epoch_signer.pub)
.work (*system.work.generate (open->hash ()))
.build ();
// Process and confirm the initial chain with the change block
nano::test::process (node, { send, open, change });
nano::test::confirm (node, { change });
ASSERT_TIMELY (5s, node.block_confirmed (change->hash ()));
auto channel = nano::test::test_channel (node);
// Request vote for the conflicting epoch block
std::vector<std::pair<nano::block_hash, nano::root>> request{ { epoch_open->hash (), epoch_open->root () } };
auto future1 = observe_confirm_ack (channel);
ASSERT_TRUE (node.aggregator.request (request, channel));
ASSERT_EQ (future1.wait_for (5s), std::future_status::ready);
auto ack1 = future1.get ();
ASSERT_EQ (ack1.vote->hashes.size (), 1);
ASSERT_EQ (ack1.vote->hashes[0], change->hash ()); // Vote for the correct alternative (change block)
ASSERT_EQ (ack1.vote->account, nano::dev::genesis_key.pub);
// Process the conflicting epoch block
nano::test::process (node, { pending, epoch_open });
nano::test::confirm (node, { pending, epoch_open });
// Workaround for vote spacing dropping requests with the same root
// FIXME: Vote spacing should use full qualified root
WAIT (1s);
// Request vote for the conflicting epoch block again
auto future2 = observe_confirm_ack (channel);
ASSERT_TRUE (node.aggregator.request (request, channel));
ASSERT_EQ (future2.wait_for (5s), std::future_status::ready);
auto ack2 = future2.get ();
ASSERT_EQ (ack2.vote->hashes.size (), 1);
ASSERT_EQ (ack2.vote->hashes[0], epoch_open->hash ()); // Vote for the epoch block
ASSERT_EQ (ack2.vote->account, nano::dev::genesis_key.pub);
}
/*
* Request for multiple cemented blocks in a chain should generate votes regardless of vote spacing
*/
TEST (request_aggregator, cemented_no_spacing)
{
nano::test::system system;
auto & node = *system.add_node ();
// Voting needs a rep key set up on the node
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
// Create a chain of 3 blocks: send1 -> send2 -> send3
nano::state_block_builder builder;
auto send1 = builder.make_block ()
.account (nano::dev::genesis_key.pub)
.previous (nano::dev::genesis->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - 1)
.link (nano::dev::genesis_key.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (nano::dev::genesis->hash ()))
.build ();
auto send2 = builder.make_block ()
.account (nano::dev::genesis_key.pub)
.previous (send1->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - 2)
.link (nano::dev::genesis_key.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (send1->hash ()))
.build ();
auto send3 = builder.make_block ()
.account (nano::dev::genesis_key.pub)
.previous (send2->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - 3)
.link (nano::dev::genesis_key.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (send2->hash ()))
.build ();
// Process and confirm all blocks in the chain
nano::test::process (node, { send1, send2, send3 });
nano::test::confirm (node, { send1, send2, send3 });
ASSERT_TRUE (node.block_confirmed (send3->hash ()));
auto channel = nano::test::test_channel (node);
// Request votes for blocks at different positions in the chain
std::vector<std::pair<nano::block_hash, nano::root>> request{
{ send1->hash (), send1->root () },
{ send2->hash (), send2->root () },
{ send3->hash (), send3->root () }
};
// Request votes for all blocks
auto future = observe_confirm_ack (channel);
ASSERT_TRUE (node.aggregator.request (request, channel));
// Wait for the votes
ASSERT_EQ (future.wait_for (5s), std::future_status::ready);
auto ack = future.get ();
// Verify we got votes for all blocks in the chain
ASSERT_EQ (ack.vote->hashes.size (), 3);
ASSERT_EQ (ack.vote->account, nano::dev::genesis_key.pub);
// Verify individual vote properties
std::set<nano::block_hash> voted_hashes;
for (auto const & hash : ack.vote->hashes)
{
voted_hashes.insert (hash);
}
// Verify we got votes for all three blocks
ASSERT_TRUE (voted_hashes.find (send1->hash ()) != voted_hashes.end ());
ASSERT_TRUE (voted_hashes.find (send2->hash ()) != voted_hashes.end ());
ASSERT_TRUE (voted_hashes.find (send3->hash ()) != voted_hashes.end ());
}

14
nano/core_test/stacktrace.cpp Normal file
View file

@ -0,0 +1,14 @@
#include <nano/lib/stacktrace.hpp>
#include <gtest/gtest.h>
// Check that the stacktrace contains the current function name
// This depends on the way testcase names are compiled by gtest
// Current name: "stacktrace_human_readable_Test::TestBody()"
TEST (stacktrace, human_readable)
{
auto stacktrace = nano::generate_stacktrace ();
std::cout << stacktrace << std::endl;
ASSERT_FALSE (stacktrace.empty ());
ASSERT_TRUE (stacktrace.find ("stacktrace_human_readable_Test") != std::string::npos);
}

28
nano/core_test/toml.cpp
View file

@ -251,6 +251,7 @@ TEST (toml_config, daemon_config_deserialize_defaults)
[node.lmdb]
[node.rocksdb]
[node.tcp]
[node.network]
[opencl]
[rpc]
[rpc.child_process]
@ -325,7 +326,6 @@ TEST (toml_config, daemon_config_deserialize_defaults)
ASSERT_EQ (conf.node.bounded_backlog.enable, defaults.node.bounded_backlog.enable);
ASSERT_EQ (conf.node.bounded_backlog.batch_size, defaults.node.bounded_backlog.batch_size);
ASSERT_EQ (conf.node.bounded_backlog.max_queued_notifications, defaults.node.bounded_backlog.max_queued_notifications);
ASSERT_EQ (conf.node.bounded_backlog.scan_rate, defaults.node.bounded_backlog.scan_rate);
ASSERT_EQ (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled);
@ -426,6 +426,14 @@ TEST (toml_config, daemon_config_deserialize_defaults)
ASSERT_EQ (conf.node.tcp.connect_timeout, defaults.node.tcp.connect_timeout);
ASSERT_EQ (conf.node.tcp.handshake_timeout, defaults.node.tcp.handshake_timeout);
ASSERT_EQ (conf.node.tcp.io_timeout, defaults.node.tcp.io_timeout);
ASSERT_EQ (conf.node.network.peer_reachout.count (), defaults.node.network.peer_reachout.count ());
ASSERT_EQ (conf.node.network.cached_peer_reachout.count (), defaults.node.network.cached_peer_reachout.count ());
ASSERT_EQ (conf.node.network.max_peers_per_ip, defaults.node.network.max_peers_per_ip);
ASSERT_EQ (conf.node.network.max_peers_per_subnetwork, defaults.node.network.max_peers_per_subnetwork);
ASSERT_EQ (conf.node.network.duplicate_filter_size, defaults.node.network.duplicate_filter_size);
ASSERT_EQ (conf.node.network.duplicate_filter_cutoff, defaults.node.network.duplicate_filter_cutoff);
ASSERT_EQ (conf.node.network.minimum_fanout, defaults.node.network.minimum_fanout);
}
/** Deserialize a node config with non-default values */
@ -657,6 +665,15 @@ TEST (toml_config, daemon_config_deserialize_no_defaults)
handshake_timeout = 999
io_timeout = 999
[node.network]
peer_reachout = 999
cached_peer_reachout = 9999
max_peers_per_ip = 99
max_peers_per_subnetwork = 99
duplicate_filter_size = 9999999
duplicate_filter_cutoff = 999
minimum_fanout = 99
[opencl]
device = 999
enable = true
@ -743,7 +760,6 @@ TEST (toml_config, daemon_config_deserialize_no_defaults)
ASSERT_NE (conf.node.bounded_backlog.enable, defaults.node.bounded_backlog.enable);
ASSERT_NE (conf.node.bounded_backlog.batch_size, defaults.node.bounded_backlog.batch_size);
ASSERT_NE (conf.node.bounded_backlog.max_queued_notifications, defaults.node.bounded_backlog.max_queued_notifications);
ASSERT_NE (conf.node.bounded_backlog.scan_rate, defaults.node.bounded_backlog.scan_rate);
ASSERT_NE (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled);
@ -846,6 +862,14 @@ TEST (toml_config, daemon_config_deserialize_no_defaults)
ASSERT_NE (conf.node.tcp.connect_timeout, defaults.node.tcp.connect_timeout);
ASSERT_NE (conf.node.tcp.handshake_timeout, defaults.node.tcp.handshake_timeout);
ASSERT_NE (conf.node.tcp.io_timeout, defaults.node.tcp.io_timeout);
ASSERT_NE (conf.node.network.peer_reachout.count (), defaults.node.network.peer_reachout.count ());
ASSERT_NE (conf.node.network.cached_peer_reachout.count (), defaults.node.network.cached_peer_reachout.count ());
ASSERT_NE (conf.node.network.max_peers_per_ip, defaults.node.network.max_peers_per_ip);
ASSERT_NE (conf.node.network.max_peers_per_subnetwork, defaults.node.network.max_peers_per_subnetwork);
ASSERT_NE (conf.node.network.duplicate_filter_size, defaults.node.network.duplicate_filter_size);
ASSERT_NE (conf.node.network.duplicate_filter_cutoff, defaults.node.network.duplicate_filter_cutoff);
ASSERT_NE (conf.node.network.minimum_fanout, defaults.node.network.minimum_fanout);
}
/** There should be no required values **/

6
nano/core_test/vote_processor.cpp
View file

@ -114,7 +114,7 @@ TEST (vote_processor, weights)
auto & node (*system.nodes[0]);
// Create representatives of different weight levels
auto const stake = node.config.online_weight_minimum.number ();
auto const stake = nano::dev::genesis->balance ().number ();
auto const level0 = stake / 5000; // 0.02%
auto const level1 = stake / 500; // 0.2%
auto const level2 = stake / 50; // 2%
@ -248,8 +248,8 @@ TEST (vote_processor, local_broadcast_without_a_representative)
ASSERT_NE (votes.end (), existing);
ASSERT_EQ (vote->timestamp (), existing->second.timestamp);
// Ensure the vote was broadcast
ASSERT_EQ (1, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out));
ASSERT_EQ (1, node.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::out));
ASSERT_TIMELY_EQ (5s, 1, node.stats.count (nano::stat::type::message, nano::stat::detail::confirm_ack, nano::stat::dir::out));
ASSERT_TIMELY_EQ (5s, 1, node.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::out));
}
// Issue that tracks last changes on this test: https://github.com/nanocurrency/nano-node/issues/3485

42
nano/core_test/wallet.cpp
View file

@ -319,48 +319,6 @@ TEST (wallet, rekey)
ASSERT_TRUE (wallet.rekey (transaction, "2"));
}
TEST (account, encode_zero)
{
nano::account number0{};
std::string str0;
number0.encode_account (str0);
/*
* Handle different lengths for "xrb_" prefixed and "nano_" prefixed accounts
*/
ASSERT_EQ ((str0.front () == 'x') ? 64 : 65, str0.size ());
ASSERT_EQ (65, str0.size ());
nano::account number1;
ASSERT_FALSE (number1.decode_account (str0));
ASSERT_EQ (number0, number1);
}
TEST (account, encode_all)
{
nano::account number0;
number0.decode_hex ("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff");
std::string str0;
number0.encode_account (str0);
/*
* Handle different lengths for "xrb_" prefixed and "nano_" prefixed accounts
*/
ASSERT_EQ ((str0.front () == 'x') ? 64 : 65, str0.size ());
nano::account number1;
ASSERT_FALSE (number1.decode_account (str0));
ASSERT_EQ (number0, number1);
}
TEST (account, encode_fail)
{
nano::account number0{};
std::string str0;
number0.encode_account (str0);
str0[16] ^= 1;
nano::account number1;
ASSERT_TRUE (number1.decode_account (str0));
}
TEST (wallet, hash_password)
{
bool init;

173
nano/core_test/websocket.cpp
View file

@ -463,6 +463,177 @@ TEST (websocket, confirmation_options_votes)
}
}
TEST (websocket, confirmation_options_linked_account)
{
nano::test::system system;
nano::node_config config = system.default_config ();
config.websocket_config.enabled = true;
config.websocket_config.port = system.get_available_port ();
auto node1 (system.add_node (config));
std::atomic<bool> ack_ready{ false };
auto task1 = ([&ack_ready, config, &node1] () {
fake_websocket_client client (node1->websocket.server->listening_port ());
client.send_message (R"json({"action": "subscribe", "topic": "confirmation", "ack": "true", "options": {"confirmation_type": "active_quorum", "include_block": "true", "include_linked_account": "true"}})json");
client.await_ack ();
ack_ready = true;
EXPECT_EQ (1, node1->websocket.server->subscriber_count (nano::websocket::topic::confirmation));
return client.get_response ();
});
auto future1 = std::async (std::launch::async, task1);
ASSERT_TIMELY (10s, ack_ready);
// Confirm a state block for an in-wallet account
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
nano::keypair key;
auto balance = nano::dev::constants.genesis_amount;
auto send_amount = node1->config.online_weight_minimum.number () + 1;
nano::block_hash previous (node1->latest (nano::dev::genesis_key.pub));
{
nano::state_block_builder builder;
balance -= send_amount;
auto send = builder
.account (nano::dev::genesis_key.pub)
.previous (previous)
.representative (nano::dev::genesis_key.pub)
.balance (balance)
.link (key.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*system.work.generate (previous))
.build ();
node1->process_active (send);
previous = send->hash ();
}
ASSERT_TIMELY_EQ (5s, future1.wait_for (0s), std::future_status::ready);
auto response1 = future1.get ();
ASSERT_TRUE (response1);
boost::property_tree::ptree event;
std::stringstream stream;
stream << response1.get ();
boost::property_tree::read_json (stream, event);
ASSERT_EQ (event.get<std::string> ("topic"), "confirmation");
try
{
boost::property_tree::ptree block_content = event.get_child ("message.block");
// Check if linked_account is present
ASSERT_EQ (1, block_content.count ("linked_account"));
// Make sure linked_account is non-zero.
ASSERT_NE ("0", block_content.get<std::string> ("linked_account"));
}
catch (std::runtime_error const & ex)
{
FAIL () << ex.what ();
}
ack_ready = false;
auto task2 = ([&ack_ready, config, &node1] () {
fake_websocket_client client (node1->websocket.server->listening_port ());
client.send_message (R"json({"action": "subscribe", "topic": "confirmation", "ack": "true", "options": {"confirmation_type": "active_quorum", "include_block": "true", "include_linked_account": "true"}})json");
client.await_ack ();
ack_ready = true;
EXPECT_EQ (1, node1->websocket.server->subscriber_count (nano::websocket::topic::confirmation));
return client.get_response ();
});
auto future2 = std::async (std::launch::async, task2);
ASSERT_TIMELY (10s, ack_ready);
// Quick-confirm a receive block
{
nano::state_block_builder builder;
balance = send_amount;
auto open = builder
.account (key.pub)
.previous (0)
.representative (nano::dev::genesis_key.pub)
.balance (balance)
.link (previous)
.sign (key.prv, key.pub)
.work (*system.work.generate (key.pub))
.build ();
node1->process_active (open);
previous = open->hash ();
}
ASSERT_TIMELY_EQ (5s, future2.wait_for (0s), std::future_status::ready);
auto response2 = future2.get ();
ASSERT_TRUE (response2);
boost::property_tree::ptree event2;
std::stringstream stream2;
stream2 << response2.get ();
boost::property_tree::read_json (stream2, event2);
ASSERT_EQ (event2.get<std::string> ("topic"), "confirmation");
try
{
boost::property_tree::ptree block_content = event2.get_child ("message.block");
// Check if linked_account is present
ASSERT_EQ (1, block_content.count ("linked_account"));
// Make sure linked_account is non-zero.
ASSERT_NE ("0", block_content.get<std::string> ("linked_account"));
}
catch (std::runtime_error const & ex)
{
FAIL () << ex.what ();
}
ack_ready = false;
auto task3 = ([&ack_ready, config, &node1] () {
fake_websocket_client client (node1->websocket.server->listening_port ());
client.send_message (R"json({"action": "subscribe", "topic": "confirmation", "ack": "true", "options": {"confirmation_type": "active_quorum", "include_block": "true", "include_linked_account": "true"}})json");
client.await_ack ();
ack_ready = true;
EXPECT_EQ (1, node1->websocket.server->subscriber_count (nano::websocket::topic::confirmation));
return client.get_response ();
});
auto future3 = std::async (std::launch::async, task3);
ASSERT_TIMELY (10s, ack_ready);
// Quick-confirm a change block
{
nano::state_block_builder builder;
auto change = builder
.account (key.pub)
.previous (previous)
.representative (key.pub)
.balance (balance)
.link (0)
.sign (key.prv, key.pub)
.work (*system.work.generate (previous))
.build ();
node1->process_active (change);
}
ASSERT_TIMELY_EQ (5s, future3.wait_for (0s), std::future_status::ready);
auto response3 = future3.get ();
ASSERT_TRUE (response3);
boost::property_tree::ptree event3;
std::stringstream stream3;
stream3 << response3.get ();
boost::property_tree::read_json (stream3, event3);
ASSERT_EQ (event3.get<std::string> ("topic"), "confirmation");
try
{
boost::property_tree::ptree block_content = event3.get_child ("message.block");
// Check if linked_account is present
ASSERT_EQ (1, block_content.count ("linked_account"));
// Make sure linked_account is zero.
ASSERT_EQ ("0", block_content.get<std::string> ("linked_account"));
}
catch (std::runtime_error const & ex)
{
FAIL () << ex.what ();
}
}
TEST (websocket, confirmation_options_sideband)
{
nano::test::system system;
@ -681,7 +852,7 @@ TEST (websocket, vote_options_type)
// Custom made votes for simplicity
auto vote = nano::test::make_vote (nano::dev::genesis_key, { nano::dev::genesis }, 0, 0);
nano::websocket::message_builder builder;
nano::websocket::message_builder builder{ node1->ledger };
auto msg (builder.vote_received (vote, nano::vote_code::replay));
node1->websocket.server->broadcast (msg);

6
nano/lib/CMakeLists.txt
View file

@ -55,6 +55,8 @@ add_library(
errors.cpp
files.hpp
files.cpp
formatting.hpp
formatting.cpp
fwd.hpp
id_dispenser.hpp
interval.hpp
@ -146,10 +148,6 @@ target_link_libraries(
Boost::property_tree
Boost::stacktrace)
if(NANO_STACKTRACE_BACKTRACE)
target_link_libraries(nano_lib backtrace)
endif()
target_compile_definitions(
nano_lib
PRIVATE -DMAJOR_VERSION_STRING=${CPACK_PACKAGE_VERSION_MAJOR}

36
nano/lib/blocks.cpp
View file

@ -472,17 +472,11 @@ void nano::send_block::serialize_json (std::string & string_a, bool single_line)
void nano::send_block::serialize_json (boost::property_tree::ptree & tree) const
{
tree.put ("type", "send");
std::string previous;
hashables.previous.encode_hex (previous);
tree.put ("previous", previous);
tree.put ("previous", hashables.previous.to_string ());
tree.put ("destination", hashables.destination.to_account ());
std::string balance;
hashables.balance.encode_hex (balance);
tree.put ("balance", balance);
std::string signature_l;
signature.encode_hex (signature_l);
tree.put ("balance", hashables.balance.to_string ());
tree.put ("work", nano::to_string_hex (work));
tree.put ("signature", signature_l);
tree.put ("signature", signature.to_string ());
}
bool nano::send_block::deserialize_json (boost::property_tree::ptree const & tree_a)
@ -836,10 +830,8 @@ void nano::open_block::serialize_json (boost::property_tree::ptree & tree) const
tree.put ("source", hashables.source.to_string ());
tree.put ("representative", hashables.representative.to_account ());
tree.put ("account", hashables.account.to_account ());
std::string signature_l;
signature.encode_hex (signature_l);
tree.put ("work", nano::to_string_hex (work));
tree.put ("signature", signature_l);
tree.put ("signature", signature.to_string ());
}
bool nano::open_block::deserialize_json (boost::property_tree::ptree const & tree_a)
@ -1105,9 +1097,7 @@ void nano::change_block::serialize_json (boost::property_tree::ptree & tree) con
tree.put ("previous", hashables.previous.to_string ());
tree.put ("representative", hashables.representative.to_account ());
tree.put ("work", nano::to_string_hex (work));
std::string signature_l;
signature.encode_hex (signature_l);
tree.put ("signature", signature_l);
tree.put ("signature", signature.to_string ());
}
bool nano::change_block::deserialize_json (boost::property_tree::ptree const & tree_a)
@ -1424,9 +1414,7 @@ void nano::state_block::serialize_json (boost::property_tree::ptree & tree) cons
tree.put ("balance", hashables.balance.to_string_dec ());
tree.put ("link", hashables.link.to_string ());
tree.put ("link_as_account", hashables.link.to_account ());
std::string signature_l;
signature.encode_hex (signature_l);
tree.put ("signature", signature_l);
tree.put ("signature", signature.to_string ());
tree.put ("work", nano::to_string_hex (work));
}
@ -1722,16 +1710,10 @@ void nano::receive_block::serialize_json (std::string & string_a, bool single_li
void nano::receive_block::serialize_json (boost::property_tree::ptree & tree) const
{
tree.put ("type", "receive");
std::string previous;
hashables.previous.encode_hex (previous);
tree.put ("previous", previous);
std::string source;
hashables.source.encode_hex (source);
tree.put ("source", source);
std::string signature_l;
signature.encode_hex (signature_l);
tree.put ("previous", hashables.previous.to_string ());
tree.put ("source", hashables.source.to_string ());
tree.put ("work", nano::to_string_hex (work));
tree.put ("signature", signature_l);
tree.put ("signature", signature.to_string ());
}
bool nano::receive_block::deserialize_json (boost::property_tree::ptree const & tree_a)

2
nano/lib/common.hpp
View file

@ -2,6 +2,7 @@
namespace boost::asio::ip
{
class address;
class tcp;
template <typename InternetProtocol>
class basic_endpoint;
@ -9,6 +10,7 @@ class basic_endpoint;
namespace nano
{
using ip_address = boost::asio::ip::address;
using endpoint = boost::asio::ip::basic_endpoint<boost::asio::ip::tcp>;
using tcp_endpoint = endpoint;
}

1
nano/lib/formatting.cpp Normal file
View file

@ -0,0 +1 @@
#include <nano/lib/formatting.hpp>

61
nano/lib/formatting.hpp Normal file
View file

@ -0,0 +1,61 @@
#pragma once
#include <nano/lib/common.hpp>
#include <nano/lib/numbers.hpp>
#include <fmt/ostream.h>
template <>
struct fmt::formatter<nano::endpoint> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::ip_address> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::uint128_t> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::uint256_t> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::uint512_t> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::uint128_union> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::uint256_union> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::uint512_union> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::hash_or_account> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::block_hash> : fmt::formatter<nano::uint256_union>
{
};
template <>
struct fmt::formatter<nano::public_key> : fmt::formatter<nano::uint256_union>
{
};
template <>
struct fmt::formatter<nano::qualified_root> : fmt::formatter<nano::uint512_union>
{
};
template <>
struct fmt::formatter<nano::root> : fmt::formatter<nano::hash_or_account>
{
};

19
nano/lib/function.hpp Normal file
View file

@ -0,0 +1,19 @@
#pragma once
#include <functional>
#include <memory>
#include <utility>
namespace nano
{
// TODO: Replace with std::move_only_function in C++23
template <typename F>
auto wrap_move_only (F && f)
{
using fn_type = decltype (std::function{ std::declval<F> () });
auto ptr = std::make_shared<std::decay_t<F>> (std::forward<F> (f));
return fn_type ([ptr] (auto &&... args) {
return (*ptr) (std::forward<decltype (args)> (args)...);
});
}
}

2
nano/lib/fwd.hpp
View file

@ -12,10 +12,12 @@ class block_details;
class block_visitor;
class container_info;
class jsonconfig;
class logger;
class mutable_block_visitor;
class network_constants;
class object_stream;
class root;
class stats;
class thread_pool;
class thread_runner;
class tomlconfig;

23
nano/lib/interval.hpp
View file

@ -1,13 +1,14 @@
#pragma once
#include <chrono>
#include <mutex>
namespace nano
{
class interval
{
public:
bool elapsed (auto target)
bool elapse (auto target)
{
auto const now = std::chrono::steady_clock::now ();
if (now - last >= target)
@ -21,4 +22,24 @@ public:
private:
std::chrono::steady_clock::time_point last{ std::chrono::steady_clock::now () };
};
class interval_mt
{
public:
bool elapse (auto target)
{
std::lock_guard guard{ mutex };
auto const now = std::chrono::steady_clock::now ();
if (now - last >= target)
{
last = now;
return true;
}
return false;
}
private:
std::mutex mutex;
std::chrono::steady_clock::time_point last{ std::chrono::steady_clock::now () };
};
}

45
nano/lib/locks.hpp
View file

@ -299,11 +299,47 @@ public:
locked * owner{ nullptr };
};
struct const_scoped_lock final
{
const_scoped_lock (locked const * owner_a) :
owner (owner_a)
{
owner->mutex.lock ();
}
~const_scoped_lock ()
{
owner->mutex.unlock ();
}
T const * operator->() const
{
return &owner->obj;
}
T const & get () const
{
return owner->obj;
}
T const & operator* () const
{
return get ();
}
locked const * owner{ nullptr };
};
scoped_lock operator->()
{
return scoped_lock (this);
}
const_scoped_lock operator->() const
{
return const_scoped_lock (this);
}
T & operator= (T const & other)
{
nano::unique_lock<nano::mutex> lk (mutex);
@ -313,6 +349,7 @@ public:
operator T () const
{
nano::unique_lock<nano::mutex> lk (mutex);
return obj;
}
@ -322,8 +359,14 @@ public:
return scoped_lock (this);
}
/** Returns a const scoped lock wrapper, allowing multiple calls to the underlying object under the same lock */
const_scoped_lock lock () const
{
return const_scoped_lock (this);
}
private:
T obj;
nano::mutex mutex;
mutable nano::mutex mutex; // Mutex needs to be mutable to allow locking in const methods
};
}

1
nano/lib/logging.hpp
View file

@ -1,5 +1,6 @@
#pragma once
#include <nano/lib/formatting.hpp>
#include <nano/lib/logging_enums.hpp>
#include <nano/lib/object_stream.hpp>
#include <nano/lib/object_stream_adapters.hpp>

5
nano/lib/logging_enums.hpp
View file

@ -39,7 +39,7 @@ enum class type
qt,
rpc,
rpc_connection,
rpc_callbacks,
http_callbacks,
rpc_request,
ipc,
ipc_server,
@ -59,7 +59,7 @@ enum class type
tcp_server,
tcp_listener,
tcp_channels,
prunning,
pruning,
conf_processor_bounded,
conf_processor_unbounded,
distributed_work,
@ -86,6 +86,7 @@ enum class type
monitor,
confirming_set,
bounded_backlog,
request_aggregator,
// bootstrap
bulk_pull_client,

168
nano/lib/numbers.cpp
View file

@ -5,6 +5,8 @@
#include <nano/lib/utility.hpp>
#include <nano/secure/common.hpp>
#include <boost/io/ios_state.hpp>
#include <crypto/ed25519-donna/ed25519.h>
#include <cryptopp/aes.h>
#include <cryptopp/modes.h>
@ -52,33 +54,40 @@ nano::public_key nano::public_key::from_node_id (std::string const & text)
return result;
}
void nano::public_key::encode_account (std::string & destination_a) const
void nano::public_key::encode_account (std::ostream & os) const
{
debug_assert (destination_a.empty ());
destination_a.reserve (65);
uint64_t check (0);
uint64_t check = 0;
blake2b_state hash;
blake2b_init (&hash, 5);
blake2b_update (&hash, bytes.data (), bytes.size ());
blake2b_final (&hash, reinterpret_cast<uint8_t *> (&check), 5);
nano::uint512_t number_l (number ());
nano::uint512_t number_l{ number () };
number_l <<= 40;
number_l |= nano::uint512_t (check);
number_l |= nano::uint512_t{ check };
// Pre-calculate all characters in reverse order
std::array<char, 60> encoded{};
for (auto i (0); i < 60; ++i)
{
uint8_t r (number_l & static_cast<uint8_t> (0x1f));
uint8_t r{ number_l & static_cast<uint8_t> (0x1f) };
number_l >>= 5;
destination_a.push_back (account_encode (r));
encoded[59 - i] = account_encode (r);
}
destination_a.append ("_onan"); // nano_
std::reverse (destination_a.begin (), destination_a.end ());
// Write prefix
os << "nano_";
// Write encoded characters
os.write (encoded.data (), encoded.size ());
}
std::string nano::public_key::to_account () const
{
std::string result;
encode_account (result);
return result;
std::stringstream stream;
encode_account (stream);
return stream.str ();
}
nano::public_key const & nano::public_key::null ()
@ -170,13 +179,6 @@ void nano::uint256_union::encrypt (nano::raw_key const & cleartext, nano::raw_ke
enc.ProcessData (bytes.data (), cleartext.bytes.data (), sizeof (cleartext.bytes));
}
std::string nano::uint256_union::to_string () const
{
std::string result;
encode_hex (result);
return result;
}
nano::uint256_union & nano::uint256_union::operator^= (nano::uint256_union const & other_a)
{
auto j (other_a.qwords.begin ());
@ -204,13 +206,11 @@ nano::uint256_union::uint256_union (std::string const & hex_a)
release_assert (!error);
}
void nano::uint256_union::encode_hex (std::string & text) const
void nano::uint256_union::encode_hex (std::ostream & stream) const
{
debug_assert (text.empty ());
std::stringstream stream;
boost::io::ios_flags_saver ifs{ stream };
stream << std::hex << std::uppercase << std::noshowbase << std::setw (64) << std::setfill ('0');
stream << number ();
text = stream.str ();
}
bool nano::uint256_union::decode_hex (std::string const & text)
@ -242,13 +242,11 @@ bool nano::uint256_union::decode_hex (std::string const & text)
return error;
}
void nano::uint256_union::encode_dec (std::string & text) const
void nano::uint256_union::encode_dec (std::ostream & stream) const
{
debug_assert (text.empty ());
std::stringstream stream;
boost::io::ios_flags_saver ifs{ stream };
stream << std::dec << std::noshowbase;
stream << number ();
text = stream.str ();
}
bool nano::uint256_union::decode_dec (std::string const & text)
@ -276,13 +274,29 @@ bool nano::uint256_union::decode_dec (std::string const & text)
return error;
}
void nano::uint512_union::encode_hex (std::string & text) const
std::string nano::uint256_union::to_string () const
{
debug_assert (text.empty ());
std::stringstream stream;
encode_hex (stream);
return stream.str ();
}
std::string nano::uint256_union::to_string_dec () const
{
std::stringstream stream;
encode_dec (stream);
return stream.str ();
}
/*
* uint512_union
*/
void nano::uint512_union::encode_hex (std::ostream & stream) const
{
boost::io::ios_flags_saver ifs{ stream };
stream << std::hex << std::uppercase << std::noshowbase << std::setw (128) << std::setfill ('0');
stream << number ();
text = stream.str ();
}
bool nano::uint512_union::decode_hex (std::string const & text)
@ -312,11 +326,15 @@ bool nano::uint512_union::decode_hex (std::string const & text)
std::string nano::uint512_union::to_string () const
{
std::string result;
encode_hex (result);
return result;
std::stringstream stream;
encode_hex (stream);
return stream.str ();
}
/*
* raw_key
*/
nano::raw_key::~raw_key ()
{
secure_wipe_memory (bytes.data (), bytes.size ());
@ -371,19 +389,21 @@ bool nano::validate_message (nano::public_key const & public_key, nano::uint256_
return validate_message (public_key, message.bytes.data (), sizeof (message.bytes), signature);
}
/*
* uint128_union
*/
nano::uint128_union::uint128_union (std::string const & string_a)
{
auto error (decode_hex (string_a));
release_assert (!error);
}
void nano::uint128_union::encode_hex (std::string & text) const
void nano::uint128_union::encode_hex (std::ostream & stream) const
{
debug_assert (text.empty ());
std::stringstream stream;
boost::io::ios_flags_saver ifs{ stream };
stream << std::hex << std::uppercase << std::noshowbase << std::setw (32) << std::setfill ('0');
stream << number ();
text = stream.str ();
}
bool nano::uint128_union::decode_hex (std::string const & text)
@ -411,13 +431,11 @@ bool nano::uint128_union::decode_hex (std::string const & text)
return error;
}
void nano::uint128_union::encode_dec (std::string & text) const
void nano::uint128_union::encode_dec (std::ostream & stream) const
{
debug_assert (text.empty ());
std::stringstream stream;
boost::io::ios_flags_saver ifs{ stream };
stream << std::dec << std::noshowbase;
stream << number ();
text = stream.str ();
}
bool nano::uint128_union::decode_dec (std::string const & text, bool decimal)
@ -525,6 +543,24 @@ bool nano::uint128_union::decode_dec (std::string const & text, nano::uint128_t
return error;
}
std::string nano::uint128_union::to_string () const
{
std::stringstream stream;
encode_hex (stream);
return stream.str ();
}
std::string nano::uint128_union::to_string_dec () const
{
std::stringstream stream;
encode_dec (stream);
return stream.str ();
}
/*
*
*/
void format_frac (std::ostringstream & stream, nano::uint128_t value, nano::uint128_t scale, int precision)
{
auto reduce = scale;
@ -651,20 +687,6 @@ std::string nano::uint128_union::format_balance (nano::uint128_t scale, int prec
return ::format_balance (number (), scale, precision, group_digits, thousands_sep, decimal_point, grouping);
}
std::string nano::uint128_union::to_string () const
{
std::string result;
encode_hex (result);
return result;
}
std::string nano::uint128_union::to_string_dec () const
{
std::string result;
encode_dec (result);
return result;
}
bool nano::hash_or_account::decode_hex (std::string const & text_a)
{
return raw.decode_hex (text_a);
@ -685,6 +707,10 @@ std::string nano::hash_or_account::to_account () const
return account.to_account ();
}
/*
*
*/
std::string nano::to_string_hex (uint64_t const value_a)
{
std::stringstream stream;
@ -738,34 +764,40 @@ std::string nano::to_string (double const value_a, int const precision_a)
return stream.str ();
}
std::ostream & nano::operator<< (std::ostream & os, const uint128_union & val)
std::ostream & nano::operator<< (std::ostream & os, const nano::uint128_union & val)
{
// TODO: Replace with streaming implementation
os << val.to_string ();
val.encode_hex (os);
return os;
}
std::ostream & nano::operator<< (std::ostream & os, const uint256_union & val)
std::ostream & nano::operator<< (std::ostream & os, const nano::uint256_union & val)
{
// TODO: Replace with streaming implementation
os << val.to_string ();
val.encode_hex (os);
return os;
}
std::ostream & nano::operator<< (std::ostream & os, const uint512_union & val)
std::ostream & nano::operator<< (std::ostream & os, const nano::uint512_union & val)
{
// TODO: Replace with streaming implementation
os << val.to_string ();
val.encode_hex (os);
return os;
}
std::ostream & nano::operator<< (std::ostream & os, const hash_or_account & val)
std::ostream & nano::operator<< (std::ostream & os, const nano::hash_or_account & val)
{
// TODO: Replace with streaming implementation
os << val.to_string ();
val.raw.encode_hex (os);
return os;
}
std::ostream & nano::operator<< (std::ostream & os, const nano::account & val)
{
val.encode_account (os);
return os;
}
/*
*
*/
#ifdef _WIN32
#pragma warning(push)
#pragma warning(disable : 4146) // warning C4146: unary minus operator applied to unsigned type, result still unsigned

53
nano/lib/numbers.hpp
View file

@ -49,9 +49,9 @@ public:
*/
explicit uint128_union (std::string const &);
void encode_hex (std::string &) const;
void encode_hex (std::ostream &) const;
bool decode_hex (std::string const &);
void encode_dec (std::string &) const;
void encode_dec (std::ostream &) const;
bool decode_dec (std::string const &, bool = false);
bool decode_dec (std::string const &, nano::uint128_t);
@ -151,9 +151,9 @@ public:
uint256_union & operator^= (uint256_union const &);
uint256_union operator^ (uint256_union const &) const;
void encode_hex (std::string &) const;
void encode_hex (std::ostream &) const;
bool decode_hex (std::string const &);
void encode_dec (std::string &) const;
void encode_dec (std::ostream &) const;
bool decode_dec (std::string const &);
void clear ()
@ -173,6 +173,7 @@ public:
}
std::string to_string () const;
std::string to_string_dec () const;
public:
union
@ -236,7 +237,7 @@ public:
static const public_key & null ();
bool decode_node_id (std::string const &);
void encode_account (std::string &) const;
void encode_account (std::ostream &) const;
bool decode_account (std::string const &);
std::string to_node_id () const;
@ -427,7 +428,7 @@ public:
return *this;
}
void encode_hex (std::string &) const;
void encode_hex (std::ostream &) const;
bool decode_hex (std::string const &);
void clear ()
@ -519,6 +520,7 @@ std::ostream & operator<< (std::ostream &, const uint128_union &);
std::ostream & operator<< (std::ostream &, const uint256_union &);
std::ostream & operator<< (std::ostream &, const uint512_union &);
std::ostream & operator<< (std::ostream &, const hash_or_account &);
std::ostream & operator<< (std::ostream &, const account &);
/**
* Convert a double to string in fixed format
@ -615,42 +617,3 @@ struct hash<::nano::uint512_union>;
template <>
struct hash<::nano::qualified_root>;
}
/*
* Formatters
*/
template <>
struct fmt::formatter<nano::uint128_union> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::uint256_union> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::uint512_union> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::hash_or_account> : fmt::ostream_formatter
{
};
template <>
struct fmt::formatter<nano::block_hash> : fmt::formatter<nano::uint256_union>
{
};
template <>
struct fmt::formatter<nano::public_key> : fmt::formatter<nano::uint256_union>
{
};
template <>
struct fmt::formatter<nano::qualified_root> : fmt::formatter<nano::uint512_union>
{
};

6
nano/lib/observer_set.hpp
View file

@ -46,6 +46,12 @@ public:
return observers.size ();
}
void clear ()
{
nano::lock_guard<nano::mutex> lock{ mutex };
observers.clear ();
}
nano::container_info container_info () const
{
nano::unique_lock<nano::mutex> lock{ mutex };

3
nano/lib/rocksdbconfig.cpp
View file

@ -4,7 +4,6 @@
nano::error nano::rocksdb_config::serialize_toml (nano::tomlconfig & toml) const
{
toml.put ("enable", enable, "Whether to use the RocksDB backend for the ledger database.\ntype:bool");
toml.put ("io_threads", io_threads, "Number of threads to use with the background compaction and flushing.\ntype:uint32");
toml.put ("read_cache", read_cache, "Amount of megabytes per table allocated to read cache. Valid range is 1 - 1024. Default is 32.\nCarefully monitor memory usage if non-default values are used\ntype:long");
toml.put ("write_cache", write_cache, "Total amount of megabytes allocated to write cache. Valid range is 1 - 256. Default is 64.\nCarefully monitor memory usage if non-default values are used\ntype:long");
@ -14,7 +13,7 @@ nano::error nano::rocksdb_config::serialize_toml (nano::tomlconfig & toml) const
nano::error nano::rocksdb_config::deserialize_toml (nano::tomlconfig & toml)
{
toml.get_optional<bool> ("enable", enable);
toml.get_optional<bool> ("enable", enable); // TODO: This setting can be removed in future versions
toml.get_optional<unsigned> ("io_threads", io_threads);
toml.get_optional<long> ("read_cache", read_cache);
toml.get_optional<long> ("write_cache", write_cache);

46
nano/lib/stats_enums.hpp
View file

@ -15,18 +15,23 @@ enum class type
test,
error,
message,
message_loopback,
block,
ledger,
ledger_notifications,
rollback,
network,
vote,
vote_processor,
vote_processor_tier,
vote_processor_overfill,
vote_rebroadcaster,
election,
election_cleanup,
election_vote,
http_callback,
http_callbacks,
http_callbacks_notified,
http_callbacks_ec,
ipc,
tcp,
tcp_server,
@ -113,6 +118,7 @@ enum class type
message_processor_type,
process_confirmed,
online_reps,
pruning,
_last // Must be the last enum
};
@ -166,6 +172,10 @@ enum class detail
other,
drop,
queued,
error,
failed,
refresh,
sent,
reset,
// processing queue
@ -285,6 +295,7 @@ enum class detail
broadcast_block_repeat,
confirm_once,
confirm_once_failed,
confirmation_request,
// election types
manual,
@ -424,11 +435,13 @@ enum class detail
cleanup_outdated,
erase_stale,
// vote generator
// vote_generator
generator_broadcasts,
generator_replies,
generator_replies_discarded,
generator_spacing,
sent_pr,
sent_non_pr,
// hinting
missing_block,
@ -459,6 +472,7 @@ enum class detail
transition_priority,
transition_priority_failed,
election_cleanup,
activate_immediately,
// active_elections
started,
@ -529,6 +543,7 @@ enum class detail
blocking_overflow,
priority_insert,
priority_set,
priority_erase,
priority_unblocked,
erase_by_threshold,
erase_by_blocking,
@ -536,6 +551,8 @@ enum class detail
deprioritize,
deprioritize_failed,
sync_dependencies,
decay_blocking,
blocking_decayed,
dependency_synced,
request_blocks,
@ -571,6 +588,10 @@ enum class detail
tier_2,
tier_3,
// ledger_notifications
notify_processed,
notify_rolled_back,
// confirming_set
notify_cemented,
notify_already_cemented,
@ -618,6 +639,7 @@ enum class detail
sample,
rep_new,
rep_update,
rep_trim,
update_online,
// error codes
@ -626,6 +648,26 @@ enum class detail
host_unreachable,
not_supported,
// http
error_resolving,
error_connecting,
error_sending,
error_completing,
bad_status,
// http_callbacks
block_confirmed,
large_backlog,
// pruning
ledger_pruning,
pruning_target,
pruned_count,
collect_targets,
// vote_rebroadcaster
rebroadcast_hashes,
_last // Must be the last enum
};

22
nano/lib/thread_roles.cpp
View file

@ -37,11 +37,14 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
case nano::thread_role::name::vote_cache_processing:
thread_role_name_string = "Vote cache proc";
break;
case nano::thread_role::name::vote_rebroadcasting:
thread_role_name_string = "Vote rebroad";
break;
case nano::thread_role::name::block_processing:
thread_role_name_string = "Blck processing";
break;
case nano::thread_role::name::block_processing_notifications:
thread_role_name_string = "Blck proc notif";
case nano::thread_role::name::ledger_notifications:
thread_role_name_string = "Ledger notif";
break;
case nano::thread_role::name::request_loop:
thread_role_name_string = "Request loop";
@ -58,6 +61,9 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
case nano::thread_role::name::voting:
thread_role_name_string = "Voting";
break;
case nano::thread_role::name::voting_final:
thread_role_name_string = "Voting final";
break;
case nano::thread_role::name::signature_checking:
thread_role_name_string = "Signature check";
break;
@ -106,12 +112,6 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
case nano::thread_role::name::bounded_backlog_scan:
thread_role_name_string = "Bounded b scan";
break;
case nano::thread_role::name::bounded_backlog_notifications:
thread_role_name_string = "Bounded b notif";
break;
case nano::thread_role::name::vote_generator_queue:
thread_role_name_string = "Voting que";
break;
case nano::thread_role::name::bootstrap:
thread_role_name_string = "Bootstrap";
break;
@ -190,6 +190,12 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
case nano::thread_role::name::monitor:
thread_role_name_string = "Monitor";
break;
case nano::thread_role::name::http_callbacks:
thread_role_name_string = "HTTP callbacks";
break;
case nano::thread_role::name::pruning:
thread_role_name_string = "Pruning";
break;
default:
debug_assert (false && "nano::thread_role::get_string unhandled thread role");
}

8
nano/lib/thread_roles.hpp
View file

@ -17,13 +17,15 @@ enum class name
message_processing,
vote_processing,
vote_cache_processing,
vote_rebroadcasting,
block_processing,
block_processing_notifications,
ledger_notifications,
request_loop,
wallet_actions,
bootstrap_initiator,
bootstrap_connections,
voting,
voting_final,
signature_checking,
rpc_request_processor,
rpc_process_container,
@ -40,8 +42,6 @@ enum class name
backlog_scan,
bounded_backlog,
bounded_backlog_scan,
bounded_backlog_notifications,
vote_generator_queue,
telemetry,
bootstrap,
bootstrap_database_scan,
@ -68,6 +68,8 @@ enum class name
vote_router,
online_reps,
monitor,
http_callbacks,
pruning,
};
std::string_view to_string (name);

2
nano/lib/uniquer.hpp
View file

@ -27,7 +27,7 @@ public:
nano::lock_guard<nano::mutex> guard{ mutex };
if (cleanup_interval.elapsed (cleanup_cutoff))
if (cleanup_interval.elapse (cleanup_cutoff))
{
cleanup ();
}

5
nano/lib/walletconfig.cpp
View file

@ -24,10 +24,7 @@ nano::error nano::wallet_config::parse (std::string const & wallet_a, std::strin
nano::error nano::wallet_config::serialize_toml (nano::tomlconfig & toml) const
{
std::string wallet_string;
wallet.encode_hex (wallet_string);
toml.put ("wallet", wallet_string, "Wallet identifier\ntype:string,hex");
toml.put ("wallet", wallet.to_string (), "Wallet identifier\ntype:string,hex");
toml.put ("account", account.to_account (), "Current wallet account\ntype:string,account");
return toml.get_error ();
}

2
nano/nano_node/daemon.cpp
View file

@ -103,7 +103,7 @@ void nano::daemon::run (std::filesystem::path const & data_path, nano::node_flag
logger.info (nano::log::type::daemon, "Starting up Nano node...");
// Print info about number of logical cores detected, those are used to decide how many IO, worker and signature checker threads to spawn
logger.info (nano::log::type::daemon, "Hardware concurrency: {} ( configured: {} )", std::thread::hardware_concurrency (), nano::hardware_concurrency ());
logger.info (nano::log::type::daemon, "Hardware concurrency: {} (configured: {})", std::thread::hardware_concurrency (), nano::hardware_concurrency ());
logger.info (nano::log::type::daemon, "File descriptors limit: {}", nano::get_file_descriptor_limit ());
// for the daemon start up, if the user hasn't specified a port in

6
nano/nano_node/entry.cpp
View file

@ -16,6 +16,7 @@
#include <nano/node/json_handler.hpp>
#include <nano/node/node.hpp>
#include <nano/node/online_reps.hpp>
#include <nano/node/pruning.hpp>
#include <nano/node/transport/inproc.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
@ -445,8 +446,7 @@ int main (int argc, char * const * argv)
using time_point = std::chrono::system_clock::time_point;
time_point ts (std::chrono::duration_cast<time_point::duration> (std::chrono::nanoseconds (i->first)));
std::time_t timestamp = std::chrono::system_clock::to_time_t (ts);
std::string weight;
i->second.encode_dec (weight);
std::string weight = i->second.to_string_dec ();
std::cout << boost::str (boost::format ("Timestamp %1% Weight %2%\n") % ctime (&timestamp) % weight);
}
}
@ -1923,7 +1923,7 @@ int main (int argc, char * const * argv)
nano::update_flags (node_flags, vm);
nano::inactive_node inactive_node (data_path, node_flags);
auto node = inactive_node.node;
node->ledger_pruning (node_flags.block_processor_batch_size != 0 ? node_flags.block_processor_batch_size : 16 * 1024, true);
node->pruning.ledger_pruning (node_flags.block_processor_batch_size != 0 ? node_flags.block_processor_batch_size : 16 * 1024, true);
}
else if (vm.count ("debug_stacktrace"))
{

17
nano/node/CMakeLists.txt
View file

@ -20,8 +20,11 @@ add_library(
backlog_scan.cpp
bandwidth_limiter.hpp
bandwidth_limiter.cpp
block_context.hpp
block_processor.hpp
block_processor.cpp
block_source.hpp
block_source.cpp
bucketing.hpp
bucketing.cpp
bounded_backlog.hpp
@ -85,6 +88,8 @@ add_library(
ipc/ipc_server.cpp
json_handler.hpp
json_handler.cpp
ledger_notifications.hpp
ledger_notifications.cpp
local_block_broadcaster.cpp
local_block_broadcaster.hpp
local_vote_history.cpp
@ -123,8 +128,8 @@ add_library(
peer_exclusion.cpp
portmapping.hpp
portmapping.cpp
process_live_dispatcher.cpp
process_live_dispatcher.hpp
pruning.hpp
pruning.cpp
recently_cemented_cache.cpp
recently_cemented_cache.hpp
recently_confirmed_cache.cpp
@ -135,6 +140,8 @@ add_library(
rep_tiers.cpp
request_aggregator.hpp
request_aggregator.cpp
rpc_callbacks.hpp
rpc_callbacks.cpp
scheduler/bucket.cpp
scheduler/bucket.hpp
scheduler/component.hpp
@ -160,6 +167,8 @@ add_library(
transport/fwd.hpp
transport/inproc.hpp
transport/inproc.cpp
transport/loopback.cpp
transport/loopback.hpp
transport/message_deserializer.hpp
transport/message_deserializer.cpp
transport/tcp_channels.hpp
@ -172,6 +181,8 @@ add_library(
transport/tcp_server.cpp
transport/tcp_socket.hpp
transport/tcp_socket.cpp
transport/test_channel.hpp
transport/test_channel.cpp
transport/traffic_type.hpp
transport/traffic_type.cpp
transport/transport.hpp
@ -184,6 +195,8 @@ add_library(
vote_generator.cpp
vote_processor.hpp
vote_processor.cpp
vote_rebroadcaster.hpp
vote_rebroadcaster.cpp
vote_router.hpp
vote_router.cpp
vote_spacing.hpp

36
nano/node/active_elections.cpp
View file

@ -7,6 +7,7 @@
#include <nano/node/confirmation_solicitor.hpp>
#include <nano/node/confirming_set.hpp>
#include <nano/node/election.hpp>
#include <nano/node/ledger_notifications.hpp>
#include <nano/node/node.hpp>
#include <nano/node/online_reps.hpp>
#include <nano/node/repcrawler.hpp>
@ -21,11 +22,11 @@
using namespace std::chrono;
nano::active_elections::active_elections (nano::node & node_a, nano::confirming_set & confirming_set_a, nano::block_processor & block_processor_a) :
nano::active_elections::active_elections (nano::node & node_a, nano::ledger_notifications & ledger_notifications_a, nano::confirming_set & confirming_set_a) :
config{ node_a.config.active_elections },
node{ node_a },
ledger_notifications{ ledger_notifications_a },
confirming_set{ confirming_set_a },
block_processor{ block_processor_a },
recently_confirmed{ config.confirmation_cache },
recently_cemented{ config.confirmation_history_size }
{
@ -55,7 +56,7 @@ nano::active_elections::active_elections (nano::node & node_a, nano::confirming_
});
// Notify elections about alternative (forked) blocks
block_processor.batch_processed.add ([this] (auto const & batch) {
ledger_notifications.blocks_processed.add ([this] (auto const & batch) {
for (auto const & [result, context] : batch)
{
if (result == nano::block_status::fork)
@ -66,7 +67,7 @@ nano::active_elections::active_elections (nano::node & node_a, nano::confirming_
});
// Stop all rolled back active transactions except initial
block_processor.rolled_back.add ([this] (auto const & blocks, auto const & rollback_root) {
ledger_notifications.blocks_rolled_back.add ([this] (auto const & blocks, auto const & rollback_root) {
for (auto const & block : blocks)
{
if (block->qualified_root () != rollback_root)
@ -405,12 +406,14 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr<
if (!recently_confirmed.exists (root))
{
result.inserted = true;
auto observe_rep_cb = [&node = node] (auto const & rep_a) {
// TODO: Is this neccessary? Move this outside of the election class
// Representative is defined as online if replying to live votes or rep_crawler queries
// Passing this callback into the election is important
// We need to observe and update the online voting weight *before* election quorum is checked
auto observe_rep_callback = [&node = node] (auto const & rep_a) {
node.online_reps.observe (rep_a);
};
result.election = nano::make_shared<nano::election> (node, block_a, nullptr, observe_rep_cb, election_behavior_a);
result.election = nano::make_shared<nano::election> (node, block_a, nullptr, observe_rep_callback, election_behavior_a);
roots.get<tag_root> ().emplace (entry{ root, result.election, std::move (erased_callback_a) });
node.vote_router.connect (hash, result.election);
@ -419,11 +422,16 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr<
count_by_behavior[result.election->behavior ()]++;
// Skip passive phase for blocks without cached votes to avoid bootstrap delays
bool active_immediately = false;
if (node.vote_cache.contains (hash))
bool activate_immediately = false;
if (!node.vote_cache.contains (hash))
{
activate_immediately = true;
}
if (activate_immediately)
{
node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::activate_immediately);
result.election->transition_active ();
active_immediately = true;
}
node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::started);
@ -434,9 +442,9 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr<
nano::log::arg{ "election", result.election });
node.logger.debug (nano::log::type::active_elections, "Started new election for block: {} (behavior: {}, active immediately: {})",
hash.to_string (),
hash,
to_string (election_behavior_a),
active_immediately);
activate_immediately);
}
else
{
@ -572,7 +580,7 @@ bool nano::active_elections::publish (std::shared_ptr<nano::block> const & block
node.vote_cache_processor.trigger (block_a->hash ());
node.stats.inc (nano::stat::type::active, nano::stat::detail::election_block_conflict);
node.logger.debug (nano::log::type::active_elections, "Block was added to an existing election: {}", block_a->hash ().to_string ());
node.logger.debug (nano::log::type::active_elections, "Block was added to an existing election: {}", block_a->hash ());
}
}
return result;

4
nano/node/active_elections.hpp
View file

@ -90,7 +90,7 @@ private: // Elections
ordered_roots roots;
public:
active_elections (nano::node &, nano::confirming_set &, nano::block_processor &);
active_elections (nano::node &, nano::ledger_notifications &, nano::confirming_set &);
~active_elections ();
void start ();
@ -144,8 +144,8 @@ private:
private: // Dependencies
active_elections_config const & config;
nano::node & node;
nano::ledger_notifications & ledger_notifications;
nano::confirming_set & confirming_set;
nano::block_processor & block_processor;
public:
nano::recently_confirmed_cache recently_confirmed;

44
nano/node/block_context.hpp Normal file
View file

@ -0,0 +1,44 @@
#pragma once
#include <nano/node/block_source.hpp>
#include <nano/secure/common.hpp>
#include <future>
namespace nano
{
class block_context
{
public:
using result_t = nano::block_status;
using callback_t = std::function<void (result_t)>;
public: // Keep fields public for simplicity
std::shared_ptr<nano::block> block;
nano::block_source source;
callback_t callback;
std::chrono::steady_clock::time_point arrival{ std::chrono::steady_clock::now () };
public:
block_context (std::shared_ptr<nano::block> block, nano::block_source source, callback_t callback = nullptr) :
block{ std::move (block) },
source{ source },
callback{ std::move (callback) }
{
debug_assert (source != nano::block_source::unknown);
}
std::future<result_t> get_future ()
{
return promise.get_future ();
}
void set_result (result_t result)
{
promise.set_value (result);
}
private:
std::promise<result_t> promise;
};
}

151
nano/node/block_processor.cpp
View file

@ -5,6 +5,7 @@
#include <nano/lib/timer.hpp>
#include <nano/node/active_elections.hpp>
#include <nano/node/block_processor.hpp>
#include <nano/node/ledger_notifications.hpp>
#include <nano/node/local_vote_history.hpp>
#include <nano/node/node.hpp>
#include <nano/node/unchecked_map.hpp>
@ -18,14 +19,14 @@
* block_processor
*/
nano::block_processor::block_processor (nano::node_config const & node_config, nano::ledger & ledger_a, nano::unchecked_map & unchecked_a, nano::stats & stats_a, nano::logger & logger_a) :
nano::block_processor::block_processor (nano::node_config const & node_config, nano::ledger & ledger_a, nano::ledger_notifications & ledger_notifications_a, nano::unchecked_map & unchecked_a, nano::stats & stats_a, nano::logger & logger_a) :
config{ node_config.block_processor },
network_params{ node_config.network_params },
ledger{ ledger_a },
ledger_notifications{ ledger_notifications_a },
unchecked{ unchecked_a },
stats{ stats_a },
logger{ logger_a },
workers{ 1, nano::thread_role::name::block_processing_notifications }
logger{ logger_a }
{
queue.max_size_query = [this] (auto const & origin) {
switch (origin.source)
@ -65,15 +66,12 @@ nano::block_processor::~block_processor ()
{
// Thread must be stopped before destruction
debug_assert (!thread.joinable ());
debug_assert (!workers.alive ());
}
void nano::block_processor::start ()
{
debug_assert (!thread.joinable ());
workers.start ();
thread = std::thread ([this] () {
nano::thread_role::set (nano::thread_role::name::block_processing);
run ();
@ -91,7 +89,6 @@ void nano::block_processor::stop ()
{
thread.join ();
}
workers.stop ();
}
// TODO: Remove and replace all checks with calls to size (block_source)
@ -121,7 +118,7 @@ bool nano::block_processor::add (std::shared_ptr<nano::block> const & block, blo
to_string (source),
channel ? channel->to_string () : "<unknown>"); // TODO: Lazy eval
return add_impl (context{ block, source, std::move (callback) }, channel);
return add_impl ({ block, source, std::move (callback) }, channel);
}
std::optional<nano::block_status> nano::block_processor::add_blocking (std::shared_ptr<nano::block> const & block, block_source const source)
@ -129,7 +126,7 @@ std::optional<nano::block_status> nano::block_processor::add_blocking (std::shar
stats.inc (nano::stat::type::block_processor, nano::stat::detail::process_blocking);
logger.debug (nano::log::type::block_processor, "Processing block (blocking): {} (source: {})", block->hash ().to_string (), to_string (source));
context ctx{ block, source };
nano::block_context ctx{ block, source };
auto future = ctx.get_future ();
add_impl (std::move (ctx));
@ -152,10 +149,10 @@ void nano::block_processor::force (std::shared_ptr<nano::block> const & block_a)
stats.inc (nano::stat::type::block_processor, nano::stat::detail::force);
logger.debug (nano::log::type::block_processor, "Forcing block: {}", block_a->hash ().to_string ());
add_impl (context{ block_a, block_source::forced });
add_impl ({ block_a, block_source::forced });
}
bool nano::block_processor::add_impl (context ctx, std::shared_ptr<nano::transport::channel> const & channel)
bool nano::block_processor::add_impl (nano::block_context ctx, std::shared_ptr<nano::transport::channel> const & channel)
{
auto const source = ctx.source;
bool added = false;
@ -175,7 +172,7 @@ bool nano::block_processor::add_impl (context ctx, std::shared_ptr<nano::transpo
return added;
}
void nano::block_processor::rollback_competitor (secure::write_transaction const & transaction, nano::block const & fork_block)
void nano::block_processor::rollback_competitor (secure::write_transaction & transaction, nano::block const & fork_block)
{
auto const hash = fork_block.hash ();
auto const successor_hash = ledger.any.block_successor (transaction, fork_block.qualified_root ());
@ -197,10 +194,13 @@ void nano::block_processor::rollback_competitor (secure::write_transaction const
logger.debug (nano::log::type::block_processor, "Blocks rolled back: {}", rollback_list.size ());
}
// Notify observers of the rolled back blocks on a background thread while not holding the ledger write lock
workers.post ([this, rollback_list = std::move (rollback_list), root = fork_block.qualified_root ()] () {
rolled_back.notify (rollback_list, root);
});
if (!rollback_list.empty ())
{
// Notify observers of the rolled back blocks on a background thread while not holding the ledger write lock
ledger_notifications.notify_rolled_back (transaction, std::move (rollback_list), fork_block.qualified_root (), [this] {
stats.inc (nano::stat::type::block_processor, nano::stat::detail::notify_rolled_back);
});
}
}
}
@ -210,55 +210,41 @@ void nano::block_processor::run ()
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
condition.wait (lock, [this] {
return stopped || !queue.empty ();
});
if (stopped)
{
return;
}
lock.unlock ();
// It's possible that ledger processing happens faster than the notifications can be processed by other components, cooldown here
ledger_notifications.wait ([this] {
stats.inc (nano::stat::type::block_processor, nano::stat::detail::cooldown);
});
lock.lock ();
if (!queue.empty ())
{
// It's possible that ledger processing happens faster than the notifications can be processed by other components, cooldown here
while (workers.queued_tasks () >= config.max_queued_notifications)
{
stats.inc (nano::stat::type::block_processor, nano::stat::detail::cooldown);
condition.wait_for (lock, 100ms, [this] { return stopped; });
if (stopped)
{
return;
}
}
if (log_interval.elapsed (15s))
if (log_interval.elapse (15s))
{
logger.info (nano::log::type::block_processor, "{} blocks (+ {} forced) in processing queue",
queue.size (),
queue.size ({ nano::block_source::forced }));
}
auto processed = process_batch (lock);
process_batch (lock);
debug_assert (!lock.owns_lock ());
lock.lock ();
// Queue notifications to be dispatched in the background
workers.post ([this, processed = std::move (processed)] () mutable {
stats.inc (nano::stat::type::block_processor, nano::stat::detail::notify);
// Set results for futures when not holding the lock
for (auto & [result, context] : processed)
{
if (context.callback)
{
context.callback (result);
}
context.set_result (result);
}
batch_processed.notify (processed);
});
}
else
{
condition.wait (lock, [this] {
return stopped || !queue.empty ();
});
}
}
}
auto nano::block_processor::next () -> context
auto nano::block_processor::next () -> nano::block_context
{
debug_assert (!mutex.try_lock ());
debug_assert (!queue.empty ()); // This should be checked before calling next
@ -273,14 +259,14 @@ auto nano::block_processor::next () -> context
release_assert (false, "next() called when no blocks are ready");
}
auto nano::block_processor::next_batch (size_t max_count) -> std::deque<context>
auto nano::block_processor::next_batch (size_t max_count) -> std::deque<nano::block_context>
{
debug_assert (!mutex.try_lock ());
debug_assert (!queue.empty ());
queue.periodic_update ();
std::deque<context> results;
std::deque<nano::block_context> results;
while (!queue.empty () && results.size () < max_count)
{
results.push_back (next ());
@ -288,7 +274,7 @@ auto nano::block_processor::next_batch (size_t max_count) -> std::deque<context>
return results;
}
auto nano::block_processor::process_batch (nano::unique_lock<nano::mutex> & lock) -> processed_batch_t
void nano::block_processor::process_batch (nano::unique_lock<nano::mutex> & lock)
{
debug_assert (lock.owns_lock ());
debug_assert (!mutex.try_lock ());
@ -307,7 +293,8 @@ auto nano::block_processor::process_batch (nano::unique_lock<nano::mutex> & lock
size_t number_of_blocks_processed = 0;
size_t number_of_forced_processed = 0;
processed_batch_t processed;
std::deque<std::pair<nano::block_status, nano::block_context>> processed;
for (auto & ctx : batch)
{
auto const hash = ctx.block->hash ();
@ -332,10 +319,13 @@ auto nano::block_processor::process_batch (nano::unique_lock<nano::mutex> & lock
logger.debug (nano::log::type::block_processor, "Processed {} blocks ({} forced) in {} {}", number_of_blocks_processed, number_of_forced_processed, timer.value ().count (), timer.unit ());
}
return processed;
// Queue notifications to be dispatched in the background
ledger_notifications.notify_processed (transaction, std::move (processed), [this] {
stats.inc (nano::stat::type::block_processor, nano::stat::detail::notify_processed);
});
}
nano::block_status nano::block_processor::process_one (secure::write_transaction const & transaction_a, context const & context, bool const forced_a)
nano::block_status nano::block_processor::process_one (secure::write_transaction const & transaction_a, nano::block_context const & context, bool const forced_a)
{
auto block = context.block;
auto const hash = block->hash ();
@ -392,41 +382,51 @@ nano::block_status nano::block_processor::process_one (secure::write_transaction
stats.inc (nano::stat::type::ledger, nano::stat::detail::old);
break;
}
// These are unexpected and indicate erroneous/malicious behavior, log debug info to highlight the issue
case nano::block_status::bad_signature:
{
logger.debug (nano::log::type::block_processor, "Block signature is invalid: {}", hash);
break;
}
case nano::block_status::negative_spend:
{
logger.debug (nano::log::type::block_processor, "Block spends negative amount: {}", hash);
break;
}
case nano::block_status::unreceivable:
{
logger.debug (nano::log::type::block_processor, "Block is unreceivable: {}", hash);
break;
}
case nano::block_status::fork:
{
stats.inc (nano::stat::type::ledger, nano::stat::detail::fork);
logger.debug (nano::log::type::block_processor, "Block is a fork: {}", hash);
break;
}
case nano::block_status::opened_burn_account:
{
logger.debug (nano::log::type::block_processor, "Block opens burn account: {}", hash);
break;
}
case nano::block_status::balance_mismatch:
{
logger.debug (nano::log::type::block_processor, "Block balance mismatch: {}", hash);
break;
}
case nano::block_status::representative_mismatch:
{
logger.debug (nano::log::type::block_processor, "Block representative mismatch: {}", hash);
break;
}
case nano::block_status::block_position:
{
logger.debug (nano::log::type::block_processor, "Block is in incorrect position: {}", hash);
break;
}
case nano::block_status::insufficient_work:
{
logger.debug (nano::log::type::block_processor, "Block has insufficient work: {}", hash);
break;
}
}
@ -441,32 +441,9 @@ nano::container_info nano::block_processor::container_info () const
info.put ("blocks", queue.size ());
info.put ("forced", queue.size ({ nano::block_source::forced }));
info.add ("queue", queue.container_info ());
info.add ("workers", workers.container_info ());
return info;
}
/*
* block_processor::context
*/
nano::block_processor::context::context (std::shared_ptr<nano::block> block, nano::block_source source_a, callback_t callback_a) :
block{ std::move (block) },
source{ source_a },
callback{ std::move (callback_a) }
{
debug_assert (source != nano::block_source::unknown);
}
auto nano::block_processor::context::get_future () -> std::future<result_t>
{
return promise.get_future ();
}
void nano::block_processor::context::set_result (result_t const & result)
{
promise.set_value (result);
}
/*
* block_processor_config
*/
@ -496,17 +473,3 @@ nano::error nano::block_processor_config::deserialize (nano::tomlconfig & toml)
return toml.get_error ();
}
/*
*
*/
std::string_view nano::to_string (nano::block_source source)
{
return nano::enum_util::name (source);
}
nano::stat::detail nano::to_stat_detail (nano::block_source type)
{
return nano::enum_util::cast<nano::stat::detail> (type);
}

75
nano/node/block_processor.hpp
View file

@ -2,6 +2,8 @@
#include <nano/lib/logging.hpp>
#include <nano/lib/thread_pool.hpp>
#include <nano/node/block_context.hpp>
#include <nano/node/block_source.hpp>
#include <nano/node/fair_queue.hpp>
#include <nano/node/fwd.hpp>
#include <nano/secure/common.hpp>
@ -14,22 +16,6 @@
namespace nano
{
enum class block_source
{
unknown = 0,
live,
live_originator,
bootstrap,
bootstrap_legacy,
unchecked,
local,
forced,
election,
};
std::string_view to_string (block_source);
nano::stat::detail to_stat_detail (block_source);
class block_processor_config final
{
public:
@ -39,6 +25,8 @@ public:
nano::error serialize (nano::tomlconfig & toml) const;
public:
size_t batch_size{ 256 };
// Maximum number of blocks to queue from network peers
size_t max_peer_queue{ 128 };
// Maximum number of blocks to queue from system components (local RPC, bootstrap)
@ -49,9 +37,6 @@ public:
size_t priority_bootstrap{ 8 };
size_t priority_local{ 16 };
size_t priority_system{ 32 };
size_t batch_size{ 256 };
size_t max_queued_notifications{ 8 };
};
/**
@ -60,31 +45,8 @@ public:
*/
class block_processor final
{
public: // Context
class context
{
public:
using result_t = nano::block_status;
using callback_t = std::function<void (result_t)>;
context (std::shared_ptr<nano::block> block, nano::block_source source, callback_t callback = nullptr);
std::shared_ptr<nano::block> block;
nano::block_source source;
callback_t callback;
std::chrono::steady_clock::time_point arrival{ std::chrono::steady_clock::now () };
std::future<result_t> get_future ();
private:
void set_result (result_t const &);
std::promise<result_t> promise;
friend class block_processor;
};
public:
block_processor (nano::node_config const &, nano::ledger &, nano::unchecked_map &, nano::stats &, nano::logger &);
block_processor (nano::node_config const &, nano::ledger &, nano::ledger_notifications &, nano::unchecked_map &, nano::stats &, nano::logger &);
~block_processor ();
void start ();
@ -100,20 +62,11 @@ public:
std::atomic<bool> flushing{ false };
public: // Events
// All processed blocks including forks, rejected etc
using processed_batch_t = std::deque<std::pair<nano::block_status, context>>;
using processed_batch_event_t = nano::observer_set<processed_batch_t>;
processed_batch_event_t batch_processed;
// Rolled back blocks <rolled back blocks, root of rollback>
using rolled_back_event_t = nano::observer_set<std::deque<std::shared_ptr<nano::block>>, nano::qualified_root>;
rolled_back_event_t rolled_back;
private: // Dependencies
block_processor_config const & config;
nano::network_params const & network_params;
nano::ledger & ledger;
nano::ledger_notifications & ledger_notifications;
nano::unchecked_map & unchecked;
nano::stats & stats;
nano::logger & logger;
@ -121,21 +74,19 @@ private: // Dependencies
private:
void run ();
// Roll back block in the ledger that conflicts with 'block'
void rollback_competitor (secure::write_transaction const &, nano::block const & block);
nano::block_status process_one (secure::write_transaction const &, context const &, bool forced = false);
processed_batch_t process_batch (nano::unique_lock<nano::mutex> &);
std::deque<context> next_batch (size_t max_count);
context next ();
bool add_impl (context, std::shared_ptr<nano::transport::channel> const & channel = nullptr);
void rollback_competitor (secure::write_transaction &, nano::block const & block);
nano::block_status process_one (secure::write_transaction const &, nano::block_context const &, bool forced = false);
void process_batch (nano::unique_lock<nano::mutex> &);
std::deque<nano::block_context> next_batch (size_t max_count);
nano::block_context next ();
bool add_impl (nano::block_context, std::shared_ptr<nano::transport::channel> const & channel = nullptr);
private:
nano::fair_queue<context, nano::block_source> queue;
nano::fair_queue<nano::block_context, nano::block_source> queue;
bool stopped{ false };
nano::condition_variable condition;
mutable nano::mutex mutex{ mutex_identifier (mutexes::block_processor) };
std::thread thread;
nano::thread_pool workers;
};
}

13
nano/node/block_source.cpp Normal file
View file

@ -0,0 +1,13 @@
#include <nano/lib/enum_util.hpp>
#include <nano/lib/stats_enums.hpp>
#include <nano/node/block_source.hpp>
std::string_view nano::to_string (nano::block_source source)
{
return nano::enum_util::name (source);
}
nano::stat::detail nano::to_stat_detail (nano::block_source type)
{
return nano::enum_util::cast<nano::stat::detail> (type);
}

24
nano/node/block_source.hpp Normal file
View file

@ -0,0 +1,24 @@
#pragma once
#include <nano/lib/fwd.hpp>
#include <string_view>
namespace nano
{
enum class block_source
{
unknown = 0,
live,
live_originator,
bootstrap,
bootstrap_legacy,
unchecked,
local,
forced,
election,
};
std::string_view to_string (block_source);
nano::stat::detail to_stat_detail (block_source);
}

43
nano/node/bootstrap/account_sets.cpp
View file

@ -110,7 +110,20 @@ void nano::bootstrap::account_sets::priority_set (nano::account const & account,
}
}
void nano::bootstrap::account_sets::block (nano::account const & account, nano::block_hash const & dependency)
void nano::bootstrap::account_sets::priority_erase (nano::account const & account)
{
if (account.is_zero ())
{
return;
}
if (priorities.get<tag_account> ().erase (account) > 0)
{
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::priority_erase);
}
}
void nano::bootstrap::account_sets::block (nano::account const & account, nano::block_hash const & dependency, std::chrono::steady_clock::time_point now)
{
debug_assert (!account.is_zero ());
@ -121,7 +134,7 @@ void nano::bootstrap::account_sets::block (nano::account const & account, nano::
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::block);
debug_assert (blocking.get<tag_account> ().count (account) == 0);
blocking.get<tag_account> ().insert ({ account, dependency });
blocking.get<tag_account> ().insert ({ account, dependency, now });
trim_overflow ();
}
else
@ -304,6 +317,32 @@ void nano::bootstrap::account_sets::sync_dependencies ()
trim_overflow ();
}
size_t nano::bootstrap::account_sets::decay_blocking (std::chrono::steady_clock::time_point now)
{
stats.inc (nano::stat::type::bootstrap_account_sets, nano::stat::detail::decay_blocking);
auto const cutoff = now - config.blocking_decay;
// Erase all entries that are older than the cutoff
size_t result = 0;
for (auto it = blocking.get<tag_timestamp> ().begin (); it != blocking.get<tag_timestamp> ().end ();)
{
if (it->timestamp <= cutoff)
{
it = blocking.get<tag_timestamp> ().erase (it);
++result;
}
else
{
break; // Entries are sorted by timestamp, no need to continue
}
}
stats.add (nano::stat::type::bootstrap_account_sets, nano::stat::detail::blocking_decayed, result);
return result;
}
bool nano::bootstrap::account_sets::blocked (nano::account const & account) const
{
return blocking.get<tag_account> ().contains (account);

30
nano/node/bootstrap/account_sets.hpp
View file

@ -13,6 +13,7 @@
#include <boost/multi_index/sequenced_index.hpp>
#include <boost/multi_index_container.hpp>
#include <chrono>
#include <random>
namespace mi = boost::multi_index;
@ -35,20 +36,12 @@ public:
void reset ();
/**
* If an account is not blocked, increase its priority.
* If the account does not exist in priority set and is not blocked, inserts a new entry.
* Current implementation increases priority by 1.0f each increment
*/
void priority_up (nano::account const & account);
/**
* Decreases account priority
* Current implementation divides priority by 2.0f and saturates down to 1.0f.
*/
void priority_down (nano::account const & account);
void priority_set (nano::account const & account, double priority = priority_initial);
void priority_erase (nano::account const & account);
void block (nano::account const & account, nano::block_hash const & dependency);
void block (nano::account const & account, nano::block_hash const & dependency, std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now ());
void unblock (nano::account const & account, std::optional<nano::block_hash> const & hash = std::nullopt);
void timestamp_set (nano::account const & account);
@ -64,6 +57,11 @@ public:
*/
void sync_dependencies ();
/**
* Should be called periodically to remove old entries from the blocking set
*/
size_t decay_blocking (std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now ());
struct priority_result
{
nano::account account;
@ -102,8 +100,9 @@ private:
{
nano::account account;
double priority;
unsigned fails{ 0 };
std::chrono::steady_clock::time_point timestamp{};
std::chrono::steady_clock::time_point timestamp{}; // Use for cooldown, set to current time when this account is sampled
id_t id{ generate_id () }; // Uniformly distributed, used for random querying
};
@ -111,7 +110,9 @@ private:
{
nano::account account;
nano::block_hash dependency;
nano::account dependency_account{ 0 };
std::chrono::steady_clock::time_point timestamp; // Used for decaying old entries
nano::account dependency_account{ 0 }; // Account that contains the dependency block, fetched via a background dependency walker
id_t id{ generate_id () }; // Uniformly distributed, used for random querying
};
@ -122,6 +123,7 @@ private:
class tag_dependency {};
class tag_dependency_account {};
class tag_priority {};
class tag_timestamp {};
// Tracks the ongoing account priorities
using ordered_priorities = boost::multi_index_container<priority_entry,
@ -147,7 +149,9 @@ private:
mi::ordered_non_unique<mi::tag<tag_dependency_account>,
mi::member<blocking_entry, nano::account, &blocking_entry::dependency_account>>,
mi::ordered_unique<mi::tag<tag_id>,
mi::member<blocking_entry, id_t, &blocking_entry::id>>
mi::member<blocking_entry, id_t, &blocking_entry::id>>,
mi::ordered_non_unique<mi::tag<tag_timestamp>,
mi::member<blocking_entry, std::chrono::steady_clock::time_point, &blocking_entry::timestamp>>
>>;
// clang-format on

2
nano/node/bootstrap/bootstrap_config.cpp
View file

@ -11,6 +11,7 @@ nano::error nano::account_sets_config::deserialize (nano::tomlconfig & toml)
toml.get ("priorities_max", priorities_max);
toml.get ("blocking_max", blocking_max);
toml.get_duration ("cooldown", cooldown);
toml.get_duration ("blocking_decay", blocking_decay);
return toml.get_error ();
}
@ -21,6 +22,7 @@ nano::error nano::account_sets_config::serialize (nano::tomlconfig & toml) const
toml.put ("priorities_max", priorities_max, "Cutoff size limit for the priority list.\ntype:uint64");
toml.put ("blocking_max", blocking_max, "Cutoff size limit for the blocked accounts from the priority list.\ntype:uint64");
toml.put ("cooldown", cooldown.count (), "Waiting time for an account to become available.\ntype:milliseconds");
toml.put ("blocking_decay", blocking_decay.count (), "Time to wait before removing an account from the blocked list.\ntype:seconds");
return toml.get_error ();
}

5
nano/node/bootstrap/bootstrap_config.hpp
View file

@ -4,6 +4,8 @@
#include <nano/lib/timer.hpp>
#include <nano/node/bootstrap/bootstrap_server.hpp>
using namespace std::chrono_literals;
namespace nano
{
class tomlconfig;
@ -19,6 +21,7 @@ public:
std::size_t priorities_max{ 256 * 1024 };
std::size_t blocking_max{ 256 * 1024 };
std::chrono::milliseconds cooldown{ 1000 * 3 };
std::chrono::seconds blocking_decay{ 15min };
};
class frontier_scan_config final
@ -26,7 +29,7 @@ class frontier_scan_config final
public:
// TODO: Serialize & deserialize
unsigned head_parallelistm{ 128 };
unsigned head_parallelism{ 128 };
unsigned consideration_count{ 4 };
std::size_t candidates{ 1000 };
std::chrono::milliseconds cooldown{ 1000 * 5 };

25
nano/node/bootstrap/bootstrap_server.cpp
View file

@ -16,7 +16,8 @@ nano::bootstrap_server::bootstrap_server (bootstrap_server_config const & config
store{ store_a },
ledger{ ledger_a },
network_constants{ network_constants_a },
stats{ stats_a }
stats{ stats_a },
limiter{ config.limiter, /* allow bursts */ 3.0 } // TODO: Limiter bucket capacity should be at least equal to the batch size, currently it's not configurable
{
queue.max_size_query = [this] (auto const & origin) {
return config.max_queue;
@ -181,6 +182,22 @@ void nano::bootstrap_server::run ()
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
condition.wait (lock, [this] () {
return stopped || !queue.empty ();
});
// Rate limit the processing
while (!stopped && !limiter.should_pass (config.batch_size))
{
stats.inc (nano::stat::type::bootstrap_server, nano::stat::detail::cooldown);
condition.wait_for (lock, 100ms);
}
if (stopped)
{
return;
}
if (!queue.empty ())
{
stats.inc (nano::stat::type::bootstrap_server, nano::stat::detail::loop);
@ -190,10 +207,6 @@ void nano::bootstrap_server::run ()
lock.lock ();
}
else
{
condition.wait (lock, [this] () { return stopped || !queue.empty (); });
}
}
}
@ -429,6 +442,7 @@ nano::error nano::bootstrap_server_config::serialize (nano::tomlconfig & toml) c
toml.put ("max_queue", max_queue, "Maximum number of queued requests per peer. \ntype:uint64");
toml.put ("threads", threads, "Number of threads to process requests. \ntype:uint64");
toml.put ("batch_size", batch_size, "Maximum number of requests to process in a single batch. \ntype:uint64");
toml.put ("limiter", limiter, "Rate limit for processing requests. Use 0 for unlimited. \ntype:uint64");
return toml.get_error ();
}
@ -438,6 +452,7 @@ nano::error nano::bootstrap_server_config::deserialize (nano::tomlconfig & toml)
toml.get ("max_queue", max_queue);
toml.get ("threads", threads);
toml.get ("batch_size", batch_size);
toml.get ("limiter", limiter);
return toml.get_error ();
}

3
nano/node/bootstrap/bootstrap_server.hpp
View file

@ -2,6 +2,7 @@
#include <nano/lib/locks.hpp>
#include <nano/lib/observer_set.hpp>
#include <nano/lib/rate_limiting.hpp>
#include <nano/node/fair_queue.hpp>
#include <nano/node/fwd.hpp>
#include <nano/node/messages.hpp>
@ -24,6 +25,7 @@ public:
size_t max_queue{ 16 };
size_t threads{ 1 };
size_t batch_size{ 64 };
size_t limiter{ 500 };
};
/**
@ -91,6 +93,7 @@ private: // Dependencies
private:
nano::fair_queue<request_t, nano::no_value> queue;
nano::rate_limiter limiter;
std::atomic<bool> stopped{ false };
nano::condition_variable condition;

84
nano/node/bootstrap/bootstrap_service.cpp
View file

@ -6,6 +6,7 @@
#include <nano/node/block_processor.hpp>
#include <nano/node/bootstrap/bootstrap_service.hpp>
#include <nano/node/bootstrap/crawlers.hpp>
#include <nano/node/ledger_notifications.hpp>
#include <nano/node/network.hpp>
#include <nano/node/nodeconfig.hpp>
#include <nano/node/transport/transport.hpp>
@ -18,11 +19,13 @@
using namespace std::chrono_literals;
nano::bootstrap_service::bootstrap_service (nano::node_config const & node_config_a, nano::block_processor & block_processor_a, nano::ledger & ledger_a, nano::network & network_a, nano::stats & stat_a, nano::logger & logger_a) :
nano::bootstrap_service::bootstrap_service (nano::node_config const & node_config_a, nano::ledger & ledger_a, nano::ledger_notifications & ledger_notifications_a,
nano::block_processor & block_processor_a, nano::network & network_a, nano::stats & stat_a, nano::logger & logger_a) :
config{ node_config_a.bootstrap },
network_constants{ node_config_a.network_params.network },
block_processor{ block_processor_a },
ledger{ ledger_a },
ledger_notifications{ ledger_notifications_a },
block_processor{ block_processor_a },
network{ network_a },
stats{ stat_a },
logger{ logger_a },
@ -36,7 +39,8 @@ nano::bootstrap_service::bootstrap_service (nano::node_config const & node_confi
frontiers_limiter{ config.frontier_rate_limit },
workers{ 1, nano::thread_role::name::bootstrap_worker }
{
block_processor.batch_processed.add ([this] (auto const & batch) {
// Inspect all processed blocks
ledger_notifications.blocks_processed.add ([this] (auto const & batch) {
{
nano::lock_guard<nano::mutex> lock{ mutex };
@ -51,7 +55,7 @@ nano::bootstrap_service::bootstrap_service (nano::node_config const & node_confi
});
// Unblock rolled back accounts as the dependency is no longer valid
block_processor.rolled_back.add ([this] (auto const & blocks, auto const & rollback_root) {
ledger_notifications.blocks_rolled_back.add ([this] (auto const & blocks, auto const & rollback_root) {
nano::lock_guard<nano::mutex> lock{ mutex };
for (auto const & block : blocks)
{
@ -340,7 +344,16 @@ void nano::bootstrap_service::inspect (secure::transaction const & tx, nano::blo
}
}
break;
case nano::block_status::gap_epoch_open_pending:
{
// Epoch open blocks for accounts that don't have any pending blocks yet
debug_assert (block.type () == block_type::state); // Only state blocks can have epoch open pending status
const auto account = block.account_field ().value_or (0);
accounts.priority_erase (account);
}
break;
default: // No need to handle other cases
// TODO: If we receive blocks that are invalid (bad signature, fork, etc.), we should penalize the peer that sent them
break;
}
}
@ -529,33 +542,58 @@ bool nano::bootstrap_service::request (nano::account account, size_t count, std:
// Check if the account picked has blocks, if it does, start the pull from the highest block
if (auto info = ledger.store.account.get (transaction, account))
{
tag.type = query_type::blocks_by_hash;
// Probabilistically choose between requesting blocks from account frontier or confirmed frontier
// Optimistic requests start from the (possibly unconfirmed) account frontier and are vulnerable to bootstrap poisoning
// Safe requests start from the confirmed frontier and given enough time will eventually resolve forks
bool optimistic_reuest = rng.random (100) < config.optimistic_request_percentage;
if (!optimistic_reuest)
{
if (auto conf_info = ledger.store.confirmation_height.get (transaction, account))
{
stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::safe);
tag.start = conf_info->frontier;
tag.hash = conf_info->height;
}
}
if (tag.start.is_zero ())
if (optimistic_reuest) // Optimistic request case
{
stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::optimistic);
tag.type = query_type::blocks_by_hash;
tag.start = info->head;
tag.hash = info->head;
logger.debug (nano::log::type::bootstrap, "Requesting blocks for {} starting from account frontier: {} (optimistic: {})",
account,
tag.start,
optimistic_reuest);
}
else // Pessimistic (safe) request case
{
stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::safe);
if (auto conf_info = ledger.store.confirmation_height.get (transaction, account))
{
tag.type = query_type::blocks_by_hash;
tag.start = conf_info->frontier;
tag.hash = conf_info->frontier;
logger.debug (nano::log::type::bootstrap, "Requesting blocks for {} starting from confirmation frontier: {} (optimistic: {})",
account,
tag.start,
optimistic_reuest);
}
else
{
tag.type = query_type::blocks_by_account;
tag.start = account;
logger.debug (nano::log::type::bootstrap, "Requesting blocks for {} starting from account root (optimistic: {})",
account,
optimistic_reuest);
}
}
}
else
{
stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::base);
tag.type = query_type::blocks_by_account;
tag.start = account;
logger.debug (nano::log::type::bootstrap, "Requesting blocks for {}", account);
}
}
@ -569,6 +607,9 @@ bool nano::bootstrap_service::request_info (nano::block_hash hash, std::shared_p
tag.source = source;
tag.start = hash;
tag.hash = hash;
logger.debug (nano::log::type::bootstrap, "Requesting account info for: {}", hash);
return send (channel, tag);
}
@ -578,6 +619,9 @@ bool nano::bootstrap_service::request_frontiers (nano::account start, std::share
tag.type = query_type::frontiers;
tag.source = source;
tag.start = start;
logger.debug (nano::log::type::bootstrap, "Requesting frontiers starting from: {}", start);
return send (channel, tag);
}
@ -727,11 +771,14 @@ void nano::bootstrap_service::cleanup_and_sync ()
throttle.resize (compute_throttle_size ());
accounts.decay_blocking ();
auto const now = std::chrono::steady_clock::now ();
auto should_timeout = [&] (async_tag const & tag) {
return tag.cutoff < now;
};
// Erase timed out requests
auto & tags_by_order = tags.get<tag_sequenced> ();
while (!tags_by_order.empty () && should_timeout (tags_by_order.front ()))
{
@ -741,7 +788,8 @@ void nano::bootstrap_service::cleanup_and_sync ()
tags_by_order.pop_front ();
}
if (sync_dependencies_interval.elapsed (60s))
// Reinsert known dependencies into the priority set
if (sync_dependencies_interval.elapse (nano::is_dev_run () ? 1s : 60s))
{
stats.inc (nano::stat::type::bootstrap, nano::stat::detail::sync_dependencies);
accounts.sync_dependencies ();
@ -1061,6 +1109,8 @@ void nano::bootstrap_service::process_frontiers (std::deque<std::pair<nano::acco
stats.add (nano::stat::type::bootstrap_frontiers, nano::stat::detail::outdated, outdated);
stats.add (nano::stat::type::bootstrap_frontiers, nano::stat::detail::pending, pending);
logger.debug (nano::log::type::bootstrap, "Processed {} frontiers of which outdated: {}, pending: {}", frontiers.size (), outdated, pending);
nano::lock_guard<nano::mutex> guard{ mutex };
for (auto const & account : result)

5
nano/node/bootstrap/bootstrap_service.hpp
View file

@ -30,7 +30,7 @@ namespace nano
class bootstrap_service
{
public:
bootstrap_service (nano::node_config const &, nano::block_processor &, nano::ledger &, nano::network &, nano::stats &, nano::logger &);
bootstrap_service (nano::node_config const &, nano::ledger &, nano::ledger_notifications &, nano::block_processor &, nano::network &, nano::stats &, nano::logger &);
~bootstrap_service ();
void start ();
@ -67,8 +67,9 @@ public:
private: // Dependencies
bootstrap_config const & config;
nano::network_constants const & network_constants;
nano::block_processor & block_processor;
nano::ledger & ledger;
nano::ledger_notifications & ledger_notifications;
nano::block_processor & block_processor;
nano::network & network;
nano::stats & stats;
nano::logger & logger;

6
nano/node/bootstrap/frontier_scan.cpp
View file

@ -9,13 +9,13 @@ nano::bootstrap::frontier_scan::frontier_scan (frontier_scan_config const & conf
{
// Divide nano::account numeric range into consecutive and equal ranges
nano::uint256_t max_account = std::numeric_limits<nano::uint256_t>::max ();
nano::uint256_t range_size = max_account / config.head_parallelistm;
nano::uint256_t range_size = max_account / config.head_parallelism;
for (unsigned i = 0; i < config.head_parallelistm; ++i)
for (unsigned i = 0; i < config.head_parallelism; ++i)
{
// Start at 1 to avoid the burn account
nano::uint256_t start = (i == 0) ? 1 : i * range_size;
nano::uint256_t end = (i == config.head_parallelistm - 1) ? max_account : start + range_size;
nano::uint256_t end = (i == config.head_parallelism - 1) ? max_account : start + range_size;
heads.emplace_back (frontier_head{ nano::account{ start }, nano::account{ end } });
}

40
nano/node/bounded_backlog.cpp
View file

@ -4,6 +4,7 @@
#include <nano/node/block_processor.hpp>
#include <nano/node/bounded_backlog.hpp>
#include <nano/node/confirming_set.hpp>
#include <nano/node/ledger_notifications.hpp>
#include <nano/node/node.hpp>
#include <nano/node/scheduler/component.hpp>
#include <nano/secure/common.hpp>
@ -12,18 +13,17 @@
#include <nano/secure/ledger_set_confirmed.hpp>
#include <nano/secure/transaction.hpp>
nano::bounded_backlog::bounded_backlog (nano::node_config const & config_a, nano::node & node_a, nano::ledger & ledger_a, nano::bucketing & bucketing_a, nano::backlog_scan & backlog_scan_a, nano::block_processor & block_processor_a, nano::confirming_set & confirming_set_a, nano::stats & stats_a, nano::logger & logger_a) :
nano::bounded_backlog::bounded_backlog (nano::node_config const & config_a, nano::node & node_a, nano::ledger & ledger_a, nano::ledger_notifications & ledger_notifications_a, nano::bucketing & bucketing_a, nano::backlog_scan & backlog_scan_a, nano::block_processor & block_processor_a, nano::confirming_set & confirming_set_a, nano::stats & stats_a, nano::logger & logger_a) :
config{ config_a },
node{ node_a },
ledger{ ledger_a },
ledger_notifications{ ledger_notifications_a },
bucketing{ bucketing_a },
backlog_scan{ backlog_scan_a },
block_processor{ block_processor_a },
confirming_set{ confirming_set_a },
stats{ stats_a },
logger{ logger_a },
scan_limiter{ config.bounded_backlog.scan_rate },
workers{ 1, nano::thread_role::name::bounded_backlog_notifications }
scan_limiter{ config.bounded_backlog.scan_rate }
{
// Activate accounts with unconfirmed blocks
backlog_scan.batch_activated.add ([this] (auto const & batch) {
@ -47,7 +47,7 @@ nano::bounded_backlog::bounded_backlog (nano::node_config const & config_a, nano
});
// Track unconfirmed blocks
block_processor.batch_processed.add ([this] (auto const & batch) {
ledger_notifications.blocks_processed.add ([this] (auto const & batch) {
auto transaction = ledger.tx_begin_read ();
for (auto const & [result, context] : batch)
{
@ -60,7 +60,7 @@ nano::bounded_backlog::bounded_backlog (nano::node_config const & config_a, nano
});
// Remove rolled back blocks from the backlog
block_processor.rolled_back.add ([this] (auto const & blocks, auto const & rollback_root) {
ledger_notifications.blocks_rolled_back.add ([this] (auto const & blocks, auto const & rollback_root) {
nano::lock_guard<nano::mutex> guard{ mutex };
for (auto const & block : blocks)
{
@ -83,7 +83,6 @@ nano::bounded_backlog::~bounded_backlog ()
// Thread must be stopped before destruction
debug_assert (!thread.joinable ());
debug_assert (!scan_thread.joinable ());
debug_assert (!workers.alive ());
}
void nano::bounded_backlog::start ()
@ -95,8 +94,6 @@ void nano::bounded_backlog::start ()
return;
}
workers.start ();
thread = std::thread{ [this] () {
nano::thread_role::set (nano::thread_role::name::bounded_backlog);
run ();
@ -123,7 +120,6 @@ void nano::bounded_backlog::stop ()
{
scan_thread.join ();
}
workers.stop ();
}
size_t nano::bounded_backlog::index_size () const
@ -206,16 +202,14 @@ void nano::bounded_backlog::run ()
return;
}
lock.unlock ();
// Wait until all notification about the previous rollbacks are processed
while (workers.queued_tasks () >= config.bounded_backlog.max_queued_notifications)
{
ledger_notifications.wait ([this] {
stats.inc (nano::stat::type::bounded_backlog, nano::stat::detail::cooldown);
condition.wait_for (lock, 100ms, [this] { return stopped.load (); });
if (stopped)
{
return;
}
}
});
lock.lock ();
stats.inc (nano::stat::type::bounded_backlog, nano::stat::detail::loop);
@ -298,7 +292,7 @@ std::deque<nano::block_hash> nano::bounded_backlog::perform_rollbacks (std::dequ
// Here we check that the block is still OK to rollback, there could be a delay between gathering the targets and performing the rollbacks
if (auto block = ledger.any.block_get (transaction, hash))
{
logger.debug (nano::log::type::bounded_backlog, "Rolling back: {}, account: {}", hash.to_string (), block->account ().to_account ());
logger.debug (nano::log::type::bounded_backlog, "Rolling back: {}, account: {}", hash, block->account ());
std::deque<std::shared_ptr<nano::block>> rollback_list;
bool error = ledger.rollback (transaction, hash, rollback_list);
@ -310,9 +304,8 @@ std::deque<nano::block_hash> nano::bounded_backlog::perform_rollbacks (std::dequ
}
// Notify observers of the rolled back blocks on a background thread, avoid dispatching notifications when holding ledger write transaction
workers.post ([this, rollback_list = std::move (rollback_list), root = block->qualified_root ()] {
// TODO: Calling block_processor's event here is not ideal, but duplicating these events is even worse
block_processor.rolled_back.notify (rollback_list, root);
ledger_notifications.notify_rolled_back (transaction, std::move (rollback_list), block->qualified_root (), [this] {
stats.inc (nano::stat::type::bounded_backlog, nano::stat::detail::notify_rolled_back);
});
// Return early if we reached the maximum number of rollbacks
@ -420,7 +413,6 @@ nano::container_info nano::bounded_backlog::container_info () const
nano::lock_guard<nano::mutex> guard{ mutex };
nano::container_info info;
info.put ("backlog", index.size ());
info.put ("notifications", workers.queued_tasks ());
info.add ("index", index.container_info ());
return info;
}
@ -547,7 +539,6 @@ nano::error nano::bounded_backlog_config::serialize (nano::tomlconfig & toml) co
{
toml.put ("enable", enable, "Enable the bounded backlog. \ntype:bool");
toml.put ("batch_size", batch_size, "Maximum number of blocks to rollback per iteration. \ntype:uint64");
toml.put ("max_queued_notifications", max_queued_notifications, "Maximum number of queued background tasks before cooldown. \ntype:uint64");
toml.put ("scan_rate", scan_rate, "Rate limit for refreshing the backlog index. \ntype:uint64");
return toml.get_error ();
@ -557,7 +548,6 @@ nano::error nano::bounded_backlog_config::deserialize (nano::tomlconfig & toml)
{
toml.get ("enable", enable);
toml.get ("batch_size", batch_size);
toml.get ("max_queued_notifications", max_queued_notifications);
toml.get ("scan_rate", scan_rate);
return toml.get_error ();

7
nano/node/bounded_backlog.hpp
View file

@ -101,14 +101,13 @@ public:
public:
bool enable{ true };
size_t batch_size{ 32 };
size_t max_queued_notifications{ 128 };
size_t scan_rate{ 64 };
};
class bounded_backlog
{
public:
bounded_backlog (nano::node_config const &, nano::node &, nano::ledger &, nano::bucketing &, nano::backlog_scan &, nano::block_processor &, nano::confirming_set &, nano::stats &, nano::logger &);
bounded_backlog (nano::node_config const &, nano::node &, nano::ledger &, nano::ledger_notifications &, nano::bucketing &, nano::backlog_scan &, nano::block_processor &, nano::confirming_set &, nano::stats &, nano::logger &);
~bounded_backlog ();
void start ();
@ -124,9 +123,9 @@ private: // Dependencies
nano::node_config const & config;
nano::node & node;
nano::ledger & ledger;
nano::ledger_notifications & ledger_notifications;
nano::bucketing & bucketing;
nano::backlog_scan & backlog_scan;
nano::block_processor & block_processor;
nano::confirming_set & confirming_set;
nano::stats & stats;
nano::logger & logger;
@ -155,7 +154,5 @@ private:
mutable nano::mutex mutex;
std::thread thread;
std::thread scan_thread;
nano::thread_pool workers;
};
}

11
nano/node/confirming_set.cpp
View file

@ -4,16 +4,17 @@
#include <nano/lib/thread_roles.hpp>
#include <nano/node/block_processor.hpp>
#include <nano/node/confirming_set.hpp>
#include <nano/node/ledger_notifications.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
#include <nano/secure/ledger_set_confirmed.hpp>
#include <nano/store/component.hpp>
#include <nano/store/write_queue.hpp>
nano::confirming_set::confirming_set (confirming_set_config const & config_a, nano::ledger & ledger_a, nano::block_processor & block_processor_a, nano::stats & stats_a, nano::logger & logger_a) :
nano::confirming_set::confirming_set (confirming_set_config const & config_a, nano::ledger & ledger_a, nano::ledger_notifications & ledger_notifications_a, nano::stats & stats_a, nano::logger & logger_a) :
config{ config_a },
ledger{ ledger_a },
block_processor{ block_processor_a },
ledger_notifications{ ledger_notifications_a },
stats{ stats_a },
logger{ logger_a },
workers{ 1, nano::thread_role::name::confirmation_height_notifications }
@ -26,7 +27,7 @@ nano::confirming_set::confirming_set (confirming_set_config const & config_a, na
});
// Requeue blocks that failed to cement immediately due to missing ledger blocks
block_processor.batch_processed.add ([this] (auto const & batch) {
ledger_notifications.blocks_processed.add ([this] (auto const & batch) {
bool should_notify = false;
{
std::lock_guard lock{ mutex };
@ -262,12 +263,12 @@ void nano::confirming_set::run_batch (std::unique_lock<std::mutex> & lock)
if (success)
{
stats.inc (nano::stat::type::confirming_set, nano::stat::detail::cemented_hash);
logger.debug (nano::log::type::confirming_set, "Cemented block: {} (total cemented: {})", hash.to_string (), cemented_count);
logger.debug (nano::log::type::confirming_set, "Cemented block: {} (total cemented: {})", hash, cemented_count);
}
else
{
stats.inc (nano::stat::type::confirming_set, nano::stat::detail::cementing_failed);
logger.debug (nano::log::type::confirming_set, "Failed to cement block: {}", hash.to_string ());
logger.debug (nano::log::type::confirming_set, "Failed to cement block: {}", hash);
// Requeue failed blocks for processing later
// Add them to the deferred set while still holding the exclusive database write transaction to avoid block processor races

4
nano/node/confirming_set.hpp
View file

@ -52,7 +52,7 @@ class confirming_set final
friend class confirmation_height_pruned_source_Test;
public:
confirming_set (confirming_set_config const &, nano::ledger &, nano::block_processor &, nano::stats &, nano::logger &);
confirming_set (confirming_set_config const &, nano::ledger &, nano::ledger_notifications &, nano::stats &, nano::logger &);
~confirming_set ();
void start ();
@ -83,7 +83,7 @@ public: // Events
private: // Dependencies
confirming_set_config const & config;
nano::ledger & ledger;
nano::block_processor & block_processor;
nano::ledger_notifications & ledger_notifications;
nano::stats & stats;
nano::logger & logger;

58
nano/node/distributed_work.cpp
View file

@ -41,7 +41,7 @@ nano::distributed_work::~distributed_work ()
{
if (!node_l->stopped && node_l->websocket.server && node_l->websocket.server->any_subscriber (nano::websocket::topic::work))
{
nano::websocket::message_builder builder;
nano::websocket::message_builder builder{ node_l->ledger };
if (status == work_generation_status::success)
{
node_l->websocket.server->broadcast (builder.work_generation (request.version, request.root.as_block_hash (), work_result, request.difficulty, node_l->default_difficulty (request.version), elapsed.value (), winner, bad_peers));
@ -96,7 +96,8 @@ void nano::distributed_work::start ()
}
else
{
this_l->node.logger.error (nano::log::type::distributed_work, "Error resolving work peer: {}:{} ({})", peer.first, peer.second, ec.message ());
this_l->node.logger.error (nano::log::type::distributed_work, "Error resolving work peer: {}:{} ({})",
peer.first, peer.second, ec.message ());
this_l->failure ();
}
@ -169,9 +170,8 @@ void nano::distributed_work::do_request (nano::tcp_endpoint const & endpoint_a)
}
else if (ec)
{
this_l->node.logger.error (nano::log::type::distributed_work, "Work peer responded with an error {}:{} ({})",
nano::util::to_str (connection->endpoint.address ()),
connection->endpoint.port (),
this_l->node.logger.error (nano::log::type::distributed_work, "Work peer responded with an error {} ({})",
connection->endpoint,
ec.message ());
this_l->add_bad_peer (connection->endpoint);
@ -187,9 +187,8 @@ void nano::distributed_work::do_request (nano::tcp_endpoint const & endpoint_a)
}
else if (ec && ec != boost::system::errc::operation_canceled)
{
this_l->node.logger.error (nano::log::type::distributed_work, "Unable to write to work peer {}:{} ({})",
nano::util::to_str (connection->endpoint.address ()),
connection->endpoint.port (),
this_l->node.logger.error (nano::log::type::distributed_work, "Unable to write to work peer {} ({})",
connection->endpoint,
ec.message ());
this_l->add_bad_peer (connection->endpoint);
@ -199,9 +198,8 @@ void nano::distributed_work::do_request (nano::tcp_endpoint const & endpoint_a)
}
else if (ec && ec != boost::system::errc::operation_canceled)
{
this_l->node.logger.error (nano::log::type::distributed_work, "Unable to connect to work peer {}:{} ({})",
nano::util::to_str (connection->endpoint.address ()),
connection->endpoint.port (),
this_l->node.logger.error (nano::log::type::distributed_work, "Unable to connect to work peer {} ({})",
connection->endpoint,
ec.message ());
this_l->add_bad_peer (connection->endpoint);
@ -234,9 +232,8 @@ void nano::distributed_work::do_cancel (nano::tcp_endpoint const & endpoint_a)
[this_l, peer_cancel, cancelling_l] (boost::system::error_code const & ec, std::size_t bytes_transferred) {
if (ec && ec != boost::system::errc::operation_canceled)
{
this_l->node.logger.error (nano::log::type::distributed_work, "Unable to send work cancel to work peer {}:{} ({})",
nano::util::to_str (cancelling_l->endpoint.address ()),
cancelling_l->endpoint.port (),
this_l->node.logger.error (nano::log::type::distributed_work, "Unable to send work cancel to work peer {} ({})",
cancelling_l->endpoint,
ec.message ());
}
}));
@ -265,27 +262,24 @@ void nano::distributed_work::success (std::string const & body_a, nano::tcp_endp
}
else
{
node.logger.error (nano::log::type::distributed_work, "Incorrect work response from {}:{} for root {} with difficulty {}: {}",
nano::util::to_str (endpoint_a.address ()),
endpoint_a.port (),
request.root.to_string (),
node.logger.error (nano::log::type::distributed_work, "Incorrect work response from {} for root {} with difficulty {}: {}",
endpoint_a,
request.root,
nano::to_string_hex (request.difficulty),
work_text);
}
}
else
{
node.logger.error (nano::log::type::distributed_work, "Work response from {}:{} wasn't a number: {}",
nano::util::to_str (endpoint_a.address ()),
endpoint_a.port (),
node.logger.error (nano::log::type::distributed_work, "Work response from {} wasn't a number: {}",
endpoint_a,
work_text);
}
}
catch (...)
{
node.logger.error (nano::log::type::distributed_work, "Work response from {}:{} wasn't parsable: {}",
nano::util::to_str (endpoint_a.address ()),
endpoint_a.port (),
node.logger.error (nano::log::type::distributed_work, "Work response from {} wasn't parsable: {}",
endpoint_a,
body_a);
}
if (error)
@ -319,17 +313,15 @@ void nano::distributed_work::stop_once (bool const local_stop_a)
connection_l->socket.close (ec);
if (ec)
{
this_l->node.logger.error (nano::log::type::distributed_work, "Error closing socket with work peer: {}:{} ({})",
nano::util::to_str (connection_l->endpoint.address ()),
connection_l->endpoint.port (),
this_l->node.logger.error (nano::log::type::distributed_work, "Error closing socket with work peer: {} ({})",
connection_l->endpoint,
ec.message ());
}
}
else
{
this_l->node.logger.error (nano::log::type::distributed_work, "Error cancelling operation with work peer: {}:{} ({})",
nano::util::to_str (connection_l->endpoint.address ()),
connection_l->endpoint.port (),
this_l->node.logger.error (nano::log::type::distributed_work, "Error cancelling operation with work peer: {} ({})",
connection_l->endpoint,
ec.message ());
}
}
@ -347,7 +339,7 @@ void nano::distributed_work::set_once (uint64_t const work_a, std::string const
elapsed.stop ();
node.logger.info (nano::log::type::distributed_work, "Work generation for {}, with a threshold difficulty of {} (multiplier {}x) complete: {} ms",
request.root.to_string (),
request.root,
nano::to_string_hex (request.difficulty),
nano::to_string (nano::difficulty::to_multiplier (request.difficulty, node.default_difficulty (request.version)), 2),
elapsed.value ().count ());
@ -369,7 +361,7 @@ void nano::distributed_work::cancel ()
elapsed.stop ();
node.logger.info (nano::log::type::distributed_work, "Work generation for {} was cancelled after {} ms",
request.root.to_string (),
request.root,
elapsed.value ().count ());
status = work_generation_status::cancelled;
@ -397,7 +389,7 @@ void nano::distributed_work::handle_failure ()
if (!local_generation_started && !finished.exchange (true))
{
node.logger.info (nano::log::type::distributed_work, "Work peer(s) failed to generate work for root {}, retrying... (backoff: {}s)",
request.root.to_string (),
request.root,
backoff.count ());
status = work_generation_status::failure_peers;

68
nano/node/election.cpp
View file

@ -23,9 +23,9 @@ std::chrono::milliseconds nano::election::base_latency () const
* election
*/
nano::election::election (nano::node & node_a, std::shared_ptr<nano::block> const & block_a, std::function<void (std::shared_ptr<nano::block> const &)> const & confirmation_action_a, std::function<void (nano::account const &)> const & live_vote_action_a, nano::election_behavior election_behavior_a) :
nano::election::election (nano::node & node_a, std::shared_ptr<nano::block> const & block_a, std::function<void (std::shared_ptr<nano::block> const &)> const & confirmation_action_a, std::function<void (nano::account const &)> const & vote_action_a, nano::election_behavior election_behavior_a) :
confirmation_action (confirmation_action_a),
live_vote_action (live_vote_action_a),
vote_action (vote_action_a),
node (node_a),
behavior_m (election_behavior_a),
status (block_a),
@ -65,7 +65,7 @@ void nano::election::confirm_once (nano::unique_lock<nano::mutex> & lock)
nano::log::arg{ "status", extended_status });
node.logger.debug (nano::log::type::election, "Election confirmed with winner: {} (behavior: {}, state: {}, voters: {}, blocks: {}, duration: {}ms, confirmation requests: {})",
status_l.winner->hash ().to_string (),
status_l.winner->hash (),
to_string (behavior_m),
to_string (state_m),
extended_status.status.voter_count,
@ -168,12 +168,24 @@ std::chrono::milliseconds nano::election::confirm_req_time () const
void nano::election::send_confirm_req (nano::confirmation_solicitor & solicitor_a)
{
debug_assert (!mutex.try_lock ());
if (confirm_req_time () < (std::chrono::steady_clock::now () - last_req))
{
if (!solicitor_a.add (*this))
{
last_req = std::chrono::steady_clock::now ();
++confirmation_request_count;
node.stats.inc (nano::stat::type::election, nano::stat::detail::confirmation_request);
node.logger.debug (nano::log::type::election, "Sent confirmation request for root: {} (behavior: {}, state: {}, voters: {}, blocks: {}, duration: {}ms, confirmation requests: {})",
qualified_root,
to_string (behavior_m),
to_string (state_m),
status.voter_count,
status.block_count,
duration ().count (),
confirmation_request_count.load ());
}
}
}
@ -196,9 +208,10 @@ bool nano::election::transition_priority ()
behavior_m = nano::election_behavior::priority;
last_vote = std::chrono::steady_clock::time_point{}; // allow new outgoing votes immediately
node.logger.debug (nano::log::type::election, "Transitioned election behavior to priority from {} for root: {}",
node.logger.debug (nano::log::type::election, "Transitioned election behavior to priority from {} for root: {} (duration: {}ms)",
to_string (behavior_m),
qualified_root.to_string ());
qualified_root,
duration ().count ());
return true;
}
@ -252,10 +265,18 @@ void nano::election::broadcast_block (nano::confirmation_solicitor & solicitor_a
{
if (!solicitor_a.broadcast (*this))
{
node.stats.inc (nano::stat::type::election, last_block_hash.is_zero () ? nano::stat::detail::broadcast_block_initial : nano::stat::detail::broadcast_block_repeat);
last_block = std::chrono::steady_clock::now ();
last_block_hash = status.winner->hash ();
node.stats.inc (nano::stat::type::election, last_block_hash.is_zero () ? nano::stat::detail::broadcast_block_initial : nano::stat::detail::broadcast_block_repeat);
node.logger.debug (nano::log::type::election, "Broadcasting current winner: {} for root: {} (behavior: {}, state: {}, voters: {}, blocks: {}, duration: {}ms)",
status.winner->hash (),
qualified_root,
to_string (behavior_m),
to_string (state_m),
status.voter_count,
status.block_count,
duration ().count ());
}
}
}
@ -481,7 +502,8 @@ std::shared_ptr<nano::block> nano::election::find (nano::block_hash const & hash
nano::vote_code nano::election::vote (nano::account const & rep, uint64_t timestamp_a, nano::block_hash const & block_hash_a, nano::vote_source vote_source_a)
{
auto weight = node.ledger.weight (rep);
auto const weight = node.ledger.weight (rep);
if (!node.network_params.network.is_dev_network () && weight <= node.minimum_principal_weight ())
{
return vote_code::indeterminate;
@ -517,11 +539,8 @@ nano::vote_code nano::election::vote (nano::account const & rep, uint64_t timest
}
}
// Update voter list entry
last_votes[rep] = { std::chrono::steady_clock::now (), timestamp_a, block_hash_a };
if (vote_source_a != vote_source::cache)
{
live_vote_action (rep);
}
node.stats.inc (nano::stat::type::election, nano::stat::detail::vote);
node.stats.inc (nano::stat::type::election_vote, to_stat_detail (vote_source_a));
@ -536,6 +555,20 @@ nano::vote_code nano::election::vote (nano::account const & rep, uint64_t timest
nano::log::arg{ "vote_source", vote_source_a },
nano::log::arg{ "weight", weight });
node.logger.debug (nano::log::type::election, "Vote received for: {} from: {} root: {} (final: {}, weight: {}, source: {})",
block_hash_a,
rep,
qualified_root,
nano::vote::is_final_timestamp (timestamp_a),
weight,
to_string (vote_source_a));
// This must execute before calculating the vote tally to ensure accurate online weight and quorum numbers are used
if (vote_action)
{
vote_action (rep);
}
if (!confirmed_locked ())
{
confirm_if_quorum (lock);
@ -810,11 +843,10 @@ bool nano::election::contains (nano::block_hash const & hash) const
return last_blocks.contains (hash);
}
// TODO: Remove the need for .to_string () calls
void nano::election::operator() (nano::object_stream & obs) const
{
obs.write ("id", id);
obs.write ("qualified_root", qualified_root.to_string ());
obs.write ("qualified_root", qualified_root);
obs.write ("behavior", behavior_m);
obs.write ("height", height);
obs.write ("status", current_status ());
@ -822,7 +854,7 @@ void nano::election::operator() (nano::object_stream & obs) const
void nano::election_extended_status::operator() (nano::object_stream & obs) const
{
obs.write ("winner", status.winner->hash ().to_string ());
obs.write ("winner", status.winner->hash ());
obs.write ("tally_amount", status.tally.to_string_dec ());
obs.write ("final_tally_amount", status.final_tally.to_string_dec ());
obs.write ("confirmation_request_count", status.confirmation_request_count);
@ -833,8 +865,8 @@ void nano::election_extended_status::operator() (nano::object_stream & obs) cons
obs.write_range ("votes", votes, [] (auto const & entry, nano::object_stream & obs) {
auto & [account, info] = entry;
obs.write ("account", account.to_account ());
obs.write ("hash", info.hash.to_string ());
obs.write ("account", account);
obs.write ("hash", info.hash);
obs.write ("final", nano::vote::is_final_timestamp (info.timestamp));
obs.write ("timestamp", info.timestamp);
obs.write ("time", info.time.time_since_epoch ().count ());
@ -847,7 +879,7 @@ void nano::election_extended_status::operator() (nano::object_stream & obs) cons
obs.write_range ("tally", tally, [] (auto const & entry, nano::object_stream & obs) {
auto & [amount, block] = entry;
obs.write ("hash", block->hash ().to_string ());
obs.write ("hash", block->hash ());
obs.write ("amount", amount);
});
}

4
nano/node/election.hpp
View file

@ -64,8 +64,10 @@ class election final : public std::enable_shared_from_this<election>
private:
// Minimum time between broadcasts of the current winner of an election, as a backup to requesting confirmations
std::chrono::milliseconds base_latency () const;
// Callbacks
std::function<void (std::shared_ptr<nano::block> const &)> confirmation_action;
std::function<void (nano::account const &)> live_vote_action;
std::function<void (nano::account const &)> vote_action;
private: // State management
static unsigned constexpr passive_duration_factor = 5;

2
nano/node/epoch_upgrader.cpp
View file

@ -65,7 +65,7 @@ void nano::epoch_upgrader::upgrade_impl (nano::raw_key const & prv_a, nano::epoc
bool fork (result == nano::block_status::fork);
logger.error (nano::log::type::epoch_upgrader, "Failed to upgrade account {} (valid signature: {}, valid work: {}, fork: {})",
account_a.to_account (),
account_a,
valid_signature,
valid_work,
fork);

7
nano/node/fwd.hpp
View file

@ -19,6 +19,7 @@ class bootstrap_service;
class confirming_set;
class election;
class election_status;
class ledger_notifications;
class local_block_broadcaster;
class local_vote_history;
class logger;
@ -29,18 +30,19 @@ class node_config;
class node_flags;
class node_observers;
class online_reps;
class pruning;
class recently_cemented_cache;
class recently_confirmed_cache;
class rep_crawler;
class rep_tiers;
class http_callbacks;
class telemetry;
class unchecked_map;
class stats;
class vote_cache;
enum class vote_code;
enum class vote_source;
class vote_generator;
class vote_processor;
class vote_rebroadcaster;
class vote_router;
class vote_spacing;
class wallets;
@ -49,6 +51,7 @@ enum class block_source;
enum class election_behavior;
enum class election_state;
enum class vote_code;
enum class vote_source;
}
namespace nano::scheduler

10
nano/node/ipc/flatbuffers_util.cpp
View file

@ -14,7 +14,7 @@ std::unique_ptr<nanoapi::BlockStateT> nano::ipc::flatbuffers_builder::from (nano
block->balance = block_a.balance ().to_string_dec ();
block->link = block_a.link_field ().value ().to_string ();
block->link_as_account = block_a.link_field ().value ().to_account ();
block_a.signature.encode_hex (block->signature);
block->signature = block_a.signature.to_string ();
block->work = nano::to_string_hex (block_a.work);
if (is_state_send_a)
@ -43,7 +43,7 @@ std::unique_ptr<nanoapi::BlockSendT> nano::ipc::flatbuffers_builder::from (nano:
block->balance = block_a.balance ().to_string_dec ();
block->destination = block_a.hashables.destination.to_account ();
block->previous = block_a.previous ().to_string ();
block_a.signature.encode_hex (block->signature);
block->signature = block_a.signature.to_string ();
block->work = nano::to_string_hex (block_a.work);
return block;
}
@ -54,7 +54,7 @@ std::unique_ptr<nanoapi::BlockReceiveT> nano::ipc::flatbuffers_builder::from (na
block->hash = block_a.hash ().to_string ();
block->source = block_a.source_field ().value ().to_string ();
block->previous = block_a.previous ().to_string ();
block_a.signature.encode_hex (block->signature);
block->signature = block_a.signature.to_string ();
block->work = nano::to_string_hex (block_a.work);
return block;
}
@ -66,7 +66,7 @@ std::unique_ptr<nanoapi::BlockOpenT> nano::ipc::flatbuffers_builder::from (nano:
block->source = block_a.source_field ().value ().to_string ();
block->account = block_a.account ().to_account ();
block->representative = block_a.representative_field ().value ().to_account ();
block_a.signature.encode_hex (block->signature);
block->signature = block_a.signature.to_string ();
block->work = nano::to_string_hex (block_a.work);
return block;
}
@ -77,7 +77,7 @@ std::unique_ptr<nanoapi::BlockChangeT> nano::ipc::flatbuffers_builder::from (nan
block->hash = block_a.hash ().to_string ();
block->previous = block_a.previous ().to_string ();
block->representative = block_a.representative_field ().value ().to_account ();
block_a.signature.encode_hex (block->signature);
block->signature = block_a.signature.to_string ();
block->work = nano::to_string_hex (block_a.work);
return block;
}

57
nano/node/json_handler.cpp
View file

@ -618,8 +618,7 @@ void nano::json_handler::account_info ()
response_l.put ("open_block", info.open_block.to_string ());
response_l.put ("representative_block", node.ledger.representative (transaction, info.head).to_string ());
nano::amount balance_l (info.balance);
std::string balance;
balance_l.encode_dec (balance);
std::string balance = balance_l.to_string_dec ();
response_l.put ("balance", balance);
@ -635,8 +634,7 @@ void nano::json_handler::account_info ()
// block_height and confirmed height are the same, so can just reuse balance
confirmed_balance_l = balance_l;
}
std::string confirmed_balance;
confirmed_balance_l.encode_dec (confirmed_balance);
std::string confirmed_balance = confirmed_balance_l.to_string_dec ();
response_l.put ("confirmed_balance", confirmed_balance);
}
@ -1137,6 +1135,19 @@ void nano::json_handler::block_info ()
{
auto account = block->account ();
response_l.put ("block_account", account.to_account ());
bool include_linked_account = request.get<bool> ("include_linked_account", false);
if (include_linked_account)
{
auto linked_account = node.ledger.linked_account (transaction, *block);
if (linked_account.has_value ())
{
response_l.put ("linked_account", linked_account.value ().to_account ());
}
else
{
response_l.put ("linked_account", "0");
}
}
auto amount = node.ledger.any.block_amount (transaction, hash);
if (amount)
{
@ -1275,6 +1286,7 @@ void nano::json_handler::blocks_info ()
bool const receive_hash = request.get<bool> ("receive_hash", false);
bool const source = request.get<bool> ("source", false);
bool const json_block_l = request.get<bool> ("json_block", false);
bool const include_linked_account = request.get<bool> ("include_linked_account", false);
bool const include_not_found = request.get<bool> ("include_not_found", false);
boost::property_tree::ptree blocks;
@ -1294,6 +1306,18 @@ void nano::json_handler::blocks_info ()
boost::property_tree::ptree entry;
auto account = block->account ();
entry.put ("block_account", account.to_account ());
if (include_linked_account)
{
auto linked_account = node.ledger.linked_account (transaction, *block);
if (linked_account.has_value ())
{
entry.put ("linked_account", linked_account.value ().to_account ());
}
else
{
entry.put ("linked_account", "0");
}
}
auto amount = node.ledger.any.block_amount (transaction, hash);
if (amount)
{
@ -2233,8 +2257,7 @@ void nano::json_handler::delegators ()
{
if (info.balance.number () >= threshold.number ())
{
std::string balance;
nano::uint128_union (info.balance).encode_dec (balance);
std::string balance = nano::uint128_union (info.balance).to_string_dec ();
nano::account const & delegator (i->first);
delegators.put (delegator.to_account (), balance);
}
@ -2648,6 +2671,7 @@ void nano::json_handler::account_history ()
if (!ec)
{
boost::property_tree::ptree history;
bool include_linked_account (request.get_optional<bool> ("include_linked_account") == true);
bool output_raw (request.get_optional<bool> ("raw") == true);
response_l.put ("account", account.to_account ());
auto block = node.ledger.any.block_get (transaction, hash);
@ -2664,6 +2688,18 @@ void nano::json_handler::account_history ()
block->visit (visitor);
if (!entry.empty ())
{
if (include_linked_account)
{
auto linked_account = node.ledger.linked_account (transaction, *block);
if (linked_account.has_value ())
{
entry.put ("linked_account", linked_account.value ().to_account ());
}
else
{
entry.put ("linked_account", "0");
}
}
entry.put ("local_timestamp", std::to_string (block->sideband ().timestamp));
entry.put ("height", std::to_string (block->sideband ().height));
entry.put ("hash", hash.to_string ());
@ -2786,8 +2822,7 @@ void nano::json_handler::ledger ()
response_a.put ("frontier", info.head.to_string ());
response_a.put ("open_block", info.open_block.to_string ());
response_a.put ("representative_block", node.ledger.representative (transaction, info.head).to_string ());
std::string balance;
nano::uint128_union (info.balance).encode_dec (balance);
std::string balance = nano::uint128_union (info.balance).to_string_dec ();
response_a.put ("balance", balance);
response_a.put ("modified_timestamp", std::to_string (info.modified));
response_a.put ("block_count", std::to_string (info.block_count));
@ -2839,8 +2874,7 @@ void nano::json_handler::ledger ()
response_a.put ("frontier", info.head.to_string ());
response_a.put ("open_block", info.open_block.to_string ());
response_a.put ("representative_block", node.ledger.representative (transaction, info.head).to_string ());
std::string balance;
(i->first).encode_dec (balance);
std::string balance = (i->first).to_string_dec ();
response_a.put ("balance", balance);
response_a.put ("modified_timestamp", std::to_string (info.modified));
response_a.put ("block_count", std::to_string (info.block_count));
@ -4702,8 +4736,7 @@ void nano::json_handler::wallet_ledger ()
entry.put ("frontier", info->head.to_string ());
entry.put ("open_block", info->open_block.to_string ());
entry.put ("representative_block", node.ledger.representative (block_transaction, info->head).to_string ());
std::string balance;
nano::uint128_union (info->balance).encode_dec (balance);
std::string balance = nano::uint128_union (info->balance).to_string_dec ();
entry.put ("balance", balance);
entry.put ("modified_timestamp", std::to_string (info->modified));
entry.put ("block_count", std::to_string (info->block_count));

138
nano/node/ledger_notifications.cpp Normal file
View file

@ -0,0 +1,138 @@
#include <nano/lib/thread_roles.hpp>
#include <nano/node/ledger_notifications.hpp>
#include <nano/node/nodeconfig.hpp>
#include <nano/secure/transaction.hpp>
nano::ledger_notifications::ledger_notifications (nano::node_config const & config, nano::stats & stats, nano::logger & logger) :
config{ config },
stats{ stats },
logger{ logger }
{
}
nano::ledger_notifications::~ledger_notifications ()
{
debug_assert (!thread.joinable ());
}
void nano::ledger_notifications::start ()
{
debug_assert (!thread.joinable ());
thread = std::thread{ [this] () {
nano::thread_role::set (nano::thread_role::name::ledger_notifications);
run ();
} };
}
void nano::ledger_notifications::stop ()
{
{
nano::lock_guard<nano::mutex> guard{ mutex };
stopped = true;
}
condition.notify_all ();
if (thread.joinable ())
{
thread.join ();
}
}
void nano::ledger_notifications::wait (std::function<void ()> cooldown_action)
{
nano::unique_lock<nano::mutex> lock{ mutex };
condition.wait (lock, [this, &cooldown_action] {
bool predicate = stopped || notifications.size () < config.max_ledger_notifications;
if (!predicate && cooldown_action)
{
cooldown_action ();
}
return predicate;
});
}
void nano::ledger_notifications::notify_processed (nano::secure::write_transaction & transaction, processed_batch_t processed, std::function<void ()> callback)
{
{
nano::lock_guard<nano::mutex> guard{ mutex };
notifications.emplace_back (transaction.get_future (), nano::wrap_move_only ([this, processed = std::move (processed), callback = std::move (callback)] () mutable {
stats.inc (nano::stat::type::ledger_notifications, nano::stat::detail::notify_processed);
// Set results for futures when not holding the lock
for (auto & [result, context] : processed)
{
if (context.callback)
{
context.callback (result);
}
context.set_result (result);
}
blocks_processed.notify (processed);
if (callback)
{
callback ();
}
}));
}
condition.notify_all ();
}
void nano::ledger_notifications::notify_rolled_back (nano::secure::write_transaction & transaction, rolled_back_batch_t batch, nano::qualified_root rollback_root, std::function<void ()> callback)
{
{
nano::lock_guard<nano::mutex> guard{ mutex };
notifications.emplace_back (transaction.get_future (), nano::wrap_move_only ([this, batch = std::move (batch), rollback_root, callback = std::move (callback)] () {
stats.inc (nano::stat::type::ledger_notifications, nano::stat::detail::notify_rolled_back);
blocks_rolled_back.notify (batch, rollback_root);
if (callback)
{
callback ();
}
}));
}
condition.notify_all ();
}
void nano::ledger_notifications::run ()
{
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
condition.wait (lock, [this] {
return stopped || !notifications.empty ();
});
if (stopped)
{
return;
}
while (!notifications.empty ())
{
auto notification = std::move (notifications.front ());
notifications.pop_front ();
lock.unlock ();
auto & [future, callback] = notification;
future.wait (); // Wait for the associated transaction to be committed
callback (); // Notify observers
condition.notify_all (); // Notify waiting threads about possible vacancy
lock.lock ();
}
}
}
nano::container_info nano::ledger_notifications::container_info () const
{
nano::lock_guard<nano::mutex> guard{ mutex };
nano::container_info info;
info.put ("notifications", notifications.size ());
return info;
}

66
nano/node/ledger_notifications.hpp Normal file
View file

@ -0,0 +1,66 @@
#pragma once
#include <nano/lib/function.hpp>
#include <nano/lib/observer_set.hpp>
#include <nano/node/block_context.hpp>
#include <nano/node/fwd.hpp>
#include <nano/secure/common.hpp>
#include <deque>
#include <functional>
#include <future>
#include <thread>
namespace nano
{
class ledger_notifications
{
public: // Events
// All processed blocks including forks, rejected etc
using processed_batch_t = std::deque<std::pair<nano::block_status, nano::block_context>>;
using processed_batch_event_t = nano::observer_set<processed_batch_t>;
processed_batch_event_t blocks_processed;
// Rolled back blocks <rolled back blocks, root of rollback>
using rolled_back_batch_t = std::deque<std::shared_ptr<nano::block>>;
using rolled_back_event_t = nano::observer_set<std::deque<std::shared_ptr<nano::block>>, nano::qualified_root>;
rolled_back_event_t blocks_rolled_back;
public:
ledger_notifications (nano::node_config const &, nano::stats &, nano::logger &);
~ledger_notifications ();
void start ();
void stop ();
/* Components should cooperate to ensure that the notification queue does not grow indefinitely */
void wait (std::function<void ()> cooldown_action = nullptr);
/*
* Write transactions are passed to ensure that notifications are queued in the correct order, which is the same as the order of write transactions
* However, we cannot dispatch notifications before the write transaction is committed otherwise the notified components may not see the changes
* It's an important subtlety and the reason for additional complexity in this and transaction classes
*/
void notify_processed (nano::secure::write_transaction &, processed_batch_t batch, std::function<void ()> callback = nullptr);
void notify_rolled_back (nano::secure::write_transaction &, rolled_back_batch_t batch, nano::qualified_root rollback_root, std::function<void ()> callback = nullptr);
nano::container_info container_info () const;
private: // Dependencies
nano::node_config const & config;
nano::stats & stats;
nano::logger & logger;
private:
void run ();
private:
using entry = std::pair<std::shared_future<void>, std::function<void ()>>; // <transaction commited future, notification callback>
std::deque<entry> notifications;
std::thread thread;
nano::condition_variable condition;
mutable nano::mutex mutex;
bool stopped{ false };
};
}

13
nano/node/local_block_broadcaster.cpp
View file

@ -3,6 +3,7 @@
#include <nano/lib/utility.hpp>
#include <nano/node/block_processor.hpp>
#include <nano/node/confirming_set.hpp>
#include <nano/node/ledger_notifications.hpp>
#include <nano/node/local_block_broadcaster.hpp>
#include <nano/node/network.hpp>
#include <nano/node/node.hpp>
@ -10,10 +11,10 @@
#include <boost/range/iterator_range.hpp>
nano::local_block_broadcaster::local_block_broadcaster (local_block_broadcaster_config const & config_a, nano::node & node_a, nano::block_processor & block_processor_a, nano::network & network_a, nano::confirming_set & confirming_set_a, nano::stats & stats_a, nano::logger & logger_a, bool enabled_a) :
nano::local_block_broadcaster::local_block_broadcaster (local_block_broadcaster_config const & config_a, nano::node & node_a, nano::ledger_notifications & ledger_notifications_a, nano::network & network_a, nano::confirming_set & confirming_set_a, nano::stats & stats_a, nano::logger & logger_a, bool enabled_a) :
config{ config_a },
node{ node_a },
block_processor{ block_processor_a },
ledger_notifications{ ledger_notifications_a },
network{ network_a },
confirming_set{ confirming_set_a },
stats{ stats_a },
@ -26,7 +27,7 @@ nano::local_block_broadcaster::local_block_broadcaster (local_block_broadcaster_
return;
}
block_processor.batch_processed.add ([this] (auto const & batch) {
ledger_notifications.blocks_processed.add ([this] (auto const & batch) {
bool should_notify = false;
for (auto const & [result, context] : batch)
{
@ -56,7 +57,7 @@ nano::local_block_broadcaster::local_block_broadcaster (local_block_broadcaster_
}
});
block_processor.rolled_back.add ([this] (auto const & blocks, auto const & rollback_root) {
ledger_notifications.blocks_rolled_back.add ([this] (auto const & blocks, auto const & rollback_root) {
nano::lock_guard<nano::mutex> guard{ mutex };
for (auto const & block : blocks)
{
@ -127,7 +128,7 @@ void nano::local_block_broadcaster::run ()
{
stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::loop);
if (cleanup_interval.elapsed (config.cleanup_interval))
if (cleanup_interval.elapse (config.cleanup_interval))
{
cleanup (lock);
debug_assert (lock.owns_lock ());
@ -191,7 +192,7 @@ void nano::local_block_broadcaster::run_broadcasts (nano::unique_lock<nano::mute
}
logger.debug (nano::log::type::local_block_broadcaster, "Broadcasting block: {} (rebroadcasts so far: {})",
entry.block->hash ().to_string (),
entry.block->hash (),
entry.rebroadcasts);
stats.inc (nano::stat::type::local_block_broadcaster, nano::stat::detail::broadcast, nano::stat::dir::out);

4
nano/node/local_block_broadcaster.hpp
View file

@ -53,7 +53,7 @@ public:
class local_block_broadcaster final
{
public:
local_block_broadcaster (local_block_broadcaster_config const &, nano::node &, nano::block_processor &, nano::network &, nano::confirming_set &, nano::stats &, nano::logger &, bool enabled = false);
local_block_broadcaster (local_block_broadcaster_config const &, nano::node &, nano::ledger_notifications &, nano::network &, nano::confirming_set &, nano::stats &, nano::logger &, bool enabled = false);
~local_block_broadcaster ();
void start ();
@ -73,7 +73,7 @@ private:
private: // Dependencies
local_block_broadcaster_config const & config;
nano::node & node;
nano::block_processor & block_processor;
nano::ledger_notifications & ledger_notifications;
nano::network & network;
nano::confirming_set & confirming_set;
nano::stats & stats;

26
nano/node/make_store.cpp
View file

@ -1,14 +1,32 @@
#include <nano/lib/logging.hpp>
#include <nano/node/make_store.hpp>
#include <nano/node/nodeconfig.hpp>
#include <nano/store/lmdb/lmdb.hpp>
#include <nano/store/rocksdb/rocksdb.hpp>
std::unique_ptr<nano::store::component> nano::make_store (nano::logger & logger, std::filesystem::path const & path, nano::ledger_constants & constants, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade)
std::unique_ptr<nano::store::component> nano::make_store (nano::logger & logger, std::filesystem::path const & path, nano::ledger_constants & constants, bool read_only, bool add_db_postfix, nano::node_config node_config)
{
if (rocksdb_config.enable)
auto decide_backend = [&] () -> nano::database_backend {
if (node_config.rocksdb_config.enable && node_config.database_backend == nano::database_backend::lmdb)
{
logger.warn (nano::log::type::config, "Use of deprecated `[node.rocksdb].enable` setting detected in config file, defaulting to RocksDB backend.\nPlease edit config-node.toml and use the new '[node].database_backend' for future compatibility.");
return nano::database_backend::rocksdb;
}
return node_config.database_backend;
};
auto backend = decide_backend ();
switch (backend)
{
return std::make_unique<nano::store::rocksdb::component> (logger, add_db_postfix ? path / "rocksdb" : path, constants, rocksdb_config, read_only);
case nano::database_backend::lmdb:
{
return std::make_unique<nano::store::lmdb::component> (logger, add_db_postfix ? path / "data.ldb" : path, constants, node_config.diagnostics_config.txn_tracking, node_config.block_processor_batch_max_time, node_config.lmdb_config, node_config.backup_before_upgrade);
}
case nano::database_backend::rocksdb:
{
return std::make_unique<nano::store::rocksdb::component> (logger, add_db_postfix ? path / "rocksdb" : path, constants, node_config.rocksdb_config, read_only);
}
}
return std::make_unique<nano::store::lmdb::component> (logger, add_db_postfix ? path / "data.ldb" : path, constants, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_config_a, backup_before_upgrade);
release_assert (false); // Must be handled above
}

3
nano/node/make_store.hpp
View file

@ -4,6 +4,7 @@
#include <nano/lib/lmdbconfig.hpp>
#include <nano/lib/logging.hpp>
#include <nano/lib/rocksdbconfig.hpp>
#include <nano/node/nodeconfig.hpp>
#include <chrono>
@ -22,5 +23,5 @@ class component;
namespace nano
{
std::unique_ptr<nano::store::component> make_store (nano::logger &, std::filesystem::path const & path, nano::ledger_constants & constants, bool open_read_only = false, bool add_db_postfix = true, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false);
std::unique_ptr<nano::store::component> make_store (nano::logger &, std::filesystem::path const & path, nano::ledger_constants & constants, bool read_only = false, bool add_db_postfix = true, nano::node_config node_config = nano::node_config{});
}

143
nano/node/network.cpp
View file

@ -247,72 +247,109 @@ void nano::network::send_keepalive_self (std::shared_ptr<nano::transport::channe
channel->send (message, nano::transport::traffic_type::keepalive);
}
void nano::network::flood_message (nano::message const & message, nano::transport::traffic_type type, float scale) const
size_t nano::network::flood_message (nano::message const & message, nano::transport::traffic_type type, float scale) const
{
for (auto const & channel : list (fanout (scale)))
auto channels = list (fanout (scale), [type] (auto const & channel) {
return !channel->max (type); // Only use channels that are not full for this traffic type
});
size_t result = 0;
for (auto const & channel : channels)
{
channel->send (message, type);
bool sent = channel->send (message, type);
result += sent;
}
return result;
}
void nano::network::flood_keepalive (float scale) const
size_t nano::network::flood_keepalive (float scale) const
{
nano::keepalive message{ node.network_params.network };
random_fill (message.peers);
flood_message (message, nano::transport::traffic_type::keepalive, scale);
return flood_message (message, nano::transport::traffic_type::keepalive, scale);
}
void nano::network::flood_keepalive_self (float scale) const
size_t nano::network::flood_keepalive_self (float scale) const
{
nano::keepalive message{ node.network_params.network };
fill_keepalive_self (message.peers);
flood_message (message, nano::transport::traffic_type::keepalive, scale);
return flood_message (message, nano::transport::traffic_type::keepalive, scale);
}
void nano::network::flood_block (std::shared_ptr<nano::block> const & block, nano::transport::traffic_type type) const
size_t nano::network::flood_block (std::shared_ptr<nano::block> const & block, nano::transport::traffic_type type) const
{
nano::publish message{ node.network_params.network, block };
flood_message (message, type);
return flood_message (message, type);
}
void nano::network::flood_block_initial (std::shared_ptr<nano::block> const & block) const
size_t nano::network::flood_block_initial (std::shared_ptr<nano::block> const & block) const
{
nano::publish message{ node.network_params.network, block, /* is_originator */ true };
size_t result = 0;
for (auto const & rep : node.rep_crawler.principal_representatives ())
{
rep.channel->send (message, nano::transport::traffic_type::block_broadcast_initial);
bool sent = rep.channel->send (message, nano::transport::traffic_type::block_broadcast_initial);
result += sent;
}
for (auto & peer : list_non_pr (fanout (1.0)))
{
peer->send (message, nano::transport::traffic_type::block_broadcast_initial);
bool sent = peer->send (message, nano::transport::traffic_type::block_broadcast_initial);
result += sent;
}
return result;
}
void nano::network::flood_vote (std::shared_ptr<nano::vote> const & vote, float scale, bool rebroadcasted) const
size_t nano::network::flood_vote_rebroadcasted (std::shared_ptr<nano::vote> const & vote, float scale) const
{
nano::confirm_ack message{ node.network_params.network, vote, rebroadcasted };
for (auto & channel : list (fanout (scale)))
nano::confirm_ack message{ node.network_params.network, vote, /* rebroadcasted */ true };
auto const type = nano::transport::traffic_type::vote_rebroadcast;
auto channels = list (fanout (scale), [type] (auto const & channel) {
return !channel->max (type); // Only use channels that are not full for this traffic type
});
size_t result = 0;
for (auto & channel : channels)
{
channel->send (message, rebroadcasted ? nano::transport::traffic_type::vote_rebroadcast : nano::transport::traffic_type::vote);
bool sent = channel->send (message, type);
result += sent;
}
return result;
}
void nano::network::flood_vote_non_pr (std::shared_ptr<nano::vote> const & vote, float scale, bool rebroadcasted) const
size_t nano::network::flood_vote_non_pr (std::shared_ptr<nano::vote> const & vote, float scale) const
{
nano::confirm_ack message{ node.network_params.network, vote, rebroadcasted };
for (auto & channel : list_non_pr (fanout (scale)))
nano::confirm_ack message{ node.network_params.network, vote };
auto const type = transport::traffic_type::vote;
auto channels = list_non_pr (fanout (scale), [type] (auto const & channel) {
return !channel->max (type); // Only use channels that are not full for this traffic type
});
size_t result = 0;
for (auto & channel : channels)
{
channel->send (message, rebroadcasted ? nano::transport::traffic_type::vote_rebroadcast : nano::transport::traffic_type::vote);
bool sent = channel->send (message, type);
result += sent;
}
return result;
}
void nano::network::flood_vote_pr (std::shared_ptr<nano::vote> const & vote, bool rebroadcasted) const
size_t nano::network::flood_vote_pr (std::shared_ptr<nano::vote> const & vote) const
{
nano::confirm_ack message{ node.network_params.network, vote, rebroadcasted };
nano::confirm_ack message{ node.network_params.network, vote };
auto const type = nano::transport::traffic_type::vote;
size_t result = 0;
for (auto const & channel : node.rep_crawler.principal_representatives ())
{
channel.channel->send (message, rebroadcasted ? nano::transport::traffic_type::vote_rebroadcast : nano::transport::traffic_type::vote);
bool sent = channel.channel->send (message, type);
result += sent;
}
return result;
}
void nano::network::flood_block_many (std::deque<std::shared_ptr<nano::block>> blocks, nano::transport::traffic_type type, std::chrono::milliseconds delay, std::function<void ()> callback) const
@ -357,12 +394,12 @@ bool nano::network::merge_peer (nano::endpoint const & peer)
if (track_reachout (peer))
{
node.stats.inc (nano::stat::type::network, nano::stat::detail::merge_peer);
node.logger.debug (nano::log::type::network, "Initiating peer merge: {}", fmt::streamed (peer));
node.logger.debug (nano::log::type::network, "Initiating peer merge: {}", peer);
bool started = tcp_channels.start_tcp (peer);
if (!started)
{
node.stats.inc (nano::stat::type::tcp, nano::stat::detail::merge_peer_failed);
node.logger.debug (nano::log::type::network, "Peer merge failed: {}", fmt::streamed (peer));
node.logger.debug (nano::log::type::network, "Peer merge failed: {}", peer);
}
return started;
}
@ -397,9 +434,9 @@ bool nano::network::track_reachout (nano::endpoint const & endpoint_a)
return tcp_channels.track_reachout (endpoint_a);
}
std::deque<std::shared_ptr<nano::transport::channel>> nano::network::list (std::size_t max_count, uint8_t minimum_version) const
std::deque<std::shared_ptr<nano::transport::channel>> nano::network::list (std::size_t max_count, channel_filter filter) const
{
auto result = tcp_channels.list (minimum_version);
auto result = tcp_channels.list (filter);
nano::random_pool_shuffle (result.begin (), result.end ()); // Randomize returned peer order
if (max_count > 0 && result.size () > max_count)
{
@ -408,9 +445,9 @@ std::deque<std::shared_ptr<nano::transport::channel>> nano::network::list (std::
return result;
}
std::deque<std::shared_ptr<nano::transport::channel>> nano::network::list_non_pr (std::size_t max_count, uint8_t minimum_version) const
std::deque<std::shared_ptr<nano::transport::channel>> nano::network::list_non_pr (std::size_t max_count, channel_filter filter) const
{
auto result = tcp_channels.list (minimum_version);
auto result = tcp_channels.list (filter);
auto partition_point = std::partition (result.begin (), result.end (),
[this] (std::shared_ptr<nano::transport::channel> const & channel) {
@ -427,10 +464,21 @@ std::deque<std::shared_ptr<nano::transport::channel>> nano::network::list_non_pr
return result;
}
std::deque<std::shared_ptr<nano::transport::channel>> nano::network::list (std::size_t max_count, uint8_t minimum_version) const
{
return list (max_count, [minimum_version] (auto const & channel) { return channel->get_network_version () >= minimum_version; });
}
std::deque<std::shared_ptr<nano::transport::channel>> nano::network::list_non_pr (std::size_t max_count, uint8_t minimum_version) const
{
return list_non_pr (max_count, [minimum_version] (auto const & channel) { return channel->get_network_version () >= minimum_version; });
}
// Simulating with sqrt_broadcast_simulate shows we only need to broadcast to sqrt(total_peers) random peers in order to successfully publish to everyone with high probability
std::size_t nano::network::fanout (float scale) const
{
return static_cast<std::size_t> (std::ceil (scale * size_sqrt ()));
auto fanout_l = std::max (static_cast<float> (config.minimum_fanout), size_log ());
return static_cast<std::size_t> (std::ceil (scale * fanout_l));
}
std::unordered_set<std::shared_ptr<nano::transport::channel>> nano::network::random_set (std::size_t max_count, uint8_t minimum_version) const
@ -519,9 +567,10 @@ std::size_t nano::network::size () const
return tcp_channels.size ();
}
float nano::network::size_sqrt () const
float nano::network::size_log () const
{
return static_cast<float> (std::sqrt (size ()));
auto size_l = std::max (static_cast<size_t> (1u), size ()); // Clamp size to domain of std::log
return static_cast<float> (std::log (size_l));
}
bool nano::network::empty () const
@ -735,4 +784,34 @@ nano::container_info nano::syn_cookies::container_info () const
info.put ("syn_cookies", cookies.size ());
info.put ("syn_cookies_per_ip", cookies_per_ip.size ());
return info;
}
/*
* network_config
*/
nano::error nano::network_config::serialize (nano::tomlconfig & toml) const
{
toml.put ("peer_reachout", peer_reachout.count (), "Time between attempts to reach out to peers. \ntype:milliseconds");
toml.put ("cached_peer_reachout", cached_peer_reachout.count (), "Time between attempts to reach out to cached peers. \ntype:milliseconds");
toml.put ("max_peers_per_ip", max_peers_per_ip, "Maximum number of peers allowed from a single IP address. \ntype:size_t");
toml.put ("max_peers_per_subnetwork", max_peers_per_subnetwork, "Maximum number of peers allowed from the same subnetwork. \ntype:size_t");
toml.put ("duplicate_filter_size", duplicate_filter_size, "Size of the duplicate detection filter. \ntype:size_t");
toml.put ("duplicate_filter_cutoff", duplicate_filter_cutoff, "Time in seconds before a duplicate entry expires. \ntype:uint64");
toml.put ("minimum_fanout", minimum_fanout, "Minimum number of peers to fan out messages to. \ntype:size_t");
return toml.get_error ();
}
nano::error nano::network_config::deserialize (nano::tomlconfig & toml)
{
toml.get_duration ("peer_reachout", peer_reachout);
toml.get_duration ("cached_peer_reachout", cached_peer_reachout);
toml.get ("max_peers_per_ip", max_peers_per_ip);
toml.get ("max_peers_per_subnetwork", max_peers_per_subnetwork);
toml.get ("duplicate_filter_size", duplicate_filter_size);
toml.get ("duplicate_filter_cutoff", duplicate_filter_cutoff);
toml.get ("minimum_fanout", minimum_fanout);
return toml.get_error ();
}

32
nano/node/network.hpp
View file

@ -66,7 +66,8 @@ public:
}
}
// TODO: Serialization & deserialization
nano::error deserialize (nano::tomlconfig &);
nano::error serialize (nano::tomlconfig &) const;
public:
std::chrono::milliseconds peer_reachout{ 250ms };
@ -79,6 +80,8 @@ public:
size_t duplicate_filter_size{ 1024 * 1024 };
uint64_t duplicate_filter_cutoff{ 60 };
size_t minimum_fanout{ 2 };
};
class network final
@ -92,16 +95,16 @@ public:
nano::endpoint endpoint () const;
void flood_message (nano::message const &, nano::transport::traffic_type, float scale = 1.0f) const;
void flood_keepalive (float scale = 1.0f) const;
void flood_keepalive_self (float scale = 0.5f) const;
void flood_vote (std::shared_ptr<nano::vote> const &, float scale, bool rebroadcasted = false) const;
void flood_vote_pr (std::shared_ptr<nano::vote> const &, bool rebroadcasted = false) const;
void flood_vote_non_pr (std::shared_ptr<nano::vote> const &, float scale, bool rebroadcasted = false) const;
size_t flood_message (nano::message const &, nano::transport::traffic_type, float scale = 1.0f) const;
size_t flood_keepalive (float scale = 1.0f) const;
size_t flood_keepalive_self (float scale = 0.5f) const;
size_t flood_vote_pr (std::shared_ptr<nano::vote> const &) const;
size_t flood_vote_non_pr (std::shared_ptr<nano::vote> const &, float scale) const;
size_t flood_vote_rebroadcasted (std::shared_ptr<nano::vote> const &, float scale) const;
// Flood block to all PRs and a random selection of non-PRs
void flood_block_initial (std::shared_ptr<nano::block> const &) const;
size_t flood_block_initial (std::shared_ptr<nano::block> const &) const;
// Flood block to a random selection of peers
void flood_block (std::shared_ptr<nano::block> const &, nano::transport::traffic_type) const;
size_t flood_block (std::shared_ptr<nano::block> const &, nano::transport::traffic_type) const;
void flood_block_many (std::deque<std::shared_ptr<nano::block>>, nano::transport::traffic_type, std::chrono::milliseconds delay = 10ms, std::function<void ()> callback = nullptr) const;
void send_keepalive (std::shared_ptr<nano::transport::channel> const &) const;
@ -118,8 +121,13 @@ public:
// Should we reach out to this endpoint with a keepalive message? If yes, register a new reachout attempt
bool track_reachout (nano::endpoint const &);
std::deque<std::shared_ptr<nano::transport::channel>> list (std::size_t max_count = 0, uint8_t minimum_version = 0) const;
std::deque<std::shared_ptr<nano::transport::channel>> list_non_pr (std::size_t max_count, uint8_t minimum_version = 0) const;
using channel_filter = std::function<bool (std::shared_ptr<nano::transport::channel> const &)>;
std::deque<std::shared_ptr<nano::transport::channel>> list (std::size_t max_count = 0, channel_filter = nullptr) const;
std::deque<std::shared_ptr<nano::transport::channel>> list_non_pr (std::size_t max_count = 0, channel_filter = nullptr) const;
std::deque<std::shared_ptr<nano::transport::channel>> list (std::size_t max_count, uint8_t minimum_version) const;
std::deque<std::shared_ptr<nano::transport::channel>> list_non_pr (std::size_t max_count, uint8_t minimum_version) const;
// Desired fanout for a given scale
std::size_t fanout (float scale = 1.0f) const;
@ -134,7 +142,7 @@ public:
nano::tcp_endpoint bootstrap_peer ();
void cleanup (std::chrono::steady_clock::time_point const & cutoff);
std::size_t size () const;
float size_sqrt () const;
float size_log () const;
bool empty () const;
void erase (nano::transport::channel const &);
/** Disconnects and adds peer to exclusion list */

397
nano/node/node.cpp
View file

@ -20,6 +20,7 @@
#include <nano/node/daemonconfig.hpp>
#include <nano/node/election_status.hpp>
#include <nano/node/endpoint.hpp>
#include <nano/node/ledger_notifications.hpp>
#include <nano/node/local_block_broadcaster.hpp>
#include <nano/node/local_vote_history.hpp>
#include <nano/node/make_store.hpp>
@ -29,16 +30,20 @@
#include <nano/node/online_reps.hpp>
#include <nano/node/peer_history.hpp>
#include <nano/node/portmapping.hpp>
#include <nano/node/pruning.hpp>
#include <nano/node/request_aggregator.hpp>
#include <nano/node/rpc_callbacks.hpp>
#include <nano/node/scheduler/component.hpp>
#include <nano/node/scheduler/hinted.hpp>
#include <nano/node/scheduler/manual.hpp>
#include <nano/node/scheduler/optimistic.hpp>
#include <nano/node/scheduler/priority.hpp>
#include <nano/node/telemetry.hpp>
#include <nano/node/transport/loopback.hpp>
#include <nano/node/transport/tcp_listener.hpp>
#include <nano/node/vote_generator.hpp>
#include <nano/node/vote_processor.hpp>
#include <nano/node/vote_rebroadcaster.hpp>
#include <nano/node/vote_router.hpp>
#include <nano/node/websocket.hpp>
#include <nano/secure/ledger.hpp>
@ -105,7 +110,7 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
work{ work_a },
distributed_work_impl{ std::make_unique<nano::distributed_work_factory> (*this) },
distributed_work{ *distributed_work_impl },
store_impl{ nano::make_store (logger, application_path_a, network_params.ledger, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_config, config_a.backup_before_upgrade) },
store_impl{ nano::make_store (logger, application_path_a, network_params.ledger, flags.read_only, true, config_a) },
store{ *store_impl },
unchecked_impl{ std::make_unique<nano::unchecked_map> (config.max_unchecked_blocks, stats, flags.disable_block_processor_unchecked_deletion) },
unchecked{ *unchecked_impl },
@ -113,8 +118,10 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
wallets_store{ *wallets_store_impl },
wallets_impl{ std::make_unique<nano::wallets> (wallets_store.init_error (), *this) },
wallets{ *wallets_impl },
ledger_impl{ std::make_unique<nano::ledger> (store, stats, network_params.ledger, flags_a.generate_cache, config_a.representative_vote_weight_minimum.number ()) },
ledger_impl{ std::make_unique<nano::ledger> (store, network_params.ledger, stats, logger, flags_a.generate_cache, config_a.representative_vote_weight_minimum.number ()) },
ledger{ *ledger_impl },
ledger_notifications_impl{ std::make_unique<nano::ledger_notifications> (config, stats, logger) },
ledger_notifications{ *ledger_notifications_impl },
outbound_limiter_impl{ std::make_unique<nano::bandwidth_limiter> (config) },
outbound_limiter{ *outbound_limiter_impl },
message_processor_impl{ std::make_unique<nano::message_processor> (config.message_processor, *this) },
@ -124,6 +131,7 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
//
network_impl{ std::make_unique<nano::network> (*this, config.peering_port.has_value () ? *config.peering_port : 0) },
network{ *network_impl },
loopback_channel{ std::make_shared<nano::transport::loopback_channel> (*this) },
telemetry_impl{ std::make_unique<nano::telemetry> (flags, *this, network, observers, network_params, stats) },
telemetry{ *telemetry_impl },
// BEWARE: `bootstrap` takes `network.port` instead of `config.peering_port` because when the user doesn't specify
@ -137,13 +145,13 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
tcp_listener{ *tcp_listener_impl },
port_mapping_impl{ std::make_unique<nano::port_mapping> (*this) },
port_mapping{ *port_mapping_impl },
block_processor_impl{ std::make_unique<nano::block_processor> (config, ledger, unchecked, stats, logger) },
block_processor_impl{ std::make_unique<nano::block_processor> (config, ledger, ledger_notifications, unchecked, stats, logger) },
block_processor{ *block_processor_impl },
confirming_set_impl{ std::make_unique<nano::confirming_set> (config.confirming_set, ledger, block_processor, stats, logger) },
confirming_set_impl{ std::make_unique<nano::confirming_set> (config.confirming_set, ledger, ledger_notifications, stats, logger) },
confirming_set{ *confirming_set_impl },
bucketing_impl{ std::make_unique<nano::bucketing> () },
bucketing{ *bucketing_impl },
active_impl{ std::make_unique<nano::active_elections> (*this, confirming_set, block_processor) },
active_impl{ std::make_unique<nano::active_elections> (*this, ledger_notifications, confirming_set) },
active{ *active_impl },
online_reps_impl{ std::make_unique<nano::online_reps> (config, ledger, stats, logger) },
online_reps{ *online_reps_impl },
@ -165,41 +173,43 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
vote_processor{ *vote_processor_impl },
vote_cache_processor_impl{ std::make_unique<nano::vote_cache_processor> (config.vote_processor, vote_router, vote_cache, stats, logger) },
vote_cache_processor{ *vote_cache_processor_impl },
generator_impl{ std::make_unique<nano::vote_generator> (config, *this, ledger, wallets, vote_processor, history, network, stats, logger, /* non-final */ false) },
generator_impl{ std::make_unique<nano::vote_generator> (config, *this, ledger, wallets, vote_processor, history, network, stats, logger, /* non-final */ false, loopback_channel) },
generator{ *generator_impl },
final_generator_impl{ std::make_unique<nano::vote_generator> (config, *this, ledger, wallets, vote_processor, history, network, stats, logger, /* final */ true) },
final_generator_impl{ std::make_unique<nano::vote_generator> (config, *this, ledger, wallets, vote_processor, history, network, stats, logger, /* final */ true, loopback_channel) },
final_generator{ *final_generator_impl },
scheduler_impl{ std::make_unique<nano::scheduler::component> (config, *this, ledger, bucketing, block_processor, active, online_reps, vote_cache, confirming_set, stats, logger) },
scheduler_impl{ std::make_unique<nano::scheduler::component> (config, *this, ledger, ledger_notifications, bucketing, active, online_reps, vote_cache, confirming_set, stats, logger) },
scheduler{ *scheduler_impl },
aggregator_impl{ std::make_unique<nano::request_aggregator> (config.request_aggregator, *this, stats, generator, final_generator, history, ledger, wallets, vote_router) },
aggregator_impl{ std::make_unique<nano::request_aggregator> (config.request_aggregator, *this, generator, final_generator, history, ledger, wallets, vote_router) },
aggregator{ *aggregator_impl },
backlog_scan_impl{ std::make_unique<nano::backlog_scan> (config.backlog_scan, ledger, stats) },
backlog_scan{ *backlog_scan_impl },
backlog_impl{ std::make_unique<nano::bounded_backlog> (config, *this, ledger, bucketing, backlog_scan, block_processor, confirming_set, stats, logger) },
backlog_impl{ std::make_unique<nano::bounded_backlog> (config, *this, ledger, ledger_notifications, bucketing, backlog_scan, block_processor, confirming_set, stats, logger) },
backlog{ *backlog_impl },
bootstrap_server_impl{ std::make_unique<nano::bootstrap_server> (config.bootstrap_server, store, ledger, network_params.network, stats) },
bootstrap_server{ *bootstrap_server_impl },
bootstrap_impl{ std::make_unique<nano::bootstrap_service> (config, block_processor, ledger, network, stats, logger) },
bootstrap_impl{ std::make_unique<nano::bootstrap_service> (config, ledger, ledger_notifications, block_processor, network, stats, logger) },
bootstrap{ *bootstrap_impl },
websocket_impl{ std::make_unique<nano::websocket_server> (config.websocket_config, observers, wallets, ledger, io_ctx, logger) },
websocket_impl{ std::make_unique<nano::websocket_server> (config.websocket_config, *this, observers, wallets, ledger, io_ctx, logger) },
websocket{ *websocket_impl },
epoch_upgrader_impl{ std::make_unique<nano::epoch_upgrader> (*this, ledger, store, network_params, logger) },
epoch_upgrader{ *epoch_upgrader_impl },
local_block_broadcaster_impl{ std::make_unique<nano::local_block_broadcaster> (config.local_block_broadcaster, *this, block_processor, network, confirming_set, stats, logger, !flags.disable_block_processor_republishing) },
local_block_broadcaster_impl{ std::make_unique<nano::local_block_broadcaster> (config.local_block_broadcaster, *this, ledger_notifications, network, confirming_set, stats, logger, !flags.disable_block_processor_republishing) },
local_block_broadcaster{ *local_block_broadcaster_impl },
process_live_dispatcher_impl{ std::make_unique<nano::process_live_dispatcher> (ledger, scheduler.priority, vote_cache, websocket) },
process_live_dispatcher{ *process_live_dispatcher_impl },
peer_history_impl{ std::make_unique<nano::peer_history> (config.peer_history, store, network, logger, stats) },
peer_history{ *peer_history_impl },
monitor_impl{ std::make_unique<nano::monitor> (config.monitor, *this) },
monitor{ *monitor_impl },
http_callbacks_impl{ std::make_unique<nano::http_callbacks> (*this) },
http_callbacks{ *http_callbacks_impl },
pruning_impl{ std::make_unique<nano::pruning> (config, flags, ledger, stats, logger) },
pruning{ *pruning_impl },
vote_rebroadcaster_impl{ std::make_unique<nano::vote_rebroadcaster> (vote_router, network, wallets, stats, logger) },
vote_rebroadcaster{ *vote_rebroadcaster_impl },
startup_time{ std::chrono::steady_clock::now () },
node_seq{ seq }
{
logger.debug (nano::log::type::node, "Constructing node...");
process_live_dispatcher.connect (block_processor);
vote_cache.rep_weight_query = [this] (nano::account const & rep) {
return ledger.weight (rep);
};
@ -214,34 +224,55 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
}
});
// Republish vote if it is new and the node does not host a principal representative (or close to)
vote_router.vote_processed.add ([this] (std::shared_ptr<nano::vote> const & vote, nano::vote_source source, std::unordered_map<nano::block_hash, nano::vote_code> const & results) {
bool processed = std::any_of (results.begin (), results.end (), [] (auto const & result) {
return result.second == nano::vote_code::vote;
});
if (processed)
{
auto const reps = wallets.reps ();
if (!reps.have_half_rep () && !reps.exists (vote->account))
{
network.flood_vote (vote, 0.5f, /* rebroadcasted */ true);
}
}
});
// Do some cleanup due to this block never being processed by confirmation height processor
confirming_set.cementing_failed.add ([this] (auto const & hash) {
active.recently_confirmed.erase (hash);
});
// Announce new blocks via websocket
ledger_notifications.blocks_processed.add ([this] (auto const & batch) {
auto const transaction = ledger.tx_begin_read ();
for (auto const & [result, context] : batch)
{
if (result == nano::block_status::progress)
{
if (websocket.server && websocket.server->any_subscriber (nano::websocket::topic::new_unconfirmed_block))
{
websocket.server->broadcast (nano::websocket::message_builder (ledger).new_block_arrived (*context.block));
}
}
}
});
// Do some cleanup of rolled back blocks
block_processor.rolled_back.add ([this] (auto const & blocks, auto const & rollback_root) {
ledger_notifications.blocks_rolled_back.add ([this] (auto const & blocks, auto const & rollback_root) {
for (auto const & block : blocks)
{
history.erase (block->root ());
}
});
// Representative is defined as online if replying to live votes or rep crawler queries
observers.vote.add ([this] (std::shared_ptr<nano::vote> vote, std::shared_ptr<nano::transport::channel> const & channel, nano::vote_source source, nano::vote_code code) {
release_assert (vote != nullptr);
release_assert (channel != nullptr);
debug_assert (code != nano::vote_code::invalid);
// Track rep weight voting on live elections
bool should_observe = (code != nano::vote_code::indeterminate);
// Ignore republished votes when rep crawling
if (source == nano::vote_source::live)
{
should_observe |= rep_crawler.process (vote, channel);
}
if (should_observe)
{
online_reps.observe (vote->account);
}
});
if (!init_error ())
{
wallets.observer = [this] (bool active) {
@ -250,90 +281,11 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
network.disconnect_observer = [this] () {
observers.disconnect.notify ();
};
if (!config.callback_address.empty ())
{
observers.blocks.add ([this] (nano::election_status const & status_a, std::vector<nano::vote_with_weight_info> const & votes_a, nano::account const & account_a, nano::amount const & amount_a, bool is_state_send_a, bool is_state_epoch_a) {
auto block_a (status_a.winner);
if ((status_a.type == nano::election_status_type::active_confirmed_quorum || status_a.type == nano::election_status_type::active_confirmation_height))
{
auto node_l (shared_from_this ());
io_ctx.post ([node_l, block_a, account_a, amount_a, is_state_send_a, is_state_epoch_a] () {
boost::property_tree::ptree event;
event.add ("account", account_a.to_account ());
event.add ("hash", block_a->hash ().to_string ());
std::string block_text;
block_a->serialize_json (block_text);
event.add ("block", block_text);
event.add ("amount", amount_a.to_string_dec ());
if (is_state_send_a)
{
event.add ("is_send", is_state_send_a);
event.add ("subtype", "send");
}
// Subtype field
else if (block_a->type () == nano::block_type::state)
{
if (block_a->is_change ())
{
event.add ("subtype", "change");
}
else if (is_state_epoch_a)
{
debug_assert (amount_a == 0 && node_l->ledger.is_epoch_link (block_a->link_field ().value ()));
event.add ("subtype", "epoch");
}
else
{
event.add ("subtype", "receive");
}
}
std::stringstream ostream;
boost::property_tree::write_json (ostream, event);
ostream.flush ();
auto body (std::make_shared<std::string> (ostream.str ()));
auto address (node_l->config.callback_address);
auto port (node_l->config.callback_port);
auto target (std::make_shared<std::string> (node_l->config.callback_target));
auto resolver (std::make_shared<boost::asio::ip::tcp::resolver> (node_l->io_ctx));
resolver->async_resolve (boost::asio::ip::tcp::resolver::query (address, std::to_string (port)), [node_l, address, port, target, body, resolver] (boost::system::error_code const & ec, boost::asio::ip::tcp::resolver::iterator i_a) {
if (!ec)
{
node_l->do_rpc_callback (i_a, address, port, target, body, resolver);
}
else
{
node_l->logger.error (nano::log::type::rpc_callbacks, "Error resolving callback: {}:{} ({})", address, port, ec.message ());
node_l->stats.inc (nano::stat::type::error, nano::stat::detail::http_callback, nano::stat::dir::out);
}
});
});
}
});
}
observers.channel_connected.add ([this] (std::shared_ptr<nano::transport::channel> const & channel) {
network.send_keepalive_self (channel);
});
observers.vote.add ([this] (std::shared_ptr<nano::vote> vote, std::shared_ptr<nano::transport::channel> const & channel, nano::vote_source source, nano::vote_code code) {
debug_assert (vote != nullptr);
debug_assert (code != nano::vote_code::invalid);
if (channel == nullptr)
{
return; // Channel expired when waiting for vote to be processed
}
// Ignore republished votes
if (source == nano::vote_source::live)
{
bool active_in_rep_crawler = rep_crawler.process (vote, channel);
if (active_in_rep_crawler)
{
// Representative is defined as online if replying to live votes or rep_crawler queries
online_reps.observe (vote->account);
}
}
});
// Cancelling local work generation
observers.work_cancel.add ([this] (nano::root const & root_a) {
this->work.cancel (root_a);
@ -347,6 +299,7 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
logger.info (nano::log::type::node, "Active network: {}", network_label);
logger.info (nano::log::type::node, "Database backend: {}", store.vendor_get ());
logger.info (nano::log::type::node, "Data path: {}", application_path.string ());
logger.info (nano::log::type::node, "Ledger path: {}", store.get_database_path ().string ());
logger.info (nano::log::type::node, "Work pool threads: {} ({})", work.threads.size (), (work.opencl ? "OpenCL" : "CPU"));
logger.info (nano::log::type::node, "Work peers: {}", config.work_peers.size ());
logger.info (nano::log::type::node, "Node ID: {}", node_id.pub.to_node_id ());
@ -407,18 +360,16 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
{
auto const bootstrap_weights = get_bootstrap_weights ();
ledger.bootstrap_weight_max_blocks = bootstrap_weights.first;
ledger.bootstrap_weights = bootstrap_weights.second;
logger.info (nano::log::type::node, "Initial bootstrap height: {}", ledger.bootstrap_weight_max_blocks);
logger.info (nano::log::type::node, "Current ledger height: {}", ledger.block_count ());
logger.info (nano::log::type::node, "Initial bootstrap height: {:>10}", ledger.bootstrap_weight_max_blocks);
logger.info (nano::log::type::node, "Current ledger height: {:>10}", ledger.block_count ());
// Use bootstrap weights if initial bootstrap is not completed
const bool use_bootstrap_weight = ledger.block_count () < bootstrap_weights.first;
const bool use_bootstrap_weight = !ledger.bootstrap_height_reached ();
if (use_bootstrap_weight)
{
logger.info (nano::log::type::node, "Using predefined representative weights, since block count is less than bootstrap threshold");
ledger.bootstrap_weights = bootstrap_weights.second;
logger.info (nano::log::type::node, "******************************************** Bootstrap weights ********************************************");
// Sort the weights
@ -463,6 +414,11 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
}
});
}
else
{
logger.error (nano::log::type::node, "Failed to initialize node");
}
node_initialized_latch.count_down ();
}
@ -472,68 +428,6 @@ nano::node::~node ()
stop ();
}
// TODO: Move to a separate class
void nano::node::do_rpc_callback (boost::asio::ip::tcp::resolver::iterator i_a, std::string const & address, uint16_t port, std::shared_ptr<std::string> const & target, std::shared_ptr<std::string> const & body, std::shared_ptr<boost::asio::ip::tcp::resolver> const & resolver)
{
if (i_a != boost::asio::ip::tcp::resolver::iterator{})
{
auto node_l (shared_from_this ());
auto sock (std::make_shared<boost::asio::ip::tcp::socket> (node_l->io_ctx));
sock->async_connect (i_a->endpoint (), [node_l, target, body, sock, address, port, i_a, resolver] (boost::system::error_code const & ec) mutable {
if (!ec)
{
auto req (std::make_shared<boost::beast::http::request<boost::beast::http::string_body>> ());
req->method (boost::beast::http::verb::post);
req->target (*target);
req->version (11);
req->insert (boost::beast::http::field::host, address);
req->insert (boost::beast::http::field::content_type, "application/json");
req->body () = *body;
req->prepare_payload ();
boost::beast::http::async_write (*sock, *req, [node_l, sock, address, port, req, i_a, target, body, resolver] (boost::system::error_code const & ec, std::size_t bytes_transferred) mutable {
if (!ec)
{
auto sb (std::make_shared<boost::beast::flat_buffer> ());
auto resp (std::make_shared<boost::beast::http::response<boost::beast::http::string_body>> ());
boost::beast::http::async_read (*sock, *sb, *resp, [node_l, sb, resp, sock, address, port, i_a, target, body, resolver] (boost::system::error_code const & ec, std::size_t bytes_transferred) mutable {
if (!ec)
{
if (boost::beast::http::to_status_class (resp->result ()) == boost::beast::http::status_class::successful)
{
node_l->stats.inc (nano::stat::type::http_callback, nano::stat::detail::initiate, nano::stat::dir::out);
}
else
{
node_l->logger.error (nano::log::type::rpc_callbacks, "Callback to {}:{} failed [status: {}]", address, port, nano::util::to_str (resp->result ()));
node_l->stats.inc (nano::stat::type::error, nano::stat::detail::http_callback, nano::stat::dir::out);
}
}
else
{
node_l->logger.error (nano::log::type::rpc_callbacks, "Unable to complete callback: {}:{} ({})", address, port, ec.message ());
node_l->stats.inc (nano::stat::type::error, nano::stat::detail::http_callback, nano::stat::dir::out);
};
});
}
else
{
node_l->logger.error (nano::log::type::rpc_callbacks, "Unable to send callback: {}:{} ({})", address, port, ec.message ());
node_l->stats.inc (nano::stat::type::error, nano::stat::detail::http_callback, nano::stat::dir::out);
}
});
}
else
{
node_l->logger.error (nano::log::type::rpc_callbacks, "Unable to connect to callback address({}): {}:{} ({})", address, i_a->endpoint ().address ().to_string (), port, ec.message ());
node_l->stats.inc (nano::stat::type::error, nano::stat::detail::http_callback, nano::stat::dir::out);
++i_a;
node_l->do_rpc_callback (i_a, address, port, target, body, resolver);
}
});
}
}
bool nano::node::copy_with_compaction (std::filesystem::path const & destination)
{
return store.copy_db (destination);
@ -585,7 +479,7 @@ void nano::node::process_active (std::shared_ptr<nano::block> const & incoming)
[[nodiscard]] nano::block_status nano::node::process (secure::write_transaction const & transaction, std::shared_ptr<nano::block> block)
{
auto status = ledger.process (transaction, block);
logger.debug (nano::log::type::node, "Directly processed block: {} (status: {})", block->hash ().to_string (), to_string (status));
logger.debug (nano::log::type::node, "Directly processed block: {} (status: {})", block->hash (), to_string (status));
return status;
}
@ -610,13 +504,6 @@ void nano::node::start ()
network.start ();
message_processor.start ();
if (flags.enable_pruning)
{
auto this_l (shared ());
workers.post ([this_l] () {
this_l->ongoing_ledger_pruning ();
});
}
if (!flags.disable_rep_crawler)
{
rep_crawler.start ();
@ -658,6 +545,7 @@ void nano::node::start ()
rep_tiers.start ();
vote_processor.start ();
vote_cache_processor.start ();
ledger_notifications.start ();
block_processor.start ();
active.start ();
generator.start ();
@ -677,6 +565,9 @@ void nano::node::start ()
vote_router.start ();
online_reps.start ();
monitor.start ();
http_callbacks.start ();
pruning.start ();
vote_rebroadcaster.start ();
add_initial_peers ();
}
@ -713,6 +604,7 @@ void nano::node::stop ()
generator.stop ();
final_generator.stop ();
confirming_set.stop ();
ledger_notifications.stop ();
telemetry.stop ();
websocket.stop ();
bootstrap_server.stop ();
@ -724,6 +616,9 @@ void nano::node::stop ()
message_processor.stop ();
network.stop ();
monitor.stop ();
http_callbacks.stop ();
pruning.stop ();
vote_rebroadcaster.stop ();
bootstrap_workers.stop ();
wallet_workers.stop ();
@ -819,110 +714,6 @@ void nano::node::search_receivable_all ()
});
}
bool nano::node::collect_ledger_pruning_targets (std::deque<nano::block_hash> & pruning_targets_a, nano::account & last_account_a, uint64_t const batch_read_size_a, uint64_t const max_depth_a, uint64_t const cutoff_time_a)
{
uint64_t read_operations (0);
bool finish_transaction (false);
auto transaction = ledger.tx_begin_read ();
for (auto i (store.confirmation_height.begin (transaction, last_account_a)), n (store.confirmation_height.end (transaction)); i != n && !finish_transaction;)
{
++read_operations;
auto const & account (i->first);
nano::block_hash hash (i->second.frontier);
uint64_t depth (0);
while (!hash.is_zero () && depth < max_depth_a)
{
auto block = ledger.any.block_get (transaction, hash);
if (block != nullptr)
{
if (block->sideband ().timestamp > cutoff_time_a || depth == 0)
{
hash = block->previous ();
}
else
{
break;
}
}
else
{
release_assert (depth != 0);
hash = 0;
}
if (++depth % batch_read_size_a == 0)
{
// FIXME: This is triggering an assertion where the iterator is still used after transaction is refreshed
transaction.refresh ();
}
}
if (!hash.is_zero ())
{
pruning_targets_a.push_back (hash);
}
read_operations += depth;
if (read_operations >= batch_read_size_a)
{
last_account_a = inc_sat (account.number ());
finish_transaction = true;
}
else
{
++i;
}
}
return !finish_transaction || last_account_a.is_zero ();
}
void nano::node::ledger_pruning (uint64_t const batch_size_a, bool bootstrap_weight_reached_a)
{
uint64_t const max_depth (config.max_pruning_depth != 0 ? config.max_pruning_depth : std::numeric_limits<uint64_t>::max ());
uint64_t const cutoff_time (bootstrap_weight_reached_a ? nano::seconds_since_epoch () - config.max_pruning_age.count () : std::numeric_limits<uint64_t>::max ());
uint64_t pruned_count (0);
uint64_t transaction_write_count (0);
nano::account last_account (1); // 0 Burn account is never opened. So it can be used to break loop
std::deque<nano::block_hash> pruning_targets;
bool target_finished (false);
while ((transaction_write_count != 0 || !target_finished) && !stopped)
{
// Search pruning targets
while (pruning_targets.size () < batch_size_a && !target_finished && !stopped)
{
target_finished = collect_ledger_pruning_targets (pruning_targets, last_account, batch_size_a * 2, max_depth, cutoff_time);
}
// Pruning write operation
transaction_write_count = 0;
if (!pruning_targets.empty () && !stopped)
{
auto write_transaction = ledger.tx_begin_write (nano::store::writer::pruning);
while (!pruning_targets.empty () && transaction_write_count < batch_size_a && !stopped)
{
auto const & pruning_hash (pruning_targets.front ());
auto account_pruned_count (ledger.pruning_action (write_transaction, pruning_hash, batch_size_a));
transaction_write_count += account_pruned_count;
pruning_targets.pop_front ();
}
pruned_count += transaction_write_count;
logger.debug (nano::log::type::prunning, "Pruned blocks: {}", pruned_count);
}
}
logger.debug (nano::log::type::prunning, "Total recently pruned block count: {}", pruned_count);
}
void nano::node::ongoing_ledger_pruning ()
{
auto bootstrap_weight_reached (ledger.block_count () >= ledger.bootstrap_weight_max_blocks);
ledger_pruning (flags.block_processor_batch_size != 0 ? flags.block_processor_batch_size : 2 * 1024, bootstrap_weight_reached);
auto const ledger_pruning_interval (bootstrap_weight_reached ? config.max_pruning_age : std::min (config.max_pruning_age, std::chrono::seconds (15 * 60)));
auto this_l (shared ());
workers.post_delayed (ledger_pruning_interval, [this_l] () {
this_l->workers.post ([this_l] () {
this_l->ongoing_ledger_pruning ();
});
});
}
uint64_t nano::node::default_difficulty (nano::work_version const version_a) const
{
uint64_t result{ std::numeric_limits<uint64_t>::max () };
@ -1204,6 +995,9 @@ nano::container_info nano::node::container_info () const
info.add ("bandwidth", outbound_limiter.container_info ());
info.add ("backlog_scan", backlog_scan.container_info ());
info.add ("bounded_backlog", backlog.container_info ());
info.add ("http_callbacks", http_callbacks.container_info ());
info.add ("pruning", pruning.container_info ());
info.add ("vote_rebroadcaster", vote_rebroadcaster.container_info ());
return info;
}
@ -1213,29 +1007,36 @@ nano::container_info nano::node::container_info () const
nano::keypair nano::load_or_create_node_id (std::filesystem::path const & application_path)
{
auto & logger = nano::default_logger ();
logger.info (nano::log::type::init, "Using data directory: {}", application_path.string ());
auto node_private_key_path = application_path / "node_id_private.key";
std::ifstream ifs (node_private_key_path.c_str ());
if (ifs.good ())
{
nano::default_logger ().info (nano::log::type::init, "Reading node id from: '{}'", node_private_key_path.string ());
std::string node_private_key;
ifs >> node_private_key;
release_assert (node_private_key.size () == 64);
nano::keypair kp = nano::keypair (node_private_key);
logger.info (nano::log::type::init, "Loaded local node ID: {}",
kp.pub.to_node_id ());
return kp;
}
else
{
// no node_id found, generate new one
nano::default_logger ().info (nano::log::type::init, "Generating a new node id, saving to: '{}'", node_private_key_path.string ());
nano::keypair kp;
std::ofstream ofs (node_private_key_path.c_str (), std::ofstream::out | std::ofstream::trunc);
ofs << kp.prv.to_string () << std::endl
<< std::flush;
ofs.close ();
release_assert (!ofs.fail ());
logger.info (nano::log::type::init, "Generated new local node ID: {}",
kp.pub.to_node_id ());
return kp;
}
}
}

16
nano/node/node.hpp
View file

@ -13,7 +13,6 @@
#include <nano/node/nodeconfig.hpp>
#include <nano/node/online_reps.hpp>
#include <nano/node/portmapping.hpp>
#include <nano/node/process_live_dispatcher.hpp>
#include <nano/node/rep_tiers.hpp>
#include <nano/node/repcrawler.hpp>
#include <nano/node/transport/tcp_server.hpp>
@ -61,9 +60,6 @@ public:
nano::uint128_t minimum_principal_weight ();
void backup_wallet ();
void search_receivable_all ();
bool collect_ledger_pruning_targets (std::deque<nano::block_hash> &, nano::account &, uint64_t const, uint64_t const, uint64_t const);
void ledger_pruning (uint64_t const, bool);
void ongoing_ledger_pruning ();
// The default difficulty updates to base only when the first epoch_2 block is processed
uint64_t default_difficulty (nano::work_version const) const;
uint64_t default_receive_difficulty (nano::work_version const) const;
@ -84,7 +80,6 @@ public:
uint64_t block_count () const;
uint64_t cemented_count () const;
void do_rpc_callback (boost::asio::ip::tcp::resolver::iterator i_a, std::string const &, uint16_t, std::shared_ptr<std::string> const &, std::shared_ptr<std::string> const &, std::shared_ptr<boost::asio::ip::tcp::resolver> const &);
bool online () const;
bool init_error () const;
std::pair<uint64_t, std::unordered_map<nano::account, nano::uint128_t>> get_bootstrap_weights () const;
@ -135,12 +130,15 @@ public:
nano::wallets & wallets;
std::unique_ptr<nano::ledger> ledger_impl;
nano::ledger & ledger;
std::unique_ptr<nano::ledger_notifications> ledger_notifications_impl;
nano::ledger_notifications & ledger_notifications;
std::unique_ptr<nano::bandwidth_limiter> outbound_limiter_impl;
nano::bandwidth_limiter & outbound_limiter;
std::unique_ptr<nano::message_processor> message_processor_impl;
nano::message_processor & message_processor;
std::unique_ptr<nano::network> network_impl;
nano::network & network;
std::shared_ptr<nano::transport::channel> loopback_channel;
std::unique_ptr<nano::telemetry> telemetry_impl;
nano::telemetry & telemetry;
std::unique_ptr<nano::transport::tcp_listener> tcp_listener_impl;
@ -197,12 +195,16 @@ public:
nano::epoch_upgrader & epoch_upgrader;
std::unique_ptr<nano::local_block_broadcaster> local_block_broadcaster_impl;
nano::local_block_broadcaster & local_block_broadcaster;
std::unique_ptr<nano::process_live_dispatcher> process_live_dispatcher_impl;
nano::process_live_dispatcher & process_live_dispatcher;
std::unique_ptr<nano::peer_history> peer_history_impl;
nano::peer_history & peer_history;
std::unique_ptr<nano::monitor> monitor_impl;
nano::monitor & monitor;
std::unique_ptr<nano::http_callbacks> http_callbacks_impl;
nano::http_callbacks & http_callbacks;
std::unique_ptr<nano::pruning> pruning_impl;
nano::pruning & pruning;
std::unique_ptr<nano::vote_rebroadcaster> vote_rebroadcaster_impl;
nano::vote_rebroadcaster & vote_rebroadcaster;
public:
std::chrono::steady_clock::time_point const startup_time;

73
nano/node/nodeconfig.cpp
View file

@ -250,6 +250,10 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
tcp.serialize (tcp_l);
toml.put_child ("tcp", tcp_l);
nano::tomlconfig network_l;
network.serialize (network_l);
toml.put_child ("network", network_l);
nano::tomlconfig request_aggregator_l;
request_aggregator.serialize (request_aggregator_l);
toml.put_child ("request_aggregator", request_aggregator_l);
@ -391,6 +395,12 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
tcp.deserialize (config_l);
}
if (toml.has_key ("network"))
{
auto config_l = toml.get_required_child ("network");
network.deserialize (config_l);
}
if (toml.has_key ("request_aggregator"))
{
auto config_l = toml.get_required_child ("request_aggregator");
@ -537,6 +547,19 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
toml.get<bool> ("allow_local_peers", allow_local_peers);
toml.get<unsigned> (signature_checker_threads_key, signature_checker_threads);
if (toml.has_key ("database_backend"))
{
auto backend_str = toml.get<std::string> ("database_backend");
if (auto backend = parse_database_backend (backend_str))
{
database_backend = backend.value ();
}
else
{
toml.get_error ().set ("Unknown database_backend type: " + backend_str);
}
}
if (toml.has_key ("lmdb"))
{
auto lmdb_config_l (toml.get_required_child ("lmdb"));
@ -679,3 +702,53 @@ std::optional<unsigned> nano::node_config::env_io_threads ()
}();
return value;
}
std::optional<nano::database_backend> nano::node_config::env_database_backend ()
{
static auto const backend = [] () -> std::optional<nano::database_backend> {
if (auto value = nano::env::get<std::string> ("NANO_BACKEND"))
{
auto backend = parse_database_backend (*value);
if (backend.has_value ())
{
std::cerr << "Default database backend overridden by NANO_BACKEND environment variable: " << to_string (*backend) << std::endl;
}
else
{
std::cerr << "Unknown database backend in NANO_BACKEND environment variable: " << *value << std::endl;
}
return backend;
}
return std::nullopt;
}();
return backend;
}
/*
* database_backend
*/
std::string nano::to_string (nano::database_backend const value)
{
switch (value)
{
case nano::database_backend::lmdb:
return "lmdb";
case nano::database_backend::rocksdb:
return "rocksdb";
}
release_assert (false);
}
std::optional<nano::database_backend> nano::parse_database_backend (std::string const & value)
{
if (value == "lmdb")
{
return nano::database_backend::lmdb;
}
if (value == "rocksdb")
{
return nano::database_backend::rocksdb;
}
return {};
}

14
nano/node/nodeconfig.hpp
View file

@ -43,6 +43,15 @@ namespace nano
{
class tomlconfig;
enum class database_backend
{
lmdb,
rocksdb
};
std::string to_string (database_backend);
std::optional<database_backend> parse_database_backend (std::string const &);
/**
* Node configuration
*/
@ -133,7 +142,9 @@ public:
uint64_t max_pruning_depth{ 0 };
nano::rocksdb_config rocksdb_config;
nano::lmdb_config lmdb_config;
nano::database_backend database_backend{ env_database_backend ().value_or (nano::database_backend::lmdb) };
bool enable_upnp{ true };
std::size_t max_ledger_notifications{ 8 };
public:
nano::vote_cache_config vote_cache;
@ -156,7 +167,8 @@ public:
/** Entry is ignored if it cannot be parsed as a valid address:port */
void deserialize_address (std::string const &, std::vector<std::pair<std::string, uint16_t>> &) const;
private:
public:
static std::optional<nano::database_backend> env_database_backend ();
static std::optional<unsigned> env_io_threads ();
};

40
nano/node/online_reps.cpp
View file

@ -32,7 +32,7 @@ void nano::online_reps::start ()
nano::lock_guard<nano::mutex> lock{ mutex };
cached_trended = trended_l;
logger.info (nano::log::type::online_reps, "Initial trended weight: {}", fmt::streamed (cached_trended));
logger.info (nano::log::type::online_reps, "Initial trended weight: {}", cached_trended);
}
thread = std::thread ([this] () {
@ -66,26 +66,40 @@ void nano::online_reps::observe (nano::account const & rep)
stats.inc (nano::stat::type::online_reps, new_insert ? nano::stat::detail::rep_new : nano::stat::detail::rep_update);
bool trimmed = trim ();
// Update current online weight if anything changed
if (new_insert || trimmed)
if (new_insert)
{
stats.inc (nano::stat::type::online_reps, nano::stat::detail::update_online);
logger.debug (nano::log::type::online_reps, "Observed new representative: {}", rep.to_account ());
cached_online = calculate_online ();
}
}
}
bool nano::online_reps::trim ()
void nano::online_reps::trim ()
{
debug_assert (!mutex.try_lock ());
auto now = std::chrono::steady_clock::now ();
auto cutoff = reps.get<tag_time> ().lower_bound (now - config.network_params.node.weight_interval);
auto trimmed = reps.get<tag_time> ().begin () != cutoff;
reps.get<tag_time> ().erase (reps.get<tag_time> ().begin (), cutoff);
return trimmed;
auto const now = std::chrono::steady_clock::now ();
auto const cutoff = now - config.network_params.node.weight_interval * 2;
while (reps.get<tag_time> ().begin () != reps.get<tag_time> ().end ())
{
auto oldest = reps.get<tag_time> ().begin ();
if (oldest->time < cutoff)
{
stats.inc (nano::stat::type::online_reps, nano::stat::detail::rep_trim);
logger.debug (nano::log::type::online_reps, "Removing representative: {}, last observed: {}s ago",
oldest->account.to_account (),
nano::log::seconds_delta (oldest->time, now));
reps.get<tag_time> ().erase (oldest);
}
else
{
break; // Entries are ordered by timestamp, break early
}
}
}
void nano::online_reps::run ()
@ -100,6 +114,7 @@ void nano::online_reps::run ()
});
if (!stopped)
{
trim ();
lock.unlock ();
sample ();
lock.lock ();
@ -125,7 +140,8 @@ void nano::online_reps::sample ()
nano::lock_guard<nano::mutex> lock{ mutex };
cached_trended = trended_l;
}
logger.info (nano::log::type::online_reps, "Updated trended weight: {}", fmt::streamed (trended_l));
logger.info (nano::log::type::online_reps, "Updated trended weight: {}", trended_l);
}
nano::uint128_t nano::online_reps::calculate_online () const
@ -281,12 +297,14 @@ void nano::online_reps::force_online_weight (nano::uint128_t const & online_weig
release_assert (nano::is_dev_run ());
nano::lock_guard<nano::mutex> lock{ mutex };
cached_online = online_weight;
logger.debug (nano::log::type::online_reps, "Forced online weight: {}", online_weight);
}
void nano::online_reps::force_sample ()
{
release_assert (nano::is_dev_run ());
sample ();
logger.debug (nano::log::type::online_reps, "Forced sample call");
}
nano::container_info nano::online_reps::container_info () const

4
nano/node/online_reps.hpp
View file

@ -40,7 +40,9 @@ public:
nano::uint128_t delta () const;
/** List of online representatives, both the currently sampling ones and the ones observed in the previous sampling period */
std::vector<nano::account> list ();
void clear ();
nano::container_info container_info () const;
public:
@ -57,7 +59,7 @@ private:
void run ();
/** Called periodically to sample online weight */
void sample ();
bool trim ();
void trim ();
/** Remove old records from the database */
void trim_trended (nano::store::write_transaction const &);
/** Iterate over all database samples and remove invalid records. This is meant to clean potential leftovers from previous versions. */

4
nano/node/peer_history.cpp
View file

@ -92,7 +92,7 @@ void nano::peer_history::run_one ()
if (!exists)
{
stats.inc (nano::stat::type::peer_history, nano::stat::detail::inserted);
logger.debug (nano::log::type::peer_history, "Saved new peer: {}", fmt::streamed (endpoint));
logger.debug (nano::log::type::peer_history, "Saved new peer: {}", endpoint);
}
else
{
@ -116,7 +116,7 @@ void nano::peer_history::run_one ()
stats.inc (nano::stat::type::peer_history, nano::stat::detail::erased);
logger.debug (nano::log::type::peer_history, "Erased peer: {} (not seen for {}s)",
fmt::streamed (endpoint.endpoint ()),
endpoint.endpoint (),
nano::log::seconds_delta (timestamp));
}
}

50
nano/node/process_live_dispatcher.cpp
View file

@ -1,50 +0,0 @@
#include <nano/lib/blocks.hpp>
#include <nano/node/block_processor.hpp>
#include <nano/node/process_live_dispatcher.hpp>
#include <nano/node/scheduler/priority.hpp>
#include <nano/node/vote_cache.hpp>
#include <nano/node/websocket.hpp>
#include <nano/secure/common.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/transaction.hpp>
#include <nano/store/component.hpp>
nano::process_live_dispatcher::process_live_dispatcher (nano::ledger & ledger, nano::scheduler::priority & scheduler, nano::vote_cache & vote_cache, nano::websocket_server & websocket) :
ledger{ ledger },
scheduler{ scheduler },
vote_cache{ vote_cache },
websocket{ websocket }
{
}
void nano::process_live_dispatcher::connect (nano::block_processor & block_processor)
{
block_processor.batch_processed.add ([this] (auto const & batch) {
auto const transaction = ledger.tx_begin_read ();
for (auto const & [result, context] : batch)
{
debug_assert (context.block != nullptr);
inspect (result, *context.block, transaction);
}
});
}
void nano::process_live_dispatcher::inspect (nano::block_status const & result, nano::block const & block, secure::transaction const & transaction)
{
switch (result)
{
case nano::block_status::progress:
process_live (block, transaction);
break;
default:
break;
}
}
void nano::process_live_dispatcher::process_live (nano::block const & block, secure::transaction const & transaction)
{
if (websocket.server && websocket.server->any_subscriber (nano::websocket::topic::new_unconfirmed_block))
{
websocket.server->broadcast (nano::websocket::message_builder ().new_block_arrived (block));
}
}

39
nano/node/process_live_dispatcher.hpp
View file

@ -1,39 +0,0 @@
#pragma once
namespace nano::secure
{
class transaction;
}
namespace nano
{
class ledger;
class vote_cache;
class websocket_server;
class block_processor;
class process_return;
class block;
namespace scheduler
{
class priority;
}
// Observes confirmed blocks and dispatches the process_live function.
class process_live_dispatcher
{
public:
process_live_dispatcher (nano::ledger &, nano::scheduler::priority &, nano::vote_cache &, nano::websocket_server &);
void connect (nano::block_processor & block_processor);
private:
// Block_processor observer
void inspect (nano::block_status const & result, nano::block const & block, secure::transaction const & transaction);
void process_live (nano::block const & block, secure::transaction const & transaction);
nano::ledger & ledger;
nano::scheduler::priority & scheduler;
nano::vote_cache & vote_cache;
nano::websocket_server & websocket;
};
}

152
nano/node/pruning.cpp Normal file
View file

@ -0,0 +1,152 @@
#include <nano/node/node.hpp>
#include <nano/node/pruning.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_any.hpp>
nano::pruning::pruning (nano::node_config const & config_a, nano::node_flags const & flags_a, nano::ledger & ledger_a, nano::stats & stats_a, nano::logger & logger_a) :
config{ config_a },
flags{ flags_a },
ledger{ ledger_a },
stats{ stats_a },
logger{ logger_a },
workers{ /* single threaded */ 1, nano::thread_role::name::pruning }
{
}
nano::pruning::~pruning ()
{
// Must be stopped before destruction
debug_assert (stopped);
}
void nano::pruning::start ()
{
if (flags.enable_pruning)
{
workers.start ();
workers.post ([this] () {
ongoing_ledger_pruning ();
});
}
}
void nano::pruning::stop ()
{
stopped = true;
workers.stop ();
}
void nano::pruning::ongoing_ledger_pruning ()
{
auto bootstrap_weight_reached (ledger.block_count () >= ledger.bootstrap_weight_max_blocks);
ledger_pruning (flags.block_processor_batch_size != 0 ? flags.block_processor_batch_size : 2 * 1024, bootstrap_weight_reached);
auto const ledger_pruning_interval (bootstrap_weight_reached ? config.max_pruning_age : std::min (config.max_pruning_age, std::chrono::seconds (15 * 60)));
workers.post_delayed (ledger_pruning_interval, [this] () {
workers.post ([this] () {
ongoing_ledger_pruning ();
});
});
}
void nano::pruning::ledger_pruning (uint64_t const batch_size_a, bool bootstrap_weight_reached_a)
{
stats.inc (nano::stat::type::pruning, nano::stat::detail::ledger_pruning);
uint64_t const max_depth (config.max_pruning_depth != 0 ? config.max_pruning_depth : std::numeric_limits<uint64_t>::max ());
uint64_t const cutoff_time (bootstrap_weight_reached_a ? nano::seconds_since_epoch () - config.max_pruning_age.count () : std::numeric_limits<uint64_t>::max ());
uint64_t pruned_count (0);
uint64_t transaction_write_count (0);
nano::account last_account (1); // 0 Burn account is never opened. So it can be used to break loop
std::deque<nano::block_hash> pruning_targets;
bool target_finished (false);
while ((transaction_write_count != 0 || !target_finished) && !stopped)
{
// Search pruning targets
while (pruning_targets.size () < batch_size_a && !target_finished && !stopped)
{
stats.inc (nano::stat::type::pruning, nano::stat::detail::collect_targets);
target_finished = collect_ledger_pruning_targets (pruning_targets, last_account, batch_size_a * 2, max_depth, cutoff_time);
}
// Pruning write operation
transaction_write_count = 0;
if (!pruning_targets.empty () && !stopped)
{
auto write_transaction = ledger.tx_begin_write (nano::store::writer::pruning);
while (!pruning_targets.empty () && transaction_write_count < batch_size_a && !stopped)
{
stats.inc (nano::stat::type::pruning, nano::stat::detail::pruning_target);
auto const & pruning_hash (pruning_targets.front ());
auto account_pruned_count (ledger.pruning_action (write_transaction, pruning_hash, batch_size_a));
transaction_write_count += account_pruned_count;
pruning_targets.pop_front ();
stats.add (nano::stat::type::pruning, nano::stat::detail::pruned_count, account_pruned_count);
}
pruned_count += transaction_write_count;
logger.debug (nano::log::type::pruning, "Pruned blocks: {}", pruned_count);
}
}
logger.debug (nano::log::type::pruning, "Total recently pruned block count: {}", pruned_count);
}
bool nano::pruning::collect_ledger_pruning_targets (std::deque<nano::block_hash> & pruning_targets_a, nano::account & last_account_a, uint64_t const batch_read_size_a, uint64_t const max_depth_a, uint64_t const cutoff_time_a)
{
uint64_t read_operations (0);
bool finish_transaction (false);
auto transaction = ledger.tx_begin_read ();
for (auto i (ledger.store.confirmation_height.begin (transaction, last_account_a)), n (ledger.store.confirmation_height.end (transaction)); i != n && !finish_transaction;)
{
++read_operations;
auto const & account (i->first);
nano::block_hash hash (i->second.frontier);
uint64_t depth (0);
while (!hash.is_zero () && depth < max_depth_a)
{
auto block = ledger.any.block_get (transaction, hash);
if (block != nullptr)
{
if (block->sideband ().timestamp > cutoff_time_a || depth == 0)
{
hash = block->previous ();
}
else
{
break;
}
}
else
{
release_assert (depth != 0);
hash = 0;
}
if (++depth % batch_read_size_a == 0)
{
// FIXME: This is triggering an assertion where the iterator is still used after transaction is refreshed
transaction.refresh ();
}
}
if (!hash.is_zero ())
{
pruning_targets_a.push_back (hash);
}
read_operations += depth;
if (read_operations >= batch_read_size_a)
{
last_account_a = inc_sat (account.number ());
finish_transaction = true;
}
else
{
++i;
}
}
return !finish_transaction || last_account_a.is_zero ();
}
nano::container_info nano::pruning::container_info () const
{
return workers.container_info ();
}

39
nano/node/pruning.hpp Normal file
View file

@ -0,0 +1,39 @@
#pragma once
#include <nano/lib/numbers.hpp>
#include <nano/node/fwd.hpp>
#include <deque>
#include <thread>
namespace nano
{
class pruning final
{
public:
pruning (nano::node_config const &, nano::node_flags const &, nano::ledger &, nano::stats &, nano::logger &);
~pruning ();
void start ();
void stop ();
nano::container_info container_info () const;
void ongoing_ledger_pruning ();
void ledger_pruning (uint64_t batch_size, bool bootstrap_weight_reached);
bool collect_ledger_pruning_targets (std::deque<nano::block_hash> & pruning_targets_out, nano::account & last_account_out, uint64_t batch_read_size, uint64_t max_depth, uint64_t cutoff_time);
private: // Dependencies
nano::node_config const & config;
nano::node_flags const & flags;
nano::ledger & ledger;
nano::stats & stats;
nano::logger & logger;
private:
void run ();
std::atomic<bool> stopped{ false };
nano::thread_pool workers;
};
}

2
nano/node/rep_tiers.cpp
View file

@ -86,7 +86,7 @@ void nano::rep_tiers::run ()
void nano::rep_tiers::calculate_tiers ()
{
auto stake = online_reps.trended ();
auto rep_amounts = ledger.cache.rep_weights.get_rep_amounts ();
auto rep_amounts = ledger.rep_weights_snapshot ();
decltype (representatives_1) representatives_1_l;
decltype (representatives_2) representatives_2_l;

67
nano/node/repcrawler.cpp
View file

@ -3,6 +3,7 @@
#include <nano/node/online_reps.hpp>
#include <nano/node/repcrawler.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/ledger_set_confirmed.hpp>
#include <nano/secure/vote.hpp>
#include <ranges>
@ -37,6 +38,11 @@ void nano::rep_crawler::start ()
{
debug_assert (!thread.joinable ());
if (node.flags.disable_rep_crawler)
{
return;
}
thread = std::thread{ [this] () {
nano::thread_role::set (nano::thread_role::name::rep_crawler);
run ();
@ -91,8 +97,8 @@ void nano::rep_crawler::validate_and_process (nano::unique_lock<nano::mutex> & l
if (rep_weight < minimum)
{
logger.debug (nano::log::type::rep_crawler, "Ignoring vote from account: {} with too little voting weight: {}",
vote->account.to_account (),
fmt::streamed (rep_weight));
vote->account,
rep_weight);
continue; // Skip this vote
}
@ -129,11 +135,16 @@ void nano::rep_crawler::validate_and_process (nano::unique_lock<nano::mutex> & l
if (inserted)
{
logger.info (nano::log::type::rep_crawler, "Found representative: {} at: {}", vote->account.to_account (), channel->to_string ());
logger.info (nano::log::type::rep_crawler, "Found representative: {} at: {}",
vote->account.to_account (),
channel->to_string ());
}
if (updated)
{
logger.warn (nano::log::type::rep_crawler, "Updated representative: {} at: {} (was at: {})", vote->account.to_account (), channel->to_string (), prev_channel->to_string ());
logger.warn (nano::log::type::rep_crawler, "Updated representative: {} at: {} (was at: {})",
vote->account.to_account (),
channel->to_string (),
prev_channel->to_string ());
}
}
}
@ -208,6 +219,13 @@ void nano::rep_crawler::run ()
lock.lock ();
}
// Query local representative
{
lock.unlock ();
query (node.loopback_channel);
lock.lock ();
}
debug_assert (lock.owns_lock ());
}
}
@ -220,8 +238,8 @@ void nano::rep_crawler::cleanup ()
erase_if (reps, [this] (rep_entry const & rep) {
if (!rep.channel->alive ())
{
logger.info (nano::log::type::rep_crawler, "Evicting representative: {} with dead channel at: {}", rep.account.to_account (), rep.channel->to_string ());
stats.inc (nano::stat::type::rep_crawler, nano::stat::detail::channel_dead);
logger.info (nano::log::type::rep_crawler, "Evicting representative: {} with dead channel at: {}", rep.account, rep.channel->to_string ());
return true; // Erase
}
return false;
@ -233,13 +251,13 @@ void nano::rep_crawler::cleanup ()
{
if (query.replies == 0)
{
logger.debug (nano::log::type::rep_crawler, "Aborting unresponsive query for block: {} from: {}", query.hash.to_string (), query.channel->to_string ());
stats.inc (nano::stat::type::rep_crawler, nano::stat::detail::query_timeout);
logger.debug (nano::log::type::rep_crawler, "Aborting unresponsive query for block: {} from: {}", query.hash, query.channel->to_string ());
}
else
{
logger.debug (nano::log::type::rep_crawler, "Completion of query with: {} replies for block: {} from: {}", query.replies, query.hash.to_string (), query.channel->to_string ());
stats.inc (nano::stat::type::rep_crawler, nano::stat::detail::query_completion);
logger.debug (nano::log::type::rep_crawler, "Completion of query with: {} replies for block: {} from: {}", query.replies, query.hash, query.channel->to_string ());
}
return true; // Erase
}
@ -286,22 +304,32 @@ std::deque<std::shared_ptr<nano::transport::channel>> nano::rep_crawler::prepare
return { random_peers.begin (), random_peers.end () };
}
auto nano::rep_crawler::prepare_query_target () const -> std::optional<hash_root_t>
auto nano::rep_crawler::prepare_query_target () const -> hash_root_t
{
constexpr int max_attempts = 10;
constexpr int max_attempts = 32;
auto transaction = node.ledger.tx_begin_read ();
auto random_blocks = node.ledger.random_blocks (transaction, max_attempts);
for (auto const & block : random_blocks)
{
if (!active.recently_confirmed.exists (block->hash ()))
// Avoid blocks that could still have live votes coming in
if (active.recently_confirmed.exists (block->hash ()))
{
return std::make_pair (block->hash (), block->root ());
continue;
}
// Nodes will not respond to queries for blocks that are not confirmed
if (!node.ledger.confirmed.block_exists (transaction, block->hash ()))
{
continue;
}
return std::make_pair (block->hash (), block->root ());
}
return std::nullopt;
// If no suitable block was found, query genesis
return std::make_pair (node.network_params.ledger.genesis->hash (), node.network_params.ledger.genesis->root ());
}
bool nano::rep_crawler::track_rep_request (hash_root_t hash_root, std::shared_ptr<nano::transport::channel> const & channel)
@ -329,14 +357,7 @@ bool nano::rep_crawler::track_rep_request (hash_root_t hash_root, std::shared_pt
void nano::rep_crawler::query (std::deque<std::shared_ptr<nano::transport::channel>> const & target_channels)
{
auto maybe_hash_root = prepare_query_target ();
if (!maybe_hash_root)
{
logger.debug (nano::log::type::rep_crawler, "No block to query");
stats.inc (nano::stat::type::rep_crawler, nano::stat::detail::query_target_failed);
return;
}
auto hash_root = *maybe_hash_root;
auto hash_root = prepare_query_target ();
nano::lock_guard<nano::mutex> lock{ mutex };
@ -345,8 +366,8 @@ void nano::rep_crawler::query (std::deque<std::shared_ptr<nano::transport::chann
bool tracked = track_rep_request (hash_root, channel);
if (tracked)
{
logger.debug (nano::log::type::rep_crawler, "Sending query for block: {} to: {}", hash_root.first.to_string (), channel->to_string ());
stats.inc (nano::stat::type::rep_crawler, nano::stat::detail::query_sent);
logger.debug (nano::log::type::rep_crawler, "Sending query for block: {} to: {}", hash_root.first, channel->to_string ());
auto const & [hash, root] = hash_root;
nano::confirm_req req{ network_constants, hash, root };
@ -357,8 +378,8 @@ void nano::rep_crawler::query (std::deque<std::shared_ptr<nano::transport::chann
}
else
{
logger.debug (nano::log::type::rep_crawler, "Ignoring duplicate query for block: {} to: {}", hash_root.first.to_string (), channel->to_string ());
stats.inc (nano::stat::type::rep_crawler, nano::stat::detail::query_duplicate);
logger.debug (nano::log::type::rep_crawler, "Ignoring duplicate query for block: {} to: {}", hash_root.first, channel->to_string ());
}
}
}
@ -393,8 +414,8 @@ bool nano::rep_crawler::process (std::shared_ptr<nano::vote> const & vote, std::
});
if (found)
{
logger.debug (nano::log::type::rep_crawler, "Processing response for block: {} from: {}", target_hash.to_string (), channel->to_string ());
stats.inc (nano::stat::type::rep_crawler, nano::stat::detail::response);
logger.debug (nano::log::type::rep_crawler, "Processing response for block: {} from: {}", target_hash, channel->to_string ());
// Track response time
stats.sample (nano::stat::sample::rep_response_time, nano::log::milliseconds_delta (it->time), { 0, config.query_timeout.count () });

Some files were not shown because too many files have changed in this diff Show more