From 2e838b448587258d7b6b26e8de4e983510bc7f30 Mon Sep 17 00:00:00 2001 From: theohax <81556890+theohax@users.noreply.github.com> Date: Tue, 6 Jul 2021 16:45:25 +0300 Subject: [PATCH] Depth-first search ledger walking (#3324) * Senders discovery draft * DFS draft without checkpoints * DFS algo improvement attempt * DFS algo ++ * Latest DFS changes * Self-contain DFS traversal algorithm and introduce unit tests for it * Take out test code * Fix build * Address code review; use dependent_blocks_visitor to track block dependencies * Add more ledger_walker unit tests * Formatting * Add diskhash as a git submodule and dependency of nano-node Signed-off-by: theohax * Use diskhash in the ledger walker implementation Signed-off-by: theohax * Fix formatting Signed-off-by: theohax * Add todo note Signed-off-by: theohax * Fix format Signed-off-by: theohax * Adding submodules through get, not just .gitmodules * Build diskhash with CMake instead of make Signed-off-by: theohax * Use hybrid diskhash/in-memory-hash for ledger forward and backwards walking * Build diskhash as static lib instead of shared * Make ledger walker's diskhash key size larger to accomodate stringified uint64s * Address code review -- move null block check to call site Co-authored-by: clemahieu --- .gitmodules | 3 + CMakeLists.txt | 4 + diskhash | 1 + nano/core_test/CMakeLists.txt | 1 + nano/core_test/ledger_walker.cpp | 220 +++++++++++++++++++++++++++++++ nano/node/CMakeLists.txt | 3 + nano/node/ledger_walker.cpp | 175 ++++++++++++++++++++++++ nano/node/ledger_walker.hpp | 60 +++++++++ 8 files changed, 467 insertions(+) create mode 160000 diskhash create mode 100644 nano/core_test/ledger_walker.cpp create mode 100644 nano/node/ledger_walker.cpp create mode 100644 nano/node/ledger_walker.hpp diff --git a/.gitmodules b/.gitmodules index 47b40b9e..a59cedb4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -27,3 +27,6 @@ path = rocksdb url = https://github.com/nanocurrency/rocksdb.git branch = 6.13.3 +[submodule "diskhash"] + path = diskhash + url = https://github.com/luispedro/diskhash.git diff --git a/CMakeLists.txt b/CMakeLists.txt index c279bfce..dbacf14f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -378,6 +378,10 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules") find_package(Boost 1.70.0 REQUIRED COMPONENTS filesystem log log_setup thread program_options system) +# diskhash +add_library(diskhash STATIC ${CMAKE_SOURCE_DIR}/diskhash/src/diskhash.c) +include_directories(diskhash/src) + # RocksDB include_directories(rocksdb/include) set(USE_RTTI diff --git a/diskhash b/diskhash new file mode 160000 index 00000000..4fe2547b --- /dev/null +++ b/diskhash @@ -0,0 +1 @@ +Subproject commit 4fe2547bad3e1bc95e9a9d5df12a6e3913b27574 diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index 220dfb8d..65b3cb69 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable( gap_cache.cpp ipc.cpp ledger.cpp + ledger_walker.cpp locks.cpp logger.cpp message.cpp diff --git a/nano/core_test/ledger_walker.cpp b/nano/core_test/ledger_walker.cpp new file mode 100644 index 00000000..cf8b60a1 --- /dev/null +++ b/nano/core_test/ledger_walker.cpp @@ -0,0 +1,220 @@ +#include +#include +#include + +#include + +#include + +using namespace std::chrono_literals; + +TEST (ledger_walker, genesis_block) +{ + nano::system system{}; + const auto node = system.add_node (); + + nano::ledger_walker ledger_walker{ node->ledger }; + + std::size_t walked_blocks_count = 0; + ledger_walker.walk_backward (nano::genesis_hash, + [&] (const auto & block) { + ++walked_blocks_count; + EXPECT_EQ (block->hash (), nano::genesis_hash); + }); + + EXPECT_EQ (walked_blocks_count, 1); + + walked_blocks_count = 0; + ledger_walker.walk (nano::genesis_hash, + [&] (const auto & block) { + ++walked_blocks_count; + EXPECT_EQ (block->hash (), nano::genesis_hash); + }); + + EXPECT_EQ (walked_blocks_count, 1); +} + +namespace nano +{ +TEST (ledger_walker, genesis_account_longer) +{ + nano::system system{}; + nano::node_config node_config (nano::get_available_port (), system.logging); + node_config.enable_voting = true; + node_config.receive_minimum = 1; + + const auto node = system.add_node (node_config); + + nano::ledger_walker ledger_walker{ node->ledger }; + EXPECT_TRUE (ledger_walker.walked_blocks.empty ()); + EXPECT_EQ (1, ledger_walker.walked_blocks.bucket_count ()); + EXPECT_TRUE (ledger_walker.blocks_to_walk.empty ()); + + const auto get_number_of_walked_blocks = [&ledger_walker] (const auto & start_block_hash) { + std::size_t walked_blocks_count = 0; + ledger_walker.walk_backward (start_block_hash, + [&] (const auto & block) { + ++walked_blocks_count; + }); + + return walked_blocks_count; + }; + + const auto transaction = node->ledger.store.tx_begin_read (); + nano::account_info genesis_account_info{}; + ASSERT_FALSE (node->ledger.store.account.get (transaction, nano::nano_dev_account, genesis_account_info)); + EXPECT_EQ (get_number_of_walked_blocks (genesis_account_info.open_block), 1); + EXPECT_EQ (get_number_of_walked_blocks (genesis_account_info.head), 1); + + system.wallet (0)->insert_adhoc (nano::dev_genesis_key.prv); + for (auto itr = 1; itr <= 5; ++itr) + { + const auto send = system.wallet (0)->send_action (nano::dev_genesis_key.pub, nano::dev_genesis_key.pub, 1); + ASSERT_TRUE (send); + EXPECT_EQ (get_number_of_walked_blocks (send->hash ()), 1 + itr * 2 - 1); + ASSERT_TIMELY (3s, 1 + itr * 2 == node->ledger.cache.cemented_count); + ASSERT_FALSE (node->ledger.store.account.get (transaction, nano::nano_dev_account, genesis_account_info)); + // TODO: check issue with account head + // EXPECT_EQ(get_number_of_walked_blocks (genesis_account_info.head), 1 + itr * 2); + } + + EXPECT_TRUE (ledger_walker.walked_blocks.empty ()); + EXPECT_EQ (1, ledger_walker.walked_blocks.bucket_count ()); + EXPECT_TRUE (ledger_walker.blocks_to_walk.empty ()); +} + +} + +TEST (ledger_walker, cross_account) +{ + nano::system system{}; + nano::node_config node_config (nano::get_available_port (), system.logging); + node_config.enable_voting = true; + node_config.receive_minimum = 1; + + const auto node = system.add_node (node_config); + + system.wallet (0)->insert_adhoc (nano::dev_genesis_key.prv); + ASSERT_TRUE (system.wallet (0)->send_action (nano::dev_genesis_key.pub, nano::dev_genesis_key.pub, 1)); + ASSERT_TIMELY (3s, 3 == node->ledger.cache.cemented_count); + + nano::keypair key{}; + system.wallet (0)->insert_adhoc (key.prv); + ASSERT_TRUE (system.wallet (0)->send_action (nano::dev_genesis_key.pub, key.pub, 1)); + ASSERT_TIMELY (3s, 5 == node->ledger.cache.cemented_count); + + const auto transaction = node->ledger.store.tx_begin_read (); + nano::account_info account_info{}; + ASSERT_FALSE (node->ledger.store.account.get (transaction, key.pub, account_info)); + + // TODO: check issue with account head + // const auto first = node->ledger.store.block_get_no_sideband(transaction, account_info.head); + // const auto second = node->ledger.store.block_get_no_sideband(transaction, first->previous()); + // const auto third = node->ledger.store.block_get_no_sideband(transaction, second->previous()); + // const auto fourth = node->ledger.store.block_get_no_sideband(transaction, third->previous()); + // const auto fifth = node->ledger.store.block_get_no_sideband(transaction, fourth->previous()); + // + // const auto expected_blocks_to_walk = { first, second, third, fourth, fifth }; + // auto expected_blocks_to_walk_itr = expected_blocks_to_walk.begin(); + // + // nano::ledger_walker ledger_walker{ node->ledger }; + // ledger_walker.walk_backward (account_info.block_count, [&] (const auto & block) { + // if (expected_blocks_to_walk_itr == expected_blocks_to_walk.end()) + // { + // EXPECT_TRUE(false); + // return false; + // } + // + // EXPECT_EQ((*expected_blocks_to_walk_itr++)->hash(), block->hash()); + // return true; + // }); + // + // EXPECT_EQ(expected_blocks_to_walk_itr, expected_blocks_to_walk.end()); +} + +TEST (ledger_walker, ladder_geometry) +{ + nano::system system{}; + + nano::node_config node_config (nano::get_available_port (), system.logging); + node_config.enable_voting = true; + node_config.receive_minimum = 1; + + const auto node = system.add_node (node_config); + std::array keys{}; + + system.wallet (0)->insert_adhoc (nano::dev_genesis_key.prv); + for (auto itr = 0; itr != keys.size (); ++itr) + { + system.wallet (0)->insert_adhoc (keys[itr].prv); + const auto block = system.wallet (0)->send_action (nano::dev_genesis_key.pub, keys[itr].pub, 1000); + ASSERT_TIMELY (3s, 1 + (itr + 1) * 2 == node->ledger.cache.cemented_count); + } + + std::vector amounts_to_send (10); + std::iota (amounts_to_send.begin (), amounts_to_send.end (), 1); + + const nano::account * last_destination{}; + for (auto itr = 0; itr != amounts_to_send.size (); ++itr) + { + const auto source_index = itr % keys.size (); + const auto destination_index = (source_index + 1) % keys.size (); + last_destination = &keys[destination_index].pub; + + const auto send = system.wallet (0)->send_action (keys[source_index].pub, keys[destination_index].pub, amounts_to_send[itr]); + ASSERT_TRUE (send); + + ASSERT_TIMELY (3s, 1 + keys.size () * 2 + (itr + 1) * 2 == node->ledger.cache.cemented_count); + } + + ASSERT_TRUE (last_destination); + const auto transaction = node->ledger.store.tx_begin_read (); + nano::account_info last_destination_info{}; + const auto last_destination_read_error = node->ledger.store.account.get (transaction, *last_destination, last_destination_info); + ASSERT_FALSE (last_destination_read_error); + + // This is how we expect chains to look like (for 3 accounts and 10 amounts to be sent) + // k1: 1000 SEND 3 SEND 6 SEND 9 SEND + // k2: 1000 1 SEND 4 SEND 7 SEND 10 + // k3: 1000 2 SEND 5 SEND 8 SEND + + std::vector amounts_expected_backwards{ 10, 9, 8, 5, 4, 3, 1000, 1, 1000, 2, 1000, 6, 7 }; + auto amounts_expected_backwards_itr = amounts_expected_backwards.cbegin (); + + nano::ledger_walker ledger_walker{ node->ledger }; + ledger_walker.walk_backward (last_destination_info.head, + [&] (const auto & block) { + if (block->sideband ().details.is_receive) + { + nano::amount previous_balance{}; + if (!block->previous ().is_zero ()) + { + const auto previous_block = node->ledger.store.block_get_no_sideband (transaction, block->previous ()); + previous_balance = previous_block->balance (); + } + + EXPECT_EQ (*amounts_expected_backwards_itr++, block->balance ().number () - previous_balance.number ()); + } + }); + + EXPECT_EQ (amounts_expected_backwards_itr, amounts_expected_backwards.cend ()); + + auto amounts_expected_itr = amounts_expected_backwards.crbegin (); + + ledger_walker.walk (last_destination_info.head, + [&] (const auto & block) { + if (block->sideband ().details.is_receive) + { + nano::amount previous_balance{}; + if (!block->previous ().is_zero ()) + { + const auto previous_block = node->ledger.store.block_get_no_sideband (transaction, block->previous ()); + previous_balance = previous_block->balance (); + } + + EXPECT_EQ (*amounts_expected_itr++, block->balance ().number () - previous_balance.number ()); + } + }); + + EXPECT_EQ (amounts_expected_itr, amounts_expected_backwards.crend ()); +} diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index d00f6a78..5d3afb2e 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -76,6 +76,8 @@ add_library( ipc/ipc_server.cpp json_handler.hpp json_handler.cpp + ledger_walker.hpp + ledger_walker.cpp lmdb/lmdb.hpp lmdb/lmdb.cpp lmdb/lmdb_env.hpp @@ -164,6 +166,7 @@ target_link_libraries( Boost::thread Boost::boost rocksdb + diskhash ${CMAKE_DL_LIBS} ${psapi_lib}) diff --git a/nano/node/ledger_walker.cpp b/nano/node/ledger_walker.cpp new file mode 100644 index 00000000..0616dec3 --- /dev/null +++ b/nano/node/ledger_walker.cpp @@ -0,0 +1,175 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +nano::ledger_walker::ledger_walker (nano::ledger const & ledger_a) : + ledger{ ledger_a }, + use_in_memory_walked_blocks{ true }, + walked_blocks{}, + walked_blocks_disk{}, + blocks_to_walk{} +{ + debug_assert (!ledger.store.init_error ()); +} + +void nano::ledger_walker::walk_backward (nano::block_hash const & start_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a) +{ + const auto transaction = ledger.store.tx_begin_read (); + + enqueue_block (start_block_hash_a); + while (!blocks_to_walk.empty ()) + { + const auto block = dequeue_block (transaction); + if (!should_visit_callback_a (block)) + { + continue; + } + + visitor_callback_a (block); + for (const auto & hash : ledger.dependent_blocks (transaction, *block)) + { + if (!hash.is_zero ()) + { + const auto block = ledger.store.block_get (transaction, hash); + if (block) + { + enqueue_block (ledger.store.block_get (transaction, hash)); + } + } + } + } + + clear_queue (); +} + +void nano::ledger_walker::walk (nano::block_hash const & end_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a) +{ + std::uint64_t last_walked_block_order_index = 0; + dht::DiskHash walked_blocks_order{ nano::unique_path ().c_str (), static_cast (std::to_string (std::numeric_limits::max ()).size ()) + 1, dht::DHOpenRW }; + + walk_backward (end_block_hash_a, + should_visit_callback_a, + [&] (const auto & block) { + walked_blocks_order.insert (std::to_string (++last_walked_block_order_index).c_str (), block->hash ()); + }); + + const auto transaction = ledger.store.tx_begin_read (); + for (auto walked_block_order_index = last_walked_block_order_index; walked_block_order_index != 0; --walked_block_order_index) + { + const auto * block_hash = walked_blocks_order.lookup (std::to_string (walked_block_order_index).c_str ()); + if (!block_hash) + { + debug_assert (false); + continue; + } + + const auto block = ledger.store.block_get (transaction, *block_hash); + if (!block) + { + debug_assert (false); + continue; + } + + visitor_callback_a (block); + } +} + +void nano::ledger_walker::walk_backward (nano::block_hash const & start_block_hash_a, visitor_callback const & visitor_callback_a) +{ + walk_backward ( + start_block_hash_a, + [&] (const auto & /* block */) { + return true; + }, + visitor_callback_a); +} + +void nano::ledger_walker::walk (nano::block_hash const & end_block_hash_a, visitor_callback const & visitor_callback_a) +{ + walk ( + end_block_hash_a, + [&] (const auto & /* block */) { + return true; + }, + visitor_callback_a); +} + +void nano::ledger_walker::enqueue_block (nano::block_hash block_hash_a) +{ + if (add_to_walked_blocks (block_hash_a)) + { + blocks_to_walk.emplace (std::move (block_hash_a)); + } +} + +void nano::ledger_walker::enqueue_block (std::shared_ptr const & block_a) +{ + debug_assert (block_a); + enqueue_block (block_a->hash ()); +} + +bool nano::ledger_walker::add_to_walked_blocks (nano::block_hash const & block_hash_a) +{ + if (use_in_memory_walked_blocks) + { + if (walked_blocks.size () < in_memory_block_count) + { + return walked_blocks.emplace (block_hash_a).second; + } + + use_in_memory_walked_blocks = false; + + debug_assert (!walked_blocks_disk.has_value ()); + walked_blocks_disk.emplace (nano::unique_path ().c_str (), sizeof (nano::block_hash::bytes) + 1, dht::DHOpenRW); + + for (const auto & walked_block_hash : walked_blocks) + { + if (!add_to_walked_blocks_disk (walked_block_hash)) + { + debug_assert (false); + } + } + + decltype (walked_blocks){}.swap (walked_blocks); + } + + return add_to_walked_blocks_disk (block_hash_a); +} + +bool nano::ledger_walker::add_to_walked_blocks_disk (nano::block_hash const & block_hash_a) +{ + debug_assert (!use_in_memory_walked_blocks); + debug_assert (walked_blocks_disk.has_value ()); + + std::array block_hash_key{}; + std::copy (block_hash_a.chars.cbegin (), + block_hash_a.chars.cend (), + block_hash_key.begin ()); + + return walked_blocks_disk->insert (block_hash_key.data (), true); +} + +void nano::ledger_walker::clear_queue () +{ + use_in_memory_walked_blocks = true; + + decltype (walked_blocks){}.swap (walked_blocks); + walked_blocks_disk.reset (); + + decltype (blocks_to_walk){}.swap (blocks_to_walk); +} + +std::shared_ptr nano::ledger_walker::dequeue_block (nano::transaction const & transaction_a) +{ + auto block = ledger.store.block_get (transaction_a, blocks_to_walk.top ()); + blocks_to_walk.pop (); + + return block; +} diff --git a/nano/node/ledger_walker.hpp b/nano/node/ledger_walker.hpp new file mode 100644 index 00000000..ecc494f2 --- /dev/null +++ b/nano/node/ledger_walker.hpp @@ -0,0 +1,60 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace nano +{ +class block; +class ledger; +class transaction; + +/** Walks the ledger starting from a start block and applying a depth-first search algorithm */ +class ledger_walker final +{ +public: + using should_visit_callback = std::function const &)>; + using visitor_callback = std::function const &)>; + + explicit ledger_walker (nano::ledger const & ledger_a); + + /** Start traversing (in a backwards direction -- towards genesis) from \p start_block_hash_a until \p should_visit_callback_a returns false, calling \p visitor_callback_a at each block. Prefer 'walk' instead, if possible. */ + void walk_backward (nano::block_hash const & start_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a); + + /** Start traversing (in a forward direction -- towards end_block_hash_a) from first block (genesis onwards) where \p should_visit_a returns true until \p end_block_hash_a, calling \p visitor_callback at each block. Prefer this one, instead of 'walk_backwards', if possible. */ + void walk (nano::block_hash const & end_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a); + + /** Methods similar to walk_backward and walk, but that do not offer the possibility of providing a user-defined should_visit_callback function. */ + void walk_backward (nano::block_hash const & start_block_hash_a, visitor_callback const & visitor_callback_a); + void walk (nano::block_hash const & end_block_hash_a, visitor_callback const & visitor_callback_a); + + /** How many blocks will be held in the in-memory hash before using the disk hash for walking. */ + // TODO TSB: make this 65536 + static constexpr std::size_t in_memory_block_count = 0; + +private: + nano::ledger const & ledger; + bool use_in_memory_walked_blocks; + std::unordered_set walked_blocks; + std::optional> walked_blocks_disk; + std::stack blocks_to_walk; + + void enqueue_block (nano::block_hash block_hash_a); + void enqueue_block (std::shared_ptr const & block_a); + bool add_to_walked_blocks (nano::block_hash const & block_hash_a); + bool add_to_walked_blocks_disk (nano::block_hash const & block_hash_a); + void clear_queue (); + std::shared_ptr dequeue_block (nano::transaction const & transaction_a); + + friend class ledger_walker_genesis_account_longer_Test; +}; + +}