Depth-first search ledger walking (#3324)

* Senders discovery draft

* DFS draft without checkpoints

* DFS algo improvement attempt

* DFS algo ++

* Latest DFS changes

* Self-contain DFS traversal algorithm and introduce unit tests for it

* Take out test code

* Fix build

* Address code review; use dependent_blocks_visitor to track block dependencies

* Add more ledger_walker unit tests

* Formatting

* Add diskhash as a git submodule and dependency of nano-node

Signed-off-by: theohax <theo@nano.org>

* Use diskhash in the ledger walker implementation

Signed-off-by: theohax <theo@nano.org>

* Fix formatting

Signed-off-by: theohax <theo@nano.org>

* Add todo note

Signed-off-by: theohax <theo@nano.org>

* Fix format

Signed-off-by: theohax <theo@nano.org>

* Adding submodules through get, not just .gitmodules

* Build diskhash with CMake instead of make

Signed-off-by: theohax <theo@nano.org>

* Use hybrid diskhash/in-memory-hash for ledger forward and backwards walking

* Build diskhash as static lib instead of shared

* Make ledger walker's diskhash key size larger to accomodate stringified uint64s

* Address code review -- move null block check to call site

Co-authored-by: clemahieu <clemahieu@gmail.com>
This commit is contained in:
theohax 2021-07-06 16:45:25 +03:00 committed by GitHub
commit 2e838b4485
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 467 additions and 0 deletions

3
.gitmodules vendored
View file

@ -27,3 +27,6 @@
path = rocksdb
url = https://github.com/nanocurrency/rocksdb.git
branch = 6.13.3
[submodule "diskhash"]
path = diskhash
url = https://github.com/luispedro/diskhash.git

View file

@ -378,6 +378,10 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules")
find_package(Boost 1.70.0 REQUIRED COMPONENTS filesystem log log_setup thread
program_options system)
# diskhash
add_library(diskhash STATIC ${CMAKE_SOURCE_DIR}/diskhash/src/diskhash.c)
include_directories(diskhash/src)
# RocksDB
include_directories(rocksdb/include)
set(USE_RTTI

1
diskhash Submodule

@ -0,0 +1 @@
Subproject commit 4fe2547bad3e1bc95e9a9d5df12a6e3913b27574

View file

@ -20,6 +20,7 @@ add_executable(
gap_cache.cpp
ipc.cpp
ledger.cpp
ledger_walker.cpp
locks.cpp
logger.cpp
message.cpp

View file

@ -0,0 +1,220 @@
#include <nano/node/ledger_walker.hpp>
#include <nano/node/testing.hpp>
#include <nano/test_common/testutil.hpp>
#include <gtest/gtest.h>
#include <numeric>
using namespace std::chrono_literals;
TEST (ledger_walker, genesis_block)
{
nano::system system{};
const auto node = system.add_node ();
nano::ledger_walker ledger_walker{ node->ledger };
std::size_t walked_blocks_count = 0;
ledger_walker.walk_backward (nano::genesis_hash,
[&] (const auto & block) {
++walked_blocks_count;
EXPECT_EQ (block->hash (), nano::genesis_hash);
});
EXPECT_EQ (walked_blocks_count, 1);
walked_blocks_count = 0;
ledger_walker.walk (nano::genesis_hash,
[&] (const auto & block) {
++walked_blocks_count;
EXPECT_EQ (block->hash (), nano::genesis_hash);
});
EXPECT_EQ (walked_blocks_count, 1);
}
namespace nano
{
TEST (ledger_walker, genesis_account_longer)
{
nano::system system{};
nano::node_config node_config (nano::get_available_port (), system.logging);
node_config.enable_voting = true;
node_config.receive_minimum = 1;
const auto node = system.add_node (node_config);
nano::ledger_walker ledger_walker{ node->ledger };
EXPECT_TRUE (ledger_walker.walked_blocks.empty ());
EXPECT_EQ (1, ledger_walker.walked_blocks.bucket_count ());
EXPECT_TRUE (ledger_walker.blocks_to_walk.empty ());
const auto get_number_of_walked_blocks = [&ledger_walker] (const auto & start_block_hash) {
std::size_t walked_blocks_count = 0;
ledger_walker.walk_backward (start_block_hash,
[&] (const auto & block) {
++walked_blocks_count;
});
return walked_blocks_count;
};
const auto transaction = node->ledger.store.tx_begin_read ();
nano::account_info genesis_account_info{};
ASSERT_FALSE (node->ledger.store.account.get (transaction, nano::nano_dev_account, genesis_account_info));
EXPECT_EQ (get_number_of_walked_blocks (genesis_account_info.open_block), 1);
EXPECT_EQ (get_number_of_walked_blocks (genesis_account_info.head), 1);
system.wallet (0)->insert_adhoc (nano::dev_genesis_key.prv);
for (auto itr = 1; itr <= 5; ++itr)
{
const auto send = system.wallet (0)->send_action (nano::dev_genesis_key.pub, nano::dev_genesis_key.pub, 1);
ASSERT_TRUE (send);
EXPECT_EQ (get_number_of_walked_blocks (send->hash ()), 1 + itr * 2 - 1);
ASSERT_TIMELY (3s, 1 + itr * 2 == node->ledger.cache.cemented_count);
ASSERT_FALSE (node->ledger.store.account.get (transaction, nano::nano_dev_account, genesis_account_info));
// TODO: check issue with account head
// EXPECT_EQ(get_number_of_walked_blocks (genesis_account_info.head), 1 + itr * 2);
}
EXPECT_TRUE (ledger_walker.walked_blocks.empty ());
EXPECT_EQ (1, ledger_walker.walked_blocks.bucket_count ());
EXPECT_TRUE (ledger_walker.blocks_to_walk.empty ());
}
}
TEST (ledger_walker, cross_account)
{
nano::system system{};
nano::node_config node_config (nano::get_available_port (), system.logging);
node_config.enable_voting = true;
node_config.receive_minimum = 1;
const auto node = system.add_node (node_config);
system.wallet (0)->insert_adhoc (nano::dev_genesis_key.prv);
ASSERT_TRUE (system.wallet (0)->send_action (nano::dev_genesis_key.pub, nano::dev_genesis_key.pub, 1));
ASSERT_TIMELY (3s, 3 == node->ledger.cache.cemented_count);
nano::keypair key{};
system.wallet (0)->insert_adhoc (key.prv);
ASSERT_TRUE (system.wallet (0)->send_action (nano::dev_genesis_key.pub, key.pub, 1));
ASSERT_TIMELY (3s, 5 == node->ledger.cache.cemented_count);
const auto transaction = node->ledger.store.tx_begin_read ();
nano::account_info account_info{};
ASSERT_FALSE (node->ledger.store.account.get (transaction, key.pub, account_info));
// TODO: check issue with account head
// const auto first = node->ledger.store.block_get_no_sideband(transaction, account_info.head);
// const auto second = node->ledger.store.block_get_no_sideband(transaction, first->previous());
// const auto third = node->ledger.store.block_get_no_sideband(transaction, second->previous());
// const auto fourth = node->ledger.store.block_get_no_sideband(transaction, third->previous());
// const auto fifth = node->ledger.store.block_get_no_sideband(transaction, fourth->previous());
//
// const auto expected_blocks_to_walk = { first, second, third, fourth, fifth };
// auto expected_blocks_to_walk_itr = expected_blocks_to_walk.begin();
//
// nano::ledger_walker ledger_walker{ node->ledger };
// ledger_walker.walk_backward (account_info.block_count, [&] (const auto & block) {
// if (expected_blocks_to_walk_itr == expected_blocks_to_walk.end())
// {
// EXPECT_TRUE(false);
// return false;
// }
//
// EXPECT_EQ((*expected_blocks_to_walk_itr++)->hash(), block->hash());
// return true;
// });
//
// EXPECT_EQ(expected_blocks_to_walk_itr, expected_blocks_to_walk.end());
}
TEST (ledger_walker, ladder_geometry)
{
nano::system system{};
nano::node_config node_config (nano::get_available_port (), system.logging);
node_config.enable_voting = true;
node_config.receive_minimum = 1;
const auto node = system.add_node (node_config);
std::array<nano::keypair, 3> keys{};
system.wallet (0)->insert_adhoc (nano::dev_genesis_key.prv);
for (auto itr = 0; itr != keys.size (); ++itr)
{
system.wallet (0)->insert_adhoc (keys[itr].prv);
const auto block = system.wallet (0)->send_action (nano::dev_genesis_key.pub, keys[itr].pub, 1000);
ASSERT_TIMELY (3s, 1 + (itr + 1) * 2 == node->ledger.cache.cemented_count);
}
std::vector<nano::uint128_t> amounts_to_send (10);
std::iota (amounts_to_send.begin (), amounts_to_send.end (), 1);
const nano::account * last_destination{};
for (auto itr = 0; itr != amounts_to_send.size (); ++itr)
{
const auto source_index = itr % keys.size ();
const auto destination_index = (source_index + 1) % keys.size ();
last_destination = &keys[destination_index].pub;
const auto send = system.wallet (0)->send_action (keys[source_index].pub, keys[destination_index].pub, amounts_to_send[itr]);
ASSERT_TRUE (send);
ASSERT_TIMELY (3s, 1 + keys.size () * 2 + (itr + 1) * 2 == node->ledger.cache.cemented_count);
}
ASSERT_TRUE (last_destination);
const auto transaction = node->ledger.store.tx_begin_read ();
nano::account_info last_destination_info{};
const auto last_destination_read_error = node->ledger.store.account.get (transaction, *last_destination, last_destination_info);
ASSERT_FALSE (last_destination_read_error);
// This is how we expect chains to look like (for 3 accounts and 10 amounts to be sent)
// k1: 1000 SEND 3 SEND 6 SEND 9 SEND
// k2: 1000 1 SEND 4 SEND 7 SEND 10
// k3: 1000 2 SEND 5 SEND 8 SEND
std::vector<nano::uint128_t> amounts_expected_backwards{ 10, 9, 8, 5, 4, 3, 1000, 1, 1000, 2, 1000, 6, 7 };
auto amounts_expected_backwards_itr = amounts_expected_backwards.cbegin ();
nano::ledger_walker ledger_walker{ node->ledger };
ledger_walker.walk_backward (last_destination_info.head,
[&] (const auto & block) {
if (block->sideband ().details.is_receive)
{
nano::amount previous_balance{};
if (!block->previous ().is_zero ())
{
const auto previous_block = node->ledger.store.block_get_no_sideband (transaction, block->previous ());
previous_balance = previous_block->balance ();
}
EXPECT_EQ (*amounts_expected_backwards_itr++, block->balance ().number () - previous_balance.number ());
}
});
EXPECT_EQ (amounts_expected_backwards_itr, amounts_expected_backwards.cend ());
auto amounts_expected_itr = amounts_expected_backwards.crbegin ();
ledger_walker.walk (last_destination_info.head,
[&] (const auto & block) {
if (block->sideband ().details.is_receive)
{
nano::amount previous_balance{};
if (!block->previous ().is_zero ())
{
const auto previous_block = node->ledger.store.block_get_no_sideband (transaction, block->previous ());
previous_balance = previous_block->balance ();
}
EXPECT_EQ (*amounts_expected_itr++, block->balance ().number () - previous_balance.number ());
}
});
EXPECT_EQ (amounts_expected_itr, amounts_expected_backwards.crend ());
}

View file

@ -76,6 +76,8 @@ add_library(
ipc/ipc_server.cpp
json_handler.hpp
json_handler.cpp
ledger_walker.hpp
ledger_walker.cpp
lmdb/lmdb.hpp
lmdb/lmdb.cpp
lmdb/lmdb_env.hpp
@ -164,6 +166,7 @@ target_link_libraries(
Boost::thread
Boost::boost
rocksdb
diskhash
${CMAKE_DL_LIBS}
${psapi_lib})

175
nano/node/ledger_walker.cpp Normal file
View file

@ -0,0 +1,175 @@
#include <nano/lib/blocks.hpp>
#include <nano/lib/errors.hpp>
#include <nano/node/ledger_walker.hpp>
#include <nano/secure/blockstore.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/utility.hpp>
#include <algorithm>
#include <limits>
#include <utility>
nano::ledger_walker::ledger_walker (nano::ledger const & ledger_a) :
ledger{ ledger_a },
use_in_memory_walked_blocks{ true },
walked_blocks{},
walked_blocks_disk{},
blocks_to_walk{}
{
debug_assert (!ledger.store.init_error ());
}
void nano::ledger_walker::walk_backward (nano::block_hash const & start_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a)
{
const auto transaction = ledger.store.tx_begin_read ();
enqueue_block (start_block_hash_a);
while (!blocks_to_walk.empty ())
{
const auto block = dequeue_block (transaction);
if (!should_visit_callback_a (block))
{
continue;
}
visitor_callback_a (block);
for (const auto & hash : ledger.dependent_blocks (transaction, *block))
{
if (!hash.is_zero ())
{
const auto block = ledger.store.block_get (transaction, hash);
if (block)
{
enqueue_block (ledger.store.block_get (transaction, hash));
}
}
}
}
clear_queue ();
}
void nano::ledger_walker::walk (nano::block_hash const & end_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a)
{
std::uint64_t last_walked_block_order_index = 0;
dht::DiskHash<nano::block_hash> walked_blocks_order{ nano::unique_path ().c_str (), static_cast<int> (std::to_string (std::numeric_limits<std::uint64_t>::max ()).size ()) + 1, dht::DHOpenRW };
walk_backward (end_block_hash_a,
should_visit_callback_a,
[&] (const auto & block) {
walked_blocks_order.insert (std::to_string (++last_walked_block_order_index).c_str (), block->hash ());
});
const auto transaction = ledger.store.tx_begin_read ();
for (auto walked_block_order_index = last_walked_block_order_index; walked_block_order_index != 0; --walked_block_order_index)
{
const auto * block_hash = walked_blocks_order.lookup (std::to_string (walked_block_order_index).c_str ());
if (!block_hash)
{
debug_assert (false);
continue;
}
const auto block = ledger.store.block_get (transaction, *block_hash);
if (!block)
{
debug_assert (false);
continue;
}
visitor_callback_a (block);
}
}
void nano::ledger_walker::walk_backward (nano::block_hash const & start_block_hash_a, visitor_callback const & visitor_callback_a)
{
walk_backward (
start_block_hash_a,
[&] (const auto & /* block */) {
return true;
},
visitor_callback_a);
}
void nano::ledger_walker::walk (nano::block_hash const & end_block_hash_a, visitor_callback const & visitor_callback_a)
{
walk (
end_block_hash_a,
[&] (const auto & /* block */) {
return true;
},
visitor_callback_a);
}
void nano::ledger_walker::enqueue_block (nano::block_hash block_hash_a)
{
if (add_to_walked_blocks (block_hash_a))
{
blocks_to_walk.emplace (std::move (block_hash_a));
}
}
void nano::ledger_walker::enqueue_block (std::shared_ptr<nano::block> const & block_a)
{
debug_assert (block_a);
enqueue_block (block_a->hash ());
}
bool nano::ledger_walker::add_to_walked_blocks (nano::block_hash const & block_hash_a)
{
if (use_in_memory_walked_blocks)
{
if (walked_blocks.size () < in_memory_block_count)
{
return walked_blocks.emplace (block_hash_a).second;
}
use_in_memory_walked_blocks = false;
debug_assert (!walked_blocks_disk.has_value ());
walked_blocks_disk.emplace (nano::unique_path ().c_str (), sizeof (nano::block_hash::bytes) + 1, dht::DHOpenRW);
for (const auto & walked_block_hash : walked_blocks)
{
if (!add_to_walked_blocks_disk (walked_block_hash))
{
debug_assert (false);
}
}
decltype (walked_blocks){}.swap (walked_blocks);
}
return add_to_walked_blocks_disk (block_hash_a);
}
bool nano::ledger_walker::add_to_walked_blocks_disk (nano::block_hash const & block_hash_a)
{
debug_assert (!use_in_memory_walked_blocks);
debug_assert (walked_blocks_disk.has_value ());
std::array<decltype (nano::block_hash::chars)::value_type, sizeof (nano::block_hash::bytes) + 1> block_hash_key{};
std::copy (block_hash_a.chars.cbegin (),
block_hash_a.chars.cend (),
block_hash_key.begin ());
return walked_blocks_disk->insert (block_hash_key.data (), true);
}
void nano::ledger_walker::clear_queue ()
{
use_in_memory_walked_blocks = true;
decltype (walked_blocks){}.swap (walked_blocks);
walked_blocks_disk.reset ();
decltype (blocks_to_walk){}.swap (blocks_to_walk);
}
std::shared_ptr<nano::block> nano::ledger_walker::dequeue_block (nano::transaction const & transaction_a)
{
auto block = ledger.store.block_get (transaction_a, blocks_to_walk.top ());
blocks_to_walk.pop ();
return block;
}

View file

@ -0,0 +1,60 @@
#pragma once
#include <nano/lib/numbers.hpp>
#include <cstddef>
#include <functional>
#include <memory>
#include <optional>
#include <stack>
#include <unordered_set>
#include <diskhash.hpp>
namespace nano
{
class block;
class ledger;
class transaction;
/** Walks the ledger starting from a start block and applying a depth-first search algorithm */
class ledger_walker final
{
public:
using should_visit_callback = std::function<bool (std::shared_ptr<nano::block> const &)>;
using visitor_callback = std::function<void (std::shared_ptr<nano::block> const &)>;
explicit ledger_walker (nano::ledger const & ledger_a);
/** Start traversing (in a backwards direction -- towards genesis) from \p start_block_hash_a until \p should_visit_callback_a returns false, calling \p visitor_callback_a at each block. Prefer 'walk' instead, if possible. */
void walk_backward (nano::block_hash const & start_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a);
/** Start traversing (in a forward direction -- towards end_block_hash_a) from first block (genesis onwards) where \p should_visit_a returns true until \p end_block_hash_a, calling \p visitor_callback at each block. Prefer this one, instead of 'walk_backwards', if possible. */
void walk (nano::block_hash const & end_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a);
/** Methods similar to walk_backward and walk, but that do not offer the possibility of providing a user-defined should_visit_callback function. */
void walk_backward (nano::block_hash const & start_block_hash_a, visitor_callback const & visitor_callback_a);
void walk (nano::block_hash const & end_block_hash_a, visitor_callback const & visitor_callback_a);
/** How many blocks will be held in the in-memory hash before using the disk hash for walking. */
// TODO TSB: make this 65536
static constexpr std::size_t in_memory_block_count = 0;
private:
nano::ledger const & ledger;
bool use_in_memory_walked_blocks;
std::unordered_set<nano::block_hash> walked_blocks;
std::optional<dht::DiskHash<bool>> walked_blocks_disk;
std::stack<nano::block_hash> blocks_to_walk;
void enqueue_block (nano::block_hash block_hash_a);
void enqueue_block (std::shared_ptr<nano::block> const & block_a);
bool add_to_walked_blocks (nano::block_hash const & block_hash_a);
bool add_to_walked_blocks_disk (nano::block_hash const & block_hash_a);
void clear_queue ();
std::shared_ptr<nano::block> dequeue_block (nano::transaction const & transaction_a);
friend class ledger_walker_genesis_account_longer_Test;
};
}