Minecon724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 1

Create nano::unchecked_map ADT which is the interface to the unchecked table. Rather than directly making modifications to the unchecked table, this ADT abstracts away the details of where this information is stored. nano::unchecked_map will queue write/trigger operations for processing in a background thread. This means nano::unchecked_map::put no longer requires a database transaction to call. This also changes the semantics of the unchecked put operations as they're no longer blocking, fix up many tests that made this assumption. (#3553)

Browse source 
Co-authored-by: clemahieu <clemahieu@gmail.com>
This commit is contained in:
Thiago Silva 2022-02-03 10:17:33 -03:00 • committed by GitHub
commit 8b42872977
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 627 additions and 350 deletions
Download patch file Download diff file Expand all files Collapse all files

1
nano/core_test/CMakeLists.txt
View file

@ -42,6 +42,7 @@ add_executable(
toml.cpp
timer.cpp
uint256_union.cpp
unchecked_map.cpp
utility.cpp
vote_processor.cpp
voting.cpp

229
nano/core_test/block_store.cpp
View file

@ -22,6 +22,8 @@
#include <stdlib.h>
using namespace std::chrono_literals;
namespace
{
void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height, nano::block_hash const & rep_block);
@ -347,109 +349,152 @@ TEST (block_store, genesis)
ASSERT_EQ (nano::dev::genesis->account (), nano::dev::genesis_key.pub);
}
TEST (bootstrap, simple)
// This test checks for basic operations in the unchecked table such as putting a new block, retrieving it, and
// deleting it from the database
TEST (unchecked, simple)
{
nano::logger_mt logger;
nano::system system{};
nano::logger_mt logger{};
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
std::shared_ptr<nano::block> block1 = std::make_shared<nano::send_block> (0, 1, 2, nano::keypair ().prv, 4, 5);
auto transaction (store->tx_begin_write ());
auto block2 (store->unchecked.get (transaction, block1->previous ()));
ASSERT_TRUE (block2.empty ());
store->unchecked.put (transaction, block1->previous (), block1);
auto block3 (store->unchecked.get (transaction, block1->previous ()));
ASSERT_FALSE (block3.empty ());
ASSERT_EQ (*block1, *(block3[0].block));
store->unchecked.del (transaction, nano::unchecked_key (block1->previous (), block1->hash ()));
auto block4 (store->unchecked.get (transaction, block1->previous ()));
ASSERT_TRUE (block4.empty ());
std::shared_ptr<nano::block> block = std::make_shared<nano::send_block> (0, 1, 2, nano::keypair ().prv, 4, 5);
// Asserts the block wasn't added yet to the unchecked table
auto block_listing1 = unchecked.get (store->tx_begin_read (), block->previous ());
ASSERT_TRUE (block_listing1.empty ());
// Enqueues a block to be saved on the unchecked table
unchecked.put (block->previous (), block);
// Waits for the block to get written in the database
auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) {
return unchecked.get (transaction_a, block_hash_a).size () > 0;
};
ASSERT_TIMELY (5s, check_block_is_listed (store->tx_begin_read (), block->previous ()));
auto transaction = store->tx_begin_write ();
// Retrieves the block from the database
auto block_listing2 = unchecked.get (transaction, block->previous ());
ASSERT_FALSE (block_listing2.empty ());
// Asserts the added block is equal to the retrieved one
ASSERT_EQ (*block, *(block_listing2[0].block));
// Deletes the block from the database
unchecked.del (transaction, nano::unchecked_key (block->previous (), block->hash ()));
// Asserts the block is deleted
auto block_listing3 = unchecked.get (transaction, block->previous ());
ASSERT_TRUE (block_listing3.empty ());
}
// This test ensures the unchecked table is able to receive more than one block
TEST (unchecked, multiple)
{
nano::system system{};
if (nano::rocksdb_config::using_rocksdb_in_tests ())
{
// Don't test this in rocksdb mode
return;
}
nano::logger_mt logger;
nano::logger_mt logger{};
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
std::shared_ptr<nano::block> block1 = std::make_shared<nano::send_block> (4, 1, 2, nano::keypair ().prv, 4, 5);
auto transaction (store->tx_begin_write ());
auto block2 (store->unchecked.get (transaction, block1->previous ()));
ASSERT_TRUE (block2.empty ());
store->unchecked.put (transaction, block1->previous (), block1);
store->unchecked.put (transaction, block1->source (), block1);
auto block3 (store->unchecked.get (transaction, block1->previous ()));
ASSERT_FALSE (block3.empty ());
auto block4 (store->unchecked.get (transaction, block1->source ()));
ASSERT_FALSE (block4.empty ());
std::shared_ptr<nano::block> block = std::make_shared<nano::send_block> (4, 1, 2, nano::keypair ().prv, 4, 5);
// Asserts the block wasn't added yet to the unchecked table
auto block_listing1 = unchecked.get (store->tx_begin_read (), block->previous ());
ASSERT_TRUE (block_listing1.empty ());
// Enqueues the first block
unchecked.put (block->previous (), block);
// Enqueues a second block
unchecked.put (block->source (), block);
auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) {
return unchecked.get (transaction_a, block_hash_a).size () > 0;
};
// Waits for and asserts the first block gets saved in the database
ASSERT_TIMELY (5s, check_block_is_listed (store->tx_begin_read (), block->previous ()));
// Waits for and asserts the second block gets saved in the database
ASSERT_TIMELY (5s, check_block_is_listed (store->tx_begin_read (), block->source ()));
}
// This test ensures that a block can't occur twice in the unchecked table.
TEST (unchecked, double_put)
{
nano::logger_mt logger;
nano::system system{};
nano::logger_mt logger{};
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
std::shared_ptr<nano::block> block1 = std::make_shared<nano::send_block> (4, 1, 2, nano::keypair ().prv, 4, 5);
auto transaction (store->tx_begin_write ());
auto block2 (store->unchecked.get (transaction, block1->previous ()));
ASSERT_TRUE (block2.empty ());
store->unchecked.put (transaction, block1->previous (), block1);
store->unchecked.put (transaction, block1->previous (), block1);
auto block3 (store->unchecked.get (transaction, block1->previous ()));
ASSERT_EQ (block3.size (), 1);
std::shared_ptr<nano::block> block = std::make_shared<nano::send_block> (4, 1, 2, nano::keypair ().prv, 4, 5);
// Asserts the block wasn't added yet to the unchecked table
auto block_listing1 = unchecked.get (store->tx_begin_read (), block->previous ());
ASSERT_TRUE (block_listing1.empty ());
// Enqueues the block to be saved in the unchecked table
unchecked.put (block->previous (), block);
// Enqueues the block again in an attempt to have it there twice
unchecked.put (block->previous (), block);
auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) {
return unchecked.get (transaction_a, block_hash_a).size () > 0;
};
// Waits for and asserts the block was added at least once
ASSERT_TIMELY (5s, check_block_is_listed (store->tx_begin_read (), block->previous ()));
// Asserts the block was added at most once -- this is objective of this test.
auto block_listing2 = unchecked.get (store->tx_begin_read (), block->previous ());
ASSERT_EQ (block_listing2.size (), 1);
}
// Tests that recurrent get calls return the correct values
TEST (unchecked, multiple_get)
{
nano::logger_mt logger;
nano::system system{};
nano::logger_mt logger{};
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
// Instantiates three blocks
std::shared_ptr<nano::block> block1 = std::make_shared<nano::send_block> (4, 1, 2, nano::keypair ().prv, 4, 5);
std::shared_ptr<nano::block> block2 = std::make_shared<nano::send_block> (3, 1, 2, nano::keypair ().prv, 4, 5);
std::shared_ptr<nano::block> block3 = std::make_shared<nano::send_block> (5, 1, 2, nano::keypair ().prv, 4, 5);
{
auto transaction (store->tx_begin_write ());
store->unchecked.put (transaction, block1->previous (), block1); // unchecked1
store->unchecked.put (transaction, block1->hash (), block1); // unchecked2
store->unchecked.put (transaction, block2->previous (), block2); // unchecked3
store->unchecked.put (transaction, block1->previous (), block2); // unchecked1
store->unchecked.put (transaction, block1->hash (), block2); // unchecked2
store->unchecked.put (transaction, block3->previous (), block3);
store->unchecked.put (transaction, block3->hash (), block3); // unchecked4
store->unchecked.put (transaction, block1->previous (), block3); // unchecked1
}
auto transaction (store->tx_begin_read ());
auto unchecked_count (store->unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 8);
// Add the blocks' info to the unchecked table
unchecked.put (block1->previous (), block1); // unchecked1
unchecked.put (block1->hash (), block1); // unchecked2
unchecked.put (block2->previous (), block2); // unchecked3
unchecked.put (block1->previous (), block2); // unchecked1
unchecked.put (block1->hash (), block2); // unchecked2
unchecked.put (block3->previous (), block3);
unchecked.put (block3->hash (), block3); // unchecked4
unchecked.put (block1->previous (), block3); // unchecked1
// Waits for the blocks to get saved in the database
ASSERT_TIMELY (5s, 8 == unchecked.count (store->tx_begin_read ()));
std::vector<nano::block_hash> unchecked1;
auto unchecked1_blocks (store->unchecked.get (transaction, block1->previous ()));
// Asserts the entries will be found for the provided key
auto transaction = store->tx_begin_read ();
auto unchecked1_blocks = unchecked.get (transaction, block1->previous ());
ASSERT_EQ (unchecked1_blocks.size (), 3);
for (auto & i : unchecked1_blocks)
{
unchecked1.push_back (i.block->hash ());
}
// Asserts the payloads where correclty saved
ASSERT_TRUE (std::find (unchecked1.begin (), unchecked1.end (), block1->hash ()) != unchecked1.end ());
ASSERT_TRUE (std::find (unchecked1.begin (), unchecked1.end (), block2->hash ()) != unchecked1.end ());
ASSERT_TRUE (std::find (unchecked1.begin (), unchecked1.end (), block3->hash ()) != unchecked1.end ());
std::vector<nano::block_hash> unchecked2;
auto unchecked2_blocks (store->unchecked.get (transaction, block1->hash ()));
// Asserts the entries will be found for the provided key
auto unchecked2_blocks = unchecked.get (transaction, block1->hash ());
ASSERT_EQ (unchecked2_blocks.size (), 2);
for (auto & i : unchecked2_blocks)
{
unchecked2.push_back (i.block->hash ());
}
// Asserts the payloads where correctly saved
ASSERT_TRUE (std::find (unchecked2.begin (), unchecked2.end (), block1->hash ()) != unchecked2.end ());
ASSERT_TRUE (std::find (unchecked2.begin (), unchecked2.end (), block2->hash ()) != unchecked2.end ());
auto unchecked3 (store->unchecked.get (transaction, block2->previous ()));
// Asserts the entry is found by the key and the payload is saved
auto unchecked3 = unchecked.get (transaction, block2->previous ());
ASSERT_EQ (unchecked3.size (), 1);
ASSERT_EQ (unchecked3[0].block->hash (), block2->hash ());
auto unchecked4 (store->unchecked.get (transaction, block3->hash ()));
// Asserts the entry is found by the key and the payload is saved
auto unchecked4 = unchecked.get (transaction, block3->hash ());
ASSERT_EQ (unchecked4.size (), 1);
ASSERT_EQ (unchecked4[0].block->hash (), block3->hash ());
auto unchecked5 (store->unchecked.get (transaction, block2->hash ()));
// Asserts no entry is found for a block that wasn't added
auto unchecked5 = unchecked.get (transaction, block2->hash ());
ASSERT_EQ (unchecked5.size (), 0);
}
@ -480,30 +525,10 @@ TEST (block_store, empty_bootstrap)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
auto transaction (store->tx_begin_read ());
auto [begin, end] = store->unchecked.full_range (transaction);
ASSERT_EQ (end, begin);
}
TEST (block_store, one_bootstrap)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
std::shared_ptr<nano::block> block1 = std::make_shared<nano::send_block> (0, 1, 2, nano::keypair ().prv, 4, 5);
auto transaction (store->tx_begin_write ());
store->unchecked.put (transaction, block1->hash (), block1);
auto begin (store->unchecked.begin (transaction));
auto end (store->unchecked.end ());
ASSERT_NE (end, begin);
auto hash1 (begin->first.key ());
ASSERT_EQ (block1->hash (), hash1);
auto blocks (store->unchecked.get (transaction, hash1));
ASSERT_EQ (1, blocks.size ());
auto block2 (blocks[0].block);
ASSERT_EQ (*block1, *block2);
++begin;
auto [begin, end] = unchecked.full_range (transaction);
ASSERT_EQ (end, begin);
}
@ -919,42 +944,26 @@ TEST (block_store, pruned_random)
TEST (block_store, DISABLED_change_dupsort) // Unchecked is no longer dupsort table
{
auto path (nano::unique_path ());
nano::logger_mt logger;
nano::mdb_store store (logger, path, nano::dev::constants);
nano::logger_mt logger{};
nano::mdb_store store{ logger, path, nano::dev::constants };
nano::unchecked_map unchecked{ store, false };
auto transaction (store.tx_begin_write ());
ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked_handle, 1));
ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE, &store.unchecked_handle));
std::shared_ptr<nano::block> send1 = std::make_shared<nano::send_block> (0, 0, 0, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
std::shared_ptr<nano::block> send2 = std::make_shared<nano::send_block> (1, 0, 0, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
ASSERT_NE (send1->hash (), send2->hash ());
store.unchecked.put (transaction, send1->hash (), send1);
store.unchecked.put (transaction, send1->hash (), send2);
{
auto iterator1 (store.unchecked.begin (transaction));
++iterator1;
ASSERT_EQ (store.unchecked.end (), iterator1);
}
unchecked.put (send1->hash (), send1);
unchecked.put (send1->hash (), send2);
ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked_handle, 0));
mdb_dbi_close (store.env, store.unchecked_handle);
ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE | MDB_DUPSORT, &store.unchecked_handle));
store.unchecked.put (transaction, send1->hash (), send1);
store.unchecked.put (transaction, send1->hash (), send2);
{
auto iterator1 (store.unchecked.begin (transaction));
++iterator1;
ASSERT_EQ (store.unchecked.end (), iterator1);
}
unchecked.put (send1->hash (), send1);
unchecked.put (send1->hash (), send2);
ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked_handle, 1));
ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE | MDB_DUPSORT, &store.unchecked_handle));
store.unchecked.put (transaction, send1->hash (), send1);
store.unchecked.put (transaction, send1->hash (), send2);
{
auto iterator1 (store.unchecked.begin (transaction));
++iterator1;
ASSERT_NE (store.unchecked.end (), iterator1);
++iterator1;
ASSERT_EQ (store.unchecked.end (), iterator1);
}
unchecked.put (send1->hash (), send1);
unchecked.put (send1->hash (), send2);
}
TEST (block_store, state_block)
@ -2017,18 +2026,28 @@ TEST (block_store, rocksdb_force_test_env_variable)
namespace nano
{
// This thest ensures the tombstone_count is increased when there is a delete. The tombstone_count is part of a flush
// logic bound to the way RocksDB is used by the node.
TEST (rocksdb_block_store, tombstone_count)
{
if (nano::rocksdb_config::using_rocksdb_in_tests ())
{
nano::logger_mt logger;
nano::system system{};
nano::logger_mt logger{};
auto store = std::make_unique<nano::rocksdb_store> (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
auto transaction = store->tx_begin_write ();
std::shared_ptr<nano::block> block1 = std::make_shared<nano::send_block> (0, 1, 2, nano::keypair ().prv, 4, 5);
store->unchecked.put (transaction, block1->previous (), block1);
std::shared_ptr<nano::block> block = std::make_shared<nano::send_block> (0, 1, 2, nano::keypair ().prv, 4, 5);
// Enqueues a block to be saved in the database
unchecked.put (block->previous (), block);
auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) {
return unchecked.get (transaction_a, block_hash_a).size () > 0;
};
// Waits for the block to get saved
ASSERT_TIMELY (5s, check_block_is_listed (store->tx_begin_read (), block->previous ()));
ASSERT_EQ (store->tombstone_map.at (nano::tables::unchecked).num_since_last_flush.load (), 0);
store->unchecked.del (transaction, nano::unchecked_key (block1->previous (), block1->hash ()));
// Perorms a delete and checks for the tombstone counter
unchecked.del (store->tx_begin_write (), nano::unchecked_key (block->previous (), block->hash ()));
ASSERT_EQ (store->tombstone_map.at (nano::tables::unchecked).num_since_last_flush.load (), 1);
}
}

21
nano/core_test/bootstrap.cpp
View file

@ -924,11 +924,10 @@ TEST (bootstrap_processor, DISABLED_lazy_unclear_state_link)
node2->bootstrap_initiator.bootstrap_lazy (receive->hash ());
// Check processed blocks
ASSERT_TIMELY (10s, !node2->bootstrap_initiator.in_progress ());
node2->block_processor.flush ();
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send2->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (open->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (receive->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send2->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (open->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (receive->hash ()));
ASSERT_EQ (0, node2->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in));
}
@ -976,10 +975,9 @@ TEST (bootstrap_processor, lazy_unclear_state_link_not_existing)
node2->bootstrap_initiator.bootstrap_lazy (send2->hash ());
// Check processed blocks
ASSERT_TIMELY (15s, !node2->bootstrap_initiator.in_progress ());
node2->block_processor.flush ();
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (open->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send2->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (open->hash ()));
ASSERT_TIMELY (5s, node2->ledger.block_or_pruned_exists (send2->hash ()));
ASSERT_EQ (1, node2->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in));
}
@ -1038,7 +1036,6 @@ TEST (bootstrap_processor, DISABLED_lazy_destinations)
node2->bootstrap_initiator.bootstrap_lazy (send2->hash ());
// Check processed blocks
ASSERT_TIMELY (10s, !node2->bootstrap_initiator.in_progress ());
node2->block_processor.flush ();
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send2->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (open->hash ()));
@ -1127,7 +1124,7 @@ TEST (bootstrap_processor, lazy_pruning_missing_block)
ASSERT_FALSE (node2->ledger.block_or_pruned_exists (state_open->hash ()));
{
auto transaction (node2->store.tx_begin_read ());
ASSERT_TRUE (node2->store.unchecked.exists (transaction, nano::unchecked_key (send2->root ().as_block_hash (), send2->hash ())));
ASSERT_TRUE (node2->unchecked.exists (transaction, nano::unchecked_key (send2->root ().as_block_hash (), send2->hash ())));
}
// Insert missing block
node2->process_active (send1);
@ -1838,7 +1835,7 @@ TEST (bulk, DISABLED_genesis_pruning)
ASSERT_EQ (1, node2->ledger.cache.block_count);
{
auto transaction (node2->store.tx_begin_write ());
node2->store.unchecked.clear (transaction);
node2->unchecked.clear (transaction);
}
// Insert pruned blocks
node2->process_active (send1);

57
nano/core_test/confirmation_height.cpp
View file

@ -219,24 +219,24 @@ TEST (confirmation_height, multiple_accounts)
TEST (confirmation_height, gap_bootstrap)
{
auto test_mode = [] (nano::confirmation_height_mode mode_a) {
nano::system system;
nano::node_flags node_flags;
nano::system system{};
nano::node_flags node_flags{};
node_flags.confirmation_height_processor_mode = mode_a;
auto & node1 = *system.add_node (node_flags);
nano::keypair destination;
auto send1 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
nano::keypair destination{};
auto send1 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node1.work_generate_blocking (*send1);
auto send2 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send2 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node1.work_generate_blocking (*send2);
auto send3 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send3 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node1.work_generate_blocking (*send3);
auto open1 (std::make_shared<nano::open_block> (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0));
auto open1 = std::make_shared<nano::open_block> (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0);
node1.work_generate_blocking (*open1);
// Receive
auto receive1 (std::make_shared<nano::receive_block> (open1->hash (), send2->hash (), destination.prv, destination.pub, 0));
auto receive1 = std::make_shared<nano::receive_block> (open1->hash (), send2->hash (), destination.prv, destination.pub, 0);
node1.work_generate_blocking (*receive1);
auto receive2 (std::make_shared<nano::receive_block> (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0));
auto receive2 = std::make_shared<nano::receive_block> (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0);
node1.work_generate_blocking (*receive2);
node1.block_processor.add (send1);
@ -249,12 +249,16 @@ TEST (confirmation_height, gap_bootstrap)
// Receive 2 comes in on the live network, however the chain has not been finished so it gets added to unchecked
node1.process_active (receive2);
node1.block_processor.flush ();
// Waits for the unchecked_map to process the 4 blocks added to the block_processor, saving them in the unchecked table
auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) {
return !node1.unchecked.get (transaction_a, block_hash_a).empty ();
};
ASSERT_TIMELY (15s, check_block_is_listed (node1.store.tx_begin_read (), receive2->previous ()));
// Confirmation heights should not be updated
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.store.unchecked.count (transaction));
auto unchecked_count (node1.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 2);
nano::confirmation_height_info confirmation_height_info;
@ -265,14 +269,11 @@ TEST (confirmation_height, gap_bootstrap)
// Now complete the chain where the block comes in on the bootstrap network.
node1.block_processor.add (open1);
node1.block_processor.flush ();
ASSERT_TIMELY (10s, node1.unchecked.count (node1.store.tx_begin_read ()) == 0);
// Confirmation height should be unchanged and unchecked should now be 0
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 0);
auto transaction = node1.store.tx_begin_read ();
nano::confirmation_height_info confirmation_height_info;
ASSERT_FALSE (node1.store.confirmation_height.get (transaction, nano::dev::genesis_key.pub, confirmation_height_info));
ASSERT_EQ (1, confirmation_height_info.height);
@ -296,10 +297,10 @@ TEST (confirmation_height, gap_bootstrap)
TEST (confirmation_height, gap_live)
{
auto test_mode = [] (nano::confirmation_height_mode mode_a) {
nano::system system;
nano::node_flags node_flags;
nano::system system{};
nano::node_flags node_flags{};
node_flags.confirmation_height_processor_mode = mode_a;
nano::node_config node_config (nano::get_available_port (), system.logging);
nano::node_config node_config{ nano::get_available_port (), system.logging };
node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
auto node = system.add_node (node_config, node_flags);
node_config.peering_port = nano::get_available_port ();
@ -309,18 +310,18 @@ TEST (confirmation_height, gap_live)
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
system.wallet (1)->insert_adhoc (destination.prv);
auto send1 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 1, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send1 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 1, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node->work_generate_blocking (*send1);
auto send2 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send2 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node->work_generate_blocking (*send2);
auto send3 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send3 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), send2->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 3, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node->work_generate_blocking (*send3);
auto open1 (std::make_shared<nano::open_block> (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0));
auto open1 = std::make_shared<nano::open_block> (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0);
node->work_generate_blocking (*open1);
auto receive1 (std::make_shared<nano::receive_block> (open1->hash (), send2->hash (), destination.prv, destination.pub, 0));
auto receive1 = std::make_shared<nano::receive_block> (open1->hash (), send2->hash (), destination.prv, destination.pub, 0);
node->work_generate_blocking (*receive1);
auto receive2 (std::make_shared<nano::receive_block> (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0));
auto receive2 = std::make_shared<nano::receive_block> (receive1->hash (), send3->hash (), destination.prv, destination.pub, 0);
node->work_generate_blocking (*receive2);
node->block_processor.add (send1);
@ -355,11 +356,11 @@ TEST (confirmation_height, gap_live)
ASSERT_TIMELY (10s, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out) == 6);
// This should confirm the open block and the source of the receive blocks
auto transaction (node->store.tx_begin_read ());
auto unchecked_count (node->store.unchecked.count (transaction));
auto transaction = node->store.tx_begin_read ();
auto unchecked_count = node->unchecked.count (transaction);
ASSERT_EQ (unchecked_count, 0);
nano::confirmation_height_info confirmation_height_info;
nano::confirmation_height_info confirmation_height_info{};
ASSERT_TRUE (node->ledger.block_confirmed (transaction, receive2->hash ()));
ASSERT_FALSE (node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub, confirmation_height_info));
ASSERT_EQ (4, confirmation_height_info.height);

2
nano/core_test/gap_cache.cpp
View file

@ -103,7 +103,7 @@ TEST (gap_cache, two_dependencies)
ASSERT_EQ (2, node1.gap_cache.size ());
node1.block_processor.add (send1, nano::seconds_since_epoch ());
node1.block_processor.flush ();
ASSERT_EQ (0, node1.gap_cache.size ());
ASSERT_TIMELY (5s, node1.gap_cache.size () == 0);
auto transaction (node1.store.tx_begin_read ());
ASSERT_TRUE (node1.store.block.exists (transaction, send1->hash ()));
ASSERT_TRUE (node1.store.block.exists (transaction, send2->hash ()));

153
nano/core_test/ledger.cpp
View file

@ -2454,11 +2454,11 @@ TEST (ledger, successor_epoch)
TEST (ledger, epoch_open_pending)
{
nano::block_builder builder;
nano::system system (1);
auto & node1 (*system.nodes[0]);
nano::block_builder builder{};
nano::system system{ 1 };
auto & node1 = *system.nodes[0];
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
nano::keypair key1;
nano::keypair key1{};
auto epoch_open = builder.state ()
.account (key1.pub)
.previous (0)
@ -2468,14 +2468,15 @@ TEST (ledger, epoch_open_pending)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (*pool.generate (key1.pub))
.build_shared ();
auto process_result (node1.ledger.process (node1.store.tx_begin_write (), *epoch_open));
auto process_result = node1.ledger.process (node1.store.tx_begin_write (), *epoch_open);
ASSERT_EQ (nano::process_result::gap_epoch_open_pending, process_result.code);
ASSERT_EQ (nano::signature_verification::valid_epoch, process_result.verified);
node1.block_processor.add (epoch_open);
node1.block_processor.flush ();
// Waits for the block to get saved in the database
ASSERT_TIMELY (10s, 1 == node1.unchecked.count (node1.store.tx_begin_read ()));
ASSERT_FALSE (node1.ledger.block_or_pruned_exists (epoch_open->hash ()));
// Open block should be inserted into unchecked
auto blocks (node1.store.unchecked.get (node1.store.tx_begin_read (), nano::hash_or_account (epoch_open->account ()).hash));
auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), nano::hash_or_account (epoch_open->account ()).hash);
ASSERT_EQ (blocks.size (), 1);
ASSERT_EQ (blocks[0].block->full_hash (), epoch_open->full_hash ());
ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid_epoch);
@ -2490,8 +2491,7 @@ TEST (ledger, epoch_open_pending)
.work (*pool.generate (nano::dev::genesis->hash ()))
.build_shared ();
node1.block_processor.add (send1);
node1.block_processor.flush ();
ASSERT_TRUE (node1.ledger.block_or_pruned_exists (epoch_open->hash ()));
ASSERT_TIMELY (10s, node1.ledger.block_or_pruned_exists (epoch_open->hash ()));
}
TEST (ledger, block_hash_account_conflict)
@ -2657,27 +2657,21 @@ TEST (ledger, unchecked_epoch)
auto epoch1 (std::make_shared<nano::state_block> (destination.pub, open1->hash (), destination.pub, nano::Gxrb_ratio, node1.ledger.epoch_link (nano::epoch::epoch_1), nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
node1.work_generate_blocking (*epoch1);
node1.block_processor.add (epoch1);
node1.block_processor.flush ();
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 1);
ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction));
auto blocks (node1.store.unchecked.get (transaction, epoch1->previous ()));
// Waits for the epoch1 block to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (10s, 1 == node1.unchecked.count (node1.store.tx_begin_read ()));
auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), epoch1->previous ());
ASSERT_EQ (blocks.size (), 1);
ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid_epoch);
}
node1.block_processor.add (send1);
node1.block_processor.add (open1);
node1.block_processor.flush ();
ASSERT_TIMELY (5s, node1.store.block.exists (node1.store.tx_begin_read (), epoch1->hash ()));
{
auto transaction (node1.store.tx_begin_read ());
ASSERT_TRUE (node1.store.block.exists (transaction, epoch1->hash ()));
auto unchecked_count (node1.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 0);
ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction));
nano::account_info info;
ASSERT_FALSE (node1.store.account.get (transaction, destination.pub, info));
// Waits for the last blocks to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (10s, 0 == node1.unchecked.count (node1.store.tx_begin_read ()));
nano::account_info info{};
ASSERT_FALSE (node1.store.account.get (node1.store.tx_begin_read (), destination.pub, info));
ASSERT_EQ (info.epoch (), nano::epoch::epoch_1);
}
}
@ -2701,32 +2695,29 @@ TEST (ledger, unchecked_epoch_invalid)
node1.work_generate_blocking (*epoch2);
node1.block_processor.add (epoch1);
node1.block_processor.add (epoch2);
node1.block_processor.flush ();
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 2);
ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction));
auto blocks (node1.store.unchecked.get (transaction, epoch1->previous ()));
// Waits for the last blocks to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (10s, 2 == node1.unchecked.count (node1.store.tx_begin_read ()));
auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), epoch1->previous ());
ASSERT_EQ (blocks.size (), 2);
ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid);
ASSERT_EQ (blocks[1].verified, nano::signature_verification::valid);
}
node1.block_processor.add (send1);
node1.block_processor.add (open1);
node1.block_processor.flush ();
// Waits for the last blocks to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (10s, node1.store.block.exists (node1.store.tx_begin_read (), epoch2->hash ()));
{
auto transaction (node1.store.tx_begin_read ());
auto transaction = node1.store.tx_begin_read ();
ASSERT_FALSE (node1.store.block.exists (transaction, epoch1->hash ()));
ASSERT_TRUE (node1.store.block.exists (transaction, epoch2->hash ()));
ASSERT_TRUE (node1.active.empty ());
auto unchecked_count (node1.store.unchecked.count (transaction));
auto unchecked_count = node1.unchecked.count (transaction);
ASSERT_EQ (unchecked_count, 0);
ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction));
nano::account_info info;
ASSERT_EQ (unchecked_count, node1.unchecked.count (transaction));
nano::account_info info{};
ASSERT_FALSE (node1.store.account.get (transaction, destination.pub, info));
ASSERT_NE (info.epoch (), nano::epoch::epoch_1);
auto epoch2_store (node1.store.block.get (transaction, epoch2->hash ()));
auto epoch2_store = node1.store.block.get (transaction, epoch2->hash ());
ASSERT_NE (nullptr, epoch2_store);
ASSERT_EQ (nano::epoch::epoch_0, epoch2_store->sideband ().details.epoch);
ASSERT_TRUE (epoch2_store->sideband ().details.is_send);
@ -2750,74 +2741,58 @@ TEST (ledger, unchecked_open)
open2->signature.bytes[0] ^= 1;
node1.block_processor.add (open1);
node1.block_processor.add (open2);
node1.block_processor.flush ();
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 1);
ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction));
auto blocks (node1.store.unchecked.get (transaction, open1->source ()));
// Waits for the last blocks to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (10s, 1 == node1.unchecked.count (node1.store.tx_begin_read ()));
auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), open1->source ());
ASSERT_EQ (blocks.size (), 1);
ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid);
}
node1.block_processor.add (send1);
node1.block_processor.flush ();
{
auto transaction (node1.store.tx_begin_read ());
ASSERT_TRUE (node1.store.block.exists (transaction, open1->hash ()));
auto unchecked_count (node1.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 0);
ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction));
}
// Waits for the send1 block to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (10s, node1.store.block.exists (node1.store.tx_begin_read (), open1->hash ()));
ASSERT_EQ (0, node1.unchecked.count (node1.store.tx_begin_read ()));
}
TEST (ledger, unchecked_receive)
{
nano::system system (1);
auto & node1 (*system.nodes[0]);
nano::keypair destination;
auto send1 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
nano::system system{ 1 };
auto & node1 = *system.nodes[0];
nano::keypair destination{};
auto send1 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), nano::dev::genesis->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node1.work_generate_blocking (*send1);
auto send2 (std::make_shared<nano::state_block> (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0));
auto send2 = std::make_shared<nano::state_block> (nano::dev::genesis->account (), send1->hash (), nano::dev::genesis->account (), nano::dev::constants.genesis_amount - 2 * nano::Gxrb_ratio, destination.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
node1.work_generate_blocking (*send2);
auto open1 (std::make_shared<nano::open_block> (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0));
auto open1 = std::make_shared<nano::open_block> (send1->hash (), destination.pub, destination.pub, destination.prv, destination.pub, 0);
node1.work_generate_blocking (*open1);
auto receive1 (std::make_shared<nano::receive_block> (open1->hash (), send2->hash (), destination.prv, destination.pub, 0));
auto receive1 = std::make_shared<nano::receive_block> (open1->hash (), send2->hash (), destination.prv, destination.pub, 0);
node1.work_generate_blocking (*receive1);
node1.block_processor.add (send1);
node1.block_processor.add (receive1);
node1.block_processor.flush ();
auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) {
return !node1.unchecked.get (transaction_a, block_hash_a).empty ();
};
// Previous block for receive1 is unknown, signature cannot be validated
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 1);
ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction));
auto blocks (node1.store.unchecked.get (transaction, receive1->previous ()));
// Waits for the last blocks to pass through block_processor and unchecked.put queues
ASSERT_TIMELY (15s, check_block_is_listed (node1.store.tx_begin_read (), receive1->previous ()));
auto blocks = node1.unchecked.get (node1.store.tx_begin_read (), receive1->previous ());
ASSERT_EQ (blocks.size (), 1);
ASSERT_EQ (blocks[0].verified, nano::signature_verification::unknown);
}
// Waits for the open1 block to pass through block_processor and unchecked.put queues
node1.block_processor.add (open1);
node1.block_processor.flush ();
ASSERT_TIMELY (15s, check_block_is_listed (node1.store.tx_begin_read (), receive1->source ()));
// Previous block for receive1 is known, signature was validated
{
auto transaction (node1.store.tx_begin_read ());
auto unchecked_count (node1.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 1);
ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction));
auto blocks (node1.store.unchecked.get (transaction, receive1->source ()));
auto transaction = node1.store.tx_begin_read ();
auto blocks (node1.unchecked.get (transaction, receive1->source ()));
ASSERT_EQ (blocks.size (), 1);
ASSERT_EQ (blocks[0].verified, nano::signature_verification::valid);
}
node1.block_processor.add (send2);
node1.block_processor.flush ();
{
auto transaction (node1.store.tx_begin_read ());
ASSERT_TRUE (node1.store.block.exists (transaction, receive1->hash ()));
auto unchecked_count (node1.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 0);
ASSERT_EQ (unchecked_count, node1.store.unchecked.count (transaction));
}
ASSERT_TIMELY (10s, node1.store.block.exists (node1.store.tx_begin_read (), receive1->hash ()));
ASSERT_EQ (0, node1.unchecked.count (node1.store.tx_begin_read ()));
}
TEST (ledger, confirmation_height_not_updated)
@ -3621,13 +3596,14 @@ TEST (ledger, hash_root_random)
TEST (ledger, migrate_lmdb_to_rocksdb)
{
auto path (nano::unique_path ());
nano::logger_mt logger;
auto path = nano::unique_path ();
nano::logger_mt logger{};
boost::asio::ip::address_v6 address (boost::asio::ip::make_address_v6 ("::ffff:127.0.0.1"));
uint16_t port = 100;
nano::mdb_store store (logger, path / "data.ldb", nano::dev::constants);
nano::stat stats;
nano::ledger ledger (store, stats, nano::dev::constants);
nano::mdb_store store{ logger, path / "data.ldb", nano::dev::constants };
nano::unchecked_map unchecked{ store, false };
nano::stat stats{};
nano::ledger ledger{ store, stats, nano::dev::constants };
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
std::shared_ptr<nano::block> send = nano::state_block_builder ()
@ -3644,7 +3620,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
auto version = 99;
{
auto transaction (store.tx_begin_write ());
auto transaction = store.tx_begin_write ();
store.initialize (transaction, ledger.cache);
ASSERT_FALSE (store.init_error ());
@ -3657,7 +3633,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
store.pending.put (transaction, nano::pending_key (nano::dev::genesis->account (), send->hash ()), nano::pending_info (nano::dev::genesis->account (), 100, nano::epoch::epoch_0));
store.pruned.put (transaction, send->hash ());
store.unchecked.put (transaction, nano::dev::genesis->hash (), send);
store.version.put (transaction, version);
send->sideband_set ({});
store.block.put (transaction, send->hash (), *send);
@ -3667,10 +3642,11 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
auto error = ledger.migrate_lmdb_to_rocksdb (path);
ASSERT_FALSE (error);
nano::rocksdb_store rocksdb_store (logger, path / "rocksdb", nano::dev::constants);
nano::rocksdb_store rocksdb_store{ logger, path / "rocksdb", nano::dev::constants };
nano::unchecked_map rocksdb_unchecked{ rocksdb_store, false };
auto rocksdb_transaction (rocksdb_store.tx_begin_read ());
nano::pending_info pending_info;
nano::pending_info pending_info{};
ASSERT_FALSE (rocksdb_store.pending.get (rocksdb_transaction, nano::pending_key (nano::dev::genesis->account (), send->hash ()), pending_info));
for (auto i = rocksdb_store.online_weight.begin (rocksdb_transaction); i != rocksdb_store.online_weight.end (); ++i)
@ -3693,11 +3669,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
ASSERT_EQ (confirmation_height_info.frontier, send->hash ());
ASSERT_TRUE (rocksdb_store.final_vote.get (rocksdb_transaction, nano::root (send->previous ())).size () == 1);
ASSERT_EQ (rocksdb_store.final_vote.get (rocksdb_transaction, nano::root (send->previous ()))[0], nano::block_hash (2));
auto unchecked_infos = rocksdb_store.unchecked.get (rocksdb_transaction, nano::dev::genesis->hash ());
ASSERT_EQ (unchecked_infos.size (), 1);
ASSERT_EQ (unchecked_infos.front ().account, nano::dev::genesis->account ());
ASSERT_EQ (*unchecked_infos.front ().block, *send);
}
TEST (ledger, unconfirmed_frontiers)

35
nano/core_test/node.cpp
View file

@ -2961,7 +2961,7 @@ TEST (node, block_processor_signatures)
// Invalid signature to unchecked
{
auto transaction (node1.store.tx_begin_write ());
node1.store.unchecked.put (transaction, send5->previous (), send5);
node1.unchecked.put (send5->previous (), send5);
}
auto receive1 = builder.make_block ()
.account (key1.pub)
@ -3289,11 +3289,11 @@ TEST (node, peer_cache_restart)
TEST (node, unchecked_cleanup)
{
nano::system system;
nano::node_flags node_flags;
nano::system system{};
nano::node_flags node_flags{};
node_flags.disable_unchecked_cleanup = true;
nano::keypair key;
auto & node (*system.add_node (node_flags));
nano::keypair key{};
auto & node = *system.add_node (node_flags);
auto open = nano::state_block_builder ()
.account (key.pub)
.previous (0)
@ -3312,32 +3312,18 @@ TEST (node, unchecked_cleanup)
// Should be cleared after unchecked cleanup
ASSERT_FALSE (node.network.publish_filter.apply (bytes.data (), bytes.size ()));
node.process_active (open);
node.block_processor.flush ();
// Waits for the open block to get saved in the database
ASSERT_TIMELY (15s, 1 == node.unchecked.count (node.store.tx_begin_read ()));
node.config.unchecked_cutoff_time = std::chrono::seconds (2);
{
auto transaction (node.store.tx_begin_read ());
auto unchecked_count (node.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 1);
ASSERT_EQ (unchecked_count, node.store.unchecked.count (transaction));
}
ASSERT_EQ (1, node.unchecked.count (node.store.tx_begin_read ()));
std::this_thread::sleep_for (std::chrono::seconds (1));
node.unchecked_cleanup ();
ASSERT_TRUE (node.network.publish_filter.apply (bytes.data (), bytes.size ()));
{
auto transaction (node.store.tx_begin_read ());
auto unchecked_count (node.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 1);
ASSERT_EQ (unchecked_count, node.store.unchecked.count (transaction));
}
ASSERT_EQ (1, node.unchecked.count (node.store.tx_begin_read ()));
std::this_thread::sleep_for (std::chrono::seconds (2));
node.unchecked_cleanup ();
ASSERT_FALSE (node.network.publish_filter.apply (bytes.data (), bytes.size ()));
{
auto transaction (node.store.tx_begin_read ());
auto unchecked_count (node.store.unchecked.count (transaction));
ASSERT_EQ (unchecked_count, 0);
ASSERT_EQ (unchecked_count, node.store.unchecked.count (transaction));
}
ASSERT_EQ (0, node.unchecked.count (node.store.tx_begin_read ()));
}
/** This checks that a node can be opened (without being blocked) when a write lock is held elsewhere */
@ -3415,7 +3401,6 @@ TEST (node, bidirectional_tcp)
.work (*node1->work_generate_blocking (nano::dev::genesis->hash ()))
.build_shared ();
node1->process_active (send1);
node1->block_processor.flush ();
ASSERT_TIMELY (10s, node1->ledger.block_or_pruned_exists (send1->hash ()) && node2->ledger.block_or_pruned_exists (send1->hash ()));
// Test block confirmation from node 1 (add representative to node 1)
system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);

2
nano/core_test/telemetry.cpp
View file

@ -686,7 +686,7 @@ TEST (telemetry, remove_peer_invalid_signature)
// (Implementation detail) So that messages are not just discarded when requests were not sent.
node->telemetry->recent_or_initial_request_telemetry_data.emplace (channel->get_endpoint (), nano::telemetry_data (), std::chrono::steady_clock::now (), true);
auto telemetry_data = nano::local_telemetry_data (node->ledger, node->network, node->config.bandwidth_limit, node->network_params, node->startup_time, node->default_difficulty (nano::work_version::work_1), node->node_id);
auto telemetry_data = nano::local_telemetry_data (node->ledger, node->network, node->unchecked, node->config.bandwidth_limit, node->network_params, node->startup_time, node->default_difficulty (nano::work_version::work_1), node->node_id);
// Change anything so that the signed message is incorrect
telemetry_data.block_count = 0;
auto telemetry_ack = nano::telemetry_ack{ nano::dev::network_params.network, telemetry_data };

81
nano/core_test/unchecked_map.cpp Normal file
View file

@ -0,0 +1,81 @@
#include <nano/lib/blockbuilders.hpp>
#include <nano/lib/logger_mt.hpp>
#include <nano/node/unchecked_map.hpp>
#include <nano/secure/store.hpp>
#include <nano/secure/utility.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>
#include <gtest/gtest.h>
#include <memory>
using namespace std::chrono_literals;
namespace
{
class context
{
public:
context () :
store{ nano::make_store (logger, nano::unique_path (), nano::dev::constants) },
unchecked{ *store, false }
{
}
nano::logger_mt logger;
std::unique_ptr<nano::store> store;
nano::unchecked_map unchecked;
};
std::shared_ptr<nano::block> block ()
{
nano::block_builder builder;
return builder.state ()
.account (nano::dev::genesis_key.pub)
.previous (nano::dev::genesis->hash ())
.representative (nano::dev::genesis_key.pub)
.balance (nano::dev::constants.genesis_amount - 1)
.link (nano::dev::genesis_key.pub)
.sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub)
.work (0)
.build_shared ();
}
}
TEST (unchecked_map, construction)
{
context context;
}
TEST (unchecked_map, put_one)
{
context context;
nano::unchecked_info info{ block (), nano::dev::genesis_key.pub, nano::seconds_since_epoch () };
context.unchecked.put (info.block->previous (), info);
}
TEST (block_store, one_bootstrap)
{
nano::system system{};
nano::logger_mt logger{};
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
nano::unchecked_map unchecked{ *store, false };
ASSERT_TRUE (!store->init_error ());
auto block1 = std::make_shared<nano::send_block> (0, 1, 2, nano::keypair ().prv, 4, 5);
unchecked.put (block1->hash (), nano::unchecked_info{ block1 });
auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) {
return unchecked.get (transaction_a, block_hash_a).size () > 0;
};
// Waits for the block1 to get saved in the database
ASSERT_TIMELY (10s, check_block_is_listed (store->tx_begin_read (), block1->hash ()));
auto transaction = store->tx_begin_read ();
auto [begin, end] = unchecked.full_range (transaction);
ASSERT_NE (end, begin);
auto hash1 = begin->first.key ();
ASSERT_EQ (block1->hash (), hash1);
auto blocks = unchecked.get (transaction, hash1);
ASSERT_EQ (1, blocks.size ());
auto block2 = blocks[0].block;
ASSERT_EQ (*block1, *block2);
++begin;
ASSERT_EQ (end, begin);
}

2
nano/lib/threading.cpp
View file

@ -86,6 +86,8 @@ std::string nano::thread_role::get_string (nano::thread_role::name role)
break;
case nano::thread_role::name::election_scheduler:
thread_role_name_string = "Election Sched";
case nano::thread_role::name::unchecked:
thread_role_name_string = "Unchecked";
}
/*

3
nano/lib/threading.hpp
View file

@ -40,7 +40,8 @@ namespace thread_role
state_block_signature_verification,
epoch_upgrader,
db_parallel_traversal,
election_scheduler
election_scheduler,
unchecked,
};
/*
* Get/Set the identifier for the current thread

6
nano/nano_node/entry.cpp
View file

@ -438,7 +438,7 @@ int main (int argc, char * const * argv)
}
// Check all unchecked keys for matching frontier hashes. Indicates an issue with process_batch algorithm
for (auto [i, n] = node->store.unchecked.full_range (transaction); i != n; ++i)
for (auto [i, n] = node->unchecked.full_range (transaction); i != n; ++i)
{
auto it = frontier_hashes.find (i->first.key ());
if (it != frontier_hashes.cend ())
@ -1004,7 +1004,7 @@ int main (int argc, char * const * argv)
if (timer_l.after_deadline (std::chrono::seconds (15)))
{
timer_l.restart ();
std::cout << boost::str (boost::format ("%1% (%2%) blocks processed (unchecked), %3% remaining") % node->ledger.cache.block_count % node->store.unchecked.count (node->store.tx_begin_read ()) % node->block_processor.size ()) << std::endl;
std::cout << boost::str (boost::format ("%1% (%2%) blocks processed (unchecked), %3% remaining") % node->ledger.cache.block_count % node->unchecked.count (node->store.tx_begin_read ()) % node->block_processor.size ()) << std::endl;
}
}
@ -1853,7 +1853,7 @@ int main (int argc, char * const * argv)
if (timer_l.after_deadline (std::chrono::seconds (60)))
{
timer_l.restart ();
std::cout << boost::str (boost::format ("%1% (%2%) blocks processed (unchecked)") % node.node->ledger.cache.block_count % node.node->store.unchecked.count (node.node->store.tx_begin_read ())) << std::endl;
std::cout << boost::str (boost::format ("%1% (%2%) blocks processed (unchecked)") % node.node->ledger.cache.block_count % node.node->unchecked.count (node.node->store.tx_begin_read ())) << std::endl;
}
}

2
nano/node/CMakeLists.txt
View file

@ -136,6 +136,8 @@ add_library(
transport/transport.cpp
transport/udp.hpp
transport/udp.cpp
unchecked_map.cpp
unchecked_map.hpp
vote_processor.hpp
vote_processor.cpp
voting.hpp

21
nano/node/blockprocessor.cpp
View file

@ -380,10 +380,8 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction
{
info_a.modified = nano::seconds_since_epoch ();
}
node.store.unchecked.put (transaction_a, block->previous (), info_a);
node.unchecked.put (block->previous (), info_a);
events_a.events.emplace_back ([this, hash] (nano::transaction const & /* unused */) { this->node.gap_cache.add (hash); });
node.stats.inc (nano::stat::type::ledger, nano::stat::detail::gap_previous);
break;
}
@ -398,10 +396,8 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction
{
info_a.modified = nano::seconds_since_epoch ();
}
node.store.unchecked.put (transaction_a, node.ledger.block_source (transaction_a, *(block)), info_a);
node.unchecked.put (node.ledger.block_source (transaction_a, *(block)), info_a);
events_a.events.emplace_back ([this, hash] (nano::transaction const & /* unused */) { this->node.gap_cache.add (hash); });
node.stats.inc (nano::stat::type::ledger, nano::stat::detail::gap_source);
break;
}
@ -416,8 +412,7 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction
{
info_a.modified = nano::seconds_since_epoch ();
}
node.store.unchecked.put (transaction_a, block->account (), info_a); // Specific unchecked key starting with epoch open block account public key
node.unchecked.put (block->account (), info_a); // Specific unchecked key starting with epoch open block account public key
node.stats.inc (nano::stat::type::ledger, nano::stat::detail::gap_source);
break;
}
@ -515,15 +510,7 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction
void nano::block_processor::queue_unchecked (nano::write_transaction const & transaction_a, nano::hash_or_account const & hash_or_account_a)
{
auto unchecked_blocks (node.store.unchecked.get (transaction_a, hash_or_account_a.hash));
for (auto & info : unchecked_blocks)
{
if (!node.flags.disable_block_processor_unchecked_deletion)
{
node.store.unchecked.del (transaction_a, nano::unchecked_key (hash_or_account_a, info.block->hash ()));
}
add (info);
}
node.unchecked.trigger (hash_or_account_a);
node.gap_cache.erase (hash_or_account_a.hash);
}

4
nano/node/cli.cpp
View file

@ -230,7 +230,7 @@ bool copy_database (boost::filesystem::path const & data_path, boost::program_op
auto & store (node.node->store);
if (vm.count ("unchecked_clear"))
{
node.node->store.unchecked.clear (store.tx_begin_write ());
node.node->unchecked.clear (store.tx_begin_write ());
}
if (vm.count ("clear_send_ids"))
{
@ -508,7 +508,7 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
if (!node.node->init_error ())
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.unchecked.clear (transaction);
node.node->unchecked.clear (transaction);
std::cout << "Unchecked blocks deleted" << std::endl;
}
else

12
nano/node/json_handler.cpp
View file

@ -1358,7 +1358,7 @@ void nano::json_handler::block_account ()
void nano::json_handler::block_count ()
{
response_l.put ("count", std::to_string (node.ledger.cache.block_count));
response_l.put ("unchecked", std::to_string (node.store.unchecked.count (node.store.tx_begin_read ())));
response_l.put ("unchecked", std::to_string (node.unchecked.count (node.store.tx_begin_read ())));
response_l.put ("cemented", std::to_string (node.ledger.cache.cemented_count));
if (node.flags.enable_pruning)
{
@ -3878,7 +3878,7 @@ void nano::json_handler::telemetry ()
if (address.is_loopback () && port == rpc_l->node.network.endpoint ().port ())
{
// Requesting telemetry metrics locally
auto telemetry_data = nano::local_telemetry_data (rpc_l->node.ledger, rpc_l->node.network, rpc_l->node.config.bandwidth_limit, rpc_l->node.network_params, rpc_l->node.startup_time, rpc_l->node.default_difficulty (nano::work_version::work_1), rpc_l->node.node_id);
auto telemetry_data = nano::local_telemetry_data (rpc_l->node.ledger, rpc_l->node.network, rpc_l->node.unchecked, rpc_l->node.config.bandwidth_limit, rpc_l->node.network_params, rpc_l->node.startup_time, rpc_l->node.default_difficulty (nano::work_version::work_1), rpc_l->node.node_id);
nano::jsonconfig config_l;
auto const should_ignore_identification_metrics = false;
@ -4026,7 +4026,7 @@ void nano::json_handler::unchecked ()
{
boost::property_tree::ptree unchecked;
auto transaction (node.store.tx_begin_read ());
for (auto [i, n] = node.store.unchecked.full_range (transaction); i != n && unchecked.size () < count; ++i)
for (auto [i, n] = node.unchecked.full_range (transaction); i != n && unchecked.size () < count; ++i)
{
nano::unchecked_info const & info (i->second);
if (json_block_l)
@ -4051,7 +4051,7 @@ void nano::json_handler::unchecked_clear ()
{
node.workers.push_task (create_worker_task ([] (std::shared_ptr<nano::json_handler> const & rpc_l) {
auto transaction (rpc_l->node.store.tx_begin_write ({ tables::unchecked }));
rpc_l->node.store.unchecked.clear (transaction);
rpc_l->node.unchecked.clear (transaction);
rpc_l->response_l.put ("success", "");
rpc_l->response_errors ();
}));
@ -4064,7 +4064,7 @@ void nano::json_handler::unchecked_get ()
if (!ec)
{
auto transaction (node.store.tx_begin_read ());
for (auto [i, n] = node.store.unchecked.full_range (transaction); i != n; ++i)
for (auto [i, n] = node.unchecked.full_range (transaction); i != n; ++i)
{
nano::unchecked_key const & key (i->first);
if (key.hash == hash)
@ -4112,7 +4112,7 @@ void nano::json_handler::unchecked_keys ()
{
boost::property_tree::ptree unchecked;
auto transaction (node.store.tx_begin_read ());
for (auto [i, n] = node.store.unchecked.equal_range (transaction, key); i != n && unchecked.size () < count; ++i)
for (auto [i, n] = node.unchecked.equal_range (transaction, key); i != n && unchecked.size () < count; ++i)
{
boost::property_tree::ptree entry;
nano::unchecked_info const & info (i->second);

2
nano/node/network.cpp
View file

@ -538,7 +538,7 @@ public:
nano::telemetry_ack telemetry_ack{ node.network_params.network };
if (!node.flags.disable_providing_telemetry_metrics)
{
auto telemetry_data = nano::local_telemetry_data (node.ledger, node.network, node.config.bandwidth_limit, node.network_params, node.startup_time, node.default_difficulty (nano::work_version::work_1), node.node_id);
auto telemetry_data = nano::local_telemetry_data (node.ledger, node.network, node.unchecked, node.config.bandwidth_limit, node.network_params, node.startup_time, node.default_difficulty (nano::work_version::work_1), node.node_id);
telemetry_ack = nano::telemetry_ack{ node.network_params.network, telemetry_data };
}
channel->send (telemetry_ack, nullptr, nano::buffer_drop_policy::no_socket_drop);

13
nano/node/node.cpp
View file

@ -93,6 +93,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co
logger (config_a.logging.min_time_between_log_output),
store_impl (nano::make_store (logger, application_path_a, network_params.ledger, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_config, config_a.backup_before_upgrade)),
store (*store_impl),
unchecked{ store, flags.disable_block_processor_unchecked_deletion },
wallets_store_impl (std::make_unique<nano::mdb_wallets_store> (application_path_a / "wallets.ldb", config_a.lmdb_config)),
wallets_store (*wallets_store_impl),
gap_cache (*this),
@ -129,6 +130,9 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co
startup_time (std::chrono::steady_clock::now ()),
node_seq (seq)
{
unchecked.satisfied = [this] (nano::unchecked_info const & info) {
this->block_processor.add (info);
};
if (!init_error ())
{
telemetry->start ();
@ -413,7 +417,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co
if (!flags.disable_unchecked_drop && !use_bootstrap_weight && !flags.read_only)
{
auto const transaction (store.tx_begin_write ({ tables::unchecked }));
store.unchecked.clear (transaction);
unchecked.clear (transaction);
logger.always_log ("Dropping unchecked blocks");
}
}
@ -677,6 +681,7 @@ void nano::node::stop ()
// Cancels ongoing work generation tasks, which may be blocking other threads
// No tasks may wait for work generation in I/O threads, or termination signal capturing will be unable to call node::stop()
distributed_work.stop ();
unchecked.stop ();
block_processor.stop ();
aggregator.stop ();
vote_processor.stop ();
@ -941,7 +946,7 @@ void nano::node::unchecked_cleanup ()
auto const now (nano::seconds_since_epoch ());
auto const transaction (store.tx_begin_read ());
// Max 1M records to clean, max 2 minutes reading to prevent slow i/o systems issues
for (auto [i, n] = store.unchecked.full_range (transaction); i != n && cleaning_list.size () < 1024 * 1024 && nano::seconds_since_epoch () - now < 120; ++i)
for (auto [i, n] = unchecked.full_range (transaction); i != n && cleaning_list.size () < 1024 * 1024 && nano::seconds_since_epoch () - now < 120; ++i)
{
nano::unchecked_key const & key (i->first);
nano::unchecked_info const & info (i->second);
@ -965,9 +970,9 @@ void nano::node::unchecked_cleanup ()
{
auto key (cleaning_list.front ());
cleaning_list.pop_front ();
if (store.unchecked.exists (transaction, key))
if (unchecked.exists (transaction, key))
{
store.unchecked.del (transaction, key);
unchecked.del (transaction, key);
}
}
}

2
nano/node/node.hpp
View file

@ -22,6 +22,7 @@
#include <nano/node/request_aggregator.hpp>
#include <nano/node/signatures.hpp>
#include <nano/node/telemetry.hpp>
#include <nano/node/unchecked_map.hpp>
#include <nano/node/vote_processor.hpp>
#include <nano/node/wallet.hpp>
#include <nano/node/write_database_queue.hpp>
@ -165,6 +166,7 @@ public:
nano::logger_mt logger;
std::unique_ptr<nano::store> store_impl;
nano::store & store;
nano::unchecked_map unchecked;
std::unique_ptr<nano::wallets_store> wallets_store_impl;
nano::wallets_store & wallets_store;
nano::gap_cache gap_cache;

5
nano/node/telemetry.cpp
View file

@ -4,6 +4,7 @@
#include <nano/node/nodeconfig.hpp>
#include <nano/node/telemetry.hpp>
#include <nano/node/transport/transport.hpp>
#include <nano/node/unchecked_map.hpp>
#include <nano/secure/buffer.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/store.hpp>
@ -628,7 +629,7 @@ nano::telemetry_data nano::consolidate_telemetry_data (std::vector<nano::telemet
return consolidated_data;
}
nano::telemetry_data nano::local_telemetry_data (nano::ledger const & ledger_a, nano::network & network_a, uint64_t bandwidth_limit_a, nano::network_params const & network_params_a, std::chrono::steady_clock::time_point statup_time_a, uint64_t active_difficulty_a, nano::keypair const & node_id_a)
nano::telemetry_data nano::local_telemetry_data (nano::ledger const & ledger_a, nano::network & network_a, nano::unchecked_map const & unchecked, uint64_t bandwidth_limit_a, nano::network_params const & network_params_a, std::chrono::steady_clock::time_point statup_time_a, uint64_t active_difficulty_a, nano::keypair const & node_id_a)
{
nano::telemetry_data telemetry_data;
telemetry_data.node_id = node_id_a.pub;
@ -637,7 +638,7 @@ nano::telemetry_data nano::local_telemetry_data (nano::ledger const & ledger_a,
telemetry_data.bandwidth_cap = bandwidth_limit_a;
telemetry_data.protocol_version = network_params_a.network.protocol_version;
telemetry_data.uptime = std::chrono::duration_cast<std::chrono::seconds> (std::chrono::steady_clock::now () - statup_time_a).count ();
telemetry_data.unchecked_count = ledger_a.store.unchecked.count (ledger_a.store.tx_begin_read ());
telemetry_data.unchecked_count = unchecked.count (ledger_a.store.tx_begin_read ());
telemetry_data.genesis_block = network_params_a.ledger.genesis->hash ();
telemetry_data.peer_count = nano::narrow_cast<decltype (telemetry_data.peer_count)> (network_a.size ());
telemetry_data.account_count = ledger_a.cache.account_count;

3
nano/node/telemetry.hpp
View file

@ -20,6 +20,7 @@ class network;
class stat;
class ledger;
class thread_pool;
class unchecked_map;
namespace transport
{
class channel;
@ -149,5 +150,5 @@ private:
std::unique_ptr<nano::container_info_component> collect_container_info (telemetry & telemetry, std::string const & name);
nano::telemetry_data consolidate_telemetry_data (std::vector<telemetry_data> const & telemetry_data);
nano::telemetry_data local_telemetry_data (nano::ledger const & ledger_a, nano::network &, uint64_t, nano::network_params const &, std::chrono::steady_clock::time_point, uint64_t, nano::keypair const &);
nano::telemetry_data local_telemetry_data (nano::ledger const & ledger_a, nano::network &, nano::unchecked_map const &, uint64_t, nano::network_params const &, std::chrono::steady_clock::time_point, uint64_t, nano::keypair const &);
}

154
nano/node/unchecked_map.cpp Normal file
View file

@ -0,0 +1,154 @@
#include <nano/lib/locks.hpp>
#include <nano/lib/threading.hpp>
#include <nano/node/unchecked_map.hpp>
#include <nano/secure/store.hpp>
#include <boost/range/join.hpp>
nano::unchecked_map::unchecked_map (nano::store & store, bool const & disable_delete) :
store{ store },
disable_delete{ disable_delete },
thread{ [this] () { run (); } }
{
}
nano::unchecked_map::~unchecked_map ()
{
stop ();
thread.join ();
}
void nano::unchecked_map::put (nano::hash_or_account const & dependency, nano::unchecked_info const & info)
{
nano::unique_lock<nano::mutex> lock{ mutex };
buffer.push_back (std::make_pair (dependency, info));
lock.unlock ();
condition.notify_all (); // Notify run ()
}
auto nano::unchecked_map::equal_range (nano::transaction const & transaction, nano::block_hash const & dependency) -> std::pair<iterator, iterator>
{
return store.unchecked.equal_range (transaction, dependency);
}
auto nano::unchecked_map::full_range (nano::transaction const & transaction) -> std::pair<iterator, iterator>
{
return store.unchecked.full_range (transaction);
}
std::vector<nano::unchecked_info> nano::unchecked_map::get (nano::transaction const & transaction, nano::block_hash const & hash)
{
return store.unchecked.get (transaction, hash);
}
bool nano::unchecked_map::exists (nano::transaction const & transaction, nano::unchecked_key const & key) const
{
return store.unchecked.exists (transaction, key);
}
void nano::unchecked_map::del (nano::write_transaction const & transaction, nano::unchecked_key const & key)
{
store.unchecked.del (transaction, key);
}
void nano::unchecked_map::clear (nano::write_transaction const & transaction)
{
store.unchecked.clear (transaction);
}
size_t nano::unchecked_map::count (nano::transaction const & transaction) const
{
return store.unchecked.count (transaction);
}
void nano::unchecked_map::stop ()
{
if (!stopped.exchange (true))
{
condition.notify_all (); // Notify flush (), run ()
}
}
void nano::unchecked_map::flush ()
{
nano::unique_lock<nano::mutex> lock{ mutex };
condition.wait (lock, [this] () {
return stopped || (buffer.empty () && back_buffer.empty () && !writing_back_buffer);
});
}
void nano::unchecked_map::trigger (nano::hash_or_account const & dependency)
{
nano::unique_lock<nano::mutex> lock{ mutex };
buffer.push_back (dependency);
debug_assert (buffer.back ().which () == 1); // which stands for "query".
lock.unlock ();
condition.notify_all (); // Notify run ()
}
nano::unchecked_map::item_visitor::item_visitor (unchecked_map & unchecked, nano::write_transaction const & transaction) :
unchecked{ unchecked },
transaction{ transaction }
{
}
void nano::unchecked_map::item_visitor::operator() (insert const & item)
{
auto const & [dependency, info] = item;
unchecked.store.unchecked.put (transaction, dependency, info);
}
void nano::unchecked_map::item_visitor::operator() (query const & item)
{
auto [i, n] = unchecked.store.unchecked.equal_range (transaction, item.hash);
std::deque<nano::unchecked_key> delete_queue;
for (; i != n; ++i)
{
auto const & key = i->first;
auto const & info = i->second;
delete_queue.push_back (key);
unchecked.satisfied (info);
}
if (!unchecked.disable_delete)
{
for (auto const & key : delete_queue)
{
unchecked.del (transaction, key);
}
}
}
void nano::unchecked_map::write_buffer (decltype (buffer) const & back_buffer)
{
auto transaction = store.tx_begin_write ();
item_visitor visitor{ *this, transaction };
for (auto const & item : back_buffer)
{
boost::apply_visitor (visitor, item);
}
}
void nano::unchecked_map::run ()
{
nano::thread_role::set (nano::thread_role::name::unchecked);
nano::unique_lock<nano::mutex> lock{ mutex };
while (!stopped)
{
if (!buffer.empty ())
{
back_buffer.swap (buffer);
writing_back_buffer = true;
lock.unlock ();
write_buffer (back_buffer);
lock.lock ();
writing_back_buffer = false;
back_buffer.clear ();
}
else
{
condition.notify_all (); // Notify flush ()
condition.wait (lock, [this] () {
return stopped || !buffer.empty ();
});
}
}
}

65
nano/node/unchecked_map.hpp Normal file
View file

@ -0,0 +1,65 @@
#pragma once
#include <nano/lib/locks.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/secure/store.hpp>
#include <atomic>
#include <thread>
#include <unordered_map>
namespace nano
{
class store;
class transaction;
class unchecked_info;
class unchecked_key;
class write_transaction;
class unchecked_map
{
public:
using iterator = nano::unchecked_store::iterator;
public:
unchecked_map (nano::store & store, bool const & do_delete);
~unchecked_map ();
void put (nano::hash_or_account const & dependency, nano::unchecked_info const & info);
std::pair<iterator, iterator> equal_range (nano::transaction const & transaction, nano::block_hash const & dependency);
std::pair<iterator, iterator> full_range (nano::transaction const & transaction);
std::vector<nano::unchecked_info> get (nano::transaction const &, nano::block_hash const &);
bool exists (nano::transaction const & transaction, nano::unchecked_key const & key) const;
void del (nano::write_transaction const & transaction, nano::unchecked_key const & key);
void clear (nano::write_transaction const & transaction);
size_t count (nano::transaction const & transaction) const;
void stop ();
void flush ();
public: // Trigger requested dependencies
void trigger (nano::hash_or_account const & dependency);
std::function<void (nano::unchecked_info const &)> satisfied{ [] (nano::unchecked_info const &) {} };
private:
using insert = std::pair<nano::hash_or_account, nano::unchecked_info>;
using query = nano::hash_or_account;
class item_visitor : boost::static_visitor<>
{
public:
item_visitor (unchecked_map & unchecked, nano::write_transaction const & transaction);
void operator() (insert const & item);
void operator() (query const & item);
unchecked_map & unchecked;
nano::write_transaction const & transaction;
};
void run ();
nano::store & store;
bool const & disable_delete;
std::deque<boost::variant<insert, query>> buffer;
std::deque<boost::variant<insert, query>> back_buffer;
bool writing_back_buffer{ false };
std::atomic<bool> stopped{ false };
nano::condition_variable condition;
nano::mutex mutex;
std::thread thread;
void write_buffer (decltype (buffer) const & back_buffer);
};
}

4
nano/qt/qt.cpp
View file

@ -923,7 +923,7 @@ std::string nano_qt::status::text ()
std::string count_string;
{
auto size (wallet.wallet_m->wallets.node.ledger.cache.block_count.load ());
unchecked = wallet.wallet_m->wallets.node.store.unchecked.count (wallet.wallet_m->wallets.node.store.tx_begin_read ());
unchecked = wallet.wallet_m->wallets.node.unchecked.count (wallet.wallet_m->wallets.node.store.tx_begin_read ());
cemented = wallet.wallet_m->wallets.node.ledger.cache.cemented_count.load ();
count_string = std::to_string (size);
}
@ -959,7 +959,7 @@ std::string nano_qt::status::text ()
result += ", Blocks: ";
if (unchecked != 0 && wallet.node.bootstrap_initiator.in_progress ())
{
count_string += "\nUnchecked: " + std::to_string (unchecked) + ", Cemented: " + std::to_string (cemented);
count_string += ", Queued: " + std::to_string (unchecked);
}
if (wallet.node.flags.enable_pruning)

67
nano/rpc_test/rpc.cpp
View file

@ -5010,17 +5010,19 @@ TEST (rpc, stats_clear)
ASSERT_LE (node->stats.last_reset ().count (), 5);
}
// Tests the RPC command returns the correct data for the unchecked blocks
TEST (rpc, unchecked)
{
nano::system system;
nano::system system{};
auto node = add_ipc_enabled_node (system);
auto const rpc_ctx = add_rpc (system, node);
nano::keypair key;
auto open (std::make_shared<nano::state_block> (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub)));
auto open2 (std::make_shared<nano::state_block> (key.pub, 0, key.pub, 2, key.pub, key.prv, key.pub, *system.work.generate (key.pub)));
nano::keypair key{};
auto open = std::make_shared<nano::state_block> (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub));
auto open2 = std::make_shared<nano::state_block> (key.pub, 0, key.pub, 2, key.pub, key.prv, key.pub, *system.work.generate (key.pub));
node->process_active (open);
node->process_active (open2);
node->block_processor.flush ();
// Waits for the last block of the queue to get saved in the database
ASSERT_TIMELY (10s, 2 == node->unchecked.count (node->store.tx_begin_read ()));
boost::property_tree::ptree request;
request.put ("action", "unchecked");
request.put ("count", 2);
@ -5041,16 +5043,18 @@ TEST (rpc, unchecked)
}
}
// Tests the RPC command returns the correct data for the unchecked blocks
TEST (rpc, unchecked_get)
{
nano::system system;
nano::system system{};
auto node = add_ipc_enabled_node (system);
auto const rpc_ctx = add_rpc (system, node);
nano::keypair key;
auto open (std::make_shared<nano::state_block> (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub)));
nano::keypair key{};
auto open = std::make_shared<nano::state_block> (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub));
node->process_active (open);
node->block_processor.flush ();
boost::property_tree::ptree request;
// Waits for the open block to get saved in the database
ASSERT_TIMELY (10s, 1 == node->unchecked.count (node->store.tx_begin_read ()));
boost::property_tree::ptree request{};
request.put ("action", "unchecked_get");
request.put ("hash", open->hash ().to_string ());
{
@ -5071,21 +5075,20 @@ TEST (rpc, unchecked_get)
TEST (rpc, unchecked_clear)
{
nano::system system;
nano::system system{};
auto node = add_ipc_enabled_node (system);
auto const rpc_ctx = add_rpc (system, node);
nano::keypair key;
auto open (std::make_shared<nano::state_block> (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub)));
nano::keypair key{};
auto open = std::make_shared<nano::state_block> (key.pub, 0, key.pub, 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub));
node->process_active (open);
node->block_processor.flush ();
boost::property_tree::ptree request;
{
ASSERT_EQ (node->store.unchecked.count (node->store.tx_begin_read ()), 1);
}
boost::property_tree::ptree request{};
// Waits for the open block to get saved in the database
ASSERT_TIMELY (10s, 1 == node->unchecked.count (node->store.tx_begin_read ()));
request.put ("action", "unchecked_clear");
auto response (wait_response (system, rpc_ctx, request));
auto response = wait_response (system, rpc_ctx, request);
ASSERT_TIMELY (10s, node->store.unchecked.count (node->store.tx_begin_read ()) == 0);
// Waits for the open block to get saved in the database
ASSERT_TIMELY (10s, 0 == node->unchecked.count (node->store.tx_begin_read ()));
}
TEST (rpc, unopened)
@ -5807,19 +5810,19 @@ TEST (rpc, epoch_upgrade_multithreaded)
TEST (rpc, account_lazy_start)
{
nano::system system;
nano::node_flags node_flags;
nano::system system{};
nano::node_flags node_flags{};
node_flags.disable_legacy_bootstrap = true;
auto node1 = system.add_node (node_flags);
nano::keypair key;
nano::keypair key{};
// Generating test chain
auto send1 (std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::dev::constants.genesis_amount - nano::Gxrb_ratio, key.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, *system.work.generate (nano::dev::genesis->hash ())));
auto send1 = std::make_shared<nano::state_block> (nano::dev::genesis_key.pub, nano::dev::genesis->hash (), nano::dev::genesis_key.pub, nano::dev::constants.genesis_amount - nano::Gxrb_ratio, key.pub, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, *system.work.generate (nano::dev::genesis->hash ()));
ASSERT_EQ (nano::process_result::progress, node1->process (*send1).code);
auto open (std::make_shared<nano::open_block> (send1->hash (), key.pub, key.pub, key.prv, key.pub, *system.work.generate (key.pub)));
auto open = std::make_shared<nano::open_block> (send1->hash (), key.pub, key.pub, key.prv, key.pub, *system.work.generate (key.pub));
ASSERT_EQ (nano::process_result::progress, node1->process (*open).code);
// Start lazy bootstrap with account
nano::node_config node_config (nano::get_available_port (), system.logging);
nano::node_config node_config{ nano::get_available_port (), system.logging };
node_config.ipc_config.transport_tcp.enabled = true;
node_config.ipc_config.transport_tcp.port = nano::get_available_port ();
auto node2 = system.add_node (node_config, node_flags);
@ -5828,15 +5831,17 @@ TEST (rpc, account_lazy_start)
boost::property_tree::ptree request;
request.put ("action", "account_info");
request.put ("account", key.pub.to_account ());
auto response (wait_response (system, rpc_ctx, request));
boost::optional<std::string> account_error (response.get_optional<std::string> ("error"));
auto response = wait_response (system, rpc_ctx, request);
boost::optional<std::string> account_error{ response.get_optional<std::string> ("error") };
ASSERT_TRUE (account_error.is_initialized ());
// Check processed blocks
ASSERT_TIMELY (10s, !node2->bootstrap_initiator.in_progress ());
node2->block_processor.flush ();
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TRUE (node2->ledger.block_or_pruned_exists (open->hash ()));
// needs timed assert because the writing (put) operation is done by a different
// thread, it might not get done before DB get operation.
ASSERT_TIMELY (10s, node2->ledger.block_or_pruned_exists (send1->hash ()));
ASSERT_TIMELY (10s, node2->ledger.block_or_pruned_exists (open->hash ()));
}
TEST (rpc, receive)

10
nano/secure/ledger.cpp
View file

@ -1438,15 +1438,6 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data
}
});
store.unchecked.for_each_par (
[&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) {
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::unchecked }));
rocksdb_store->unchecked.put (rocksdb_transaction, i->first.previous, i->second);
}
});
store.pending.for_each_par (
[&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) {
for (; i != n; ++i)
@ -1517,7 +1508,6 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data
}
// Compare counts
error |= store.unchecked.count (lmdb_transaction) != rocksdb_store->unchecked.count (rocksdb_transaction);
error |= store.peer.count (lmdb_transaction) != rocksdb_store->peer.count (rocksdb_transaction);
error |= store.pruned.count (lmdb_transaction) != rocksdb_store->pruned.count (rocksdb_transaction);
error |= store.final_vote.count (lmdb_transaction) != rocksdb_store->final_vote.count (rocksdb_transaction);

7
nano/secure/store.hpp
View file

@ -815,6 +815,7 @@ public:
virtual uint64_t account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0;
};
class unchecked_map;
/**
* Store manager
*/
@ -844,7 +845,11 @@ public:
frontier_store & frontier;
account_store & account;
pending_store & pending;
private:
unchecked_store & unchecked;
public:
online_weight_store & online_weight;
pruned_store & pruned;
peer_store & peer;
@ -870,6 +875,8 @@ public:
virtual nano::read_transaction tx_begin_read () const = 0;
virtual std::string vendor_get () const = 0;
friend class unchecked_map;
};
std::unique_ptr<nano::store> make_store (nano::logger_mt & logger, boost::filesystem::path const & path, nano::ledger_constants & constants, bool open_read_only = false, bool add_db_postfix = false, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false);

14
nano/slow_test/node.cpp
View file

@ -2,6 +2,7 @@
#include <nano/lib/threading.hpp>
#include <nano/node/election.hpp>
#include <nano/node/transport/udp.hpp>
#include <nano/node/unchecked_map.hpp>
#include <nano/test_common/network.hpp>
#include <nano/test_common/system.hpp>
#include <nano/test_common/testutil.hpp>
@ -420,17 +421,16 @@ TEST (peer_container, random_set)
// Can take up to 2 hours
TEST (store, unchecked_load)
{
nano::system system (1);
auto & node (*system.nodes[0]);
nano::system system{ 1 };
auto & node = *system.nodes[0];
std::shared_ptr<nano::block> block = std::make_shared<nano::send_block> (0, 0, 0, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, 0);
constexpr auto num_unchecked = 1000000;
constexpr auto num_unchecked = 1'000'000;
for (auto i (0); i < num_unchecked; ++i)
{
auto transaction (node.store.tx_begin_write ());
node.store.unchecked.put (transaction, i, block);
node.unchecked.put (i, block);
}
auto transaction (node.store.tx_begin_read ());
ASSERT_EQ (num_unchecked, node.store.unchecked.count (transaction));
// Waits for all the blocks to get saved in the database
ASSERT_TIMELY (8000s, num_unchecked == node.unchecked.count (node.store.tx_begin_read ()));
}
TEST (store, vote_load)