Minecon724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

[RocksDB] Tests not reading account count from store correctly (#2623)

Browse source
This commit is contained in:
Wesley Shillingford 2020-03-03 17:44:18 +00:00 • committed by GitHub
commit a1ad2fa3c5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 61 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

55
nano/core_test/node_telemetry.cpp
View file

@ -335,7 +335,7 @@ TEST (node_telemetry, many_nodes)
{
nano::system system;
// The telemetry responses can timeout if using a large number of nodes under sanitizers, so lower the number.
const auto num_nodes = (is_sanitizer_build || nano::running_within_valgrind ()) ? 4 : 10; //3; // 10;
const auto num_nodes = (is_sanitizer_build || nano::running_within_valgrind ()) ? 4 : 10;
nano::node_flags node_flags;
node_flags.disable_ongoing_telemetry_requests = true;
for (auto i = 0; i < num_nodes; ++i)
@ -343,7 +343,23 @@ TEST (node_telemetry, many_nodes)
nano::node_config node_config (nano::get_available_port (), system.logging);
// Make a metric completely different for each node so we can check afterwards that there are no duplicates
node_config.bandwidth_limit = 100000 + i;
system.add_node (node_config, node_flags);
auto node = std::make_shared<nano::node> (system.io_ctx, nano::unique_path (), system.alarm, node_config, system.work, node_flags);
node->start ();
system.nodes.push_back (node);
}
// Merge peers after creating nodes as some backends (RocksDB) can take a while to initialize nodes (Windows/Debug for instance)
// and timeouts can occur between nodes while starting up many nodes synchronously.
for (auto const & node : system.nodes)
{
for (auto const & other_node : system.nodes)
{
if (node != other_node)
{
node->network.merge_peer (other_node->network.endpoint ());
}
}
}
wait_peer_connections (system);
@ -755,17 +771,30 @@ namespace
{
void wait_peer_connections (nano::system & system_a)
{
system_a.deadline_set (10s);
auto peer_count = 0;
auto num_nodes = system_a.nodes.size ();
while (peer_count != num_nodes * (num_nodes - 1))
{
ASSERT_NO_ERROR (system_a.poll ());
peer_count = std::accumulate (system_a.nodes.cbegin (), system_a.nodes.cend (), 0, [](auto total, auto const & node) {
auto transaction = node->store.tx_begin_read ();
return total += node->store.peer_count (transaction);
});
}
auto wait_peer_count = [&system_a](bool in_memory) {
auto num_nodes = system_a.nodes.size ();
system_a.deadline_set (20s);
auto peer_count = 0;
while (peer_count != num_nodes * (num_nodes - 1))
{
ASSERT_NO_ERROR (system_a.poll ());
peer_count = std::accumulate (system_a.nodes.cbegin (), system_a.nodes.cend (), 0, [in_memory](auto total, auto const & node) {
if (in_memory)
{
return total += node->network.size ();
}
else
{
auto transaction = node->store.tx_begin_read ();
return total += node->store.peer_count (transaction);
}
});
}
};
// Do a pre-pass with in-memory containers to reduce IO if still in the process of connecting to peers
wait_peer_count (true);
wait_peer_count (false);
}
void compare_default_test_result_data (nano::telemetry_data const & telemetry_data_a, nano::node const & node_server_a)

3
nano/nano_node/entry.cpp
View file

@ -276,8 +276,7 @@ int main (int argc, char * const * argv)
else if (vm.count ("debug_account_count"))
{
nano::inactive_node node (data_path);
auto transaction (node.node->store.tx_begin_read ());
std::cout << boost::str (boost::format ("Frontier count: %1%\n") % node.node->store.account_count (transaction));
std::cout << boost::str (boost::format ("Frontier count: %1%\n") % node.node->ledger.cache.account_count);
}
else if (vm.count ("debug_mass_activity"))
{

13
nano/node/rocksdb/rocksdb.cpp
View file

@ -366,7 +366,7 @@ uint64_t nano::rocksdb_store::count (nano::transaction const & transaction_a, ro
size_t nano::rocksdb_store::count (nano::transaction const & transaction_a, tables table_a) const
{
size_t sum = 0;
// Some column families are small enough that they can just be iterated, rather than doing extra io caching counts
// Some column families are small enough (except unchecked) that they can just be iterated, rather than doing extra io caching counts
if (table_a == tables::peers)
{
for (auto i (peers_begin (transaction_a)), n (peers_end ()); i != n; ++i)
@ -381,6 +381,7 @@ size_t nano::rocksdb_store::count (nano::transaction const & transaction_a, tabl
++sum;
}
}
// This should only be used during initialization as can be expensive during bootstrapping
else if (table_a == tables::unchecked)
{
for (auto i (unchecked_begin (transaction_a)), n (unchecked_end ()); i != n; ++i)
@ -388,8 +389,18 @@ size_t nano::rocksdb_store::count (nano::transaction const & transaction_a, tabl
++sum;
}
}
// This should only be used in tests
else if (table_a == tables::accounts)
{
debug_assert (network_constants ().is_test_network ());
for (auto i (latest_begin (transaction_a)), n (latest_end ()); i != n; ++i)
{
++sum;
}
}
else
{
debug_assert (is_caching_counts (table_a));
return count (transaction_a, table_to_column_family (table_a));
}

6
nano/secure/blockstore.hpp
View file

@ -688,9 +688,9 @@ public:
virtual size_t account_count (nano::transaction const &) = 0;
virtual void confirmation_height_clear (nano::write_transaction const &, nano::account const &, uint64_t) = 0;
virtual void confirmation_height_clear (nano::write_transaction const &) = 0;
virtual nano::store_iterator<nano::account, nano::account_info> latest_begin (nano::transaction const &, nano::account const &) = 0;
virtual nano::store_iterator<nano::account, nano::account_info> latest_begin (nano::transaction const &) = 0;
virtual nano::store_iterator<nano::account, nano::account_info> latest_end () = 0;
virtual nano::store_iterator<nano::account, nano::account_info> latest_begin (nano::transaction const &, nano::account const &) const = 0;
virtual nano::store_iterator<nano::account, nano::account_info> latest_begin (nano::transaction const &) const = 0;
virtual nano::store_iterator<nano::account, nano::account_info> latest_end () const = 0;
virtual void pending_put (nano::write_transaction const &, nano::pending_key const &, nano::pending_info const &) = 0;
virtual void pending_del (nano::write_transaction const &, nano::pending_key const &) = 0;

6
nano/secure/blockstore_partial.hpp
View file

@ -354,7 +354,7 @@ public:
return nano::store_iterator<uint64_t, nano::amount> (nullptr);
}
nano::store_iterator<nano::account, nano::account_info> latest_end () override
nano::store_iterator<nano::account, nano::account_info> latest_end () const override
{
return nano::store_iterator<nano::account, nano::account_info> (nullptr);
}
@ -723,12 +723,12 @@ public:
return exists (transaction_a, tables::confirmation_height, nano::db_val<Val> (account_a));
}
nano::store_iterator<nano::account, nano::account_info> latest_begin (nano::transaction const & transaction_a, nano::account const & account_a) override
nano::store_iterator<nano::account, nano::account_info> latest_begin (nano::transaction const & transaction_a, nano::account const & account_a) const override
{
return make_iterator<nano::account, nano::account_info> (transaction_a, tables::accounts, nano::db_val<Val> (account_a));
}
nano::store_iterator<nano::account, nano::account_info> latest_begin (nano::transaction const & transaction_a) override
nano::store_iterator<nano::account, nano::account_info> latest_begin (nano::transaction const & transaction_a) const override
{
return make_iterator<nano::account, nano::account_info> (transaction_a, tables::accounts);
}