Minecon724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Online weight store break up (#3316)

Browse source 
* Moves online weight store out of block store class
* Renames the function names
This commit is contained in:
Thiago Silva 2021-06-02 10:37:25 -03:00 • committed by GitHub
commit 1a3857fd4e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 135 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

34
nano/core_test/block_store.cpp
View file

@ -1171,32 +1171,32 @@ TEST (block_store, online_weight)
ASSERT_FALSE (store->init_error ());
{
auto transaction (store->tx_begin_write ());
ASSERT_EQ (0, store->online_weight_count (transaction));
ASSERT_EQ (store->online_weight_end (), store->online_weight_begin (transaction));
ASSERT_EQ (store->online_weight_end (), store->online_weight_rbegin (transaction));
store->online_weight_put (transaction, 1, 2);
store->online_weight_put (transaction, 3, 4);
ASSERT_EQ (0, store->online_weight.count (transaction));
ASSERT_EQ (store->online_weight.end (), store->online_weight.begin (transaction));
ASSERT_EQ (store->online_weight.end (), store->online_weight.rbegin (transaction));
store->online_weight.put (transaction, 1, 2);
store->online_weight.put (transaction, 3, 4);
}
{
auto transaction (store->tx_begin_write ());
ASSERT_EQ (2, store->online_weight_count (transaction));
auto item (store->online_weight_begin (transaction));
ASSERT_NE (store->online_weight_end (), item);
ASSERT_EQ (2, store->online_weight.count (transaction));
auto item (store->online_weight.begin (transaction));
ASSERT_NE (store->online_weight.end (), item);
ASSERT_EQ (1, item->first);
ASSERT_EQ (2, item->second.number ());
auto item_last (store->online_weight_rbegin (transaction));
ASSERT_NE (store->online_weight_end (), item_last);
auto item_last (store->online_weight.rbegin (transaction));
ASSERT_NE (store->online_weight.end (), item_last);
ASSERT_EQ (3, item_last->first);
ASSERT_EQ (4, item_last->second.number ());
store->online_weight_del (transaction, 1);
ASSERT_EQ (1, store->online_weight_count (transaction));
ASSERT_EQ (store->online_weight_begin (transaction), store->online_weight_rbegin (transaction));
store->online_weight_del (transaction, 3);
store->online_weight.del (transaction, 1);
ASSERT_EQ (1, store->online_weight.count (transaction));
ASSERT_EQ (store->online_weight.begin (transaction), store->online_weight.rbegin (transaction));
store->online_weight.del (transaction, 3);
}
auto transaction (store->tx_begin_read ());
ASSERT_EQ (0, store->online_weight_count (transaction));
ASSERT_EQ (store->online_weight_end (), store->online_weight_begin (transaction));
ASSERT_EQ (store->online_weight_end (), store->online_weight_rbegin (transaction));
ASSERT_EQ (0, store->online_weight.count (transaction));
ASSERT_EQ (store->online_weight.end (), store->online_weight.begin (transaction));
ASSERT_EQ (store->online_weight.end (), store->online_weight.rbegin (transaction));
}
TEST (block_store, pruned_blocks)

6
nano/core_test/ledger.cpp
View file

@ -3862,7 +3862,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
// Lower the database to the max version unsupported for upgrades
store.confirmation_height_put (transaction, nano::genesis_account, { 2, send->hash () });
store.online_weight_put (transaction, 100, nano::amount (2));
store.online_weight.put (transaction, 100, nano::amount (2));
store.frontier.put (transaction, nano::block_hash (2), nano::account (5));
store.peer_put (transaction, endpoint_key);
@ -3884,13 +3884,13 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
nano::pending_info pending_info;
ASSERT_FALSE (rocksdb_store.pending.get (rocksdb_transaction, nano::pending_key (nano::genesis_account, send->hash ()), pending_info));
for (auto i = rocksdb_store.online_weight_begin (rocksdb_transaction); i != rocksdb_store.online_weight_end (); ++i)
for (auto i = rocksdb_store.online_weight.begin (rocksdb_transaction); i != rocksdb_store.online_weight.end (); ++i)
{
ASSERT_EQ (i->first, 100);
ASSERT_EQ (i->second, 2);
}
ASSERT_EQ (rocksdb_store.online_weight_count (rocksdb_transaction), 1);
ASSERT_EQ (rocksdb_store.online_weight.count (rocksdb_transaction), 1);
auto block1 = rocksdb_store.block_get (rocksdb_transaction, send->hash ());

2
nano/nano_node/entry.cpp
View file

@ -379,7 +379,7 @@ int main (int argc, char * const * argv)
auto current (node->online_reps.trended ());
std::cout << boost::str (boost::format ("Trended Weight %1%\n") % current);
auto transaction (node->store.tx_begin_read ());
for (auto i (node->store.online_weight_begin (transaction)), n (node->store.online_weight_end ()); i != n; ++i)
for (auto i (node->store.online_weight.begin (transaction)), n (node->store.online_weight.end ()); i != n; ++i)
{
using time_point = std::chrono::system_clock::time_point;
time_point ts (std::chrono::duration_cast<time_point::duration> (std::chrono::nanoseconds (i->first)));

4
nano/node/cli.cpp
View file

@ -228,7 +228,7 @@ bool copy_database (boost::filesystem::path const & data_path, boost::program_op
}
if (vm.count ("online_weight_clear"))
{
node.node->store.online_weight_clear (store.tx_begin_write ());
node.node->store.online_weight.clear (store.tx_begin_write ());
}
if (vm.count ("peer_clear"))
{
@ -527,7 +527,7 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
if (!node.node->init_error ())
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.online_weight_clear (transaction);
node.node->store.online_weight.clear (transaction);
std::cout << "Onine weight records are removed" << std::endl;
}
else

4
nano/node/lmdb/lmdb.cpp
View file

@ -190,7 +190,7 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const &
{
error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontiers) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "unchecked", flags, &unchecked) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight_handle) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &meta) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peers) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "pruned", flags, &pruned) != 0;
@ -868,7 +868,7 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const
case tables::unchecked:
return unchecked;
case tables::online_weight:
return online_weight;
return online_weight_handle;
case tables::meta:
return meta;
case tables::peers:

2
nano/node/lmdb/lmdb.hpp
View file

@ -160,7 +160,7 @@ public:
* Samples of online vote weight
* uint64_t -> nano::amount
*/
MDB_dbi online_weight{ 0 };
MDB_dbi online_weight_handle{ 0 };
/**
* Meta information about block store, such as versions.

12
nano/node/node.cpp
View file

@ -754,17 +754,17 @@ void nano::node::long_inactivity_cleanup ()
{
bool perform_cleanup = false;
auto transaction (store.tx_begin_write ({ tables::online_weight, tables::peers }));
if (store.online_weight_count (transaction) > 0)
if (store.online_weight.count (transaction) > 0)
{
auto sample (store.online_weight_rbegin (transaction));
auto n (store.online_weight_end ());
auto sample (store.online_weight.rbegin (transaction));
auto n (store.online_weight.end ());
debug_assert (sample != n);
auto const one_week_ago = static_cast<size_t> ((std::chrono::system_clock::now () - std::chrono::hours (7 * 24)).time_since_epoch ().count ());
perform_cleanup = sample->first < one_week_ago;
}
if (perform_cleanup)
{
store.online_weight_clear (transaction);
store.online_weight.clear (transaction);
store.peer_clear (transaction);
logger.always_log ("Removed records of peers and online weight after a long period of inactivity");
}
@ -816,8 +816,8 @@ void nano::node::ongoing_bootstrap ()
{
// Find last online weight sample (last active time for node)
uint64_t last_sample_time (0);
auto last_record = store.online_weight_rbegin (store.tx_begin_read ());
if (last_record != store.online_weight_end ())
auto last_record = store.online_weight.rbegin (store.tx_begin_read ());
if (last_record != store.online_weight.end ())
{
last_sample_time = last_record->first;
}

12
nano/node/online_reps.cpp
View file

@ -41,13 +41,13 @@ void nano::online_reps::sample ()
{
auto transaction (ledger.store.tx_begin_write ({ tables::online_weight }));
// Discard oldest entries
while (ledger.store.online_weight_count (transaction) >= config.network_params.node.max_weight_samples)
while (ledger.store.online_weight.count (transaction) >= config.network_params.node.max_weight_samples)
{
auto oldest (ledger.store.online_weight_begin (transaction));
debug_assert (oldest != ledger.store.online_weight_end ());
ledger.store.online_weight_del (transaction, oldest->first);
auto oldest (ledger.store.online_weight.begin (transaction));
debug_assert (oldest != ledger.store.online_weight.end ());
ledger.store.online_weight.del (transaction, oldest->first);
}
ledger.store.online_weight_put (transaction, std::chrono::system_clock::now ().time_since_epoch ().count (), online_l);
ledger.store.online_weight.put (transaction, std::chrono::system_clock::now ().time_since_epoch ().count (), online_l);
trend_l = calculate_trend (transaction);
}
lock.lock ();
@ -69,7 +69,7 @@ nano::uint128_t nano::online_reps::calculate_trend (nano::transaction & transact
std::vector<nano::uint128_t> items;
items.reserve (config.network_params.node.max_weight_samples + 1);
items.push_back (config.online_weight_minimum.number ());
for (auto i (ledger.store.online_weight_begin (transaction_a)), n (ledger.store.online_weight_end ()); i != n; ++i)
for (auto i (ledger.store.online_weight.begin (transaction_a)), n (ledger.store.online_weight.end ()); i != n; ++i)
{
items.push_back (i->second.number ());
}

2
nano/node/rocksdb/rocksdb.cpp
View file

@ -490,7 +490,7 @@ uint64_t nano::rocksdb_store::count (nano::transaction const & transaction_a, ta
}
else if (table_a == tables::online_weight)
{
for (auto i (online_weight_begin (transaction_a)), n (online_weight_end ()); i != n; ++i)
for (auto i (online_weight.begin (transaction_a)), n (online_weight.end ()); i != n; ++i)
{
++sum;
}

3
nano/secure/CMakeLists.txt
View file

@ -55,7 +55,8 @@ add_library(
versioning.cpp
working.hpp
store/frontier_store_partial.hpp
store/pending_store_partial.hpp)
store/pending_store_partial.hpp
store/online_weight_partial.hpp)
target_link_libraries(
secure

5
nano/secure/blockstore.cpp
View file

@ -105,8 +105,9 @@ bool nano::write_transaction::contains (nano::tables table_a) const
return impl->contains (table_a);
}
nano::block_store::block_store (nano::frontier_store & frontier_store_a, nano::pending_store & pending_store_a) :
nano::block_store::block_store (nano::frontier_store & frontier_store_a, nano::pending_store & pending_store_a, nano::online_weight_store & online_weight_store_a) :
frontier (frontier_store_a),
pending (pending_store_a)
pending (pending_store_a),
online_weight (online_weight_store_a)
{
}

25
nano/secure/blockstore.hpp
View file

@ -652,13 +652,28 @@ public:
virtual void for_each_par (std::function<void (nano::read_transaction const &, nano::store_iterator<nano::pending_key, nano::pending_info>, nano::store_iterator<nano::pending_key, nano::pending_info>)> const & action_a) const = 0;
};
/**
* Manages online weight storage and iteration
*/
class online_weight_store
{
public:
virtual void put (nano::write_transaction const &, uint64_t, nano::amount const &) = 0;
virtual void del (nano::write_transaction const &, uint64_t) = 0;
virtual nano::store_iterator<uint64_t, nano::amount> begin (nano::transaction const &) const = 0;
virtual nano::store_iterator<uint64_t, nano::amount> rbegin (nano::transaction const &) const = 0;
virtual nano::store_iterator<uint64_t, nano::amount> end () const = 0;
virtual size_t count (nano::transaction const &) const = 0;
virtual void clear (nano::write_transaction const &) = 0;
};
/**
* Manages block storage and iteration
*/
class block_store
{
public:
explicit block_store (nano::frontier_store &, nano::pending_store &);
explicit block_store (nano::frontier_store &, nano::pending_store &, nano::online_weight_store &);
virtual ~block_store () = default;
virtual void initialize (nano::write_transaction const &, nano::genesis const &, nano::ledger_cache &) = 0;
virtual void block_put (nano::write_transaction const &, nano::block_hash const &, nano::block const &) = 0;
@ -707,13 +722,7 @@ public:
virtual nano::store_iterator<nano::unchecked_key, nano::unchecked_info> unchecked_end () const = 0;
virtual size_t unchecked_count (nano::transaction const &) = 0;
virtual void online_weight_put (nano::write_transaction const &, uint64_t, nano::amount const &) = 0;
virtual void online_weight_del (nano::write_transaction const &, uint64_t) = 0;
virtual nano::store_iterator<uint64_t, nano::amount> online_weight_begin (nano::transaction const &) const = 0;
virtual nano::store_iterator<uint64_t, nano::amount> online_weight_rbegin (nano::transaction const &) const = 0;
virtual nano::store_iterator<uint64_t, nano::amount> online_weight_end () const = 0;
virtual size_t online_weight_count (nano::transaction const &) const = 0;
virtual void online_weight_clear (nano::write_transaction const &) = 0;
online_weight_store & online_weight;
virtual void version_put (nano::write_transaction const &, int) = 0;
virtual int version_get (nano::transaction const &) const = 0;

49
nano/secure/blockstore_partial.hpp
View file

@ -7,6 +7,7 @@
#include <nano/secure/blockstore.hpp>
#include <nano/secure/buffer.hpp>
#include <nano/secure/store/frontier_store_partial.hpp>
#include <nano/secure/store/online_weight_partial.hpp>
#include <nano/secure/store/pending_store_partial.hpp>
#include <crypto/cryptopp/words.h>
@ -38,8 +39,8 @@ template <typename Val, typename Derived_Store>
class block_store_partial : public block_store
{
nano::frontier_store_partial<Val, Derived_Store> frontier_store_partial;
nano::pending_store_partial<Val, Derived_Store> pending_store_partial;
nano::online_weight_store_partial<Val, Derived_Store> online_weight_store_partial;
friend void release_assert_success<Val, Derived_Store> (block_store_partial<Val, Derived_Store> const & block_store, const int status);
@ -49,13 +50,14 @@ public:
friend class nano::block_predecessor_set<Val, Derived_Store>;
friend class nano::frontier_store_partial<Val, Derived_Store>;
friend class nano::pending_store_partial<Val, Derived_Store>;
friend class nano::online_weight_store_partial<Val, Derived_Store>;
block_store_partial () :
block_store{ frontier_store_partial, pending_store_partial },
block_store{ frontier_store_partial, pending_store_partial, online_weight_store_partial },
frontier_store_partial{ *this },
pending_store_partial{ *this }
pending_store_partial{ *this },
online_weight_store_partial{ *this }
{
}
@ -236,11 +238,6 @@ public:
return nano::store_iterator<nano::endpoint_key, nano::no_value> (nullptr);
}
nano::store_iterator<uint64_t, nano::amount> online_weight_end () const override
{
return nano::store_iterator<uint64_t, nano::amount> (nullptr);
}
nano::store_iterator<nano::account, nano::account_info> accounts_end () const override
{
return nano::store_iterator<nano::account, nano::account_info> (nullptr);
@ -374,30 +371,6 @@ public:
return iterator != accounts_end () && nano::account (iterator->first) == account_a;
}
void online_weight_put (nano::write_transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override
{
nano::db_val<Val> value (amount_a);
auto status (put (transaction_a, tables::online_weight, time_a, value));
release_assert_success (*this, status);
}
void online_weight_del (nano::write_transaction const & transaction_a, uint64_t time_a) override
{
auto status (del (transaction_a, tables::online_weight, time_a));
release_assert_success (*this, status);
}
size_t online_weight_count (nano::transaction const & transaction_a) const override
{
return count (transaction_a, tables::online_weight);
}
void online_weight_clear (nano::write_transaction const & transaction_a) override
{
auto status (drop (transaction_a, tables::online_weight));
release_assert_success (*this, status);
}
void pruned_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override
{
auto status = put_key (transaction_a, tables::pruned, hash_a);
@ -638,11 +611,6 @@ public:
return make_iterator<nano::unchecked_key, nano::unchecked_info> (transaction_a, tables::unchecked, nano::db_val<Val> (key_a));
}
nano::store_iterator<uint64_t, nano::amount> online_weight_begin (nano::transaction const & transaction_a) const override
{
return make_iterator<uint64_t, nano::amount> (transaction_a, tables::online_weight);
}
nano::store_iterator<nano::endpoint_key, nano::no_value> peers_begin (nano::transaction const & transaction_a) const override
{
return make_iterator<nano::endpoint_key, nano::no_value> (transaction_a, tables::peers);
@ -683,11 +651,6 @@ public:
return make_iterator<nano::account, nano::account_info> (transaction_a, tables::accounts, false);
}
nano::store_iterator<uint64_t, nano::amount> online_weight_rbegin (nano::transaction const & transaction_a) const override
{
return make_iterator<uint64_t, nano::amount> (transaction_a, tables::online_weight, false);
}
size_t unchecked_count (nano::transaction const & transaction_a) override
{
return count (transaction_a, tables::unchecked);

6
nano/secure/ledger.cpp
View file

@ -1504,9 +1504,9 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data
auto rocksdb_transaction (rocksdb_store->tx_begin_write ());
rocksdb_store->version_put (rocksdb_transaction, version);
for (auto i (store.online_weight_begin (lmdb_transaction)), n (store.online_weight_end ()); i != n; ++i)
for (auto i (store.online_weight.begin (lmdb_transaction)), n (store.online_weight.end ()); i != n; ++i)
{
rocksdb_store->online_weight_put (rocksdb_transaction, i->first, i->second);
rocksdb_store->online_weight.put (rocksdb_transaction, i->first, i->second);
}
for (auto i (store.peers_begin (lmdb_transaction)), n (store.peers_end ()); i != n; ++i)
@ -1519,7 +1519,7 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data
error |= store.peer_count (lmdb_transaction) != rocksdb_store->peer_count (rocksdb_transaction);
error |= store.pruned_count (lmdb_transaction) != rocksdb_store->pruned_count (rocksdb_transaction);
error |= store.final_vote_count (lmdb_transaction) != rocksdb_store->final_vote_count (rocksdb_transaction);
error |= store.online_weight_count (lmdb_transaction) != rocksdb_store->online_weight_count (rocksdb_transaction);
error |= store.online_weight.count (lmdb_transaction) != rocksdb_store->online_weight.count (rocksdb_transaction);
error |= store.version_get (lmdb_transaction) != rocksdb_store->version_get (rocksdb_transaction);
// For large tables a random key is used instead and makes sure it exists

65
nano/secure/store/online_weight_partial.hpp Normal file
View file

@ -0,0 +1,65 @@
#pragma once
#include <nano/secure/blockstore_partial.hpp>
namespace nano
{
template <typename Val, typename Derived_Store>
class block_store_partial;
template <typename Val, typename Derived_Store>
void release_assert_success (block_store_partial<Val, Derived_Store> const & block_store, const int status);
template <typename Val, typename Derived_Store>
class online_weight_store_partial : public online_weight_store
{
private:
nano::block_store_partial<Val, Derived_Store> & block_store;
friend void release_assert_success<Val, Derived_Store> (block_store_partial<Val, Derived_Store> const & block_store, const int status);
public:
explicit online_weight_store_partial (nano::block_store_partial<Val, Derived_Store> & block_store_a) :
block_store (block_store_a){};
void put (nano::write_transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override
{
nano::db_val<Val> value (amount_a);
auto status (block_store.put (transaction_a, tables::online_weight, time_a, value));
release_assert_success (block_store, status);
}
void del (nano::write_transaction const & transaction_a, uint64_t time_a) override
{
auto status (block_store.del (transaction_a, tables::online_weight, time_a));
release_assert_success (block_store, status);
}
nano::store_iterator<uint64_t, nano::amount> begin (nano::transaction const & transaction_a) const override
{
return block_store.template make_iterator<uint64_t, nano::amount> (transaction_a, tables::online_weight);
}
nano::store_iterator<uint64_t, nano::amount> rbegin (nano::transaction const & transaction_a) const override
{
return block_store.template make_iterator<uint64_t, nano::amount> (transaction_a, tables::online_weight, false);
}
nano::store_iterator<uint64_t, nano::amount> end () const override
{
return nano::store_iterator<uint64_t, nano::amount> (nullptr);
}
size_t count (nano::transaction const & transaction_a) const override
{
return block_store.count (transaction_a, tables::online_weight);
}
void clear (nano::write_transaction const & transaction_a) override
{
auto status (block_store.drop (transaction_a, tables::online_weight));
release_assert_success (block_store, status);
}
};
}