From 1a3857fd4ea38fee26607a9b788965e05b3488aa Mon Sep 17 00:00:00 2001 From: Thiago Silva <82097354+thsfs@users.noreply.github.com> Date: Wed, 2 Jun 2021 10:37:25 -0300 Subject: [PATCH] Online weight store break up (#3316) * Moves online weight store out of block store class * Renames the function names --- nano/core_test/block_store.cpp | 34 +++++------ nano/core_test/ledger.cpp | 6 +- nano/nano_node/entry.cpp | 2 +- nano/node/cli.cpp | 4 +- nano/node/lmdb/lmdb.cpp | 4 +- nano/node/lmdb/lmdb.hpp | 2 +- nano/node/node.cpp | 12 ++-- nano/node/online_reps.cpp | 12 ++-- nano/node/rocksdb/rocksdb.cpp | 2 +- nano/secure/CMakeLists.txt | 3 +- nano/secure/blockstore.cpp | 5 +- nano/secure/blockstore.hpp | 25 +++++--- nano/secure/blockstore_partial.hpp | 49 ++-------------- nano/secure/ledger.cpp | 6 +- nano/secure/store/online_weight_partial.hpp | 65 +++++++++++++++++++++ 15 files changed, 135 insertions(+), 96 deletions(-) create mode 100644 nano/secure/store/online_weight_partial.hpp diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index f2177163..c409aec6 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -1171,32 +1171,32 @@ TEST (block_store, online_weight) ASSERT_FALSE (store->init_error ()); { auto transaction (store->tx_begin_write ()); - ASSERT_EQ (0, store->online_weight_count (transaction)); - ASSERT_EQ (store->online_weight_end (), store->online_weight_begin (transaction)); - ASSERT_EQ (store->online_weight_end (), store->online_weight_rbegin (transaction)); - store->online_weight_put (transaction, 1, 2); - store->online_weight_put (transaction, 3, 4); + ASSERT_EQ (0, store->online_weight.count (transaction)); + ASSERT_EQ (store->online_weight.end (), store->online_weight.begin (transaction)); + ASSERT_EQ (store->online_weight.end (), store->online_weight.rbegin (transaction)); + store->online_weight.put (transaction, 1, 2); + store->online_weight.put (transaction, 3, 4); } { auto transaction (store->tx_begin_write ()); - ASSERT_EQ (2, store->online_weight_count (transaction)); - auto item (store->online_weight_begin (transaction)); - ASSERT_NE (store->online_weight_end (), item); + ASSERT_EQ (2, store->online_weight.count (transaction)); + auto item (store->online_weight.begin (transaction)); + ASSERT_NE (store->online_weight.end (), item); ASSERT_EQ (1, item->first); ASSERT_EQ (2, item->second.number ()); - auto item_last (store->online_weight_rbegin (transaction)); - ASSERT_NE (store->online_weight_end (), item_last); + auto item_last (store->online_weight.rbegin (transaction)); + ASSERT_NE (store->online_weight.end (), item_last); ASSERT_EQ (3, item_last->first); ASSERT_EQ (4, item_last->second.number ()); - store->online_weight_del (transaction, 1); - ASSERT_EQ (1, store->online_weight_count (transaction)); - ASSERT_EQ (store->online_weight_begin (transaction), store->online_weight_rbegin (transaction)); - store->online_weight_del (transaction, 3); + store->online_weight.del (transaction, 1); + ASSERT_EQ (1, store->online_weight.count (transaction)); + ASSERT_EQ (store->online_weight.begin (transaction), store->online_weight.rbegin (transaction)); + store->online_weight.del (transaction, 3); } auto transaction (store->tx_begin_read ()); - ASSERT_EQ (0, store->online_weight_count (transaction)); - ASSERT_EQ (store->online_weight_end (), store->online_weight_begin (transaction)); - ASSERT_EQ (store->online_weight_end (), store->online_weight_rbegin (transaction)); + ASSERT_EQ (0, store->online_weight.count (transaction)); + ASSERT_EQ (store->online_weight.end (), store->online_weight.begin (transaction)); + ASSERT_EQ (store->online_weight.end (), store->online_weight.rbegin (transaction)); } TEST (block_store, pruned_blocks) diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index bf5e6591..7e9a0f29 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -3862,7 +3862,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb) // Lower the database to the max version unsupported for upgrades store.confirmation_height_put (transaction, nano::genesis_account, { 2, send->hash () }); - store.online_weight_put (transaction, 100, nano::amount (2)); + store.online_weight.put (transaction, 100, nano::amount (2)); store.frontier.put (transaction, nano::block_hash (2), nano::account (5)); store.peer_put (transaction, endpoint_key); @@ -3884,13 +3884,13 @@ TEST (ledger, migrate_lmdb_to_rocksdb) nano::pending_info pending_info; ASSERT_FALSE (rocksdb_store.pending.get (rocksdb_transaction, nano::pending_key (nano::genesis_account, send->hash ()), pending_info)); - for (auto i = rocksdb_store.online_weight_begin (rocksdb_transaction); i != rocksdb_store.online_weight_end (); ++i) + for (auto i = rocksdb_store.online_weight.begin (rocksdb_transaction); i != rocksdb_store.online_weight.end (); ++i) { ASSERT_EQ (i->first, 100); ASSERT_EQ (i->second, 2); } - ASSERT_EQ (rocksdb_store.online_weight_count (rocksdb_transaction), 1); + ASSERT_EQ (rocksdb_store.online_weight.count (rocksdb_transaction), 1); auto block1 = rocksdb_store.block_get (rocksdb_transaction, send->hash ()); diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index 87c48209..607ed8a3 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -379,7 +379,7 @@ int main (int argc, char * const * argv) auto current (node->online_reps.trended ()); std::cout << boost::str (boost::format ("Trended Weight %1%\n") % current); auto transaction (node->store.tx_begin_read ()); - for (auto i (node->store.online_weight_begin (transaction)), n (node->store.online_weight_end ()); i != n; ++i) + for (auto i (node->store.online_weight.begin (transaction)), n (node->store.online_weight.end ()); i != n; ++i) { using time_point = std::chrono::system_clock::time_point; time_point ts (std::chrono::duration_cast (std::chrono::nanoseconds (i->first))); diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index 5524a736..c58c04e7 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -228,7 +228,7 @@ bool copy_database (boost::filesystem::path const & data_path, boost::program_op } if (vm.count ("online_weight_clear")) { - node.node->store.online_weight_clear (store.tx_begin_write ()); + node.node->store.online_weight.clear (store.tx_begin_write ()); } if (vm.count ("peer_clear")) { @@ -527,7 +527,7 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map if (!node.node->init_error ()) { auto transaction (node.node->store.tx_begin_write ()); - node.node->store.online_weight_clear (transaction); + node.node->store.online_weight.clear (transaction); std::cout << "Onine weight records are removed" << std::endl; } else diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index a4ca7895..00680253 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -190,7 +190,7 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & { error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontiers) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "unchecked", flags, &unchecked) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight_handle) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &meta) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peers) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "pruned", flags, &pruned) != 0; @@ -868,7 +868,7 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const case tables::unchecked: return unchecked; case tables::online_weight: - return online_weight; + return online_weight_handle; case tables::meta: return meta; case tables::peers: diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index d61b8ae1..fbdf015e 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -160,7 +160,7 @@ public: * Samples of online vote weight * uint64_t -> nano::amount */ - MDB_dbi online_weight{ 0 }; + MDB_dbi online_weight_handle{ 0 }; /** * Meta information about block store, such as versions. diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 85443218..4a68587c 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -754,17 +754,17 @@ void nano::node::long_inactivity_cleanup () { bool perform_cleanup = false; auto transaction (store.tx_begin_write ({ tables::online_weight, tables::peers })); - if (store.online_weight_count (transaction) > 0) + if (store.online_weight.count (transaction) > 0) { - auto sample (store.online_weight_rbegin (transaction)); - auto n (store.online_weight_end ()); + auto sample (store.online_weight.rbegin (transaction)); + auto n (store.online_weight.end ()); debug_assert (sample != n); auto const one_week_ago = static_cast ((std::chrono::system_clock::now () - std::chrono::hours (7 * 24)).time_since_epoch ().count ()); perform_cleanup = sample->first < one_week_ago; } if (perform_cleanup) { - store.online_weight_clear (transaction); + store.online_weight.clear (transaction); store.peer_clear (transaction); logger.always_log ("Removed records of peers and online weight after a long period of inactivity"); } @@ -816,8 +816,8 @@ void nano::node::ongoing_bootstrap () { // Find last online weight sample (last active time for node) uint64_t last_sample_time (0); - auto last_record = store.online_weight_rbegin (store.tx_begin_read ()); - if (last_record != store.online_weight_end ()) + auto last_record = store.online_weight.rbegin (store.tx_begin_read ()); + if (last_record != store.online_weight.end ()) { last_sample_time = last_record->first; } diff --git a/nano/node/online_reps.cpp b/nano/node/online_reps.cpp index 774b81df..89b6d986 100644 --- a/nano/node/online_reps.cpp +++ b/nano/node/online_reps.cpp @@ -41,13 +41,13 @@ void nano::online_reps::sample () { auto transaction (ledger.store.tx_begin_write ({ tables::online_weight })); // Discard oldest entries - while (ledger.store.online_weight_count (transaction) >= config.network_params.node.max_weight_samples) + while (ledger.store.online_weight.count (transaction) >= config.network_params.node.max_weight_samples) { - auto oldest (ledger.store.online_weight_begin (transaction)); - debug_assert (oldest != ledger.store.online_weight_end ()); - ledger.store.online_weight_del (transaction, oldest->first); + auto oldest (ledger.store.online_weight.begin (transaction)); + debug_assert (oldest != ledger.store.online_weight.end ()); + ledger.store.online_weight.del (transaction, oldest->first); } - ledger.store.online_weight_put (transaction, std::chrono::system_clock::now ().time_since_epoch ().count (), online_l); + ledger.store.online_weight.put (transaction, std::chrono::system_clock::now ().time_since_epoch ().count (), online_l); trend_l = calculate_trend (transaction); } lock.lock (); @@ -69,7 +69,7 @@ nano::uint128_t nano::online_reps::calculate_trend (nano::transaction & transact std::vector items; items.reserve (config.network_params.node.max_weight_samples + 1); items.push_back (config.online_weight_minimum.number ()); - for (auto i (ledger.store.online_weight_begin (transaction_a)), n (ledger.store.online_weight_end ()); i != n; ++i) + for (auto i (ledger.store.online_weight.begin (transaction_a)), n (ledger.store.online_weight.end ()); i != n; ++i) { items.push_back (i->second.number ()); } diff --git a/nano/node/rocksdb/rocksdb.cpp b/nano/node/rocksdb/rocksdb.cpp index a46bb355..384d4302 100644 --- a/nano/node/rocksdb/rocksdb.cpp +++ b/nano/node/rocksdb/rocksdb.cpp @@ -490,7 +490,7 @@ uint64_t nano::rocksdb_store::count (nano::transaction const & transaction_a, ta } else if (table_a == tables::online_weight) { - for (auto i (online_weight_begin (transaction_a)), n (online_weight_end ()); i != n; ++i) + for (auto i (online_weight.begin (transaction_a)), n (online_weight.end ()); i != n; ++i) { ++sum; } diff --git a/nano/secure/CMakeLists.txt b/nano/secure/CMakeLists.txt index b2d8ec5a..26a4502e 100644 --- a/nano/secure/CMakeLists.txt +++ b/nano/secure/CMakeLists.txt @@ -55,7 +55,8 @@ add_library( versioning.cpp working.hpp store/frontier_store_partial.hpp - store/pending_store_partial.hpp) + store/pending_store_partial.hpp + store/online_weight_partial.hpp) target_link_libraries( secure diff --git a/nano/secure/blockstore.cpp b/nano/secure/blockstore.cpp index 44e517ea..219cadfc 100644 --- a/nano/secure/blockstore.cpp +++ b/nano/secure/blockstore.cpp @@ -105,8 +105,9 @@ bool nano::write_transaction::contains (nano::tables table_a) const return impl->contains (table_a); } -nano::block_store::block_store (nano::frontier_store & frontier_store_a, nano::pending_store & pending_store_a) : +nano::block_store::block_store (nano::frontier_store & frontier_store_a, nano::pending_store & pending_store_a, nano::online_weight_store & online_weight_store_a) : frontier (frontier_store_a), - pending (pending_store_a) + pending (pending_store_a), + online_weight (online_weight_store_a) { } diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index 18be9bec..c950ed8a 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -652,13 +652,28 @@ public: virtual void for_each_par (std::function, nano::store_iterator)> const & action_a) const = 0; }; +/** + * Manages online weight storage and iteration + */ +class online_weight_store +{ +public: + virtual void put (nano::write_transaction const &, uint64_t, nano::amount const &) = 0; + virtual void del (nano::write_transaction const &, uint64_t) = 0; + virtual nano::store_iterator begin (nano::transaction const &) const = 0; + virtual nano::store_iterator rbegin (nano::transaction const &) const = 0; + virtual nano::store_iterator end () const = 0; + virtual size_t count (nano::transaction const &) const = 0; + virtual void clear (nano::write_transaction const &) = 0; +}; + /** * Manages block storage and iteration */ class block_store { public: - explicit block_store (nano::frontier_store &, nano::pending_store &); + explicit block_store (nano::frontier_store &, nano::pending_store &, nano::online_weight_store &); virtual ~block_store () = default; virtual void initialize (nano::write_transaction const &, nano::genesis const &, nano::ledger_cache &) = 0; virtual void block_put (nano::write_transaction const &, nano::block_hash const &, nano::block const &) = 0; @@ -707,13 +722,7 @@ public: virtual nano::store_iterator unchecked_end () const = 0; virtual size_t unchecked_count (nano::transaction const &) = 0; - virtual void online_weight_put (nano::write_transaction const &, uint64_t, nano::amount const &) = 0; - virtual void online_weight_del (nano::write_transaction const &, uint64_t) = 0; - virtual nano::store_iterator online_weight_begin (nano::transaction const &) const = 0; - virtual nano::store_iterator online_weight_rbegin (nano::transaction const &) const = 0; - virtual nano::store_iterator online_weight_end () const = 0; - virtual size_t online_weight_count (nano::transaction const &) const = 0; - virtual void online_weight_clear (nano::write_transaction const &) = 0; + online_weight_store & online_weight; virtual void version_put (nano::write_transaction const &, int) = 0; virtual int version_get (nano::transaction const &) const = 0; diff --git a/nano/secure/blockstore_partial.hpp b/nano/secure/blockstore_partial.hpp index c82ac0e1..203a4cd3 100644 --- a/nano/secure/blockstore_partial.hpp +++ b/nano/secure/blockstore_partial.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -38,8 +39,8 @@ template class block_store_partial : public block_store { nano::frontier_store_partial frontier_store_partial; - nano::pending_store_partial pending_store_partial; + nano::online_weight_store_partial online_weight_store_partial; friend void release_assert_success (block_store_partial const & block_store, const int status); @@ -49,13 +50,14 @@ public: friend class nano::block_predecessor_set; friend class nano::frontier_store_partial; - friend class nano::pending_store_partial; + friend class nano::online_weight_store_partial; block_store_partial () : - block_store{ frontier_store_partial, pending_store_partial }, + block_store{ frontier_store_partial, pending_store_partial, online_weight_store_partial }, frontier_store_partial{ *this }, - pending_store_partial{ *this } + pending_store_partial{ *this }, + online_weight_store_partial{ *this } { } @@ -236,11 +238,6 @@ public: return nano::store_iterator (nullptr); } - nano::store_iterator online_weight_end () const override - { - return nano::store_iterator (nullptr); - } - nano::store_iterator accounts_end () const override { return nano::store_iterator (nullptr); @@ -374,30 +371,6 @@ public: return iterator != accounts_end () && nano::account (iterator->first) == account_a; } - void online_weight_put (nano::write_transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override - { - nano::db_val value (amount_a); - auto status (put (transaction_a, tables::online_weight, time_a, value)); - release_assert_success (*this, status); - } - - void online_weight_del (nano::write_transaction const & transaction_a, uint64_t time_a) override - { - auto status (del (transaction_a, tables::online_weight, time_a)); - release_assert_success (*this, status); - } - - size_t online_weight_count (nano::transaction const & transaction_a) const override - { - return count (transaction_a, tables::online_weight); - } - - void online_weight_clear (nano::write_transaction const & transaction_a) override - { - auto status (drop (transaction_a, tables::online_weight)); - release_assert_success (*this, status); - } - void pruned_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override { auto status = put_key (transaction_a, tables::pruned, hash_a); @@ -638,11 +611,6 @@ public: return make_iterator (transaction_a, tables::unchecked, nano::db_val (key_a)); } - nano::store_iterator online_weight_begin (nano::transaction const & transaction_a) const override - { - return make_iterator (transaction_a, tables::online_weight); - } - nano::store_iterator peers_begin (nano::transaction const & transaction_a) const override { return make_iterator (transaction_a, tables::peers); @@ -683,11 +651,6 @@ public: return make_iterator (transaction_a, tables::accounts, false); } - nano::store_iterator online_weight_rbegin (nano::transaction const & transaction_a) const override - { - return make_iterator (transaction_a, tables::online_weight, false); - } - size_t unchecked_count (nano::transaction const & transaction_a) override { return count (transaction_a, tables::unchecked); diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index 19e0a465..d82df994 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -1504,9 +1504,9 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data auto rocksdb_transaction (rocksdb_store->tx_begin_write ()); rocksdb_store->version_put (rocksdb_transaction, version); - for (auto i (store.online_weight_begin (lmdb_transaction)), n (store.online_weight_end ()); i != n; ++i) + for (auto i (store.online_weight.begin (lmdb_transaction)), n (store.online_weight.end ()); i != n; ++i) { - rocksdb_store->online_weight_put (rocksdb_transaction, i->first, i->second); + rocksdb_store->online_weight.put (rocksdb_transaction, i->first, i->second); } for (auto i (store.peers_begin (lmdb_transaction)), n (store.peers_end ()); i != n; ++i) @@ -1519,7 +1519,7 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data error |= store.peer_count (lmdb_transaction) != rocksdb_store->peer_count (rocksdb_transaction); error |= store.pruned_count (lmdb_transaction) != rocksdb_store->pruned_count (rocksdb_transaction); error |= store.final_vote_count (lmdb_transaction) != rocksdb_store->final_vote_count (rocksdb_transaction); - error |= store.online_weight_count (lmdb_transaction) != rocksdb_store->online_weight_count (rocksdb_transaction); + error |= store.online_weight.count (lmdb_transaction) != rocksdb_store->online_weight.count (rocksdb_transaction); error |= store.version_get (lmdb_transaction) != rocksdb_store->version_get (rocksdb_transaction); // For large tables a random key is used instead and makes sure it exists diff --git a/nano/secure/store/online_weight_partial.hpp b/nano/secure/store/online_weight_partial.hpp new file mode 100644 index 00000000..0e2895ba --- /dev/null +++ b/nano/secure/store/online_weight_partial.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include + +namespace nano +{ +template +class block_store_partial; + +template +void release_assert_success (block_store_partial const & block_store, const int status); + +template +class online_weight_store_partial : public online_weight_store +{ +private: + nano::block_store_partial & block_store; + + friend void release_assert_success (block_store_partial const & block_store, const int status); + +public: + explicit online_weight_store_partial (nano::block_store_partial & block_store_a) : + block_store (block_store_a){}; + + void put (nano::write_transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override + { + nano::db_val value (amount_a); + auto status (block_store.put (transaction_a, tables::online_weight, time_a, value)); + release_assert_success (block_store, status); + } + + void del (nano::write_transaction const & transaction_a, uint64_t time_a) override + { + auto status (block_store.del (transaction_a, tables::online_weight, time_a)); + release_assert_success (block_store, status); + } + + nano::store_iterator begin (nano::transaction const & transaction_a) const override + { + return block_store.template make_iterator (transaction_a, tables::online_weight); + } + + nano::store_iterator rbegin (nano::transaction const & transaction_a) const override + { + return block_store.template make_iterator (transaction_a, tables::online_weight, false); + } + + nano::store_iterator end () const override + { + return nano::store_iterator (nullptr); + } + + size_t count (nano::transaction const & transaction_a) const override + { + return block_store.count (transaction_a, tables::online_weight); + } + + void clear (nano::write_transaction const & transaction_a) override + { + auto status (block_store.drop (transaction_a, tables::online_weight)); + release_assert_success (block_store, status); + } +}; + +}