diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index ecd575ea4..2f611b01f 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1445,6 +1446,48 @@ TEST (rocksdb_block_store, upgrade_v21_v22) } } +// Tests that the new rep_weight table gets filled with all +// existing representatives +TEST (mdb_block_store, upgrade_v22_to_v23) +{ + nano::logger logger; + auto const path = nano::unique_path (); + nano::account rep_a{ 123 }; + nano::account rep_b{ 456 }; + // Setting the database to its 22nd version state + { + auto store{ nano::make_store (logger, path, nano::dev::constants) }; + auto txn{ store->tx_begin_write () }; + + // Add three accounts referencing two representatives + nano::account_info info1{}; + info1.representative = rep_a; + info1.balance = 1000; + store->account.put (txn, 1, info1); + + nano::account_info info2{}; + info2.representative = rep_a; + info2.balance = 500; + store->account.put (txn, 2, info2); + + nano::account_info info3{}; + info3.representative = rep_b; + info3.balance = 42; + store->account.put (txn, 3, info3); + + store->version.put (txn, 22); + } + + // Testing the upgrade code worked + auto store{ nano::make_store (logger, path, nano::dev::constants) }; + auto txn (store->tx_begin_read ()); + ASSERT_EQ (store->version.get (txn), store->version_current); + + // The rep_weight table should contain all reps now + ASSERT_EQ (1500, store->rep_weight.get (txn, rep_a)); + ASSERT_EQ (42, store->rep_weight.get (txn, rep_b)); +} + TEST (mdb_block_store, upgrade_backup) { if (nano::rocksdb_config::using_rocksdb_in_tests ()) diff --git a/nano/secure/account_info.cpp b/nano/secure/account_info.cpp index b478f95e7..1b18ca602 100644 --- a/nano/secure/account_info.cpp +++ b/nano/secure/account_info.cpp @@ -58,3 +58,36 @@ nano::epoch nano::account_info::epoch () const { return epoch_m; } + +size_t nano::account_info_v22::db_size () const +{ + debug_assert (reinterpret_cast (this) == reinterpret_cast (&head)); + debug_assert (reinterpret_cast (&head) + sizeof (head) == reinterpret_cast (&representative)); + debug_assert (reinterpret_cast (&representative) + sizeof (representative) == reinterpret_cast (&open_block)); + debug_assert (reinterpret_cast (&open_block) + sizeof (open_block) == reinterpret_cast (&balance)); + debug_assert (reinterpret_cast (&balance) + sizeof (balance) == reinterpret_cast (&modified)); + debug_assert (reinterpret_cast (&modified) + sizeof (modified) == reinterpret_cast (&block_count)); + debug_assert (reinterpret_cast (&block_count) + sizeof (block_count) == reinterpret_cast (&epoch_m)); + return sizeof (head) + sizeof (representative) + sizeof (open_block) + sizeof (balance) + sizeof (modified) + sizeof (block_count) + sizeof (epoch_m); +} + +bool nano::account_info_v22::deserialize (nano::stream & stream_a) +{ + auto error (false); + try + { + nano::read (stream_a, head.bytes); + nano::read (stream_a, representative.bytes); + nano::read (stream_a, open_block.bytes); + nano::read (stream_a, balance.bytes); + nano::read (stream_a, modified); + nano::read (stream_a, block_count); + nano::read (stream_a, epoch_m); + } + catch (std::runtime_error const &) + { + error = true; + } + + return error; +} \ No newline at end of file diff --git a/nano/secure/account_info.hpp b/nano/secure/account_info.hpp index 422515fa8..bd7a7b0cc 100644 --- a/nano/secure/account_info.hpp +++ b/nano/secure/account_info.hpp @@ -29,4 +29,24 @@ public: uint64_t block_count{ 0 }; nano::epoch epoch_m{ nano::epoch::epoch_0 }; }; + +/** + * Account info as of DB version 22. + * This class protects the DB upgrades from future changes of the account_info class. + */ +class account_info_v22 final +{ +public: + account_info_v22 () = default; + size_t db_size () const; + bool deserialize (nano::stream &); + nano::block_hash head{ 0 }; + nano::account representative{}; + nano::block_hash open_block{ 0 }; + nano::amount balance{ 0 }; + /** Seconds since posix epoch */ + nano::seconds_t modified{ 0 }; + uint64_t block_count{ 0 }; + nano::epoch epoch_m{ nano::epoch::epoch_0 }; +}; } // namespace nano diff --git a/nano/store/component.hpp b/nano/store/component.hpp index eaa8faf95..ac5103e92 100644 --- a/nano/store/component.hpp +++ b/nano/store/component.hpp @@ -72,7 +72,7 @@ namespace store store::pending & pending; store::rep_weight & rep_weight; static int constexpr version_minimum{ 21 }; - static int constexpr version_current{ 22 }; + static int constexpr version_current{ 23 }; public: store::online_weight & online_weight; diff --git a/nano/store/db_val.hpp b/nano/store/db_val.hpp index fee58d9e0..7cc415a5f 100644 --- a/nano/store/db_val.hpp +++ b/nano/store/db_val.hpp @@ -12,6 +12,7 @@ namespace nano { class account_info; +class account_info_v22; class block; class pending_info; class pending_key; @@ -63,6 +64,8 @@ public: db_val (nano::account_info const & val_a); + db_val (nano::account_info_v22 const & val_a); + db_val (nano::pending_info const & val_a); db_val (nano::pending_key const & val_a); @@ -103,6 +106,7 @@ public: } explicit operator nano::account_info () const; + explicit operator nano::account_info_v22 () const; explicit operator block_info () const { diff --git a/nano/store/db_val_impl.hpp b/nano/store/db_val_impl.hpp index c8ee18624..f2eaf4884 100644 --- a/nano/store/db_val_impl.hpp +++ b/nano/store/db_val_impl.hpp @@ -10,6 +10,13 @@ nano::store::db_val::db_val (nano::account_info const & val_a) : db_val (val_a.db_size (), const_cast (&val_a)) { } + +template +nano::store::db_val::db_val (nano::account_info_v22 const & val_a) : + db_val (val_a.db_size (), const_cast (&val_a)) +{ +} + template nano::store::db_val::db_val (std::shared_ptr const & val_a) : buffer (std::make_shared> ()) @@ -44,6 +51,15 @@ nano::store::db_val::operator nano::account_info () const return result; } +template +nano::store::db_val::operator nano::account_info_v22 () const +{ + nano::account_info_v22 result; + debug_assert (size () == result.db_size ()); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); + return result; +} + template nano::store::db_val::operator std::shared_ptr () const { diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp index b5397b50f..179c14b40 100644 --- a/nano/store/lmdb/lmdb.cpp +++ b/nano/store/lmdb/lmdb.cpp @@ -1,6 +1,8 @@ +#include #include #include #include +#include #include #include #include @@ -206,7 +208,7 @@ void nano::store::lmdb::component::open_databases (bool & error_a, store::transa pending_store.pending_handle = pending_store.pending_v0_handle; error_a |= mdb_dbi_open (env.tx (transaction_a), "final_votes", flags, &final_vote_store.final_votes_handle) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &block_store.blocks_handle) != 0; - mdb_dbi_open (env.tx (transaction_a), "rep_weights", flags, &rep_weight_store.rep_weights_handle); + error_a |= mdb_dbi_open (env.tx (transaction_a), "rep_weights", flags, &rep_weight_store.rep_weights_handle) != 0; } bool nano::store::lmdb::component::do_upgrades (store::write_transaction & transaction_a, nano::ledger_constants & constants, bool & needs_vacuuming) @@ -224,6 +226,9 @@ bool nano::store::lmdb::component::do_upgrades (store::write_transaction & trans upgrade_v21_to_v22 (transaction_a); [[fallthrough]]; case 22: + upgrade_v22_to_v23 (transaction_a); + [[fallthrough]]; + case 23: break; default: logger.critical (nano::log::type::lmdb, "The version of the ledger ({}) is too high for this node", version_l); @@ -245,6 +250,39 @@ void nano::store::lmdb::component::upgrade_v21_to_v22 (store::write_transaction logger.info (nano::log::type::lmdb, "Upgrading database from v21 to v22 completed"); } +// Fill rep_weights table with all existing representatives and their vote weight +void nano::store::lmdb::component::upgrade_v22_to_v23 (store::write_transaction const & transaction_a) +{ + logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23..."); + auto i{ make_iterator (transaction_a, tables::accounts) }; + auto end{ store::iterator (nullptr) }; + uint64_t processed_accounts = 0; + for (; i != end; ++i) + { + if (!i->second.balance.is_zero ()) + { + nano::uint128_t total{ 0 }; + nano::store::lmdb::db_val value; + auto status = get (transaction_a, tables::rep_weights, i->second.representative, value); + if (success (status)) + { + total = nano::amount{ value }.number (); + } + total += i->second.balance.number (); + status = put (transaction_a, tables::rep_weights, i->second.representative, nano::amount{ total }); + release_assert_success (status); + } + processed_accounts++; + if (processed_accounts % 250000 == 0) + { + logger.info (nano::log::type::lmdb, "processed {} accounts", processed_accounts); + } + } + logger.info (nano::log::type::lmdb, "processed {} accounts", processed_accounts); + version.put (transaction_a, 23); + logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23 completed"); +} + /** Takes a filepath, appends '_backup_' to the end (but before any extension) and saves that file in the same directory */ void nano::store::lmdb::component::create_backup_file (nano::store::lmdb::env & env_a, std::filesystem::path const & filepath_a, nano::logger & logger) { diff --git a/nano/store/lmdb/lmdb.hpp b/nano/store/lmdb/lmdb.hpp index 90bbc8ca6..c76381886 100644 --- a/nano/store/lmdb/lmdb.hpp +++ b/nano/store/lmdb/lmdb.hpp @@ -116,6 +116,7 @@ public: private: bool do_upgrades (store::write_transaction &, nano::ledger_constants & constants, bool &); void upgrade_v21_to_v22 (store::write_transaction const &); + void upgrade_v22_to_v23 (store::write_transaction const &); void open_databases (bool &, store::transaction const &, unsigned); diff --git a/nano/store/rocksdb/rocksdb.cpp b/nano/store/rocksdb/rocksdb.cpp index 515a03772..f13cdb76b 100644 --- a/nano/store/rocksdb/rocksdb.cpp +++ b/nano/store/rocksdb/rocksdb.cpp @@ -245,6 +245,9 @@ bool nano::store::rocksdb::component::do_upgrades (store::write_transaction cons upgrade_v21_to_v22 (transaction_a); [[fallthrough]]; case 22: + upgrade_v22_to_v23 (transaction_a); + [[fallthrough]]; + case 23: break; default: logger.critical (nano::log::type::rocksdb, "The version of the ledger ({}) is too high for this node", version_l); @@ -280,6 +283,39 @@ void nano::store::rocksdb::component::upgrade_v21_to_v22 (store::write_transacti logger.info (nano::log::type::rocksdb, "Upgrading database from v21 to v22 completed"); } +// Fill rep_weights table with all existing representatives and their vote weight +void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transaction const & transaction_a) +{ + logger.info (nano::log::type::rocksdb, "Upgrading database from v22 to v23..."); + auto i{ make_iterator (transaction_a, tables::accounts) }; + auto end{ store::iterator (nullptr) }; + uint64_t processed_accounts = 0; + for (; i != end; ++i) + { + if (!i->second.balance.is_zero ()) + { + nano::uint128_t total{ 0 }; + nano::store::rocksdb::db_val value; + auto status = get (transaction_a, tables::rep_weights, i->second.representative, value); + if (success (status)) + { + total = nano::amount{ value }.number (); + } + total += i->second.balance.number (); + status = put (transaction_a, tables::rep_weights, i->second.representative, nano::amount{ total }); + release_assert_success (status); + } + processed_accounts++; + if (processed_accounts % 250000 == 0) + { + logger.info (nano::log::type::lmdb, "processed {} accounts", processed_accounts); + } + } + logger.info (nano::log::type::lmdb, "processed {} accounts", processed_accounts); + version.put (transaction_a, 23); + logger.info (nano::log::type::rocksdb, "Upgrading database from v22 to v23 completed"); +} + void nano::store::rocksdb::component::generate_tombstone_map () { tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::blocks), std::forward_as_tuple (0, 25000)); diff --git a/nano/store/rocksdb/rocksdb.hpp b/nano/store/rocksdb/rocksdb.hpp index f23145b02..f6c2d683b 100644 --- a/nano/store/rocksdb/rocksdb.hpp +++ b/nano/store/rocksdb/rocksdb.hpp @@ -154,6 +154,7 @@ private: bool do_upgrades (store::write_transaction const &); void upgrade_v21_to_v22 (store::write_transaction const &); + void upgrade_v22_to_v23 (store::write_transaction const &); void construct_column_family_mutexes (); ::rocksdb::Options get_db_options ();