dn724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Fill rep_weights table

Browse source
This commit is contained in:
Gustav Schauwecker 2024-03-11 15:31:33 +01:00
commit 5980be65cb
10 changed files with 194 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

43
nano/core_test/block_store.cpp
View file

@ -7,6 +7,7 @@
#include <nano/lib/work.hpp>
#include <nano/node/common.hpp>
#include <nano/node/make_store.hpp>
#include <nano/secure/common.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/utility.hpp>
#include <nano/store/account.hpp>
@ -1445,6 +1446,48 @@ TEST (rocksdb_block_store, upgrade_v21_v22)
}
}
// Tests that the new rep_weight table gets filled with all
// existing representatives
TEST (mdb_block_store, upgrade_v22_to_v23)
{
nano::logger logger;
auto const path = nano::unique_path ();
nano::account rep_a{ 123 };
nano::account rep_b{ 456 };
// Setting the database to its 22nd version state
{
auto store{ nano::make_store (logger, path, nano::dev::constants) };
auto txn{ store->tx_begin_write () };
// Add three accounts referencing two representatives
nano::account_info info1{};
info1.representative = rep_a;
info1.balance = 1000;
store->account.put (txn, 1, info1);
nano::account_info info2{};
info2.representative = rep_a;
info2.balance = 500;
store->account.put (txn, 2, info2);
nano::account_info info3{};
info3.representative = rep_b;
info3.balance = 42;
store->account.put (txn, 3, info3);
store->version.put (txn, 22);
}
// Testing the upgrade code worked
auto store{ nano::make_store (logger, path, nano::dev::constants) };
auto txn (store->tx_begin_read ());
ASSERT_EQ (store->version.get (txn), store->version_current);
// The rep_weight table should contain all reps now
ASSERT_EQ (1500, store->rep_weight.get (txn, rep_a));
ASSERT_EQ (42, store->rep_weight.get (txn, rep_b));
}
TEST (mdb_block_store, upgrade_backup)
{
if (nano::rocksdb_config::using_rocksdb_in_tests ())

33
nano/secure/account_info.cpp
View file

@ -58,3 +58,36 @@ nano::epoch nano::account_info::epoch () const
{
return epoch_m;
}
size_t nano::account_info_v22::db_size () const
{
debug_assert (reinterpret_cast<uint8_t const *> (this) == reinterpret_cast<uint8_t const *> (&head));
debug_assert (reinterpret_cast<uint8_t const *> (&head) + sizeof (head) == reinterpret_cast<uint8_t const *> (&representative));
debug_assert (reinterpret_cast<uint8_t const *> (&representative) + sizeof (representative) == reinterpret_cast<uint8_t const *> (&open_block));
debug_assert (reinterpret_cast<uint8_t const *> (&open_block) + sizeof (open_block) == reinterpret_cast<uint8_t const *> (&balance));
debug_assert (reinterpret_cast<uint8_t const *> (&balance) + sizeof (balance) == reinterpret_cast<uint8_t const *> (&modified));
debug_assert (reinterpret_cast<uint8_t const *> (&modified) + sizeof (modified) == reinterpret_cast<uint8_t const *> (&block_count));
debug_assert (reinterpret_cast<uint8_t const *> (&block_count) + sizeof (block_count) == reinterpret_cast<uint8_t const *> (&epoch_m));
return sizeof (head) + sizeof (representative) + sizeof (open_block) + sizeof (balance) + sizeof (modified) + sizeof (block_count) + sizeof (epoch_m);
}
bool nano::account_info_v22::deserialize (nano::stream & stream_a)
{
auto error (false);
try
{
nano::read (stream_a, head.bytes);
nano::read (stream_a, representative.bytes);
nano::read (stream_a, open_block.bytes);
nano::read (stream_a, balance.bytes);
nano::read (stream_a, modified);
nano::read (stream_a, block_count);
nano::read (stream_a, epoch_m);
}
catch (std::runtime_error const &)
{
error = true;
}
return error;
}

20
nano/secure/account_info.hpp
View file

@ -29,4 +29,24 @@ public:
uint64_t block_count{ 0 };
nano::epoch epoch_m{ nano::epoch::epoch_0 };
};
/**
* Account info as of DB version 22.
* This class protects the DB upgrades from future changes of the account_info class.
*/
class account_info_v22 final
{
public:
account_info_v22 () = default;
size_t db_size () const;
bool deserialize (nano::stream &);
nano::block_hash head{ 0 };
nano::account representative{};
nano::block_hash open_block{ 0 };
nano::amount balance{ 0 };
/** Seconds since posix epoch */
nano::seconds_t modified{ 0 };
uint64_t block_count{ 0 };
nano::epoch epoch_m{ nano::epoch::epoch_0 };
};
} // namespace nano

2
nano/store/component.hpp
View file

@ -72,7 +72,7 @@ namespace store
store::pending & pending;
store::rep_weight & rep_weight;
static int constexpr version_minimum{ 21 };
static int constexpr version_current{ 22 };
static int constexpr version_current{ 23 };
public:
store::online_weight & online_weight;

4
nano/store/db_val.hpp
View file

@ -12,6 +12,7 @@
namespace nano
{
class account_info;
class account_info_v22;
class block;
class pending_info;
class pending_key;
@ -63,6 +64,8 @@ public:
db_val (nano::account_info const & val_a);
db_val (nano::account_info_v22 const & val_a);
db_val (nano::pending_info const & val_a);
db_val (nano::pending_key const & val_a);
@ -103,6 +106,7 @@ public:
}
explicit operator nano::account_info () const;
explicit operator nano::account_info_v22 () const;
explicit operator block_info () const
{

16
nano/store/db_val_impl.hpp
View file

@ -10,6 +10,13 @@ nano::store::db_val<T>::db_val (nano::account_info const & val_a) :
db_val (val_a.db_size (), const_cast<nano::account_info *> (&val_a))
{
}
template <typename T>
nano::store::db_val<T>::db_val (nano::account_info_v22 const & val_a) :
db_val (val_a.db_size (), const_cast<nano::account_info_v22 *> (&val_a))
{
}
template <typename T>
nano::store::db_val<T>::db_val (std::shared_ptr<nano::block> const & val_a) :
buffer (std::make_shared<std::vector<uint8_t>> ())
@ -44,6 +51,15 @@ nano::store::db_val<T>::operator nano::account_info () const
return result;
}
template <typename T>
nano::store::db_val<T>::operator nano::account_info_v22 () const
{
nano::account_info_v22 result;
debug_assert (size () == result.db_size ());
std::copy (reinterpret_cast<uint8_t const *> (data ()), reinterpret_cast<uint8_t const *> (data ()) + result.db_size (), reinterpret_cast<uint8_t *> (&result));
return result;
}
template <typename T>
nano::store::db_val<T>::operator std::shared_ptr<nano::block> () const
{

40
nano/store/lmdb/lmdb.cpp
View file

@ -1,6 +1,8 @@
#include <nano/lib/numbers.hpp>
#include <nano/lib/stream.hpp>
#include <nano/lib/utility.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/secure/parallel_traversal.hpp>
#include <nano/store/lmdb/iterator.hpp>
#include <nano/store/lmdb/lmdb.hpp>
#include <nano/store/lmdb/wallet_value.hpp>
@ -206,7 +208,7 @@ void nano::store::lmdb::component::open_databases (bool & error_a, store::transa
pending_store.pending_handle = pending_store.pending_v0_handle;
error_a |= mdb_dbi_open (env.tx (transaction_a), "final_votes", flags, &final_vote_store.final_votes_handle) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &block_store.blocks_handle) != 0;
mdb_dbi_open (env.tx (transaction_a), "rep_weights", flags, &rep_weight_store.rep_weights_handle);
error_a |= mdb_dbi_open (env.tx (transaction_a), "rep_weights", flags, &rep_weight_store.rep_weights_handle) != 0;
}
bool nano::store::lmdb::component::do_upgrades (store::write_transaction & transaction_a, nano::ledger_constants & constants, bool & needs_vacuuming)
@ -224,6 +226,9 @@ bool nano::store::lmdb::component::do_upgrades (store::write_transaction & trans
upgrade_v21_to_v22 (transaction_a);
[[fallthrough]];
case 22:
upgrade_v22_to_v23 (transaction_a);
[[fallthrough]];
case 23:
break;
default:
logger.critical (nano::log::type::lmdb, "The version of the ledger ({}) is too high for this node", version_l);
@ -245,6 +250,39 @@ void nano::store::lmdb::component::upgrade_v21_to_v22 (store::write_transaction
logger.info (nano::log::type::lmdb, "Upgrading database from v21 to v22 completed");
}
// Fill rep_weights table with all existing representatives and their vote weight
void nano::store::lmdb::component::upgrade_v22_to_v23 (store::write_transaction const & transaction_a)
{
logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23...");
auto i{ make_iterator<nano::account, nano::account_info_v22> (transaction_a, tables::accounts) };
auto end{ store::iterator<nano::account, nano::account_info_v22> (nullptr) };
uint64_t processed_accounts = 0;
for (; i != end; ++i)
{
if (!i->second.balance.is_zero ())
{
nano::uint128_t total{ 0 };
nano::store::lmdb::db_val value;
auto status = get (transaction_a, tables::rep_weights, i->second.representative, value);
if (success (status))
{
total = nano::amount{ value }.number ();
}
total += i->second.balance.number ();
status = put (transaction_a, tables::rep_weights, i->second.representative, nano::amount{ total });
release_assert_success (status);
}
processed_accounts++;
if (processed_accounts % 250000 == 0)
{
logger.info (nano::log::type::lmdb, "processed {} accounts", processed_accounts);
}
}
logger.info (nano::log::type::lmdb, "processed {} accounts", processed_accounts);
version.put (transaction_a, 23);
logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23 completed");
}
/** Takes a filepath, appends '_backup_<timestamp>' to the end (but before any extension) and saves that file in the same directory */
void nano::store::lmdb::component::create_backup_file (nano::store::lmdb::env & env_a, std::filesystem::path const & filepath_a, nano::logger & logger)
{

1
nano/store/lmdb/lmdb.hpp
View file

@ -116,6 +116,7 @@ public:
private:
bool do_upgrades (store::write_transaction &, nano::ledger_constants & constants, bool &);
void upgrade_v21_to_v22 (store::write_transaction const &);
void upgrade_v22_to_v23 (store::write_transaction const &);
void open_databases (bool &, store::transaction const &, unsigned);

36
nano/store/rocksdb/rocksdb.cpp
View file

@ -245,6 +245,9 @@ bool nano::store::rocksdb::component::do_upgrades (store::write_transaction cons
upgrade_v21_to_v22 (transaction_a);
[[fallthrough]];
case 22:
upgrade_v22_to_v23 (transaction_a);
[[fallthrough]];
case 23:
break;
default:
logger.critical (nano::log::type::rocksdb, "The version of the ledger ({}) is too high for this node", version_l);
@ -280,6 +283,39 @@ void nano::store::rocksdb::component::upgrade_v21_to_v22 (store::write_transacti
logger.info (nano::log::type::rocksdb, "Upgrading database from v21 to v22 completed");
}
// Fill rep_weights table with all existing representatives and their vote weight
void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transaction const & transaction_a)
{
logger.info (nano::log::type::rocksdb, "Upgrading database from v22 to v23...");
auto i{ make_iterator<nano::account, nano::account_info_v22> (transaction_a, tables::accounts) };
auto end{ store::iterator<nano::account, nano::account_info_v22> (nullptr) };
uint64_t processed_accounts = 0;
for (; i != end; ++i)
{
if (!i->second.balance.is_zero ())
{
nano::uint128_t total{ 0 };
nano::store::rocksdb::db_val value;
auto status = get (transaction_a, tables::rep_weights, i->second.representative, value);
if (success (status))
{
total = nano::amount{ value }.number ();
}
total += i->second.balance.number ();
status = put (transaction_a, tables::rep_weights, i->second.representative, nano::amount{ total });
release_assert_success (status);
}
processed_accounts++;
if (processed_accounts % 250000 == 0)
{
logger.info (nano::log::type::lmdb, "processed {} accounts", processed_accounts);
}
}
logger.info (nano::log::type::lmdb, "processed {} accounts", processed_accounts);
version.put (transaction_a, 23);
logger.info (nano::log::type::rocksdb, "Upgrading database from v22 to v23 completed");
}
void nano::store::rocksdb::component::generate_tombstone_map ()
{
tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::blocks), std::forward_as_tuple (0, 25000));

1
nano/store/rocksdb/rocksdb.hpp
View file

@ -154,6 +154,7 @@ private:
bool do_upgrades (store::write_transaction const &);
void upgrade_v21_to_v22 (store::write_transaction const &);
void upgrade_v22_to_v23 (store::write_transaction const &);
void construct_column_family_mutexes ();
::rocksdb::Options get_db_options ();