Merge pull request #4914 from pwojcikdev/rep-weights-verify-consistency

Ledger rep weights consistency checks
This commit is contained in:
Piotr Wójcik 2025-06-22 15:44:37 +02:00 committed by GitHub
commit 1c5e8cc16e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 377 additions and 283 deletions

View file

@ -383,9 +383,8 @@ TEST (block_store, genesis)
nano::logger logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::ledger_cache ledger_cache{ store->rep_weight };
auto transaction (store->tx_begin_write ());
store->initialize (transaction, ledger_cache, nano::dev::constants);
store->initialize (transaction, nano::dev::constants);
nano::account_info info;
ASSERT_FALSE (store->account.get (transaction, nano::dev::genesis_key.pub, info));
ASSERT_EQ (nano::dev::genesis->hash (), info.head);
@ -635,7 +634,6 @@ TEST (mdb_block_store, supported_version_upgrades)
nano::stats stats{ logger };
nano::ledger ledger (store, nano::dev::constants, stats, logger);
auto transaction (store.tx_begin_write ());
store.initialize (transaction, ledger.cache, nano::dev::constants);
// Lower the database to the max version unsupported for upgrades
store.version.put (transaction, store.version_minimum - 1);
}
@ -653,7 +651,6 @@ TEST (mdb_block_store, supported_version_upgrades)
nano::stats stats{ logger };
nano::ledger ledger (store, nano::dev::constants, stats, logger);
auto transaction (store.tx_begin_write ());
store.initialize (transaction, ledger.cache, nano::dev::constants);
// Lower the database version to the minimum version supported for upgrade.
store.version.put (transaction, store.version_minimum);
}
@ -897,10 +894,8 @@ TEST (block_store, cemented_count_cache)
nano::logger logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
auto transaction (store->tx_begin_write ());
nano::stats stats{ logger };
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
store->initialize (transaction, ledger.cache, nano::dev::constants);
ASSERT_EQ (1, ledger.cemented_count ());
}
@ -921,9 +916,8 @@ TEST (block_store, pruned_random)
block->sideband_set ({});
auto hash1 (block->hash ());
{
nano::ledger_cache ledger_cache{ store->rep_weight };
auto transaction (store->tx_begin_write ());
store->initialize (transaction, ledger_cache, nano::dev::constants);
store->initialize (transaction, nano::dev::constants);
store->pruned.put (transaction, hash1);
}
auto transaction (store->tx_begin_read ());
@ -951,9 +945,8 @@ TEST (block_store, state_block)
block1->sideband_set ({});
{
nano::ledger_cache ledger_cache{ store->rep_weight };
auto transaction (store->tx_begin_write ());
store->initialize (transaction, ledger_cache, nano::dev::constants);
store->initialize (transaction, nano::dev::constants);
ASSERT_EQ (nano::block_type::state, block1->type ());
store->block.put (transaction, block1->hash (), *block1);
ASSERT_TRUE (store->block.exists (transaction, block1->hash ()));
@ -990,7 +983,6 @@ TEST (mdb_block_store, sideband_height)
nano::ledger ledger (store, nano::dev::constants, stats, logger);
nano::block_builder builder;
auto transaction = ledger.tx_begin_write ();
store.initialize (transaction, ledger.cache, nano::dev::constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
auto send = builder
.send ()

View file

@ -678,11 +678,11 @@ TEST (ledger, representation_changes)
auto store{ nano::test::make_store () };
nano::keypair key1;
nano::rep_weights rep_weights{ store->rep_weight };
ASSERT_EQ (0, rep_weights.representation_get (key1.pub));
rep_weights.representation_put (key1.pub, 1);
ASSERT_EQ (1, rep_weights.representation_get (key1.pub));
rep_weights.representation_put (key1.pub, 2);
ASSERT_EQ (2, rep_weights.representation_get (key1.pub));
ASSERT_EQ (0, rep_weights.get (key1.pub));
rep_weights.put (key1.pub, 1);
ASSERT_EQ (1, rep_weights.get (key1.pub));
rep_weights.put (key1.pub, 2);
ASSERT_EQ (2, rep_weights.get (key1.pub));
}
TEST (ledger, delete_rep_weight_of_zero)
@ -690,17 +690,23 @@ TEST (ledger, delete_rep_weight_of_zero)
auto store{ nano::test::make_store () };
nano::rep_weights rep_weights{ store->rep_weight };
auto txn{ store->tx_begin_write () };
rep_weights.representation_add (txn, 1, 100);
rep_weights.representation_add_dual (txn, 2, 100, 3, 100);
rep_weights.add (txn, 1, 100);
rep_weights.add (txn, 2, 200);
ASSERT_EQ (2, rep_weights.size ());
rep_weights.move_add_sub (txn, 2, 100, 3, 120);
ASSERT_EQ (3, rep_weights.size ());
ASSERT_EQ (3, store->rep_weight.count (txn));
// set rep weights to 0
rep_weights.representation_add (txn, 1, nano::uint128_t{ 0 } - 100);
// Reduce rep weights to 0
rep_weights.sub (txn, 1, 100);
ASSERT_EQ (2, rep_weights.size ());
ASSERT_EQ (2, store->rep_weight.count (txn));
rep_weights.representation_add_dual (txn, 2, nano::uint128_t{ 0 } - 100, 3, nano::uint128_t{ 0 } - 100);
rep_weights.move_add_sub (txn, 3, 120, 2, 100);
ASSERT_EQ (1, rep_weights.size ());
ASSERT_EQ (1, store->rep_weight.count (txn));
rep_weights.sub (txn, 2, 200);
ASSERT_EQ (0, rep_weights.size ());
ASSERT_EQ (0, store->rep_weight.count (txn));
}
@ -713,22 +719,22 @@ TEST (ledger, rep_cache_min_weight)
auto txn{ store->tx_begin_write () };
// one below min weight
rep_weights.representation_add (txn, 1, 9);
rep_weights.add (txn, 1, 9);
ASSERT_EQ (0, rep_weights.size ());
ASSERT_EQ (1, store->rep_weight.count (txn));
// exactly min weight
rep_weights.representation_add (txn, 1, 1);
rep_weights.add (txn, 1, 1);
ASSERT_EQ (1, rep_weights.size ());
ASSERT_EQ (1, store->rep_weight.count (txn));
// above min weight
rep_weights.representation_add (txn, 1, 1);
rep_weights.add (txn, 1, 1);
ASSERT_EQ (1, rep_weights.size ());
ASSERT_EQ (1, store->rep_weight.count (txn));
// fall blow min weight
rep_weights.representation_add (txn, 1, nano::uint128_t{ 0 } - 5);
rep_weights.sub (txn, 1, 5);
ASSERT_EQ (0, rep_weights.size ());
ASSERT_EQ (1, store->rep_weight.count (txn));
}
@ -738,10 +744,10 @@ TEST (ledger, representation)
auto ctx = nano::test::ledger_empty ();
auto & ledger = ctx.ledger ();
auto & store = ctx.store ();
auto & rep_weights = ledger.cache.rep_weights;
auto & rep_weights = ledger.rep_weights;
auto transaction = ledger.tx_begin_write ();
auto & pool = ctx.pool ();
ASSERT_EQ (nano::dev::constants.genesis_amount, rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount, rep_weights.get (nano::dev::genesis_key.pub));
nano::keypair key2;
nano::block_builder builder;
auto block1 = builder
@ -753,7 +759,7 @@ TEST (ledger, representation)
.work (*pool.generate (nano::dev::genesis->hash ()))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block1));
ASSERT_EQ (nano::dev::constants.genesis_amount - 100, rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount - 100, rep_weights.get (nano::dev::genesis_key.pub));
nano::keypair key3;
auto block2 = builder
.open ()
@ -764,9 +770,9 @@ TEST (ledger, representation)
.work (*pool.generate (key2.pub))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block2));
ASSERT_EQ (nano::dev::constants.genesis_amount - 100, rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.representation_get (key2.pub));
ASSERT_EQ (100, rep_weights.representation_get (key3.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount - 100, rep_weights.get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.get (key2.pub));
ASSERT_EQ (100, rep_weights.get (key3.pub));
auto block3 = builder
.send ()
.previous (block1->hash ())
@ -776,9 +782,9 @@ TEST (ledger, representation)
.work (*pool.generate (block1->hash ()))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block3));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.representation_get (key2.pub));
ASSERT_EQ (100, rep_weights.representation_get (key3.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.get (key2.pub));
ASSERT_EQ (100, rep_weights.get (key3.pub));
auto block4 = builder
.receive ()
.previous (block2->hash ())
@ -787,9 +793,9 @@ TEST (ledger, representation)
.work (*pool.generate (block2->hash ()))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block4));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.representation_get (key2.pub));
ASSERT_EQ (200, rep_weights.representation_get (key3.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.get (key2.pub));
ASSERT_EQ (200, rep_weights.get (key3.pub));
nano::keypair key4;
auto block5 = builder
.change ()
@ -799,10 +805,10 @@ TEST (ledger, representation)
.work (*pool.generate (block4->hash ()))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block5));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.representation_get (key2.pub));
ASSERT_EQ (0, rep_weights.representation_get (key3.pub));
ASSERT_EQ (200, rep_weights.representation_get (key4.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.get (key2.pub));
ASSERT_EQ (0, rep_weights.get (key3.pub));
ASSERT_EQ (200, rep_weights.get (key4.pub));
nano::keypair key5;
auto block6 = builder
.send ()
@ -813,11 +819,11 @@ TEST (ledger, representation)
.work (*pool.generate (block5->hash ()))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block6));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.representation_get (key2.pub));
ASSERT_EQ (0, rep_weights.representation_get (key3.pub));
ASSERT_EQ (100, rep_weights.representation_get (key4.pub));
ASSERT_EQ (0, rep_weights.representation_get (key5.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.get (key2.pub));
ASSERT_EQ (0, rep_weights.get (key3.pub));
ASSERT_EQ (100, rep_weights.get (key4.pub));
ASSERT_EQ (0, rep_weights.get (key5.pub));
nano::keypair key6;
auto block7 = builder
.open ()
@ -828,12 +834,12 @@ TEST (ledger, representation)
.work (*pool.generate (key5.pub))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block7));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.representation_get (key2.pub));
ASSERT_EQ (0, rep_weights.representation_get (key3.pub));
ASSERT_EQ (100, rep_weights.representation_get (key4.pub));
ASSERT_EQ (0, rep_weights.representation_get (key5.pub));
ASSERT_EQ (100, rep_weights.representation_get (key6.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.get (key2.pub));
ASSERT_EQ (0, rep_weights.get (key3.pub));
ASSERT_EQ (100, rep_weights.get (key4.pub));
ASSERT_EQ (0, rep_weights.get (key5.pub));
ASSERT_EQ (100, rep_weights.get (key6.pub));
auto block8 = builder
.send ()
.previous (block6->hash ())
@ -843,12 +849,12 @@ TEST (ledger, representation)
.work (*pool.generate (block6->hash ()))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block8));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.representation_get (key2.pub));
ASSERT_EQ (0, rep_weights.representation_get (key3.pub));
ASSERT_EQ (0, rep_weights.representation_get (key4.pub));
ASSERT_EQ (0, rep_weights.representation_get (key5.pub));
ASSERT_EQ (100, rep_weights.representation_get (key6.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.get (key2.pub));
ASSERT_EQ (0, rep_weights.get (key3.pub));
ASSERT_EQ (0, rep_weights.get (key4.pub));
ASSERT_EQ (0, rep_weights.get (key5.pub));
ASSERT_EQ (100, rep_weights.get (key6.pub));
auto block9 = builder
.receive ()
.previous (block7->hash ())
@ -857,12 +863,12 @@ TEST (ledger, representation)
.work (*pool.generate (block7->hash ()))
.build ();
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block9));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.representation_get (key2.pub));
ASSERT_EQ (0, rep_weights.representation_get (key3.pub));
ASSERT_EQ (0, rep_weights.representation_get (key4.pub));
ASSERT_EQ (0, rep_weights.representation_get (key5.pub));
ASSERT_EQ (200, rep_weights.representation_get (key6.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, rep_weights.get (key2.pub));
ASSERT_EQ (0, rep_weights.get (key3.pub));
ASSERT_EQ (0, rep_weights.get (key4.pub));
ASSERT_EQ (0, rep_weights.get (key5.pub));
ASSERT_EQ (200, rep_weights.get (key6.pub));
}
TEST (ledger, double_open)
@ -873,7 +879,6 @@ TEST (ledger, double_open)
nano::stats stats{ logger };
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
nano::keypair key2;
nano::block_builder builder;
@ -4634,8 +4639,8 @@ TEST (ledger, zero_rep)
.build ();
auto transaction = node1.ledger.tx_begin_write ();
ASSERT_EQ (nano::block_status::progress, node1.ledger.process (transaction, block1));
ASSERT_EQ (0, node1.ledger.cache.rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount, node1.ledger.cache.rep_weights.representation_get (0));
ASSERT_EQ (0, node1.ledger.rep_weights.get (nano::dev::genesis_key.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount, node1.ledger.rep_weights.get (0));
auto block2 = builder.state ()
.account (nano::dev::genesis_key.pub)
.previous (block1->hash ())
@ -4646,8 +4651,8 @@ TEST (ledger, zero_rep)
.work (*system.work.generate (block1->hash ()))
.build ();
ASSERT_EQ (nano::block_status::progress, node1.ledger.process (transaction, block2));
ASSERT_EQ (nano::dev::constants.genesis_amount, node1.ledger.cache.rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, node1.ledger.cache.rep_weights.representation_get (0));
ASSERT_EQ (nano::dev::constants.genesis_amount, node1.ledger.rep_weights.get (nano::dev::genesis_key.pub));
ASSERT_EQ (0, node1.ledger.rep_weights.get (0));
}
TEST (ledger, work_validation)
@ -4810,7 +4815,6 @@ TEST (ledger, dependents_confirmed_pruning)
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::block_builder builder;
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
nano::keypair key1;
@ -4901,10 +4905,13 @@ TEST (ledger, cache)
ASSERT_EQ (account_count, ledger.account_count ());
ASSERT_EQ (block_count, ledger.block_count ());
ASSERT_EQ (cemented_count, ledger.cemented_count ());
ASSERT_EQ (genesis_weight, ledger.cache.rep_weights.representation_get (nano::dev::genesis_key.pub));
ASSERT_EQ (genesis_weight, ledger.rep_weights.get (nano::dev::genesis_key.pub));
ASSERT_EQ (pruned_count, ledger.pruned_count ());
};
cache_check (ledger);
cache_check (nano::ledger (store, nano::dev::constants, stats, logger));
nano::keypair key;
auto const latest = ledger.any.account_head (ledger.tx_begin_read (), nano::dev::genesis_key.pub);
auto send = builder.state ()
@ -4984,7 +4991,6 @@ TEST (ledger, pruning_action)
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
nano::block_builder builder;
auto send1 = builder
@ -5070,7 +5076,6 @@ TEST (ledger, pruning_large_chain)
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
size_t send_receive_pairs (20);
auto last_hash (nano::dev::genesis->hash ());
@ -5126,7 +5131,6 @@ TEST (ledger, pruning_source_rollback)
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
nano::block_builder builder;
auto epoch1 = builder
@ -5215,7 +5219,6 @@ TEST (ledger, pruning_source_rollback_legacy)
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
nano::block_builder builder;
auto send1 = builder
@ -5330,7 +5333,6 @@ TEST (ledger, pruning_legacy_blocks)
ledger.pruning = true;
nano::keypair key1;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
nano::block_builder builder;
auto send1 = builder
@ -5416,7 +5418,6 @@ TEST (ledger, pruning_safe_functions)
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
nano::block_builder builder;
auto send1 = builder
@ -5468,7 +5469,6 @@ TEST (ledger, random_blocks)
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
ledger.pruning = true;
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
nano::block_builder builder;
auto send1 = builder
@ -5571,7 +5571,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
{
auto transaction = ledger.tx_begin_write ();
store.initialize (transaction, ledger.cache, ledger.constants);
ASSERT_FALSE (store.init_error ());
// Lower the database to the max version unsupported for upgrades

View file

@ -817,7 +817,6 @@ TEST (ledger_confirm, pruned_source)
.work (*pool.generate (key2.pub))
.build ();
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, nano::dev::constants);
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send1));
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, open1));
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send2));
@ -859,7 +858,6 @@ TEST (ledger_confirmDeathTest, rollback_added_block)
.work (*pool.generate (nano::dev::genesis->hash ()))
.build ();
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
ASSERT_DEATH_IF_SUPPORTED (ledger.confirm (transaction, send->hash ()), "");
}
}

View file

@ -2572,9 +2572,8 @@ TEST (node, dont_write_lock_node)
nano::logger logger;
auto store = nano::make_store (logger, path, nano::dev::constants, false, true);
{
nano::ledger_cache ledger_cache{ store->rep_weight };
auto transaction (store->tx_begin_write ());
store->initialize (transaction, ledger_cache, nano::dev::constants);
store->initialize (transaction, nano::dev::constants);
}
// Hold write lock open until main thread is done needing it

View file

@ -20,7 +20,6 @@ TEST (processor_service, bad_send_signature)
ASSERT_FALSE (store->init_error ());
nano::ledger ledger (*store, nano::dev::constants, system.stats, system.logger);
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
auto info1 = ledger.any.account_get (transaction, nano::dev::genesis_key.pub);
ASSERT_TRUE (info1);
@ -46,7 +45,6 @@ TEST (processor_service, bad_receive_signature)
ASSERT_FALSE (store->init_error ());
nano::ledger ledger (*store, nano::dev::constants, system.stats, system.logger);
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
auto info1 = ledger.any.account_get (transaction, nano::dev::genesis_key.pub);
ASSERT_TRUE (info1);

View file

@ -135,7 +135,7 @@ TEST (vote_processor, weights)
system.wallet (0)->send_sync (nano::dev::genesis_key.pub, key2.pub, level2);
// Wait for representatives
ASSERT_TIMELY_EQ (10s, node.ledger.cache.rep_weights.get_rep_amounts ().size (), 4);
ASSERT_TIMELY_EQ (10s, node.ledger.rep_weights.get_rep_amounts ().size (), 4);
// Wait for rep tiers to be updated
node.stats.clear ();

View file

@ -4,12 +4,14 @@
#include <boost/preprocessor/facilities/empty.hpp>
#include <boost/preprocessor/facilities/overload.hpp>
#include <string>
#include <string_view>
[[noreturn]] void assert_internal (char const * check_expr, char const * func, char const * file, unsigned int line, bool is_release_assert, std::string_view error = "");
#define release_assert_1(check) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, true)
#define release_assert_2(check, error_msg) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, true, error_msg)
#define release_assert_3(check, error_msg, extra) (check) ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, true, std::string (error_msg) + " (" + (extra) + ")")
#if !BOOST_PP_VARIADICS_MSVC
#define release_assert(...) \
BOOST_PP_OVERLOAD (release_assert_, __VA_ARGS__) \
@ -23,6 +25,7 @@
#else
#define debug_assert_1(check) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, false)
#define debug_assert_2(check, error_msg) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, false, error_msg)
#define debug_assert_3(check, error_msg, extra) (check) ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, false, std::string (error_msg) + " (" + (extra) + ")")
#if !BOOST_PP_VARIADICS_MSVC
#define debug_assert(...) \
BOOST_PP_OVERLOAD (debug_assert_, __VA_ARGS__) \

View file

@ -217,7 +217,7 @@ int main (int argc, char * const * argv)
auto const bootstrap_weights = node->get_bootstrap_weights ();
auto const & hardcoded = bootstrap_weights.second;
auto const hardcoded_height = bootstrap_weights.first;
auto const ledger_unfiltered = node->ledger.cache.rep_weights.get_rep_amounts ();
auto const ledger_unfiltered = node->ledger.rep_weights.get_rep_amounts ();
auto const ledger_height = node->ledger.block_count ();
auto get_total = [] (decltype (bootstrap_weights.second) const & reps) -> nano::uint128_union {
@ -459,7 +459,7 @@ int main (int argc, char * const * argv)
auto node = inactive_node.node;
auto transaction (node->store.tx_begin_read ());
nano::uint128_t total;
auto rep_amounts = node->ledger.cache.rep_weights.get_rep_amounts ();
auto rep_amounts = node->ledger.rep_weights.get_rep_amounts ();
std::map<nano::account, nano::uint128_t> ordered_reps (rep_amounts.begin (), rep_amounts.end ());
for (auto const & rep : ordered_reps)
{

View file

@ -368,6 +368,9 @@ void nano::block_processor::process_batch (nano::unique_lock<nano::mutex> & lock
processed.emplace_back (result, std::move (ctx));
}
// We had rocksdb issues in the past, ensure that rep weights are always consistent
ledger.rep_weights.verify_consistency ();
if (number_of_blocks_processed != 0 && timer.stop () > std::chrono::milliseconds (100))
{
logger.debug (nano::log::type::block_processor, "Processed {} blocks ({} forced) in {} {}", number_of_blocks_processed, number_of_forced_processed, timer.value ().count (), timer.unit ());

View file

@ -344,6 +344,9 @@ std::deque<nano::block_hash> nano::bounded_backlog::perform_rollbacks (std::dequ
}
}
// We had rocksdb issues in the past, ensure that rep weights are always consistent
ledger.rep_weights.verify_consistency ();
return processed;
}

View file

@ -837,7 +837,7 @@ std::vector<nano::vote_with_weight_info> nano::election::votes_with_weight () co
{
if (vote_l.first != nullptr)
{
auto amount (node.ledger.cache.rep_weights.representation_get (vote_l.first));
auto amount (node.ledger.weight (vote_l.first));
nano::vote_with_weight_info vote_info{ vote_l.first, vote_l.second.time, vote_l.second.timestamp, vote_l.second.hash, amount };
sorted_votes.emplace (std::move (amount), vote_info);
}

View file

@ -2136,7 +2136,7 @@ void nano::json_handler::confirmation_info ()
{
if (block->hash () == vote.hash)
{
auto amount (node.ledger.cache.rep_weights.representation_get (representative));
auto amount (node.ledger.weight (representative));
representatives.emplace (amount, representative);
if (vote.timestamp == std::numeric_limits<uint64_t>::max ())
{
@ -3538,7 +3538,7 @@ void nano::json_handler::representatives ()
{
bool const sorting = request.get<bool> ("sorting", false);
boost::property_tree::ptree representatives;
auto rep_amounts = node.ledger.cache.rep_weights.get_rep_amounts ();
auto rep_amounts = node.ledger.rep_weights.get_rep_amounts ();
if (!sorting) // Simple
{
std::map<nano::account, nano::uint128_t> ordered (rep_amounts.begin (), rep_amounts.end ());

View file

@ -330,20 +330,6 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
config.bandwidth_limit,
config.bandwidth_limit_burst_ratio);
// First do a pass with a read to see if any writing needs doing, this saves needing to open a write lock (and potentially blocking)
auto is_initialized (false);
{
auto const transaction (store.tx_begin_read ());
is_initialized = (store.account.begin (transaction) != store.account.end (transaction));
}
if (!is_initialized && !flags.read_only)
{
auto const transaction = store.tx_begin_write ();
// Store was empty meaning we just created it, add the genesis block
store.initialize (transaction, ledger.cache, ledger.constants);
}
if (!block_or_pruned_exists (config.network_params.ledger.genesis->hash ()))
{
logger.critical (nano::log::type::node, "Genesis block not found. This commonly indicates a configuration issue, check that the --network or --data_path command line arguments are correct, and also the ledger backend node config option. If using a read-only CLI command a ledger must already exist, start the node with --daemon first.");

View file

@ -524,7 +524,6 @@ TEST (history, short_text)
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
{
auto transaction (ledger.tx_begin_write ());
store->initialize (transaction, ledger.cache, ledger.constants);
nano::keypair key;
auto latest (ledger.any.account_head (transaction, nano::dev::genesis_key.pub));
auto send = std::make_shared<nano::send_block> (latest, nano::dev::genesis_key.pub, 0, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, *system.work.generate (latest));
@ -566,7 +565,6 @@ TEST (history, pruned_source)
// Basic pruning for legacy blocks. Previous block is pruned, source is pruned
{
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, nano::dev::constants);
auto latest (ledger.any.account_head (transaction, nano::dev::genesis_key.pub));
auto send1 = std::make_shared<nano::send_block> (latest, nano::dev::genesis_key.pub, nano::dev::constants.genesis_amount - 100, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, *system.work.generate (latest));
ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send1));

View file

@ -25,8 +25,6 @@ add_library(
generate_cache_flags.cpp
ledger.hpp
ledger.cpp
ledger_cache.hpp
ledger_cache.cpp
ledger_set_any.hpp
ledger_set_any.cpp
ledger_set_confirmed.hpp

View file

@ -1,6 +1,7 @@
#pragma once
#include <nano/lib/fwd.hpp>
#include <nano/store/fwd.hpp>
namespace nano
{

View file

@ -60,7 +60,7 @@ public:
auto info = ledger.any.account_get (transaction, pending.value ().source);
release_assert (info);
ledger.store.pending.del (transaction, key);
ledger.cache.rep_weights.representation_add (transaction, info->representative, pending.value ().amount.number ());
ledger.rep_weights.add (transaction, info->representative, pending.value ().amount);
nano::account_info new_info (block_a.hashables.previous, info->representative, info->open_block, ledger.any.block_balance (transaction, block_a.hashables.previous).value (), nano::seconds_since_epoch (), info->block_count - 1, nano::epoch::epoch_0);
ledger.update_account (transaction, pending.value ().source, *info, new_info);
ledger.store.block.del (transaction, hash);
@ -77,7 +77,7 @@ public:
auto source_account = ledger.any.block_account (transaction, block_a.hashables.source);
auto info = ledger.any.account_get (transaction, destination_account);
release_assert (info);
ledger.cache.rep_weights.representation_add (transaction, info->representative, 0 - amount);
ledger.rep_weights.sub (transaction, info->representative, amount);
nano::account_info new_info (block_a.hashables.previous, info->representative, info->open_block, ledger.any.block_balance (transaction, block_a.hashables.previous).value (), nano::seconds_since_epoch (), info->block_count - 1, nano::epoch::epoch_0);
ledger.update_account (transaction, destination_account, *info, new_info);
ledger.store.block.del (transaction, hash);
@ -91,7 +91,7 @@ public:
auto amount = ledger.any.block_amount (transaction, hash).value ().number ();
auto destination_account = block_a.account ();
auto source_account = ledger.any.block_account (transaction, block_a.hashables.source);
ledger.cache.rep_weights.representation_add (transaction, block_a.representative_field ().value (), 0 - amount);
ledger.rep_weights.sub (transaction, block_a.representative_field ().value (), amount);
nano::account_info new_info;
ledger.update_account (transaction, destination_account, new_info, new_info);
ledger.store.block.del (transaction, hash);
@ -101,15 +101,15 @@ public:
void change_block (nano::change_block const & block_a) override
{
auto hash (block_a.hash ());
auto rep_block (ledger.representative (transaction, block_a.hashables.previous));
auto rep_block_hash (ledger.representative (transaction, block_a.hashables.previous));
auto account = block_a.account ();
auto info = ledger.any.account_get (transaction, account);
release_assert (info);
auto balance = ledger.any.block_balance (transaction, block_a.hashables.previous).value ();
auto block = ledger.store.block.get (transaction, rep_block);
release_assert (block != nullptr);
auto representative = block->representative_field ().value ();
ledger.cache.rep_weights.representation_add_dual (transaction, block_a.hashables.representative, 0 - balance.number (), representative, balance.number ());
auto rep_block = ledger.store.block.get (transaction, rep_block_hash);
release_assert (rep_block != nullptr);
auto representative = rep_block->representative_field ().value ();
ledger.rep_weights.move (transaction, block_a.hashables.representative, representative, balance);
ledger.store.block.del (transaction, hash);
nano::account_info new_info (block_a.hashables.previous, representative, info->open_block, info->balance, nano::seconds_since_epoch (), info->block_count - 1, nano::epoch::epoch_0);
ledger.update_account (transaction, account, *info, new_info);
@ -156,23 +156,24 @@ public:
rep_block_hash = ledger.representative (transaction, block_a.hashables.previous);
}
nano::account representative{};
nano::account previous_representative{};
if (!rep_block_hash.is_zero ())
{
// Move existing representation & add in amount delta
auto rep_block (ledger.store.block.get (transaction, rep_block_hash));
release_assert (rep_block != nullptr);
representative = rep_block->representative_field ().value ();
ledger.cache.rep_weights.representation_add_dual (transaction, representative, previous_balance, block_a.hashables.representative, 0 - block_a.hashables.balance.number ());
previous_representative = rep_block->representative_field ().value ();
// ledger.rep_weights.representation_add_dual (transaction, representative, previous_balance, block_a.hashables.representative, 0 - block_a.hashables.balance.number ());
ledger.rep_weights.move_add_sub (transaction, block_a.hashables.representative, block_a.hashables.balance, previous_representative, previous_balance);
}
else
{
// Add in amount delta only
ledger.cache.rep_weights.representation_add (transaction, block_a.hashables.representative, 0 - block_a.hashables.balance.number ());
ledger.rep_weights.sub (transaction, block_a.hashables.representative, block_a.hashables.balance.number ());
}
auto previous_version (ledger.version (transaction, block_a.hashables.previous));
nano::account_info new_info (block_a.hashables.previous, representative, info->open_block, previous_balance, nano::seconds_since_epoch (), info->block_count - 1, previous_version);
nano::account_info new_info (block_a.hashables.previous, previous_representative, info->open_block, previous_balance, nano::seconds_since_epoch (), info->block_count - 1, previous_version);
ledger.update_account (transaction, block_a.hashables.account, *info, new_info);
auto previous (ledger.store.block.get (transaction, block_a.hashables.previous));
@ -353,12 +354,13 @@ void ledger_processor::state_block_impl (nano::state_block & block_a)
if (!info.head.is_zero ())
{
// Move existing representation & add in amount delta
ledger.cache.rep_weights.representation_add_dual (transaction, info.representative, 0 - info.balance.number (), block_a.hashables.representative, block_a.hashables.balance.number ());
// ledger.rep_weights.representation_add_dual (transaction, info.representative, 0 - info.balance.number (), block_a.hashables.representative, block_a.hashables.balance.number ());
ledger.rep_weights.move_add_sub (transaction, info.representative, info.balance, block_a.hashables.representative, block_a.hashables.balance);
}
else
{
// Add in amount delta only
ledger.cache.rep_weights.representation_add (transaction, block_a.hashables.representative, block_a.hashables.balance.number ());
ledger.rep_weights.add (transaction, block_a.hashables.representative, block_a.hashables.balance);
}
if (is_send)
@ -481,7 +483,7 @@ void ledger_processor::change_block (nano::change_block & block_a)
block_a.sideband_set (nano::block_sideband (account, 0, info->balance, info->block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */));
ledger.store.block.put (transaction, hash, block_a);
auto balance = previous->balance ();
ledger.cache.rep_weights.representation_add_dual (transaction, block_a.hashables.representative, balance.number (), info->representative, 0 - balance.number ());
ledger.rep_weights.move (transaction, info->representative, block_a.hashables.representative, balance);
nano::account_info new_info (hash, block_a.hashables.representative, info->open_block, info->balance, nano::seconds_since_epoch (), info->block_count + 1, nano::epoch::epoch_0);
ledger.update_account (transaction, account, *info, new_info);
ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::change);
@ -526,7 +528,7 @@ void ledger_processor::send_block (nano::send_block & block_a)
if (result == nano::block_status::progress)
{
auto amount (info->balance.number () - block_a.hashables.balance.number ());
ledger.cache.rep_weights.representation_add (transaction, info->representative, 0 - amount);
ledger.rep_weights.sub (transaction, info->representative, amount);
block_a.sideband_set (nano::block_sideband (account, 0, block_a.hashables.balance /* unused */, info->block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */));
ledger.store.block.put (transaction, hash, block_a);
nano::account_info new_info (hash, info->representative, info->open_block, block_a.hashables.balance, nano::seconds_since_epoch (), info->block_count + 1, nano::epoch::epoch_0);
@ -590,7 +592,7 @@ void ledger_processor::receive_block (nano::receive_block & block_a)
ledger.store.block.put (transaction, hash, block_a);
nano::account_info new_info (hash, info->representative, info->open_block, new_balance, nano::seconds_since_epoch (), info->block_count + 1, nano::epoch::epoch_0);
ledger.update_account (transaction, account, *info, new_info);
ledger.cache.rep_weights.representation_add (transaction, info->representative, pending.value ().amount.number ());
ledger.rep_weights.add (transaction, info->representative, pending.value ().amount);
ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::receive);
}
}
@ -642,7 +644,7 @@ void ledger_processor::open_block (nano::open_block & block_a)
ledger.store.block.put (transaction, hash, block_a);
nano::account_info new_info (hash, block_a.representative_field ().value (), hash, pending.value ().amount.number (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0);
ledger.update_account (transaction, block_a.hashables.account, info, new_info);
ledger.cache.rep_weights.representation_add (transaction, block_a.representative_field ().value (), pending.value ().amount.number ());
ledger.rep_weights.add (transaction, block_a.representative_field ().value (), pending.value ().amount);
ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::open);
}
}
@ -728,17 +730,18 @@ void representative_visitor::state_block (nano::state_block const & block_a)
* ledger
*/
nano::ledger::ledger (nano::store::component & store_a, nano::ledger_constants & constants_a, nano::stats & stats_a, nano::logger & logger_a, nano::generate_cache_flags const & generate_cache_flags_a, nano::uint128_t min_rep_weight_a) :
constants{ constants_a },
nano::ledger::ledger (nano::store::component & store_a, nano::ledger_constants & constants_a, nano::stats & stats_a, nano::logger & logger_a, nano::generate_cache_flags generate_cache_flags_a, nano::uint128_t min_rep_weight_a) :
store{ store_a },
cache{ store_a.rep_weight, min_rep_weight_a },
constants{ constants_a },
stats{ stats_a },
logger{ logger_a },
rep_weights{ store_a.rep_weight, min_rep_weight_a },
any_impl{ std::make_unique<ledger_set_any> (*this) },
confirmed_impl{ std::make_unique<ledger_set_confirmed> (*this) },
any{ *any_impl },
confirmed{ *confirmed_impl }
{
// TODO: Throw on error
if (!store.init_error ())
{
initialize (generate_cache_flags_a);
@ -766,14 +769,29 @@ auto nano::ledger::tx_begin_read () const -> secure::read_transaction
return secure::read_transaction{ store.tx_begin_read () };
}
void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache_flags_a)
void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache_flags)
{
debug_assert (rep_weights.empty ());
logger.info (nano::log::type::ledger, "Loading ledger, this may take a while...");
if (generate_cache_flags_a.reps || generate_cache_flags_a.account_count || generate_cache_flags_a.block_count)
bool is_initialized = false;
{
auto const transaction = store.tx_begin_read ();
is_initialized = (store.account.begin (transaction) != store.account.end (transaction));
}
if (!is_initialized && store.get_mode () != nano::store::open_mode::read_only)
{
// Store was empty meaning we just created it, add the genesis block
auto const transaction = store.tx_begin_write ();
logger.info (nano::log::type::ledger, "Initializing ledger with genesis block");
store.initialize (transaction, constants);
}
if (generate_cache_flags.account_count || generate_cache_flags.block_count)
{
store.account.for_each_par (
[this] (store::read_transaction const & /*unused*/, auto i, auto n) {
[this] (store::read_transaction const &, auto i, auto n) {
uint64_t block_count_l{ 0 };
uint64_t account_count_l{ 0 };
for (; i != n; ++i)
@ -785,22 +803,37 @@ void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache
this->cache.block_count += block_count_l;
this->cache.account_count += account_count_l;
});
}
if (generate_cache_flags.reps)
{
store.rep_weight.for_each_par (
[this] (store::read_transaction const & /*unused*/, auto i, auto n) {
[this] (store::read_transaction const &, auto i, auto n) {
nano::rep_weights rep_weights_l{ this->store.rep_weight };
for (; i != n; ++i)
{
rep_weights_l.representation_put (i->first, i->second.number ());
rep_weights_l.put (i->first, i->second.number ());
}
this->cache.rep_weights.copy_from (rep_weights_l);
this->rep_weights.append_from (rep_weights_l);
});
store.pending.for_each_par (
[this] (store::read_transaction const &, auto i, auto n) {
nano::rep_weights rep_weights_l{ this->store.rep_weight };
for (; i != n; ++i)
{
rep_weights_l.put_unused (i->second.amount.number ());
}
this->rep_weights.append_from (rep_weights_l);
});
rep_weights.verify_consistency ();
}
if (generate_cache_flags_a.cemented_count)
if (generate_cache_flags.cemented_count)
{
store.confirmation_height.for_each_par (
[this] (store::read_transaction const & /*unused*/, auto i, auto n) {
[this] (store::read_transaction const &, auto i, auto n) {
uint64_t cemented_count_l (0);
for (; i != n; ++i)
{
@ -819,7 +852,10 @@ void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache
logger.info (nano::log::type::ledger, "Cemented count: {:>11}", cache.cemented_count.load ());
logger.info (nano::log::type::ledger, "Account count: {:>11}", cache.account_count.load ());
logger.info (nano::log::type::ledger, "Pruned count: {:>11}", cache.pruned_count.load ());
logger.info (nano::log::type::ledger, "Representative count: {:>5}", cache.rep_weights.size ());
logger.info (nano::log::type::ledger, "Representative count: {:>5}", rep_weights.size ());
logger.info (nano::log::type::ledger, "Weight commited: {} | unused: {}",
nano::uint128_union{ rep_weights.get_weight_committed () }.format_balance (nano::nano_ratio, 0, true),
nano::uint128_union{ rep_weights.get_weight_unused () }.format_balance (nano::nano_ratio, 0, true));
}
bool nano::ledger::unconfirmed_exists (secure::transaction const & transaction, nano::block_hash const & hash)
@ -1005,15 +1041,15 @@ std::unordered_map<nano::account, nano::uint128_t> nano::ledger::rep_weights_sna
}
else
{
return cache.rep_weights.get_rep_amounts ();
return rep_weights.get_rep_amounts ();
}
}
nano::uint128_t nano::ledger::weight (nano::account const & account_a) const
nano::uint128_t nano::ledger::weight (nano::account const & account) const
{
if (!bootstrap_height_reached ())
{
auto weight = bootstrap_weights.find (account_a);
auto weight = bootstrap_weights.find (account);
if (weight != bootstrap_weights.end ())
{
return weight->second;
@ -1022,7 +1058,7 @@ nano::uint128_t nano::ledger::weight (nano::account const & account_a) const
}
else
{
return cache.rep_weights.representation_get (account_a);
return rep_weights.get (account);
}
}
@ -1611,6 +1647,6 @@ nano::container_info nano::ledger::container_info () const
{
nano::container_info info;
info.put ("bootstrap_weights", bootstrap_weights);
info.add ("rep_weights", cache.rep_weights.container_info ());
info.add ("rep_weights", rep_weights.container_info ());
return info;
}

View file

@ -1,35 +1,43 @@
#pragma once
#include <nano/lib/config.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/timer.hpp>
#include <nano/secure/account_info.hpp>
#include <nano/secure/fwd.hpp>
#include <nano/secure/generate_cache_flags.hpp>
#include <nano/secure/ledger_cache.hpp>
#include <nano/secure/pending_info.hpp>
#include <nano/secure/rep_weights.hpp>
#include <nano/secure/transaction.hpp>
#include <deque>
#include <map>
#include <memory>
namespace nano::store
{
class component;
}
namespace nano
{
class ledger;
class ledger_set_any;
class ledger_set_confirmed;
class ledger_cache
{
friend class ledger;
private:
std::atomic<uint64_t> cemented_count{ 0 };
std::atomic<uint64_t> block_count{ 0 };
std::atomic<uint64_t> pruned_count{ 0 };
std::atomic<uint64_t> account_count{ 0 };
};
class ledger final
{
template <typename T>
friend class receivable_iterator;
public:
ledger (nano::store::component &, nano::ledger_constants &, nano::stats &, nano::logger &, nano::generate_cache_flags const & = nano::generate_cache_flags{}, nano::uint128_t min_rep_weight_a = 0);
ledger (nano::store::component &, nano::ledger_constants &, nano::stats &, nano::logger &, nano::generate_cache_flags = {}, nano::uint128_t min_rep_weight = 0);
~ledger ();
/** Start read-write transaction */
@ -92,12 +100,15 @@ public:
public:
static nano::uint128_t const unit;
nano::ledger_constants & constants;
nano::store::component & store;
nano::ledger_cache cache;
nano::ledger_constants & constants;
nano::stats & stats;
nano::logger & logger;
nano::ledger_cache cache;
nano::rep_weights rep_weights;
public:
std::unordered_map<nano::account, nano::uint128_t> bootstrap_weights;
uint64_t bootstrap_weight_max_blocks{ 1 };

View file

@ -1,6 +0,0 @@
#include <nano/secure/ledger_cache.hpp>
nano::ledger_cache::ledger_cache (nano::store::rep_weight & rep_weight_store_a, nano::uint128_t min_rep_weight_a) :
rep_weights{ rep_weight_store_a, min_rep_weight_a }
{
}

View file

@ -1,36 +0,0 @@
#pragma once
#include <nano/lib/numbers.hpp>
#include <nano/secure/rep_weights.hpp>
#include <nano/store/rep_weight.hpp>
#include <atomic>
namespace nano
{
class ledger;
}
namespace nano::store
{
class component;
}
namespace nano
{
/* Holds an in-memory cache of various counts */
class ledger_cache
{
friend class store::component;
friend class ledger;
public:
explicit ledger_cache (nano::store::rep_weight & rep_weight_store_a, nano::uint128_t min_rep_weight_a = 0);
nano::rep_weights rep_weights;
private:
std::atomic<uint64_t> cemented_count{ 0 };
std::atomic<uint64_t> block_count{ 0 };
std::atomic<uint64_t> pruned_count{ 0 };
std::atomic<uint64_t> account_count{ 0 };
};
}

View file

@ -9,69 +9,137 @@ nano::rep_weights::rep_weights (nano::store::rep_weight & rep_weight_store_a, na
{
}
void nano::rep_weights::representation_add (store::write_transaction const & txn_a, nano::account const & rep_a, nano::uint128_t const & amount_a)
void nano::rep_weights::add (store::write_transaction const & txn, nano::account const & rep, nano::uint128_t const & amount_add)
{
auto previous_weight{ rep_weight_store.get (txn_a, rep_a) };
auto new_weight = previous_weight + amount_a;
put_store (txn_a, rep_a, previous_weight, new_weight);
std::unique_lock guard{ mutex };
put_cache (rep_a, new_weight);
auto const previous_weight = rep_weight_store.get (txn, rep);
auto const new_weight = previous_weight + amount_add;
release_assert (new_weight >= previous_weight, "new weight must be greater than or equal to previous weight");
put_store (txn, rep, previous_weight, new_weight);
std::lock_guard guard{ mutex };
put_cache (rep, new_weight);
weight_committed += amount_add;
weight_unused -= amount_add;
}
void nano::rep_weights::representation_add_dual (store::write_transaction const & txn_a, nano::account const & rep_1, nano::uint128_t const & amount_1, nano::account const & rep_2, nano::uint128_t const & amount_2)
void nano::rep_weights::sub (store::write_transaction const & txn, nano::account const & rep, nano::uint128_t const & amount_sub)
{
if (rep_1 != rep_2)
auto const previous_weight = rep_weight_store.get (txn, rep);
auto const new_weight = previous_weight - amount_sub;
release_assert (new_weight <= previous_weight, "new weight must be less than or equal to previous weight");
put_store (txn, rep, previous_weight, new_weight);
std::lock_guard guard{ mutex };
put_cache (rep, new_weight);
weight_committed -= amount_sub;
weight_unused += amount_sub;
}
void nano::rep_weights::move (store::write_transaction const & txn, nano::account const & source_rep, nano::account const & dest_rep, nano::uint128_t const & amount)
{
if (source_rep == dest_rep) // Nothing to move if reps are the same
{
auto previous_weight_1{ rep_weight_store.get (txn_a, rep_1) };
auto previous_weight_2{ rep_weight_store.get (txn_a, rep_2) };
auto new_weight_1 = previous_weight_1 + amount_1;
auto new_weight_2 = previous_weight_2 + amount_2;
put_store (txn_a, rep_1, previous_weight_1, new_weight_1);
put_store (txn_a, rep_2, previous_weight_2, new_weight_2);
std::unique_lock guard{ mutex };
put_cache (rep_1, new_weight_1);
put_cache (rep_2, new_weight_2);
return;
}
auto const previous_weight_source = rep_weight_store.get (txn, source_rep);
auto const previous_weight_dest = rep_weight_store.get (txn, dest_rep);
release_assert (previous_weight_source >= amount, "source representative must have enough weight to move");
auto const new_weight_source = previous_weight_source - amount;
auto const new_weight_dest = previous_weight_dest + amount;
release_assert (new_weight_dest >= previous_weight_dest, "new weight for destination representative must be greater than or equal to previous weight");
release_assert (new_weight_source <= previous_weight_source, "new weight for source representative must be less than or equal to previous weight");
put_store (txn, source_rep, previous_weight_source, new_weight_source);
put_store (txn, dest_rep, previous_weight_dest, new_weight_dest);
std::lock_guard guard{ mutex };
put_cache (source_rep, new_weight_source);
put_cache (dest_rep, new_weight_dest);
}
void nano::rep_weights::move_add_sub (store::write_transaction const & txn, nano::account const & source_rep, nano::uint128_t const & amount_source, nano::account const & dest_rep, nano::uint128_t const & amount_dest)
{
if (amount_source == amount_dest)
{
move (txn, source_rep, dest_rep, amount_source);
}
else if (amount_dest > amount_source)
{
move (txn, source_rep, dest_rep, amount_source);
add (txn, dest_rep, amount_dest - amount_source);
}
else if (amount_source > amount_dest)
{
move (txn, source_rep, dest_rep, amount_dest);
sub (txn, source_rep, amount_source - amount_dest);
}
else
{
representation_add (txn_a, rep_1, amount_1 + amount_2);
release_assert (false);
}
}
void nano::rep_weights::representation_put (nano::account const & account_a, nano::uint128_t const & representation_a)
void nano::rep_weights::put (nano::account const & rep, nano::uint128_t const & weight)
{
std::unique_lock guard{ mutex };
put_cache (account_a, representation_a);
std::lock_guard guard{ mutex };
put_cache (rep, weight);
weight_committed += weight;
}
nano::uint128_t nano::rep_weights::representation_get (nano::account const & account_a) const
void nano::rep_weights::put_unused (nano::uint128_t const & weight)
{
std::shared_lock lk{ mutex };
return get (account_a);
std::lock_guard guard{ mutex };
weight_unused += weight;
}
nano::uint128_t nano::rep_weights::get (nano::account const & rep) const
{
std::shared_lock guard{ mutex };
return get_impl (rep);
}
/** Makes a copy */
std::unordered_map<nano::account, nano::uint128_t> nano::rep_weights::get_rep_amounts () const
{
std::shared_lock guard{ mutex };
return rep_amounts;
}
void nano::rep_weights::copy_from (nano::rep_weights & other_a)
void nano::rep_weights::append_from (nano::rep_weights const & other)
{
std::unique_lock guard_this{ mutex };
std::shared_lock guard_other{ other_a.mutex };
for (auto const & entry : other_a.rep_amounts)
std::lock_guard guard_this{ mutex };
std::shared_lock guard_other{ other.mutex };
for (auto const & entry : other.rep_amounts)
{
auto prev_amount (get (entry.first));
auto prev_amount = get_impl (entry.first);
put_cache (entry.first, prev_amount + entry.second);
}
weight_committed += other.weight_committed;
weight_unused += other.weight_unused;
}
void nano::rep_weights::put_cache (nano::account const & account_a, nano::uint128_union const & representation_a)
void nano::rep_weights::verify_consistency () const
{
auto it = rep_amounts.find (account_a);
if (representation_a < min_weight || representation_a.is_zero ())
std::shared_lock guard{ mutex };
auto total_weight = weight_committed + weight_unused;
release_assert (total_weight == std::numeric_limits<nano::uint128_t>::max (), "total weight exceeds maximum value", to_string (weight_committed) + " + " + to_string (weight_unused));
auto cached_weight = std::accumulate (rep_amounts.begin (), rep_amounts.end (), nano::uint256_t{ 0 }, [] (nano::uint256_t sum, const auto & entry) {
return sum + entry.second;
});
release_assert (cached_weight <= weight_committed, "total cached weight must match the sum of all committed weights", to_string (cached_weight) + " <= " + to_string (weight_committed));
}
void nano::rep_weights::put_cache (nano::account const & rep, nano::uint128_union const & weight)
{
debug_assert (!mutex.try_lock ());
auto it = rep_amounts.find (rep);
if (weight < min_weight || weight.is_zero ())
{
if (it != rep_amounts.end ())
{
@ -80,36 +148,37 @@ void nano::rep_weights::put_cache (nano::account const & account_a, nano::uint12
}
else
{
auto amount = representation_a.number ();
auto amount = weight.number ();
if (it != rep_amounts.end ())
{
it->second = amount;
}
else
{
rep_amounts.emplace (account_a, amount);
rep_amounts.emplace (rep, amount);
}
}
}
void nano::rep_weights::put_store (store::write_transaction const & txn_a, nano::account const & rep_a, nano::uint128_t const & previous_weight_a, nano::uint128_t const & new_weight_a)
void nano::rep_weights::put_store (store::write_transaction const & txn, nano::account const & rep, nano::uint128_t const & previous_weight, nano::uint128_t const & new_weight)
{
if (new_weight_a.is_zero ())
debug_assert (rep_weight_store.get (txn, rep) == previous_weight);
if (new_weight.is_zero ())
{
if (!previous_weight_a.is_zero ())
if (!previous_weight.is_zero ())
{
rep_weight_store.del (txn_a, rep_a);
rep_weight_store.del (txn, rep);
}
}
else
{
rep_weight_store.put (txn_a, rep_a, new_weight_a);
rep_weight_store.put (txn, rep, new_weight);
}
}
nano::uint128_t nano::rep_weights::get (nano::account const & account_a) const
nano::uint128_t nano::rep_weights::get_impl (nano::account const & rep) const
{
auto it = rep_amounts.find (account_a);
auto it = rep_amounts.find (rep);
if (it != rep_amounts.end ())
{
return it->second;
@ -126,11 +195,32 @@ std::size_t nano::rep_weights::size () const
return rep_amounts.size ();
}
bool nano::rep_weights::empty () const
{
std::shared_lock guard{ mutex };
return rep_amounts.empty () && weight_committed.is_zero () && weight_unused.is_zero ();
}
nano::uint128_t nano::rep_weights::get_weight_committed () const
{
std::shared_lock guard{ mutex };
release_assert (weight_committed <= std::numeric_limits<nano::uint128_t>::max (), "weight committed exceeds maximum uint128_t value");
return static_cast<nano::uint128_t> (weight_committed);
}
nano::uint128_t nano::rep_weights::get_weight_unused () const
{
std::shared_lock guard{ mutex };
release_assert (weight_unused <= std::numeric_limits<nano::uint128_t>::max (), "weight unused exceeds maximum uint128_t value");
return static_cast<nano::uint128_t> (weight_unused);
}
nano::container_info nano::rep_weights::container_info () const
{
std::shared_lock guard{ mutex };
nano::container_info info;
info.put ("rep_amounts", rep_amounts);
// TODO: Info about weight_committed and weight_unused
return info;
}

View file

@ -3,6 +3,7 @@
#include <nano/lib/numbers.hpp>
#include <nano/lib/numbers_templ.hpp>
#include <nano/lib/utility.hpp>
#include <nano/secure/fwd.hpp>
#include <memory>
#include <shared_mutex>
@ -10,35 +11,52 @@
namespace nano
{
namespace store
{
class component;
class rep_weight;
class write_transaction;
}
class rep_weights
{
public:
explicit rep_weights (nano::store::rep_weight & rep_weight_store_a, nano::uint128_t min_weight_a = 0);
void representation_add (store::write_transaction const & txn_a, nano::account const & source_rep_a, nano::uint128_t const & amount_a);
void representation_add_dual (store::write_transaction const & txn_a, nano::account const & source_rep_1, nano::uint128_t const & amount_1, nano::account const & source_rep_2, nano::uint128_t const & amount_2);
nano::uint128_t representation_get (nano::account const & account_a) const;
explicit rep_weights (nano::store::rep_weight &, nano::uint128_t min_weight = 0);
/* Adds or subtracts weight to the representative */
void add (store::write_transaction const &, nano::account const & rep, nano::uint128_t const & amount_add);
void sub (store::write_transaction const &, nano::account const & rep, nano::uint128_t const & amount_sub);
/* Move weight from one representative to another */
void move (store::write_transaction const &, nano::account const & source_rep, nano::account const & dest_rep, nano::uint128_t const & amount);
/* Move weight from one representative to another while adding or subtracting the weight */
void move_add_sub (store::write_transaction const &, nano::account const & source_rep, nano::uint128_t const & amount_source, nano::account const & dest_rep, nano::uint128_t const & amount_dest);
/* Only use this method when loading rep weights from the database table */
void representation_put (nano::account const & account_a, nano::uint128_t const & representation_a);
void put (nano::account const & rep, nano::uint128_t const & weight);
void put_unused (nano::uint128_t const & weight);
void append_from (rep_weights const & other);
nano::uint128_t get (nano::account const & rep) const;
std::unordered_map<nano::account, nano::uint128_t> get_rep_amounts () const;
/* Only use this method when loading rep weights from the database table */
void copy_from (rep_weights & other_a);
size_t size () const;
nano::container_info container_info () const;
bool empty () const;
nano::uint128_t get_weight_committed () const;
nano::uint128_t get_weight_unused () const;
void verify_consistency () const;
private:
nano::store::rep_weight & rep_weight_store;
nano::uint128_t const min_weight;
mutable std::shared_mutex mutex;
std::unordered_map<nano::account, nano::uint128_t> rep_amounts;
nano::store::rep_weight & rep_weight_store;
nano::uint128_t min_weight;
void put_cache (nano::account const & account_a, nano::uint128_union const & representation_a);
void put_store (store::write_transaction const & txn_a, nano::account const & rep_a, nano::uint128_t const & previous_weight_a, nano::uint128_t const & new_weight_a);
nano::uint128_t get (nano::account const & account_a) const;
// Used for consistency checking, use higher precision types to detect overflows
nano::uint256_t weight_committed{ 0 };
nano::uint256_t weight_unused{ 0 };
private:
void put_cache (nano::account const & rep, nano::uint128_union const & weight);
void put_store (store::write_transaction const &, nano::account const & rep, nano::uint128_t const & previous_weight, nano::uint128_t const & new_weight);
nano::uint128_t get_impl (nano::account const & rep) const;
};
}

View file

@ -134,7 +134,6 @@ TEST (ledger, deep_account_compute)
nano::stats stats{ logger };
nano::ledger ledger (*store, nano::dev::constants, stats, logger);
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
nano::keypair key;
auto balance (nano::dev::constants.genesis_amount - 1);
@ -1159,7 +1158,6 @@ TEST (confirmation_height, many_accounts_send_receive_self_no_elections)
{
auto transaction = ledger.tx_begin_write ();
store->initialize (transaction, ledger.cache, ledger.constants);
// Send from genesis account to all other accounts and create open block for them
for (auto i = 0; i < num_accounts; ++i)

View file

@ -1,11 +1,13 @@
#include <nano/lib/blocks.hpp>
#include <nano/lib/enum_util.hpp>
#include <nano/lib/timer.hpp>
#include <nano/secure/ledger_cache.hpp>
#include <nano/store/account.hpp>
#include <nano/store/block.hpp>
#include <nano/store/component.hpp>
#include <nano/store/confirmation_height.hpp>
#include <nano/store/final_vote.hpp>
#include <nano/store/pending.hpp>
#include <nano/store/pruned.hpp>
#include <nano/store/rep_weight.hpp>
nano::store::component::component (nano::store::block & block_store_a, nano::store::account & account_store_a, nano::store::pending & pending_store_a, nano::store::online_weight & online_weight_store_a, nano::store::pruned & pruned_store_a, nano::store::peer & peer_store_a, nano::store::confirmation_height & confirmation_height_store_a, nano::store::final_vote & final_vote_store_a, nano::store::version & version_store_a, nano::store::rep_weight & rep_weight_a) :
@ -26,19 +28,21 @@ nano::store::component::component (nano::store::block & block_store_a, nano::sto
* If using a different store version than the latest then you may need
* to modify some of the objects in the store to be appropriate for the version before an upgrade.
*/
void nano::store::component::initialize (store::write_transaction const & transaction_a, nano::ledger_cache & ledger_cache_a, nano::ledger_constants & constants)
void nano::store::component::initialize (store::write_transaction const & transaction, nano::ledger_constants & constants)
{
debug_assert (constants.genesis->has_sideband ());
debug_assert (account.begin (transaction_a) == account.end (transaction_a));
auto hash_l (constants.genesis->hash ());
block.put (transaction_a, hash_l, *constants.genesis);
++ledger_cache_a.block_count;
confirmation_height.put (transaction_a, constants.genesis->account (), nano::confirmation_height_info{ 1, constants.genesis->hash () });
++ledger_cache_a.cemented_count;
account.put (transaction_a, constants.genesis->account (), { hash_l, constants.genesis->account (), constants.genesis->hash (), std::numeric_limits<nano::uint128_t>::max (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0 });
++ledger_cache_a.account_count;
rep_weight.put (transaction_a, constants.genesis->account (), std::numeric_limits<nano::uint128_t>::max ());
ledger_cache_a.rep_weights.representation_put (constants.genesis->account (), std::numeric_limits<nano::uint128_t>::max ());
release_assert (constants.genesis->has_sideband ());
release_assert (account.begin (transaction) == account.end (transaction));
release_assert (block.begin (transaction) == block.end (transaction));
release_assert (pending.begin (transaction) == pending.end (transaction));
release_assert (confirmation_height.begin (transaction) == confirmation_height.end (transaction));
release_assert (final_vote.begin (transaction) == final_vote.end (transaction));
release_assert (rep_weight.begin (transaction) == rep_weight.end (transaction));
release_assert (pruned.begin (transaction) == pruned.end (transaction));
block.put (transaction, constants.genesis->hash (), *constants.genesis);
confirmation_height.put (transaction, constants.genesis->account (), nano::confirmation_height_info{ 1, constants.genesis->hash () });
account.put (transaction, constants.genesis->account (), { constants.genesis->hash (), constants.genesis->account (), constants.genesis->hash (), std::numeric_limits<nano::uint128_t>::max (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0 });
rep_weight.put (transaction, constants.genesis->account (), std::numeric_limits<nano::uint128_t>::max ());
}
/*

View file

@ -37,24 +37,24 @@ namespace store
friend class mdb_block_store_upgrade_v21_v22_Test;
public:
// clang-format off
explicit component (
explicit component (
nano::store::block &,
nano::store::account &,
nano::store::pending &,
nano::store::online_weight&,
nano::store::online_weight &,
nano::store::pruned &,
nano::store::peer &,
nano::store::confirmation_height &,
nano::store::final_vote &,
nano::store::version &,
nano::store::rep_weight &
);
// clang-format on
nano::store::rep_weight &);
virtual ~component () = default;
void initialize (write_transaction const & transaction_a, nano::ledger_cache & ledger_cache_a, nano::ledger_constants & constants);
void initialize (store::write_transaction const &, nano::ledger_constants &);
virtual uint64_t count (store::transaction const & transaction_a, tables table_a) const = 0;
virtual int drop (write_transaction const & transaction_a, tables table_a) = 0;
virtual int drop (store::write_transaction const & transaction_a, tables table_a) = 0;
virtual bool not_found (int status) const = 0;
virtual bool success (int status) const = 0;
virtual std::string error_string (int status) const = 0;
@ -63,6 +63,7 @@ namespace store
store::account & account;
store::pending & pending;
store::rep_weight & rep_weight;
static int constexpr version_minimum{ 21 };
static int constexpr version_current{ 24 };

View file

@ -5,14 +5,12 @@
nano::test::ledger_context::ledger_context (std::deque<std::shared_ptr<nano::block>> && blocks) :
store_m{ nano::make_store (logger_m, nano::unique_path (), nano::dev::constants) },
stats_m{ logger_m },
ledger_m{ *store_m, nano::dev::constants, stats_m, logger_m },
blocks_m{ blocks },
pool_m{ nano::dev::network_params.network, 1 }
{
debug_assert (!store_m->init_error ());
auto tx = ledger_m.tx_begin_write ();
store_m->initialize (tx, ledger_m.cache, ledger_m.constants);
for (auto const & i : blocks_m)
{
auto process_result = ledger_m.process (tx, i);

View file

@ -4,6 +4,7 @@
#include <nano/lib/stats.hpp>
#include <nano/lib/work.hpp>
#include <nano/secure/ledger.hpp>
#include <nano/store/component.hpp>
#include <nano/store/fwd.hpp>
namespace nano::test
@ -14,6 +15,7 @@ public:
/** 'blocks' initialises the ledger with each block in-order
Blocks must all return process_result::progress when processed */
ledger_context (std::deque<std::shared_ptr<nano::block>> && blocks = std::deque<std::shared_ptr<nano::block>>{});
nano::ledger & ledger ();
nano::store::component & store ();
std::deque<std::shared_ptr<nano::block>> const & blocks () const;
@ -23,8 +25,8 @@ public:
private:
nano::logger logger_m;
nano::stats stats_m{ logger_m };
std::unique_ptr<nano::store::component> store_m;
nano::stats stats_m;
nano::ledger ledger_m;
std::deque<std::shared_ptr<nano::block>> blocks_m;
nano::work_pool pool_m;