diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 5bc4f8a50..32e65351e 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -383,9 +383,8 @@ TEST (block_store, genesis) nano::logger logger; auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); ASSERT_TRUE (!store->init_error ()); - nano::ledger_cache ledger_cache{ store->rep_weight }; auto transaction (store->tx_begin_write ()); - store->initialize (transaction, ledger_cache, nano::dev::constants); + store->initialize (transaction, nano::dev::constants); nano::account_info info; ASSERT_FALSE (store->account.get (transaction, nano::dev::genesis_key.pub, info)); ASSERT_EQ (nano::dev::genesis->hash (), info.head); @@ -635,7 +634,6 @@ TEST (mdb_block_store, supported_version_upgrades) nano::stats stats{ logger }; nano::ledger ledger (store, nano::dev::constants, stats, logger); auto transaction (store.tx_begin_write ()); - store.initialize (transaction, ledger.cache, nano::dev::constants); // Lower the database to the max version unsupported for upgrades store.version.put (transaction, store.version_minimum - 1); } @@ -653,7 +651,6 @@ TEST (mdb_block_store, supported_version_upgrades) nano::stats stats{ logger }; nano::ledger ledger (store, nano::dev::constants, stats, logger); auto transaction (store.tx_begin_write ()); - store.initialize (transaction, ledger.cache, nano::dev::constants); // Lower the database version to the minimum version supported for upgrade. store.version.put (transaction, store.version_minimum); } @@ -897,10 +894,8 @@ TEST (block_store, cemented_count_cache) nano::logger logger; auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); ASSERT_TRUE (!store->init_error ()); - auto transaction (store->tx_begin_write ()); nano::stats stats{ logger }; nano::ledger ledger (*store, nano::dev::constants, stats, logger); - store->initialize (transaction, ledger.cache, nano::dev::constants); ASSERT_EQ (1, ledger.cemented_count ()); } @@ -921,9 +916,8 @@ TEST (block_store, pruned_random) block->sideband_set ({}); auto hash1 (block->hash ()); { - nano::ledger_cache ledger_cache{ store->rep_weight }; auto transaction (store->tx_begin_write ()); - store->initialize (transaction, ledger_cache, nano::dev::constants); + store->initialize (transaction, nano::dev::constants); store->pruned.put (transaction, hash1); } auto transaction (store->tx_begin_read ()); @@ -951,9 +945,8 @@ TEST (block_store, state_block) block1->sideband_set ({}); { - nano::ledger_cache ledger_cache{ store->rep_weight }; auto transaction (store->tx_begin_write ()); - store->initialize (transaction, ledger_cache, nano::dev::constants); + store->initialize (transaction, nano::dev::constants); ASSERT_EQ (nano::block_type::state, block1->type ()); store->block.put (transaction, block1->hash (), *block1); ASSERT_TRUE (store->block.exists (transaction, block1->hash ())); @@ -990,7 +983,6 @@ TEST (mdb_block_store, sideband_height) nano::ledger ledger (store, nano::dev::constants, stats, logger); nano::block_builder builder; auto transaction = ledger.tx_begin_write (); - store.initialize (transaction, ledger.cache, nano::dev::constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; auto send = builder .send () diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index c932e27c5..739aba905 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -678,11 +678,11 @@ TEST (ledger, representation_changes) auto store{ nano::test::make_store () }; nano::keypair key1; nano::rep_weights rep_weights{ store->rep_weight }; - ASSERT_EQ (0, rep_weights.representation_get (key1.pub)); - rep_weights.representation_put (key1.pub, 1); - ASSERT_EQ (1, rep_weights.representation_get (key1.pub)); - rep_weights.representation_put (key1.pub, 2); - ASSERT_EQ (2, rep_weights.representation_get (key1.pub)); + ASSERT_EQ (0, rep_weights.get (key1.pub)); + rep_weights.put (key1.pub, 1); + ASSERT_EQ (1, rep_weights.get (key1.pub)); + rep_weights.put (key1.pub, 2); + ASSERT_EQ (2, rep_weights.get (key1.pub)); } TEST (ledger, delete_rep_weight_of_zero) @@ -690,17 +690,23 @@ TEST (ledger, delete_rep_weight_of_zero) auto store{ nano::test::make_store () }; nano::rep_weights rep_weights{ store->rep_weight }; auto txn{ store->tx_begin_write () }; - rep_weights.representation_add (txn, 1, 100); - rep_weights.representation_add_dual (txn, 2, 100, 3, 100); + rep_weights.add (txn, 1, 100); + rep_weights.add (txn, 2, 200); + ASSERT_EQ (2, rep_weights.size ()); + rep_weights.move_add_sub (txn, 2, 100, 3, 120); ASSERT_EQ (3, rep_weights.size ()); ASSERT_EQ (3, store->rep_weight.count (txn)); - // set rep weights to 0 - rep_weights.representation_add (txn, 1, nano::uint128_t{ 0 } - 100); + // Reduce rep weights to 0 + rep_weights.sub (txn, 1, 100); ASSERT_EQ (2, rep_weights.size ()); ASSERT_EQ (2, store->rep_weight.count (txn)); - rep_weights.representation_add_dual (txn, 2, nano::uint128_t{ 0 } - 100, 3, nano::uint128_t{ 0 } - 100); + rep_weights.move_add_sub (txn, 3, 120, 2, 100); + ASSERT_EQ (1, rep_weights.size ()); + ASSERT_EQ (1, store->rep_weight.count (txn)); + + rep_weights.sub (txn, 2, 200); ASSERT_EQ (0, rep_weights.size ()); ASSERT_EQ (0, store->rep_weight.count (txn)); } @@ -713,22 +719,22 @@ TEST (ledger, rep_cache_min_weight) auto txn{ store->tx_begin_write () }; // one below min weight - rep_weights.representation_add (txn, 1, 9); + rep_weights.add (txn, 1, 9); ASSERT_EQ (0, rep_weights.size ()); ASSERT_EQ (1, store->rep_weight.count (txn)); // exactly min weight - rep_weights.representation_add (txn, 1, 1); + rep_weights.add (txn, 1, 1); ASSERT_EQ (1, rep_weights.size ()); ASSERT_EQ (1, store->rep_weight.count (txn)); // above min weight - rep_weights.representation_add (txn, 1, 1); + rep_weights.add (txn, 1, 1); ASSERT_EQ (1, rep_weights.size ()); ASSERT_EQ (1, store->rep_weight.count (txn)); // fall blow min weight - rep_weights.representation_add (txn, 1, nano::uint128_t{ 0 } - 5); + rep_weights.sub (txn, 1, 5); ASSERT_EQ (0, rep_weights.size ()); ASSERT_EQ (1, store->rep_weight.count (txn)); } @@ -738,10 +744,10 @@ TEST (ledger, representation) auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); - auto & rep_weights = ledger.cache.rep_weights; + auto & rep_weights = ledger.rep_weights; auto transaction = ledger.tx_begin_write (); auto & pool = ctx.pool (); - ASSERT_EQ (nano::dev::constants.genesis_amount, rep_weights.representation_get (nano::dev::genesis_key.pub)); + ASSERT_EQ (nano::dev::constants.genesis_amount, rep_weights.get (nano::dev::genesis_key.pub)); nano::keypair key2; nano::block_builder builder; auto block1 = builder @@ -753,7 +759,7 @@ TEST (ledger, representation) .work (*pool.generate (nano::dev::genesis->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block1)); - ASSERT_EQ (nano::dev::constants.genesis_amount - 100, rep_weights.representation_get (nano::dev::genesis_key.pub)); + ASSERT_EQ (nano::dev::constants.genesis_amount - 100, rep_weights.get (nano::dev::genesis_key.pub)); nano::keypair key3; auto block2 = builder .open () @@ -764,9 +770,9 @@ TEST (ledger, representation) .work (*pool.generate (key2.pub)) .build (); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block2)); - ASSERT_EQ (nano::dev::constants.genesis_amount - 100, rep_weights.representation_get (nano::dev::genesis_key.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key2.pub)); - ASSERT_EQ (100, rep_weights.representation_get (key3.pub)); + ASSERT_EQ (nano::dev::constants.genesis_amount - 100, rep_weights.get (nano::dev::genesis_key.pub)); + ASSERT_EQ (0, rep_weights.get (key2.pub)); + ASSERT_EQ (100, rep_weights.get (key3.pub)); auto block3 = builder .send () .previous (block1->hash ()) @@ -776,9 +782,9 @@ TEST (ledger, representation) .work (*pool.generate (block1->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block3)); - ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key2.pub)); - ASSERT_EQ (100, rep_weights.representation_get (key3.pub)); + ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub)); + ASSERT_EQ (0, rep_weights.get (key2.pub)); + ASSERT_EQ (100, rep_weights.get (key3.pub)); auto block4 = builder .receive () .previous (block2->hash ()) @@ -787,9 +793,9 @@ TEST (ledger, representation) .work (*pool.generate (block2->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block4)); - ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key2.pub)); - ASSERT_EQ (200, rep_weights.representation_get (key3.pub)); + ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub)); + ASSERT_EQ (0, rep_weights.get (key2.pub)); + ASSERT_EQ (200, rep_weights.get (key3.pub)); nano::keypair key4; auto block5 = builder .change () @@ -799,10 +805,10 @@ TEST (ledger, representation) .work (*pool.generate (block4->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block5)); - ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key2.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key3.pub)); - ASSERT_EQ (200, rep_weights.representation_get (key4.pub)); + ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub)); + ASSERT_EQ (0, rep_weights.get (key2.pub)); + ASSERT_EQ (0, rep_weights.get (key3.pub)); + ASSERT_EQ (200, rep_weights.get (key4.pub)); nano::keypair key5; auto block6 = builder .send () @@ -813,11 +819,11 @@ TEST (ledger, representation) .work (*pool.generate (block5->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block6)); - ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key2.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key3.pub)); - ASSERT_EQ (100, rep_weights.representation_get (key4.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key5.pub)); + ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub)); + ASSERT_EQ (0, rep_weights.get (key2.pub)); + ASSERT_EQ (0, rep_weights.get (key3.pub)); + ASSERT_EQ (100, rep_weights.get (key4.pub)); + ASSERT_EQ (0, rep_weights.get (key5.pub)); nano::keypair key6; auto block7 = builder .open () @@ -828,12 +834,12 @@ TEST (ledger, representation) .work (*pool.generate (key5.pub)) .build (); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block7)); - ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key2.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key3.pub)); - ASSERT_EQ (100, rep_weights.representation_get (key4.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key5.pub)); - ASSERT_EQ (100, rep_weights.representation_get (key6.pub)); + ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub)); + ASSERT_EQ (0, rep_weights.get (key2.pub)); + ASSERT_EQ (0, rep_weights.get (key3.pub)); + ASSERT_EQ (100, rep_weights.get (key4.pub)); + ASSERT_EQ (0, rep_weights.get (key5.pub)); + ASSERT_EQ (100, rep_weights.get (key6.pub)); auto block8 = builder .send () .previous (block6->hash ()) @@ -843,12 +849,12 @@ TEST (ledger, representation) .work (*pool.generate (block6->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block8)); - ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key2.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key3.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key4.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key5.pub)); - ASSERT_EQ (100, rep_weights.representation_get (key6.pub)); + ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub)); + ASSERT_EQ (0, rep_weights.get (key2.pub)); + ASSERT_EQ (0, rep_weights.get (key3.pub)); + ASSERT_EQ (0, rep_weights.get (key4.pub)); + ASSERT_EQ (0, rep_weights.get (key5.pub)); + ASSERT_EQ (100, rep_weights.get (key6.pub)); auto block9 = builder .receive () .previous (block7->hash ()) @@ -857,12 +863,12 @@ TEST (ledger, representation) .work (*pool.generate (block7->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, block9)); - ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.representation_get (nano::dev::genesis_key.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key2.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key3.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key4.pub)); - ASSERT_EQ (0, rep_weights.representation_get (key5.pub)); - ASSERT_EQ (200, rep_weights.representation_get (key6.pub)); + ASSERT_EQ (nano::dev::constants.genesis_amount - 200, rep_weights.get (nano::dev::genesis_key.pub)); + ASSERT_EQ (0, rep_weights.get (key2.pub)); + ASSERT_EQ (0, rep_weights.get (key3.pub)); + ASSERT_EQ (0, rep_weights.get (key4.pub)); + ASSERT_EQ (0, rep_weights.get (key5.pub)); + ASSERT_EQ (200, rep_weights.get (key6.pub)); } TEST (ledger, double_open) @@ -873,7 +879,6 @@ TEST (ledger, double_open) nano::stats stats{ logger }; nano::ledger ledger (*store, nano::dev::constants, stats, logger); auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; nano::keypair key2; nano::block_builder builder; @@ -4634,8 +4639,8 @@ TEST (ledger, zero_rep) .build (); auto transaction = node1.ledger.tx_begin_write (); ASSERT_EQ (nano::block_status::progress, node1.ledger.process (transaction, block1)); - ASSERT_EQ (0, node1.ledger.cache.rep_weights.representation_get (nano::dev::genesis_key.pub)); - ASSERT_EQ (nano::dev::constants.genesis_amount, node1.ledger.cache.rep_weights.representation_get (0)); + ASSERT_EQ (0, node1.ledger.rep_weights.get (nano::dev::genesis_key.pub)); + ASSERT_EQ (nano::dev::constants.genesis_amount, node1.ledger.rep_weights.get (0)); auto block2 = builder.state () .account (nano::dev::genesis_key.pub) .previous (block1->hash ()) @@ -4646,8 +4651,8 @@ TEST (ledger, zero_rep) .work (*system.work.generate (block1->hash ())) .build (); ASSERT_EQ (nano::block_status::progress, node1.ledger.process (transaction, block2)); - ASSERT_EQ (nano::dev::constants.genesis_amount, node1.ledger.cache.rep_weights.representation_get (nano::dev::genesis_key.pub)); - ASSERT_EQ (0, node1.ledger.cache.rep_weights.representation_get (0)); + ASSERT_EQ (nano::dev::constants.genesis_amount, node1.ledger.rep_weights.get (nano::dev::genesis_key.pub)); + ASSERT_EQ (0, node1.ledger.rep_weights.get (0)); } TEST (ledger, work_validation) @@ -4810,7 +4815,6 @@ TEST (ledger, dependents_confirmed_pruning) nano::ledger ledger (*store, nano::dev::constants, stats, logger); ledger.pruning = true; auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::block_builder builder; nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; nano::keypair key1; @@ -4901,10 +4905,13 @@ TEST (ledger, cache) ASSERT_EQ (account_count, ledger.account_count ()); ASSERT_EQ (block_count, ledger.block_count ()); ASSERT_EQ (cemented_count, ledger.cemented_count ()); - ASSERT_EQ (genesis_weight, ledger.cache.rep_weights.representation_get (nano::dev::genesis_key.pub)); + ASSERT_EQ (genesis_weight, ledger.rep_weights.get (nano::dev::genesis_key.pub)); ASSERT_EQ (pruned_count, ledger.pruned_count ()); }; + cache_check (ledger); + cache_check (nano::ledger (store, nano::dev::constants, stats, logger)); + nano::keypair key; auto const latest = ledger.any.account_head (ledger.tx_begin_read (), nano::dev::genesis_key.pub); auto send = builder.state () @@ -4984,7 +4991,6 @@ TEST (ledger, pruning_action) nano::ledger ledger (*store, nano::dev::constants, stats, logger); ledger.pruning = true; auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; nano::block_builder builder; auto send1 = builder @@ -5070,7 +5076,6 @@ TEST (ledger, pruning_large_chain) nano::ledger ledger (*store, nano::dev::constants, stats, logger); ledger.pruning = true; auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; size_t send_receive_pairs (20); auto last_hash (nano::dev::genesis->hash ()); @@ -5126,7 +5131,6 @@ TEST (ledger, pruning_source_rollback) nano::ledger ledger (*store, nano::dev::constants, stats, logger); ledger.pruning = true; auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; nano::block_builder builder; auto epoch1 = builder @@ -5215,7 +5219,6 @@ TEST (ledger, pruning_source_rollback_legacy) nano::ledger ledger (*store, nano::dev::constants, stats, logger); ledger.pruning = true; auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; nano::block_builder builder; auto send1 = builder @@ -5330,7 +5333,6 @@ TEST (ledger, pruning_legacy_blocks) ledger.pruning = true; nano::keypair key1; auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; nano::block_builder builder; auto send1 = builder @@ -5416,7 +5418,6 @@ TEST (ledger, pruning_safe_functions) nano::ledger ledger (*store, nano::dev::constants, stats, logger); ledger.pruning = true; auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; nano::block_builder builder; auto send1 = builder @@ -5468,7 +5469,6 @@ TEST (ledger, random_blocks) nano::ledger ledger (*store, nano::dev::constants, stats, logger); ledger.pruning = true; auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; nano::block_builder builder; auto send1 = builder @@ -5571,7 +5571,6 @@ TEST (ledger, migrate_lmdb_to_rocksdb) { auto transaction = ledger.tx_begin_write (); - store.initialize (transaction, ledger.cache, ledger.constants); ASSERT_FALSE (store.init_error ()); // Lower the database to the max version unsupported for upgrades diff --git a/nano/core_test/ledger_confirm.cpp b/nano/core_test/ledger_confirm.cpp index 7cb489250..0397c4e14 100644 --- a/nano/core_test/ledger_confirm.cpp +++ b/nano/core_test/ledger_confirm.cpp @@ -817,7 +817,6 @@ TEST (ledger_confirm, pruned_source) .work (*pool.generate (key2.pub)) .build (); auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, nano::dev::constants); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send1)); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, open1)); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send2)); @@ -859,7 +858,6 @@ TEST (ledger_confirmDeathTest, rollback_added_block) .work (*pool.generate (nano::dev::genesis->hash ())) .build (); auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); ASSERT_DEATH_IF_SUPPORTED (ledger.confirm (transaction, send->hash ()), ""); } } diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index 6dad6dda6..82efcb26c 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -2572,9 +2572,8 @@ TEST (node, dont_write_lock_node) nano::logger logger; auto store = nano::make_store (logger, path, nano::dev::constants, false, true); { - nano::ledger_cache ledger_cache{ store->rep_weight }; auto transaction (store->tx_begin_write ()); - store->initialize (transaction, ledger_cache, nano::dev::constants); + store->initialize (transaction, nano::dev::constants); } // Hold write lock open until main thread is done needing it diff --git a/nano/core_test/processor_service.cpp b/nano/core_test/processor_service.cpp index 44796a949..c788e6048 100644 --- a/nano/core_test/processor_service.cpp +++ b/nano/core_test/processor_service.cpp @@ -20,7 +20,6 @@ TEST (processor_service, bad_send_signature) ASSERT_FALSE (store->init_error ()); nano::ledger ledger (*store, nano::dev::constants, system.stats, system.logger); auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; auto info1 = ledger.any.account_get (transaction, nano::dev::genesis_key.pub); ASSERT_TRUE (info1); @@ -46,7 +45,6 @@ TEST (processor_service, bad_receive_signature) ASSERT_FALSE (store->init_error ()); nano::ledger ledger (*store, nano::dev::constants, system.stats, system.logger); auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; auto info1 = ledger.any.account_get (transaction, nano::dev::genesis_key.pub); ASSERT_TRUE (info1); diff --git a/nano/core_test/vote_processor.cpp b/nano/core_test/vote_processor.cpp index 144a965d2..55e9ce043 100644 --- a/nano/core_test/vote_processor.cpp +++ b/nano/core_test/vote_processor.cpp @@ -135,7 +135,7 @@ TEST (vote_processor, weights) system.wallet (0)->send_sync (nano::dev::genesis_key.pub, key2.pub, level2); // Wait for representatives - ASSERT_TIMELY_EQ (10s, node.ledger.cache.rep_weights.get_rep_amounts ().size (), 4); + ASSERT_TIMELY_EQ (10s, node.ledger.rep_weights.get_rep_amounts ().size (), 4); // Wait for rep tiers to be updated node.stats.clear (); diff --git a/nano/lib/assert.hpp b/nano/lib/assert.hpp index 94eb20559..6d86eed71 100644 --- a/nano/lib/assert.hpp +++ b/nano/lib/assert.hpp @@ -4,12 +4,14 @@ #include #include +#include #include [[noreturn]] void assert_internal (char const * check_expr, char const * func, char const * file, unsigned int line, bool is_release_assert, std::string_view error = ""); #define release_assert_1(check) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, true) #define release_assert_2(check, error_msg) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, true, error_msg) +#define release_assert_3(check, error_msg, extra) (check) ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, true, std::string (error_msg) + " (" + (extra) + ")") #if !BOOST_PP_VARIADICS_MSVC #define release_assert(...) \ BOOST_PP_OVERLOAD (release_assert_, __VA_ARGS__) \ @@ -23,6 +25,7 @@ #else #define debug_assert_1(check) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, false) #define debug_assert_2(check, error_msg) check ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, false, error_msg) +#define debug_assert_3(check, error_msg, extra) (check) ? (void)0 : assert_internal (#check, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__, false, std::string (error_msg) + " (" + (extra) + ")") #if !BOOST_PP_VARIADICS_MSVC #define debug_assert(...) \ BOOST_PP_OVERLOAD (debug_assert_, __VA_ARGS__) \ diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index c7b69df61..3378009c2 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -217,7 +217,7 @@ int main (int argc, char * const * argv) auto const bootstrap_weights = node->get_bootstrap_weights (); auto const & hardcoded = bootstrap_weights.second; auto const hardcoded_height = bootstrap_weights.first; - auto const ledger_unfiltered = node->ledger.cache.rep_weights.get_rep_amounts (); + auto const ledger_unfiltered = node->ledger.rep_weights.get_rep_amounts (); auto const ledger_height = node->ledger.block_count (); auto get_total = [] (decltype (bootstrap_weights.second) const & reps) -> nano::uint128_union { @@ -459,7 +459,7 @@ int main (int argc, char * const * argv) auto node = inactive_node.node; auto transaction (node->store.tx_begin_read ()); nano::uint128_t total; - auto rep_amounts = node->ledger.cache.rep_weights.get_rep_amounts (); + auto rep_amounts = node->ledger.rep_weights.get_rep_amounts (); std::map ordered_reps (rep_amounts.begin (), rep_amounts.end ()); for (auto const & rep : ordered_reps) { diff --git a/nano/node/block_processor.cpp b/nano/node/block_processor.cpp index 15f3df1f9..bb51d79b0 100644 --- a/nano/node/block_processor.cpp +++ b/nano/node/block_processor.cpp @@ -368,6 +368,9 @@ void nano::block_processor::process_batch (nano::unique_lock & lock processed.emplace_back (result, std::move (ctx)); } + // We had rocksdb issues in the past, ensure that rep weights are always consistent + ledger.rep_weights.verify_consistency (); + if (number_of_blocks_processed != 0 && timer.stop () > std::chrono::milliseconds (100)) { logger.debug (nano::log::type::block_processor, "Processed {} blocks ({} forced) in {} {}", number_of_blocks_processed, number_of_forced_processed, timer.value ().count (), timer.unit ()); diff --git a/nano/node/bounded_backlog.cpp b/nano/node/bounded_backlog.cpp index 794858a11..f5a48b368 100644 --- a/nano/node/bounded_backlog.cpp +++ b/nano/node/bounded_backlog.cpp @@ -344,6 +344,9 @@ std::deque nano::bounded_backlog::perform_rollbacks (std::dequ } } + // We had rocksdb issues in the past, ensure that rep weights are always consistent + ledger.rep_weights.verify_consistency (); + return processed; } diff --git a/nano/node/election.cpp b/nano/node/election.cpp index b6857fbf3..c20739c75 100644 --- a/nano/node/election.cpp +++ b/nano/node/election.cpp @@ -837,7 +837,7 @@ std::vector nano::election::votes_with_weight () co { if (vote_l.first != nullptr) { - auto amount (node.ledger.cache.rep_weights.representation_get (vote_l.first)); + auto amount (node.ledger.weight (vote_l.first)); nano::vote_with_weight_info vote_info{ vote_l.first, vote_l.second.time, vote_l.second.timestamp, vote_l.second.hash, amount }; sorted_votes.emplace (std::move (amount), vote_info); } diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 2b2729698..575a8c546 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -2136,7 +2136,7 @@ void nano::json_handler::confirmation_info () { if (block->hash () == vote.hash) { - auto amount (node.ledger.cache.rep_weights.representation_get (representative)); + auto amount (node.ledger.weight (representative)); representatives.emplace (amount, representative); if (vote.timestamp == std::numeric_limits::max ()) { @@ -3538,7 +3538,7 @@ void nano::json_handler::representatives () { bool const sorting = request.get ("sorting", false); boost::property_tree::ptree representatives; - auto rep_amounts = node.ledger.cache.rep_weights.get_rep_amounts (); + auto rep_amounts = node.ledger.rep_weights.get_rep_amounts (); if (!sorting) // Simple { std::map ordered (rep_amounts.begin (), rep_amounts.end ()); diff --git a/nano/node/node.cpp b/nano/node/node.cpp index da87ba4ba..05b4c0d72 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -330,20 +330,6 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy config.bandwidth_limit, config.bandwidth_limit_burst_ratio); - // First do a pass with a read to see if any writing needs doing, this saves needing to open a write lock (and potentially blocking) - auto is_initialized (false); - { - auto const transaction (store.tx_begin_read ()); - is_initialized = (store.account.begin (transaction) != store.account.end (transaction)); - } - - if (!is_initialized && !flags.read_only) - { - auto const transaction = store.tx_begin_write (); - // Store was empty meaning we just created it, add the genesis block - store.initialize (transaction, ledger.cache, ledger.constants); - } - if (!block_or_pruned_exists (config.network_params.ledger.genesis->hash ())) { logger.critical (nano::log::type::node, "Genesis block not found. This commonly indicates a configuration issue, check that the --network or --data_path command line arguments are correct, and also the ledger backend node config option. If using a read-only CLI command a ledger must already exist, start the node with --daemon first."); diff --git a/nano/qt_test/qt.cpp b/nano/qt_test/qt.cpp index d02524f30..59b091de7 100644 --- a/nano/qt_test/qt.cpp +++ b/nano/qt_test/qt.cpp @@ -524,7 +524,6 @@ TEST (history, short_text) nano::ledger ledger (*store, nano::dev::constants, stats, logger); { auto transaction (ledger.tx_begin_write ()); - store->initialize (transaction, ledger.cache, ledger.constants); nano::keypair key; auto latest (ledger.any.account_head (transaction, nano::dev::genesis_key.pub)); auto send = std::make_shared (latest, nano::dev::genesis_key.pub, 0, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, *system.work.generate (latest)); @@ -566,7 +565,6 @@ TEST (history, pruned_source) // Basic pruning for legacy blocks. Previous block is pruned, source is pruned { auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, nano::dev::constants); auto latest (ledger.any.account_head (transaction, nano::dev::genesis_key.pub)); auto send1 = std::make_shared (latest, nano::dev::genesis_key.pub, nano::dev::constants.genesis_amount - 100, nano::dev::genesis_key.prv, nano::dev::genesis_key.pub, *system.work.generate (latest)); ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send1)); diff --git a/nano/secure/CMakeLists.txt b/nano/secure/CMakeLists.txt index a0cbea2a4..795d7002b 100644 --- a/nano/secure/CMakeLists.txt +++ b/nano/secure/CMakeLists.txt @@ -25,8 +25,6 @@ add_library( generate_cache_flags.cpp ledger.hpp ledger.cpp - ledger_cache.hpp - ledger_cache.cpp ledger_set_any.hpp ledger_set_any.cpp ledger_set_confirmed.hpp diff --git a/nano/secure/fwd.hpp b/nano/secure/fwd.hpp index d33606c11..50315f2d1 100644 --- a/nano/secure/fwd.hpp +++ b/nano/secure/fwd.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include namespace nano { diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index ec0566c56..776e9e224 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -60,7 +60,7 @@ public: auto info = ledger.any.account_get (transaction, pending.value ().source); release_assert (info); ledger.store.pending.del (transaction, key); - ledger.cache.rep_weights.representation_add (transaction, info->representative, pending.value ().amount.number ()); + ledger.rep_weights.add (transaction, info->representative, pending.value ().amount); nano::account_info new_info (block_a.hashables.previous, info->representative, info->open_block, ledger.any.block_balance (transaction, block_a.hashables.previous).value (), nano::seconds_since_epoch (), info->block_count - 1, nano::epoch::epoch_0); ledger.update_account (transaction, pending.value ().source, *info, new_info); ledger.store.block.del (transaction, hash); @@ -77,7 +77,7 @@ public: auto source_account = ledger.any.block_account (transaction, block_a.hashables.source); auto info = ledger.any.account_get (transaction, destination_account); release_assert (info); - ledger.cache.rep_weights.representation_add (transaction, info->representative, 0 - amount); + ledger.rep_weights.sub (transaction, info->representative, amount); nano::account_info new_info (block_a.hashables.previous, info->representative, info->open_block, ledger.any.block_balance (transaction, block_a.hashables.previous).value (), nano::seconds_since_epoch (), info->block_count - 1, nano::epoch::epoch_0); ledger.update_account (transaction, destination_account, *info, new_info); ledger.store.block.del (transaction, hash); @@ -91,7 +91,7 @@ public: auto amount = ledger.any.block_amount (transaction, hash).value ().number (); auto destination_account = block_a.account (); auto source_account = ledger.any.block_account (transaction, block_a.hashables.source); - ledger.cache.rep_weights.representation_add (transaction, block_a.representative_field ().value (), 0 - amount); + ledger.rep_weights.sub (transaction, block_a.representative_field ().value (), amount); nano::account_info new_info; ledger.update_account (transaction, destination_account, new_info, new_info); ledger.store.block.del (transaction, hash); @@ -101,15 +101,15 @@ public: void change_block (nano::change_block const & block_a) override { auto hash (block_a.hash ()); - auto rep_block (ledger.representative (transaction, block_a.hashables.previous)); + auto rep_block_hash (ledger.representative (transaction, block_a.hashables.previous)); auto account = block_a.account (); auto info = ledger.any.account_get (transaction, account); release_assert (info); auto balance = ledger.any.block_balance (transaction, block_a.hashables.previous).value (); - auto block = ledger.store.block.get (transaction, rep_block); - release_assert (block != nullptr); - auto representative = block->representative_field ().value (); - ledger.cache.rep_weights.representation_add_dual (transaction, block_a.hashables.representative, 0 - balance.number (), representative, balance.number ()); + auto rep_block = ledger.store.block.get (transaction, rep_block_hash); + release_assert (rep_block != nullptr); + auto representative = rep_block->representative_field ().value (); + ledger.rep_weights.move (transaction, block_a.hashables.representative, representative, balance); ledger.store.block.del (transaction, hash); nano::account_info new_info (block_a.hashables.previous, representative, info->open_block, info->balance, nano::seconds_since_epoch (), info->block_count - 1, nano::epoch::epoch_0); ledger.update_account (transaction, account, *info, new_info); @@ -156,23 +156,24 @@ public: rep_block_hash = ledger.representative (transaction, block_a.hashables.previous); } - nano::account representative{}; + nano::account previous_representative{}; if (!rep_block_hash.is_zero ()) { // Move existing representation & add in amount delta auto rep_block (ledger.store.block.get (transaction, rep_block_hash)); release_assert (rep_block != nullptr); - representative = rep_block->representative_field ().value (); - ledger.cache.rep_weights.representation_add_dual (transaction, representative, previous_balance, block_a.hashables.representative, 0 - block_a.hashables.balance.number ()); + previous_representative = rep_block->representative_field ().value (); + // ledger.rep_weights.representation_add_dual (transaction, representative, previous_balance, block_a.hashables.representative, 0 - block_a.hashables.balance.number ()); + ledger.rep_weights.move_add_sub (transaction, block_a.hashables.representative, block_a.hashables.balance, previous_representative, previous_balance); } else { // Add in amount delta only - ledger.cache.rep_weights.representation_add (transaction, block_a.hashables.representative, 0 - block_a.hashables.balance.number ()); + ledger.rep_weights.sub (transaction, block_a.hashables.representative, block_a.hashables.balance.number ()); } auto previous_version (ledger.version (transaction, block_a.hashables.previous)); - nano::account_info new_info (block_a.hashables.previous, representative, info->open_block, previous_balance, nano::seconds_since_epoch (), info->block_count - 1, previous_version); + nano::account_info new_info (block_a.hashables.previous, previous_representative, info->open_block, previous_balance, nano::seconds_since_epoch (), info->block_count - 1, previous_version); ledger.update_account (transaction, block_a.hashables.account, *info, new_info); auto previous (ledger.store.block.get (transaction, block_a.hashables.previous)); @@ -353,12 +354,13 @@ void ledger_processor::state_block_impl (nano::state_block & block_a) if (!info.head.is_zero ()) { // Move existing representation & add in amount delta - ledger.cache.rep_weights.representation_add_dual (transaction, info.representative, 0 - info.balance.number (), block_a.hashables.representative, block_a.hashables.balance.number ()); + // ledger.rep_weights.representation_add_dual (transaction, info.representative, 0 - info.balance.number (), block_a.hashables.representative, block_a.hashables.balance.number ()); + ledger.rep_weights.move_add_sub (transaction, info.representative, info.balance, block_a.hashables.representative, block_a.hashables.balance); } else { // Add in amount delta only - ledger.cache.rep_weights.representation_add (transaction, block_a.hashables.representative, block_a.hashables.balance.number ()); + ledger.rep_weights.add (transaction, block_a.hashables.representative, block_a.hashables.balance); } if (is_send) @@ -481,7 +483,7 @@ void ledger_processor::change_block (nano::change_block & block_a) block_a.sideband_set (nano::block_sideband (account, 0, info->balance, info->block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); ledger.store.block.put (transaction, hash, block_a); auto balance = previous->balance (); - ledger.cache.rep_weights.representation_add_dual (transaction, block_a.hashables.representative, balance.number (), info->representative, 0 - balance.number ()); + ledger.rep_weights.move (transaction, info->representative, block_a.hashables.representative, balance); nano::account_info new_info (hash, block_a.hashables.representative, info->open_block, info->balance, nano::seconds_since_epoch (), info->block_count + 1, nano::epoch::epoch_0); ledger.update_account (transaction, account, *info, new_info); ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::change); @@ -526,7 +528,7 @@ void ledger_processor::send_block (nano::send_block & block_a) if (result == nano::block_status::progress) { auto amount (info->balance.number () - block_a.hashables.balance.number ()); - ledger.cache.rep_weights.representation_add (transaction, info->representative, 0 - amount); + ledger.rep_weights.sub (transaction, info->representative, amount); block_a.sideband_set (nano::block_sideband (account, 0, block_a.hashables.balance /* unused */, info->block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); ledger.store.block.put (transaction, hash, block_a); nano::account_info new_info (hash, info->representative, info->open_block, block_a.hashables.balance, nano::seconds_since_epoch (), info->block_count + 1, nano::epoch::epoch_0); @@ -590,7 +592,7 @@ void ledger_processor::receive_block (nano::receive_block & block_a) ledger.store.block.put (transaction, hash, block_a); nano::account_info new_info (hash, info->representative, info->open_block, new_balance, nano::seconds_since_epoch (), info->block_count + 1, nano::epoch::epoch_0); ledger.update_account (transaction, account, *info, new_info); - ledger.cache.rep_weights.representation_add (transaction, info->representative, pending.value ().amount.number ()); + ledger.rep_weights.add (transaction, info->representative, pending.value ().amount); ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::receive); } } @@ -642,7 +644,7 @@ void ledger_processor::open_block (nano::open_block & block_a) ledger.store.block.put (transaction, hash, block_a); nano::account_info new_info (hash, block_a.representative_field ().value (), hash, pending.value ().amount.number (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0); ledger.update_account (transaction, block_a.hashables.account, info, new_info); - ledger.cache.rep_weights.representation_add (transaction, block_a.representative_field ().value (), pending.value ().amount.number ()); + ledger.rep_weights.add (transaction, block_a.representative_field ().value (), pending.value ().amount); ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::open); } } @@ -728,17 +730,18 @@ void representative_visitor::state_block (nano::state_block const & block_a) * ledger */ -nano::ledger::ledger (nano::store::component & store_a, nano::ledger_constants & constants_a, nano::stats & stats_a, nano::logger & logger_a, nano::generate_cache_flags const & generate_cache_flags_a, nano::uint128_t min_rep_weight_a) : - constants{ constants_a }, +nano::ledger::ledger (nano::store::component & store_a, nano::ledger_constants & constants_a, nano::stats & stats_a, nano::logger & logger_a, nano::generate_cache_flags generate_cache_flags_a, nano::uint128_t min_rep_weight_a) : store{ store_a }, - cache{ store_a.rep_weight, min_rep_weight_a }, + constants{ constants_a }, stats{ stats_a }, logger{ logger_a }, + rep_weights{ store_a.rep_weight, min_rep_weight_a }, any_impl{ std::make_unique (*this) }, confirmed_impl{ std::make_unique (*this) }, any{ *any_impl }, confirmed{ *confirmed_impl } { + // TODO: Throw on error if (!store.init_error ()) { initialize (generate_cache_flags_a); @@ -766,14 +769,29 @@ auto nano::ledger::tx_begin_read () const -> secure::read_transaction return secure::read_transaction{ store.tx_begin_read () }; } -void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache_flags_a) +void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache_flags) { + debug_assert (rep_weights.empty ()); + logger.info (nano::log::type::ledger, "Loading ledger, this may take a while..."); - if (generate_cache_flags_a.reps || generate_cache_flags_a.account_count || generate_cache_flags_a.block_count) + bool is_initialized = false; + { + auto const transaction = store.tx_begin_read (); + is_initialized = (store.account.begin (transaction) != store.account.end (transaction)); + } + if (!is_initialized && store.get_mode () != nano::store::open_mode::read_only) + { + // Store was empty meaning we just created it, add the genesis block + auto const transaction = store.tx_begin_write (); + logger.info (nano::log::type::ledger, "Initializing ledger with genesis block"); + store.initialize (transaction, constants); + } + + if (generate_cache_flags.account_count || generate_cache_flags.block_count) { store.account.for_each_par ( - [this] (store::read_transaction const & /*unused*/, auto i, auto n) { + [this] (store::read_transaction const &, auto i, auto n) { uint64_t block_count_l{ 0 }; uint64_t account_count_l{ 0 }; for (; i != n; ++i) @@ -785,22 +803,37 @@ void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache this->cache.block_count += block_count_l; this->cache.account_count += account_count_l; }); + } + if (generate_cache_flags.reps) + { store.rep_weight.for_each_par ( - [this] (store::read_transaction const & /*unused*/, auto i, auto n) { + [this] (store::read_transaction const &, auto i, auto n) { nano::rep_weights rep_weights_l{ this->store.rep_weight }; for (; i != n; ++i) { - rep_weights_l.representation_put (i->first, i->second.number ()); + rep_weights_l.put (i->first, i->second.number ()); } - this->cache.rep_weights.copy_from (rep_weights_l); + this->rep_weights.append_from (rep_weights_l); }); + + store.pending.for_each_par ( + [this] (store::read_transaction const &, auto i, auto n) { + nano::rep_weights rep_weights_l{ this->store.rep_weight }; + for (; i != n; ++i) + { + rep_weights_l.put_unused (i->second.amount.number ()); + } + this->rep_weights.append_from (rep_weights_l); + }); + + rep_weights.verify_consistency (); } - if (generate_cache_flags_a.cemented_count) + if (generate_cache_flags.cemented_count) { store.confirmation_height.for_each_par ( - [this] (store::read_transaction const & /*unused*/, auto i, auto n) { + [this] (store::read_transaction const &, auto i, auto n) { uint64_t cemented_count_l (0); for (; i != n; ++i) { @@ -819,7 +852,10 @@ void nano::ledger::initialize (nano::generate_cache_flags const & generate_cache logger.info (nano::log::type::ledger, "Cemented count: {:>11}", cache.cemented_count.load ()); logger.info (nano::log::type::ledger, "Account count: {:>11}", cache.account_count.load ()); logger.info (nano::log::type::ledger, "Pruned count: {:>11}", cache.pruned_count.load ()); - logger.info (nano::log::type::ledger, "Representative count: {:>5}", cache.rep_weights.size ()); + logger.info (nano::log::type::ledger, "Representative count: {:>5}", rep_weights.size ()); + logger.info (nano::log::type::ledger, "Weight commited: {} | unused: {}", + nano::uint128_union{ rep_weights.get_weight_committed () }.format_balance (nano::nano_ratio, 0, true), + nano::uint128_union{ rep_weights.get_weight_unused () }.format_balance (nano::nano_ratio, 0, true)); } bool nano::ledger::unconfirmed_exists (secure::transaction const & transaction, nano::block_hash const & hash) @@ -1005,15 +1041,15 @@ std::unordered_map nano::ledger::rep_weights_sna } else { - return cache.rep_weights.get_rep_amounts (); + return rep_weights.get_rep_amounts (); } } -nano::uint128_t nano::ledger::weight (nano::account const & account_a) const +nano::uint128_t nano::ledger::weight (nano::account const & account) const { if (!bootstrap_height_reached ()) { - auto weight = bootstrap_weights.find (account_a); + auto weight = bootstrap_weights.find (account); if (weight != bootstrap_weights.end ()) { return weight->second; @@ -1022,7 +1058,7 @@ nano::uint128_t nano::ledger::weight (nano::account const & account_a) const } else { - return cache.rep_weights.representation_get (account_a); + return rep_weights.get (account); } } @@ -1611,6 +1647,6 @@ nano::container_info nano::ledger::container_info () const { nano::container_info info; info.put ("bootstrap_weights", bootstrap_weights); - info.add ("rep_weights", cache.rep_weights.container_info ()); + info.add ("rep_weights", rep_weights.container_info ()); return info; } diff --git a/nano/secure/ledger.hpp b/nano/secure/ledger.hpp index d9fc78fbc..c48b1d67f 100644 --- a/nano/secure/ledger.hpp +++ b/nano/secure/ledger.hpp @@ -1,35 +1,43 @@ #pragma once +#include #include #include #include #include #include -#include #include +#include #include #include #include #include -namespace nano::store -{ -class component; -} - namespace nano { +class ledger; class ledger_set_any; class ledger_set_confirmed; +class ledger_cache +{ + friend class ledger; + +private: + std::atomic cemented_count{ 0 }; + std::atomic block_count{ 0 }; + std::atomic pruned_count{ 0 }; + std::atomic account_count{ 0 }; +}; + class ledger final { template friend class receivable_iterator; public: - ledger (nano::store::component &, nano::ledger_constants &, nano::stats &, nano::logger &, nano::generate_cache_flags const & = nano::generate_cache_flags{}, nano::uint128_t min_rep_weight_a = 0); + ledger (nano::store::component &, nano::ledger_constants &, nano::stats &, nano::logger &, nano::generate_cache_flags = {}, nano::uint128_t min_rep_weight = 0); ~ledger (); /** Start read-write transaction */ @@ -92,12 +100,15 @@ public: public: static nano::uint128_t const unit; - nano::ledger_constants & constants; nano::store::component & store; - nano::ledger_cache cache; + nano::ledger_constants & constants; nano::stats & stats; nano::logger & logger; + nano::ledger_cache cache; + nano::rep_weights rep_weights; + +public: std::unordered_map bootstrap_weights; uint64_t bootstrap_weight_max_blocks{ 1 }; diff --git a/nano/secure/ledger_cache.cpp b/nano/secure/ledger_cache.cpp deleted file mode 100644 index 16c46c0f5..000000000 --- a/nano/secure/ledger_cache.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -nano::ledger_cache::ledger_cache (nano::store::rep_weight & rep_weight_store_a, nano::uint128_t min_rep_weight_a) : - rep_weights{ rep_weight_store_a, min_rep_weight_a } -{ -} diff --git a/nano/secure/ledger_cache.hpp b/nano/secure/ledger_cache.hpp deleted file mode 100644 index 81503b6df..000000000 --- a/nano/secure/ledger_cache.hpp +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -namespace nano -{ -class ledger; -} -namespace nano::store -{ -class component; -} - -namespace nano -{ -/* Holds an in-memory cache of various counts */ -class ledger_cache -{ - friend class store::component; - friend class ledger; - -public: - explicit ledger_cache (nano::store::rep_weight & rep_weight_store_a, nano::uint128_t min_rep_weight_a = 0); - nano::rep_weights rep_weights; - -private: - std::atomic cemented_count{ 0 }; - std::atomic block_count{ 0 }; - std::atomic pruned_count{ 0 }; - std::atomic account_count{ 0 }; -}; -} diff --git a/nano/secure/rep_weights.cpp b/nano/secure/rep_weights.cpp index 7556ec4b9..3e3d72a4f 100644 --- a/nano/secure/rep_weights.cpp +++ b/nano/secure/rep_weights.cpp @@ -9,69 +9,137 @@ nano::rep_weights::rep_weights (nano::store::rep_weight & rep_weight_store_a, na { } -void nano::rep_weights::representation_add (store::write_transaction const & txn_a, nano::account const & rep_a, nano::uint128_t const & amount_a) +void nano::rep_weights::add (store::write_transaction const & txn, nano::account const & rep, nano::uint128_t const & amount_add) { - auto previous_weight{ rep_weight_store.get (txn_a, rep_a) }; - auto new_weight = previous_weight + amount_a; - put_store (txn_a, rep_a, previous_weight, new_weight); - std::unique_lock guard{ mutex }; - put_cache (rep_a, new_weight); + auto const previous_weight = rep_weight_store.get (txn, rep); + auto const new_weight = previous_weight + amount_add; + release_assert (new_weight >= previous_weight, "new weight must be greater than or equal to previous weight"); + + put_store (txn, rep, previous_weight, new_weight); + + std::lock_guard guard{ mutex }; + put_cache (rep, new_weight); + + weight_committed += amount_add; + weight_unused -= amount_add; } -void nano::rep_weights::representation_add_dual (store::write_transaction const & txn_a, nano::account const & rep_1, nano::uint128_t const & amount_1, nano::account const & rep_2, nano::uint128_t const & amount_2) +void nano::rep_weights::sub (store::write_transaction const & txn, nano::account const & rep, nano::uint128_t const & amount_sub) { - if (rep_1 != rep_2) + auto const previous_weight = rep_weight_store.get (txn, rep); + auto const new_weight = previous_weight - amount_sub; + release_assert (new_weight <= previous_weight, "new weight must be less than or equal to previous weight"); + + put_store (txn, rep, previous_weight, new_weight); + + std::lock_guard guard{ mutex }; + put_cache (rep, new_weight); + + weight_committed -= amount_sub; + weight_unused += amount_sub; +} + +void nano::rep_weights::move (store::write_transaction const & txn, nano::account const & source_rep, nano::account const & dest_rep, nano::uint128_t const & amount) +{ + if (source_rep == dest_rep) // Nothing to move if reps are the same { - auto previous_weight_1{ rep_weight_store.get (txn_a, rep_1) }; - auto previous_weight_2{ rep_weight_store.get (txn_a, rep_2) }; - auto new_weight_1 = previous_weight_1 + amount_1; - auto new_weight_2 = previous_weight_2 + amount_2; - put_store (txn_a, rep_1, previous_weight_1, new_weight_1); - put_store (txn_a, rep_2, previous_weight_2, new_weight_2); - std::unique_lock guard{ mutex }; - put_cache (rep_1, new_weight_1); - put_cache (rep_2, new_weight_2); + return; + } + + auto const previous_weight_source = rep_weight_store.get (txn, source_rep); + auto const previous_weight_dest = rep_weight_store.get (txn, dest_rep); + release_assert (previous_weight_source >= amount, "source representative must have enough weight to move"); + + auto const new_weight_source = previous_weight_source - amount; + auto const new_weight_dest = previous_weight_dest + amount; + release_assert (new_weight_dest >= previous_weight_dest, "new weight for destination representative must be greater than or equal to previous weight"); + release_assert (new_weight_source <= previous_weight_source, "new weight for source representative must be less than or equal to previous weight"); + + put_store (txn, source_rep, previous_weight_source, new_weight_source); + put_store (txn, dest_rep, previous_weight_dest, new_weight_dest); + + std::lock_guard guard{ mutex }; + put_cache (source_rep, new_weight_source); + put_cache (dest_rep, new_weight_dest); +} + +void nano::rep_weights::move_add_sub (store::write_transaction const & txn, nano::account const & source_rep, nano::uint128_t const & amount_source, nano::account const & dest_rep, nano::uint128_t const & amount_dest) +{ + if (amount_source == amount_dest) + { + move (txn, source_rep, dest_rep, amount_source); + } + else if (amount_dest > amount_source) + { + move (txn, source_rep, dest_rep, amount_source); + add (txn, dest_rep, amount_dest - amount_source); + } + else if (amount_source > amount_dest) + { + move (txn, source_rep, dest_rep, amount_dest); + sub (txn, source_rep, amount_source - amount_dest); } else { - representation_add (txn_a, rep_1, amount_1 + amount_2); + release_assert (false); } } -void nano::rep_weights::representation_put (nano::account const & account_a, nano::uint128_t const & representation_a) +void nano::rep_weights::put (nano::account const & rep, nano::uint128_t const & weight) { - std::unique_lock guard{ mutex }; - put_cache (account_a, representation_a); + std::lock_guard guard{ mutex }; + put_cache (rep, weight); + weight_committed += weight; } -nano::uint128_t nano::rep_weights::representation_get (nano::account const & account_a) const +void nano::rep_weights::put_unused (nano::uint128_t const & weight) { - std::shared_lock lk{ mutex }; - return get (account_a); + std::lock_guard guard{ mutex }; + weight_unused += weight; +} + +nano::uint128_t nano::rep_weights::get (nano::account const & rep) const +{ + std::shared_lock guard{ mutex }; + return get_impl (rep); } -/** Makes a copy */ std::unordered_map nano::rep_weights::get_rep_amounts () const { std::shared_lock guard{ mutex }; return rep_amounts; } -void nano::rep_weights::copy_from (nano::rep_weights & other_a) +void nano::rep_weights::append_from (nano::rep_weights const & other) { - std::unique_lock guard_this{ mutex }; - std::shared_lock guard_other{ other_a.mutex }; - for (auto const & entry : other_a.rep_amounts) + std::lock_guard guard_this{ mutex }; + std::shared_lock guard_other{ other.mutex }; + for (auto const & entry : other.rep_amounts) { - auto prev_amount (get (entry.first)); + auto prev_amount = get_impl (entry.first); put_cache (entry.first, prev_amount + entry.second); } + weight_committed += other.weight_committed; + weight_unused += other.weight_unused; } -void nano::rep_weights::put_cache (nano::account const & account_a, nano::uint128_union const & representation_a) +void nano::rep_weights::verify_consistency () const { - auto it = rep_amounts.find (account_a); - if (representation_a < min_weight || representation_a.is_zero ()) + std::shared_lock guard{ mutex }; + auto total_weight = weight_committed + weight_unused; + release_assert (total_weight == std::numeric_limits::max (), "total weight exceeds maximum value", to_string (weight_committed) + " + " + to_string (weight_unused)); + auto cached_weight = std::accumulate (rep_amounts.begin (), rep_amounts.end (), nano::uint256_t{ 0 }, [] (nano::uint256_t sum, const auto & entry) { + return sum + entry.second; + }); + release_assert (cached_weight <= weight_committed, "total cached weight must match the sum of all committed weights", to_string (cached_weight) + " <= " + to_string (weight_committed)); +} + +void nano::rep_weights::put_cache (nano::account const & rep, nano::uint128_union const & weight) +{ + debug_assert (!mutex.try_lock ()); + + auto it = rep_amounts.find (rep); + if (weight < min_weight || weight.is_zero ()) { if (it != rep_amounts.end ()) { @@ -80,36 +148,37 @@ void nano::rep_weights::put_cache (nano::account const & account_a, nano::uint12 } else { - auto amount = representation_a.number (); + auto amount = weight.number (); if (it != rep_amounts.end ()) { it->second = amount; } else { - rep_amounts.emplace (account_a, amount); + rep_amounts.emplace (rep, amount); } } } -void nano::rep_weights::put_store (store::write_transaction const & txn_a, nano::account const & rep_a, nano::uint128_t const & previous_weight_a, nano::uint128_t const & new_weight_a) +void nano::rep_weights::put_store (store::write_transaction const & txn, nano::account const & rep, nano::uint128_t const & previous_weight, nano::uint128_t const & new_weight) { - if (new_weight_a.is_zero ()) + debug_assert (rep_weight_store.get (txn, rep) == previous_weight); + if (new_weight.is_zero ()) { - if (!previous_weight_a.is_zero ()) + if (!previous_weight.is_zero ()) { - rep_weight_store.del (txn_a, rep_a); + rep_weight_store.del (txn, rep); } } else { - rep_weight_store.put (txn_a, rep_a, new_weight_a); + rep_weight_store.put (txn, rep, new_weight); } } -nano::uint128_t nano::rep_weights::get (nano::account const & account_a) const +nano::uint128_t nano::rep_weights::get_impl (nano::account const & rep) const { - auto it = rep_amounts.find (account_a); + auto it = rep_amounts.find (rep); if (it != rep_amounts.end ()) { return it->second; @@ -126,11 +195,32 @@ std::size_t nano::rep_weights::size () const return rep_amounts.size (); } +bool nano::rep_weights::empty () const +{ + std::shared_lock guard{ mutex }; + return rep_amounts.empty () && weight_committed.is_zero () && weight_unused.is_zero (); +} + +nano::uint128_t nano::rep_weights::get_weight_committed () const +{ + std::shared_lock guard{ mutex }; + release_assert (weight_committed <= std::numeric_limits::max (), "weight committed exceeds maximum uint128_t value"); + return static_cast (weight_committed); +} + +nano::uint128_t nano::rep_weights::get_weight_unused () const +{ + std::shared_lock guard{ mutex }; + release_assert (weight_unused <= std::numeric_limits::max (), "weight unused exceeds maximum uint128_t value"); + return static_cast (weight_unused); +} + nano::container_info nano::rep_weights::container_info () const { std::shared_lock guard{ mutex }; nano::container_info info; info.put ("rep_amounts", rep_amounts); + // TODO: Info about weight_committed and weight_unused return info; } diff --git a/nano/secure/rep_weights.hpp b/nano/secure/rep_weights.hpp index 29a2c2804..5999b7d58 100644 --- a/nano/secure/rep_weights.hpp +++ b/nano/secure/rep_weights.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -10,35 +11,52 @@ namespace nano { -namespace store -{ - class component; - class rep_weight; - class write_transaction; -} - class rep_weights { public: - explicit rep_weights (nano::store::rep_weight & rep_weight_store_a, nano::uint128_t min_weight_a = 0); - void representation_add (store::write_transaction const & txn_a, nano::account const & source_rep_a, nano::uint128_t const & amount_a); - void representation_add_dual (store::write_transaction const & txn_a, nano::account const & source_rep_1, nano::uint128_t const & amount_1, nano::account const & source_rep_2, nano::uint128_t const & amount_2); - nano::uint128_t representation_get (nano::account const & account_a) const; + explicit rep_weights (nano::store::rep_weight &, nano::uint128_t min_weight = 0); + + /* Adds or subtracts weight to the representative */ + void add (store::write_transaction const &, nano::account const & rep, nano::uint128_t const & amount_add); + void sub (store::write_transaction const &, nano::account const & rep, nano::uint128_t const & amount_sub); + + /* Move weight from one representative to another */ + void move (store::write_transaction const &, nano::account const & source_rep, nano::account const & dest_rep, nano::uint128_t const & amount); + + /* Move weight from one representative to another while adding or subtracting the weight */ + void move_add_sub (store::write_transaction const &, nano::account const & source_rep, nano::uint128_t const & amount_source, nano::account const & dest_rep, nano::uint128_t const & amount_dest); + /* Only use this method when loading rep weights from the database table */ - void representation_put (nano::account const & account_a, nano::uint128_t const & representation_a); + void put (nano::account const & rep, nano::uint128_t const & weight); + void put_unused (nano::uint128_t const & weight); + void append_from (rep_weights const & other); + + nano::uint128_t get (nano::account const & rep) const; std::unordered_map get_rep_amounts () const; - /* Only use this method when loading rep weights from the database table */ - void copy_from (rep_weights & other_a); + size_t size () const; nano::container_info container_info () const; + bool empty () const; + + nano::uint128_t get_weight_committed () const; + nano::uint128_t get_weight_unused () const; + + void verify_consistency () const; private: + nano::store::rep_weight & rep_weight_store; + nano::uint128_t const min_weight; + mutable std::shared_mutex mutex; std::unordered_map rep_amounts; - nano::store::rep_weight & rep_weight_store; - nano::uint128_t min_weight; - void put_cache (nano::account const & account_a, nano::uint128_union const & representation_a); - void put_store (store::write_transaction const & txn_a, nano::account const & rep_a, nano::uint128_t const & previous_weight_a, nano::uint128_t const & new_weight_a); - nano::uint128_t get (nano::account const & account_a) const; + + // Used for consistency checking, use higher precision types to detect overflows + nano::uint256_t weight_committed{ 0 }; + nano::uint256_t weight_unused{ 0 }; + +private: + void put_cache (nano::account const & rep, nano::uint128_union const & weight); + void put_store (store::write_transaction const &, nano::account const & rep, nano::uint128_t const & previous_weight, nano::uint128_t const & new_weight); + nano::uint128_t get_impl (nano::account const & rep) const; }; } diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index b95c69656..0d5177b05 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -134,7 +134,6 @@ TEST (ledger, deep_account_compute) nano::stats stats{ logger }; nano::ledger ledger (*store, nano::dev::constants, stats, logger); auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; nano::keypair key; auto balance (nano::dev::constants.genesis_amount - 1); @@ -1159,7 +1158,6 @@ TEST (confirmation_height, many_accounts_send_receive_self_no_elections) { auto transaction = ledger.tx_begin_write (); - store->initialize (transaction, ledger.cache, ledger.constants); // Send from genesis account to all other accounts and create open block for them for (auto i = 0; i < num_accounts; ++i) diff --git a/nano/store/component.cpp b/nano/store/component.cpp index 1af3aad7a..0dda015fd 100644 --- a/nano/store/component.cpp +++ b/nano/store/component.cpp @@ -1,11 +1,13 @@ #include #include #include -#include #include #include #include #include +#include +#include +#include #include nano::store::component::component (nano::store::block & block_store_a, nano::store::account & account_store_a, nano::store::pending & pending_store_a, nano::store::online_weight & online_weight_store_a, nano::store::pruned & pruned_store_a, nano::store::peer & peer_store_a, nano::store::confirmation_height & confirmation_height_store_a, nano::store::final_vote & final_vote_store_a, nano::store::version & version_store_a, nano::store::rep_weight & rep_weight_a) : @@ -26,19 +28,21 @@ nano::store::component::component (nano::store::block & block_store_a, nano::sto * If using a different store version than the latest then you may need * to modify some of the objects in the store to be appropriate for the version before an upgrade. */ -void nano::store::component::initialize (store::write_transaction const & transaction_a, nano::ledger_cache & ledger_cache_a, nano::ledger_constants & constants) +void nano::store::component::initialize (store::write_transaction const & transaction, nano::ledger_constants & constants) { - debug_assert (constants.genesis->has_sideband ()); - debug_assert (account.begin (transaction_a) == account.end (transaction_a)); - auto hash_l (constants.genesis->hash ()); - block.put (transaction_a, hash_l, *constants.genesis); - ++ledger_cache_a.block_count; - confirmation_height.put (transaction_a, constants.genesis->account (), nano::confirmation_height_info{ 1, constants.genesis->hash () }); - ++ledger_cache_a.cemented_count; - account.put (transaction_a, constants.genesis->account (), { hash_l, constants.genesis->account (), constants.genesis->hash (), std::numeric_limits::max (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0 }); - ++ledger_cache_a.account_count; - rep_weight.put (transaction_a, constants.genesis->account (), std::numeric_limits::max ()); - ledger_cache_a.rep_weights.representation_put (constants.genesis->account (), std::numeric_limits::max ()); + release_assert (constants.genesis->has_sideband ()); + release_assert (account.begin (transaction) == account.end (transaction)); + release_assert (block.begin (transaction) == block.end (transaction)); + release_assert (pending.begin (transaction) == pending.end (transaction)); + release_assert (confirmation_height.begin (transaction) == confirmation_height.end (transaction)); + release_assert (final_vote.begin (transaction) == final_vote.end (transaction)); + release_assert (rep_weight.begin (transaction) == rep_weight.end (transaction)); + release_assert (pruned.begin (transaction) == pruned.end (transaction)); + + block.put (transaction, constants.genesis->hash (), *constants.genesis); + confirmation_height.put (transaction, constants.genesis->account (), nano::confirmation_height_info{ 1, constants.genesis->hash () }); + account.put (transaction, constants.genesis->account (), { constants.genesis->hash (), constants.genesis->account (), constants.genesis->hash (), std::numeric_limits::max (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0 }); + rep_weight.put (transaction, constants.genesis->account (), std::numeric_limits::max ()); } /* diff --git a/nano/store/component.hpp b/nano/store/component.hpp index 1de32ea52..b4a0de0fb 100644 --- a/nano/store/component.hpp +++ b/nano/store/component.hpp @@ -37,24 +37,24 @@ namespace store friend class mdb_block_store_upgrade_v21_v22_Test; public: - // clang-format off - explicit component ( + explicit component ( nano::store::block &, nano::store::account &, nano::store::pending &, - nano::store::online_weight&, + nano::store::online_weight &, nano::store::pruned &, nano::store::peer &, nano::store::confirmation_height &, nano::store::final_vote &, nano::store::version &, - nano::store::rep_weight & - ); - // clang-format on + nano::store::rep_weight &); + virtual ~component () = default; - void initialize (write_transaction const & transaction_a, nano::ledger_cache & ledger_cache_a, nano::ledger_constants & constants); + + void initialize (store::write_transaction const &, nano::ledger_constants &); + virtual uint64_t count (store::transaction const & transaction_a, tables table_a) const = 0; - virtual int drop (write_transaction const & transaction_a, tables table_a) = 0; + virtual int drop (store::write_transaction const & transaction_a, tables table_a) = 0; virtual bool not_found (int status) const = 0; virtual bool success (int status) const = 0; virtual std::string error_string (int status) const = 0; @@ -63,6 +63,7 @@ namespace store store::account & account; store::pending & pending; store::rep_weight & rep_weight; + static int constexpr version_minimum{ 21 }; static int constexpr version_current{ 24 }; diff --git a/nano/test_common/ledger_context.cpp b/nano/test_common/ledger_context.cpp index 6ae4b0fa4..0d5725823 100644 --- a/nano/test_common/ledger_context.cpp +++ b/nano/test_common/ledger_context.cpp @@ -5,14 +5,12 @@ nano::test::ledger_context::ledger_context (std::deque> && blocks) : store_m{ nano::make_store (logger_m, nano::unique_path (), nano::dev::constants) }, - stats_m{ logger_m }, ledger_m{ *store_m, nano::dev::constants, stats_m, logger_m }, blocks_m{ blocks }, pool_m{ nano::dev::network_params.network, 1 } { debug_assert (!store_m->init_error ()); auto tx = ledger_m.tx_begin_write (); - store_m->initialize (tx, ledger_m.cache, ledger_m.constants); for (auto const & i : blocks_m) { auto process_result = ledger_m.process (tx, i); diff --git a/nano/test_common/ledger_context.hpp b/nano/test_common/ledger_context.hpp index 62b376a4f..f7bb1274c 100644 --- a/nano/test_common/ledger_context.hpp +++ b/nano/test_common/ledger_context.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace nano::test @@ -14,6 +15,7 @@ public: /** 'blocks' initialises the ledger with each block in-order Blocks must all return process_result::progress when processed */ ledger_context (std::deque> && blocks = std::deque>{}); + nano::ledger & ledger (); nano::store::component & store (); std::deque> const & blocks () const; @@ -23,8 +25,8 @@ public: private: nano::logger logger_m; + nano::stats stats_m{ logger_m }; std::unique_ptr store_m; - nano::stats stats_m; nano::ledger ledger_m; std::deque> blocks_m; nano::work_pool pool_m;