From 5dfe58097243e193454d3264c7b0be6a7b16b6ce Mon Sep 17 00:00:00 2001 From: clemahieu Date: Sun, 27 Jan 2019 20:10:28 +0700 Subject: [PATCH] Add sideband information to database. (#1554) * Add sideband information to database. This commit adds block-type-specific sideband information which eliminates expensive value computations. This information is stored in-line with the block itself, instead of in a separate table, for efficient retrieval. Since this involves rewriting all block entries the process is run iteratively in the background. Once the upgrade is complete the blocks_info table is dropped from the database. * Sideband RPC & open blocks adjustment (#1555) * Disable sideband account & height for open blocks * Add sideband information to RPC block_info * Replacing RPC block with PRC block_info * Add local timestamp to RPC account_history * Fix * Rolling back changes from different PR * Correct account & balance with sideband * Update rpc.blocks_info test * Improving log message for sideband upgrade process. Initializing timestamp to sentinal value during the upgrade process. * Adding test to show that blocks are put in the correct epoch. * Bump the version number on start of sideband upgrade in addition to at the end. --- nano/core_test/block_store.cpp | 417 ++++++++++++++++++++++++++++----- nano/core_test/rpc.cpp | 7 +- nano/core_test/versioning.cpp | 4 +- nano/lib/blocks.cpp | 58 ++--- nano/lib/blocks.hpp | 2 +- nano/lib/interface.h | 34 +-- nano/lib/utility.cpp | 3 + nano/lib/utility.hpp | 1 + nano/node/lmdb.cpp | 374 +++++++++++++++++++++++------ nano/node/lmdb.hpp | 26 +- nano/node/rpc.cpp | 40 +++- nano/node/rpc.hpp | 2 +- nano/secure/blockstore.cpp | 76 ++++++ nano/secure/blockstore.hpp | 27 ++- nano/secure/ledger.cpp | 86 +++---- 15 files changed, 869 insertions(+), 288 deletions(-) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index e6a45d07..b6473720 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -16,6 +16,31 @@ TEST (block_store, construction) ASSERT_GT (now, 1408074640); } +TEST (block_store, sideband_serialization) +{ + nano::block_sideband sideband1; + sideband1.type = nano::block_type::receive; + sideband1.account = 1; + sideband1.balance = 2; + sideband1.height = 3; + sideband1.successor = 4; + sideband1.timestamp = 5; + std::vector vector; + { + nano::vectorstream stream1 (vector); + sideband1.serialize (stream1); + } + nano::bufferstream stream2 (vector.data (), vector.size ()); + nano::block_sideband sideband2; + sideband2.type = nano::block_type::receive; + ASSERT_FALSE (sideband2.deserialize (stream2)); + ASSERT_EQ (sideband1.account, sideband2.account); + ASSERT_EQ (sideband1.balance, sideband2.balance); + ASSERT_EQ (sideband1.height, sideband2.height); + ASSERT_EQ (sideband1.successor, sideband2.successor); + ASSERT_EQ (sideband1.timestamp, sideband2.timestamp); +} + TEST (block_store, add_item) { nano::logging logging; @@ -28,7 +53,8 @@ TEST (block_store, add_item) auto latest1 (store.block_get (transaction, hash1)); ASSERT_EQ (nullptr, latest1); ASSERT_FALSE (store.block_exists (transaction, hash1)); - store.block_put (transaction, hash1, block); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + store.block_put (transaction, hash1, block, sideband); auto latest2 (store.block_get (transaction, hash1)); ASSERT_NE (nullptr, latest2); ASSERT_EQ (block, *latest2); @@ -39,6 +65,29 @@ TEST (block_store, add_item) ASSERT_EQ (nullptr, latest3); } +TEST (block_store, clear_successor) +{ + nano::logging logging; + bool init (false); + nano::mdb_store store (init, logging, nano::unique_path ()); + ASSERT_TRUE (!init); + nano::open_block block1 (0, 1, 0, nano::keypair ().prv, 0, 0); + auto transaction (store.tx_begin (true)); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + store.block_put (transaction, block1.hash (), block1, sideband); + nano::open_block block2 (0, 2, 0, nano::keypair ().prv, 0, 0); + store.block_put (transaction, block2.hash (), block2, sideband); + ASSERT_NE (nullptr, store.block_get (transaction, block1.hash (), &sideband)); + ASSERT_EQ (0, sideband.successor.number ()); + sideband.successor = block2.hash (); + store.block_put (transaction, block1.hash (), block1, sideband); + ASSERT_NE (nullptr, store.block_get (transaction, block1.hash (), &sideband)); + ASSERT_EQ (block2.hash (), sideband.successor); + store.block_successor_clear (transaction, block1.hash ()); + ASSERT_NE (nullptr, store.block_get (transaction, block1.hash (), &sideband)); + ASSERT_EQ (0, sideband.successor.number ()); +} + TEST (block_store, add_nonempty_block) { nano::logging logging; @@ -52,7 +101,8 @@ TEST (block_store, add_nonempty_block) auto transaction (store.tx_begin (true)); auto latest1 (store.block_get (transaction, hash1)); ASSERT_EQ (nullptr, latest1); - store.block_put (transaction, hash1, block); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + store.block_put (transaction, hash1, block, sideband); auto latest2 (store.block_get (transaction, hash1)); ASSERT_NE (nullptr, latest2); ASSERT_EQ (block, *latest2); @@ -77,8 +127,10 @@ TEST (block_store, add_two_items) block2.signature = nano::sign_message (key1.prv, key1.pub, hash2); auto latest2 (store.block_get (transaction, hash2)); ASSERT_EQ (nullptr, latest2); - store.block_put (transaction, hash1, block); - store.block_put (transaction, hash2, block2); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + store.block_put (transaction, hash1, block, sideband); + nano::block_sideband sideband2 (nano::block_type::open, 0, 0, 0, 0, 0); + store.block_put (transaction, hash2, block2, sideband2); auto latest3 (store.block_get (transaction, hash1)); ASSERT_NE (nullptr, latest3); ASSERT_EQ (block, *latest3); @@ -98,12 +150,14 @@ TEST (block_store, add_receive) nano::keypair key2; nano::open_block block1 (0, 1, 0, nano::keypair ().prv, 0, 0); auto transaction (store.tx_begin (true)); - store.block_put (transaction, block1.hash (), block1); + nano::block_sideband sideband1 (nano::block_type::open, 0, 0, 0, 0, 0); + store.block_put (transaction, block1.hash (), block1, sideband1); nano::receive_block block (block1.hash (), 1, nano::keypair ().prv, 2, 3); nano::block_hash hash1 (block.hash ()); auto latest1 (store.block_get (transaction, hash1)); ASSERT_EQ (nullptr, latest1); - store.block_put (transaction, hash1, block); + nano::block_sideband sideband (nano::block_type::receive, 0, 0, 0, 0, 0); + store.block_put (transaction, hash1, block, sideband); auto latest2 (store.block_get (transaction, hash1)); ASSERT_NE (nullptr, latest2); ASSERT_EQ (block, *latest2); @@ -362,7 +416,8 @@ TEST (block_store, one_block) ASSERT_TRUE (!init); nano::open_block block1 (0, 1, 0, nano::keypair ().prv, 0, 0); auto transaction (store.tx_begin (true)); - store.block_put (transaction, block1.hash (), block1); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + store.block_put (transaction, block1.hash (), block1, sideband); ASSERT_TRUE (store.block_exists (transaction, block1.hash ())); } @@ -463,11 +518,13 @@ TEST (block_store, two_block) hashes.push_back (block1.hash ()); blocks.push_back (block1); auto transaction (store.tx_begin (true)); - store.block_put (transaction, hashes[0], block1); + nano::block_sideband sideband1 (nano::block_type::open, 0, 0, 0, 0, 0); + store.block_put (transaction, hashes[0], block1, sideband1); nano::open_block block2 (0, 1, 2, nano::keypair ().prv, 0, 0); hashes.push_back (block2.hash ()); blocks.push_back (block2); - store.block_put (transaction, hashes[1], block2); + nano::block_sideband sideband2 (nano::block_type::open, 0, 0, 0, 0, 0); + store.block_put (transaction, hashes[1], block2, sideband2); ASSERT_TRUE (store.block_exists (transaction, block1.hash ())); ASSERT_TRUE (store.block_exists (transaction, block2.hash ())); } @@ -478,6 +535,7 @@ TEST (block_store, two_account) bool init (false); nano::mdb_store store (init, logging, nano::unique_path ()); ASSERT_TRUE (!init); + store.stop (); nano::account account1 (1); nano::block_hash hash1 (2); nano::account account2 (3); @@ -512,6 +570,7 @@ TEST (block_store, latest_find) bool init (false); nano::mdb_store store (init, logging, nano::unique_path ()); ASSERT_TRUE (!init); + store.stop (); nano::account account1 (1); nano::block_hash hash1 (2); nano::account account2 (3); @@ -649,8 +708,10 @@ TEST (block_store, block_replace) nano::send_block send1 (0, 0, 0, nano::keypair ().prv, 0, 1); nano::send_block send2 (0, 0, 0, nano::keypair ().prv, 0, 2); auto transaction (store.tx_begin (true)); - store.block_put (transaction, 0, send1); - store.block_put (transaction, 0, send2); + nano::block_sideband sideband1 (nano::block_type::send, 0, 0, 0, 0, 0); + store.block_put (transaction, 0, send1, sideband1); + nano::block_sideband sideband2 (nano::block_type::send, 0, 0, 0, 0, 0); + store.block_put (transaction, 0, send2, sideband2); auto block3 (store.block_get (transaction, 0)); ASSERT_NE (nullptr, block3); ASSERT_EQ (2, block3->block_work ()); @@ -666,7 +727,8 @@ TEST (block_store, block_count) ASSERT_EQ (0, store.block_count (transaction).sum ()); nano::open_block block (0, 1, 0, nano::keypair ().prv, 0, 0); nano::uint256_union hash1 (block.hash ()); - store.block_put (transaction, hash1, block); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + store.block_put (transaction, hash1, block, sideband); ASSERT_EQ (1, store.block_count (transaction).sum ()); } @@ -724,6 +786,7 @@ TEST (block_store, upgrade_v2_v3) bool init (false); nano::mdb_store store (init, logging, path); ASSERT_TRUE (!init); + store.stop (); auto transaction (store.tx_begin (true)); nano::genesis genesis; auto hash (genesis.hash ()); @@ -774,6 +837,7 @@ TEST (block_store, upgrade_v3_v4) bool init (false); nano::mdb_store store (init, logging, path); ASSERT_FALSE (init); + store.stop (); auto transaction (store.tx_begin (true)); store.version_put (transaction, 3); nano::pending_info_v3 info (key1.pub, 100, key2.pub); @@ -807,6 +871,7 @@ TEST (block_store, upgrade_v4_v5) bool init (false); nano::mdb_store store (init, logging, path); ASSERT_FALSE (init); + store.stop (); auto transaction (store.tx_begin (true)); nano::genesis genesis; nano::stat stats; @@ -859,6 +924,7 @@ TEST (block_store, upgrade_v5_v6) bool init (false); nano::mdb_store store (init, logging, path); ASSERT_FALSE (init); + store.stop (); auto transaction (store.tx_begin (true)); nano::genesis genesis; store.initialize (transaction, genesis); @@ -887,6 +953,7 @@ TEST (block_store, upgrade_v6_v7) bool init (false); nano::mdb_store store (init, logging, path); ASSERT_FALSE (init); + store.stop (); auto transaction (store.tx_begin (true)); nano::genesis genesis; store.initialize (transaction, genesis); @@ -957,6 +1024,7 @@ TEST (block_store, upgrade_v7_v8) nano::logging logging; bool init (false); nano::mdb_store store (init, logging, path); + store.stop (); auto transaction (store.tx_begin (true)); ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.unchecked, 1)); ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE, &store.unchecked)); @@ -1029,6 +1097,7 @@ TEST (block_store, upgrade_v8_v9) nano::logging logging; bool init (false); nano::mdb_store store (init, logging, path); + store.stop (); auto transaction (store.tx_begin (true)); ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.vote, 1)); ASSERT_EQ (0, mdb_dbi_open (store.env.tx (transaction), "sequence", MDB_CREATE, &store.vote)); @@ -1047,53 +1116,6 @@ TEST (block_store, upgrade_v8_v9) ASSERT_EQ (10, vote->sequence); } -TEST (block_store, upgrade_v9_v10) -{ - auto path (nano::unique_path ()); - nano::block_hash hash (0); - { - bool init (false); - nano::logging logging; - nano::mdb_store store (init, logging, path); - ASSERT_FALSE (init); - auto transaction (store.tx_begin (true)); - nano::genesis genesis; - nano::stat stats; - nano::ledger ledger (store, stats); - store.initialize (transaction, genesis); - store.version_put (transaction, 9); - nano::account_info info; - store.account_get (transaction, nano::test_genesis_key.pub, info); - nano::keypair key0; - nano::uint128_t balance (nano::genesis_amount); - hash = info.head; - for (auto i (1); i < 32; ++i) // Making 31 send blocks (+ 1 open = 32 total) - { - balance = balance - nano::Gxrb_ratio; - nano::send_block block0 (hash, key0.pub, balance, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block0).code); - hash = block0.hash (); - } - nano::block_info block_info_auto; // Checking automatic block_info creation for block 32 - store.block_info_get (transaction, hash, block_info_auto); - ASSERT_EQ (block_info_auto.account, nano::test_genesis_key.pub); - ASSERT_EQ (block_info_auto.balance.number (), balance); - ASSERT_EQ (0, mdb_drop (store.env.tx (transaction), store.blocks_info, 0)); // Cleaning blocks_info subdatabase - bool block_info_exists (store.block_info_exists (transaction, hash)); - ASSERT_EQ (block_info_exists, 0); // Checking if automatic block_info is deleted - } - nano::logging logging; - bool init (false); - nano::mdb_store store (init, logging, path); - ASSERT_FALSE (init); - auto transaction (store.tx_begin ()); - ASSERT_LT (9, store.version_get (transaction)); - nano::block_info block_info; - store.block_info_get (transaction, hash, block_info); - ASSERT_EQ (block_info.account, nano::test_genesis_key.pub); - ASSERT_EQ (block_info.balance.number (), nano::genesis_amount - nano::Gxrb_ratio * 31); -} - TEST (block_store, state_block) { nano::logging logging; @@ -1106,7 +1128,8 @@ TEST (block_store, state_block) nano::keypair key1; nano::state_block block1 (1, genesis.hash (), 3, 4, 6, key1.prv, key1.pub, 7); ASSERT_EQ (nano::block_type::state, block1.type ()); - store.block_put (transaction, block1.hash (), block1); + nano::block_sideband sideband1 (nano::block_type::state, 0, 0, 0, 0, 0); + store.block_put (transaction, block1.hash (), block1, sideband1); ASSERT_TRUE (store.block_exists (transaction, block1.hash ())); auto block2 (store.block_get (transaction, block1.hash ())); ASSERT_NE (nullptr, block2); @@ -1120,3 +1143,277 @@ TEST (block_store, state_block) ASSERT_EQ (0, count2.state_v0); ASSERT_EQ (0, count2.state_v1); } + +namespace +{ +void write_legacy_sideband (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block & block_a, nano::block_hash const & successor_a, MDB_dbi db_a) +{ + std::vector vector; + { + nano::vectorstream stream (vector); + block_a.serialize (stream); + nano::write (stream, successor_a); + } + MDB_val val ({ vector.size (), vector.data () }); + auto hash (block_a.hash ()); + auto status2 (mdb_put (store_a.env.tx (transaction_a), db_a, nano::mdb_val (hash), &val, 0)); + ASSERT_EQ (0, status2); + nano::block_sideband sideband; + auto block2 (store_a.block_get (transaction_a, block_a.hash (), &sideband)); + ASSERT_NE (nullptr, block2); + ASSERT_EQ (std::numeric_limits::max (), sideband.height); +}; +} + +TEST (block_store, upgrade_sideband_genesis) +{ + bool error (false); + nano::genesis genesis; + auto path (nano::unique_path ()); + { + nano::logging logging; + nano::mdb_store store (error, logging, path); + ASSERT_FALSE (error); + store.stop (); + auto transaction (store.tx_begin (true)); + store.version_put (transaction, 11); + store.initialize (transaction, genesis); + nano::block_sideband sideband; + auto genesis_block (store.block_get (transaction, genesis.hash (), &sideband)); + ASSERT_NE (nullptr, genesis_block); + ASSERT_EQ (0, sideband.height); + write_legacy_sideband (store, transaction, *genesis_block, 0, store.open_blocks); + auto genesis_block2 (store.block_get (transaction, genesis.hash (), &sideband)); + ASSERT_NE (nullptr, genesis_block); + ASSERT_EQ (std::numeric_limits::max (), sideband.height); + } + nano::logging logging; + nano::mdb_store store (error, logging, path); + ASSERT_FALSE (error); + auto done (false); + auto iterations (0); + while (!done) + { + std::this_thread::sleep_for (std::chrono::milliseconds (10)); + auto transaction (store.tx_begin (false)); + done = store.full_sideband (transaction); + ASSERT_LT (iterations, 200); + ++iterations; + } + auto transaction (store.tx_begin_read ()); + nano::block_sideband sideband; + auto genesis_block (store.block_get (transaction, genesis.hash (), &sideband)); + ASSERT_NE (nullptr, genesis_block); + ASSERT_EQ (0, sideband.height); +} + +TEST (block_store, upgrade_sideband_two_blocks) +{ + bool error (false); + nano::genesis genesis; + nano::block_hash hash2; + auto path (nano::unique_path ()); + { + nano::logging logging; + nano::mdb_store store (error, logging, path); + ASSERT_FALSE (error); + store.stop (); + nano::stat stat; + nano::ledger ledger (store, stat); + auto transaction (store.tx_begin (true)); + store.version_put (transaction, 11); + store.initialize (transaction, genesis); + nano::state_block block (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0); + hash2 = block.hash (); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block).code); + write_legacy_sideband (store, transaction, *genesis.open, hash2, store.open_blocks); + write_legacy_sideband (store, transaction, block, 0, store.state_blocks_v0); + } + nano::logging logging; + nano::mdb_store store (error, logging, path); + ASSERT_FALSE (error); + auto done (false); + auto iterations (0); + while (!done) + { + std::this_thread::sleep_for (std::chrono::milliseconds (10)); + auto transaction (store.tx_begin (false)); + done = store.full_sideband (transaction); + ASSERT_LT (iterations, 200); + ++iterations; + } + auto transaction (store.tx_begin_read ()); + nano::block_sideband sideband; + auto genesis_block (store.block_get (transaction, genesis.hash (), &sideband)); + ASSERT_NE (nullptr, genesis_block); + ASSERT_EQ (0, sideband.height); + nano::block_sideband sideband2; + auto block2 (store.block_get (transaction, hash2, &sideband2)); + ASSERT_NE (nullptr, block2); + ASSERT_EQ (1, sideband2.height); +} + +TEST (block_store, upgrade_sideband_two_accounts) +{ + bool error (false); + nano::genesis genesis; + nano::block_hash hash2; + nano::block_hash hash3; + nano::keypair key; + auto path (nano::unique_path ()); + { + nano::logging logging; + nano::mdb_store store (error, logging, path); + ASSERT_FALSE (error); + store.stop (); + nano::stat stat; + nano::ledger ledger (store, stat); + auto transaction (store.tx_begin (true)); + store.version_put (transaction, 11); + store.initialize (transaction, genesis); + nano::state_block block1 (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0); + hash2 = block1.hash (); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block1).code); + nano::state_block block2 (key.pub, 0, nano::test_genesis_key.pub, nano::Gxrb_ratio, hash2, key.prv, key.pub, 0); + hash3 = block2.hash (); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block2).code); + write_legacy_sideband (store, transaction, *genesis.open, hash2, store.open_blocks); + write_legacy_sideband (store, transaction, block1, 0, store.state_blocks_v0); + write_legacy_sideband (store, transaction, block2, 0, store.state_blocks_v0); + } + nano::logging logging; + nano::mdb_store store (error, logging, path); + ASSERT_FALSE (error); + auto done (false); + auto iterations (0); + while (!done) + { + std::this_thread::sleep_for (std::chrono::milliseconds (10)); + auto transaction (store.tx_begin (false)); + done = store.full_sideband (transaction); + ASSERT_LT (iterations, 200); + ++iterations; + } + auto transaction (store.tx_begin_read ()); + nano::block_sideband sideband; + auto genesis_block (store.block_get (transaction, genesis.hash (), &sideband)); + ASSERT_NE (nullptr, genesis_block); + ASSERT_EQ (0, sideband.height); + nano::block_sideband sideband2; + auto block2 (store.block_get (transaction, hash2, &sideband2)); + ASSERT_NE (nullptr, block2); + ASSERT_EQ (1, sideband2.height); + nano::block_sideband sideband3; + auto block3 (store.block_get (transaction, hash3, &sideband3)); + ASSERT_NE (nullptr, block3); + ASSERT_EQ (0, sideband3.height); +} + +TEST (block_store, insert_after_legacy) +{ + nano::logging logging; + bool error (false); + nano::genesis genesis; + nano::mdb_store store (error, logging, nano::unique_path ()); + ASSERT_FALSE (error); + store.stop (); + nano::stat stat; + nano::ledger ledger (store, stat); + auto transaction (store.tx_begin (true)); + store.version_put (transaction, 11); + store.initialize (transaction, genesis); + write_legacy_sideband (store, transaction, *genesis.open, 0, store.open_blocks); + nano::state_block block (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block).code); +} + +TEST (block_store, upgrade_sideband_rollback_old) +{ + nano::logging logging; + bool error (false); + nano::genesis genesis; + nano::mdb_store store (error, logging, nano::unique_path ()); + ASSERT_FALSE (error); + store.stop (); + nano::stat stat; + nano::ledger ledger (store, stat); + auto transaction (store.tx_begin (true)); + store.version_put (transaction, 11); + store.initialize (transaction, genesis); + nano::send_block block1 (genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block1).code); + nano::send_block block2 (block1.hash (), nano::test_genesis_key.pub, nano::genesis_amount - 2 * nano::Gxrb_ratio, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block2).code); + write_legacy_sideband (store, transaction, *genesis.open, block1.hash (), store.open_blocks); + write_legacy_sideband (store, transaction, block1, block2.hash (), store.send_blocks); + write_legacy_sideband (store, transaction, block2, 0, store.send_blocks); + ASSERT_TRUE (store.block_exists (transaction, block2.hash ())); + ledger.rollback (transaction, block2.hash ()); + ASSERT_FALSE (store.block_exists (transaction, block2.hash ())); +} + +// Account for an open block should be retrievable +TEST (block_store, legacy_account_computed) +{ + nano::logging logging; + bool init (false); + nano::mdb_store store (init, logging, nano::unique_path ()); + ASSERT_TRUE (!init); + store.stop (); + nano::stat stats; + nano::ledger ledger (store, stats); + nano::genesis genesis; + auto transaction (store.tx_begin (true)); + store.initialize (transaction, genesis); + store.version_put (transaction, 11); + write_legacy_sideband (store, transaction, *genesis.open, 0, store.open_blocks); + ASSERT_EQ (nano::genesis_account, ledger.account (transaction, genesis.hash ())); +} + +TEST (block_store, upgrade_sideband_epoch) +{ + bool error (false); + nano::genesis genesis; + nano::block_hash hash2; + auto path (nano::unique_path ()); + { + nano::logging logging; + nano::mdb_store store (error, logging, path); + ASSERT_FALSE (error); + store.stop (); + nano::stat stat; + nano::ledger ledger (store, stat, 42, nano::test_genesis_key.pub); + auto transaction (store.tx_begin (true)); + store.version_put (transaction, 11); + store.initialize (transaction, genesis); + nano::state_block block1 (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount, 42, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0); + hash2 = block1.hash (); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block1).code); + ASSERT_EQ (nano::epoch::epoch_1, store.block_version (transaction, hash2)); + write_legacy_sideband (store, transaction, *genesis.open, hash2, store.open_blocks); + write_legacy_sideband (store, transaction, block1, 0, store.state_blocks_v1); + } + nano::logging logging; + nano::mdb_store store (error, logging, path); + nano::stat stat; + nano::ledger ledger (store, stat, 42, nano::test_genesis_key.pub); + ASSERT_FALSE (error); + auto done (false); + auto iterations (0); + while (!done) + { + std::this_thread::sleep_for (std::chrono::milliseconds (10)); + auto transaction (store.tx_begin (false)); + done = store.full_sideband (transaction); + ASSERT_LT (iterations, 200); + ++iterations; + } + auto transaction (store.tx_begin_write ()); + ASSERT_EQ (nano::epoch::epoch_1, store.block_version (transaction, hash2)); + nano::block_sideband sideband; + auto block1 (store.block_get (transaction, hash2, &sideband)); + ASSERT_NE (std::numeric_limits::max (), sideband.height); + nano::state_block block2 (nano::test_genesis_key.pub, hash2, nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block2).code); + ASSERT_EQ (nano::epoch::epoch_1, store.block_version (transaction, block2.hash ())); +} diff --git a/nano/core_test/rpc.cpp b/nano/core_test/rpc.cpp index a797958f..8ec520c5 100644 --- a/nano/core_test/rpc.cpp +++ b/nano/core_test/rpc.cpp @@ -3224,13 +3224,12 @@ TEST (rpc, blocks_info) ASSERT_FALSE (pending.is_initialized ()); boost::optional source (blocks.second.get_optional ("source_account")); ASSERT_FALSE (source.is_initialized ()); - boost::optional balance (blocks.second.get_optional ("balance")); - ASSERT_FALSE (balance.is_initialized ()); + std::string balance_text (blocks.second.get ("balance")); + ASSERT_EQ (nano::genesis_amount.convert_to (), balance_text); } // Test for optional values request.put ("source", "true"); request.put ("pending", "1"); - request.put ("balance", "true"); test_response response2 (request, rpc, system.io_ctx); system.deadline_set (5s); while (response2.status == 0) @@ -3244,8 +3243,6 @@ TEST (rpc, blocks_info) ASSERT_EQ ("0", source); std::string pending (blocks.second.get ("pending")); ASSERT_EQ ("0", pending); - std::string balance_text (blocks.second.get ("balance")); - ASSERT_EQ (nano::genesis_amount.convert_to (), balance_text); } } diff --git a/nano/core_test/versioning.cpp b/nano/core_test/versioning.cpp index bf212eb1..eda329e2 100644 --- a/nano/core_test/versioning.cpp +++ b/nano/core_test/versioning.cpp @@ -14,8 +14,10 @@ TEST (versioning, account_info_v1) auto error (false); nano::mdb_store store (error, logging, file); ASSERT_FALSE (error); + store.stop (); auto transaction (store.tx_begin (true)); - store.block_put (transaction, open.hash (), open); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0); + store.block_put (transaction, open.hash (), open, sideband); auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), v1.val (), 0)); ASSERT_EQ (0, status); store.version_put (transaction, 1); diff --git a/nano/lib/blocks.cpp b/nano/lib/blocks.cpp index 237dd891..2095f62f 100644 --- a/nano/lib/blocks.cpp +++ b/nano/lib/blocks.cpp @@ -53,13 +53,41 @@ bool nano::from_string_hex (std::string const & value_a, uint64_t & target_a) return error; } -std::string nano::block::to_json () +std::string nano::block::to_json () const { std::string result; serialize_json (result); return result; } +size_t nano::block::size (nano::block_type type_a) +{ + size_t result (0); + switch (type_a) + { + case nano::block_type::invalid: + case nano::block_type::not_a_block: + assert (false); + break; + case nano::block_type::send: + result = nano::send_block::size; + break; + case nano::block_type::receive: + result = nano::receive_block::size; + break; + case nano::block_type::change: + result = nano::change_block::size; + break; + case nano::block_type::open: + result = nano::open_block::size; + break; + case nano::block_type::state: + result = nano::state_block::size; + break; + } + return result; +} + nano::block_hash nano::block::hash () const { nano::uint256_union result; @@ -1427,34 +1455,6 @@ hashables (error_a, tree_a) } } -size_t nano::block::size (nano::block_type type_a) -{ - size_t result (0); - switch (type_a) - { - case nano::block_type::invalid: - case nano::block_type::not_a_block: - assert (false); - break; - case nano::block_type::send: - result = nano::send_block::size; - break; - case nano::block_type::receive: - result = nano::receive_block::size; - break; - case nano::block_type::change: - result = nano::change_block::size; - break; - case nano::block_type::open: - result = nano::open_block::size; - break; - case nano::block_type::state: - result = nano::state_block::size; - break; - } - return result; -} - void nano::receive_block::hash (blake2b_state & hash_a) const { hashables.hash (hash_a); diff --git a/nano/lib/blocks.hpp b/nano/lib/blocks.hpp index cf0cca13..e5988f65 100644 --- a/nano/lib/blocks.hpp +++ b/nano/lib/blocks.hpp @@ -47,7 +47,7 @@ public: nano::block_hash hash () const; // Return a digest of hashables and non-hashables in this block. nano::block_hash full_hash () const; - std::string to_json (); + std::string to_json () const; virtual void hash (blake2b_state &) const = 0; virtual uint64_t block_work () const = 0; virtual void block_work_set (uint64_t) = 0; diff --git a/nano/lib/interface.h b/nano/lib/interface.h index e1be3822..edb8595e 100644 --- a/nano/lib/interface.h +++ b/nano/lib/interface.h @@ -11,48 +11,36 @@ typedef unsigned char * xrb_uint512; // 64byte array for signatures typedef void * xrb_transaction; // clang-format off // Convert amount bytes 'source' to a 40 byte null-terminated decimal string 'destination' -[[deprecated]] -void xrb_uint128_to_dec (const xrb_uint128 source, char * destination); +[[deprecated]] void xrb_uint128_to_dec (const xrb_uint128 source, char * destination); // Convert public/private key bytes 'source' to a 65 byte null-terminated hex string 'destination' -[[deprecated]] -void xrb_uint256_to_string (const xrb_uint256 source, char * destination); +[[deprecated]] void xrb_uint256_to_string (const xrb_uint256 source, char * destination); // Convert public key bytes 'source' to a 66 byte non-null-terminated account string 'destination' -[[deprecated]] -void xrb_uint256_to_address (xrb_uint256 source, char * destination); +[[deprecated]] void xrb_uint256_to_address (xrb_uint256 source, char * destination); // Convert public/private key bytes 'source' to a 129 byte null-terminated hex string 'destination' -[[deprecated]] -void xrb_uint512_to_string (const xrb_uint512 source, char * destination); +[[deprecated]] void xrb_uint512_to_string (const xrb_uint512 source, char * destination); // Convert 39 byte decimal string 'source' to a byte array 'destination' // Return 0 on success, nonzero on error -[[deprecated]] -int xrb_uint128_from_dec (const char * source, xrb_uint128 destination); +[[deprecated]] int xrb_uint128_from_dec (const char * source, xrb_uint128 destination); // Convert 64 byte hex string 'source' to a byte array 'destination' // Return 0 on success, nonzero on error -[[deprecated]] -int xrb_uint256_from_string (const char * source, xrb_uint256 destination); +[[deprecated]] int xrb_uint256_from_string (const char * source, xrb_uint256 destination); // Convert 128 byte hex string 'source' to a byte array 'destination' // Return 0 on success, nonzero on error -[[deprecated]] -int xrb_uint512_from_string (const char * source, xrb_uint512 destination); +[[deprecated]] int xrb_uint512_from_string (const char * source, xrb_uint512 destination); // Check if the null-terminated string 'account' is a valid xrb account number // Return 0 on correct, nonzero on invalid -[[deprecated]] -int xrb_valid_address (const char * account); +[[deprecated]] int xrb_valid_address (const char * account); // Create a new random number in to 'destination' -[[deprecated]] -void xrb_generate_random (xrb_uint256 destination); +[[deprecated]] void xrb_generate_random (xrb_uint256 destination); // Retrieve the deterministic private key for 'seed' at 'index' -[[deprecated]] -void xrb_seed_key (const xrb_uint256 seed, int index, xrb_uint256); +[[deprecated]] void xrb_seed_key (const xrb_uint256 seed, int index, xrb_uint256); // Derive the public key 'pub' from 'key' -[[deprecated]] -void xrb_key_account (xrb_uint256 key, xrb_uint256 pub); +[[deprecated]] void xrb_key_account (xrb_uint256 key, xrb_uint256 pub); // Sign 'transaction' using 'private_key' and write to 'signature' -[[deprecated]] char * xrb_sign_transaction (const char * transaction, const xrb_uint256 private_key); // Generate work for 'transaction' [[deprecated]] diff --git a/nano/lib/utility.cpp b/nano/lib/utility.cpp index 97f67d96..69c5d630 100644 --- a/nano/lib/utility.cpp +++ b/nano/lib/utility.cpp @@ -58,6 +58,9 @@ namespace thread_role case nano::thread_role::name::signature_checking: thread_role_name_string = "Signature check"; break; + case nano::thread_role::name::slow_db_upgrade: + thread_role_name_string = "Slow db upgrade"; + break; } /* diff --git a/nano/lib/utility.hpp b/nano/lib/utility.hpp index ef0b96c2..f3ade70e 100644 --- a/nano/lib/utility.hpp +++ b/nano/lib/utility.hpp @@ -42,6 +42,7 @@ namespace thread_role bootstrap_initiator, voting, signature_checking, + slow_db_upgrade, }; /* * Get/Set the identifier for the current thread diff --git a/nano/node/lmdb.cpp b/nano/node/lmdb.cpp index a18ca996..031296fc 100644 --- a/nano/node/lmdb.cpp +++ b/nano/node/lmdb.cpp @@ -659,24 +659,6 @@ template class nano::mdb_iterator; template class nano::mdb_iterator, nano::mdb_val::no_value>; -nano::store_iterator nano::mdb_store::block_info_begin (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, blocks_info, nano::mdb_val (hash_a))); - return result; -} - -nano::store_iterator nano::mdb_store::block_info_begin (nano::transaction const & transaction_a) -{ - nano::store_iterator result (std::make_unique> (transaction_a, blocks_info)); - return result; -} - -nano::store_iterator nano::mdb_store::block_info_end () -{ - nano::store_iterator result (nullptr); - return result; -} - nano::store_iterator nano::mdb_store::representation_begin (nano::transaction const & transaction_a) { nano::store_iterator result (std::make_unique> (transaction_a, representation)); @@ -735,8 +717,10 @@ blocks_info (0), representation (0), unchecked (0), vote (0), -meta (0) +meta (0), +stopped (false) { + auto slow_upgrade (false); if (!error_a) { auto transaction (tx_begin_write ()); @@ -751,16 +735,40 @@ meta (0) error_a |= mdb_dbi_open (env.tx (transaction), "state_v1", MDB_CREATE, &state_blocks_v1) != 0; error_a |= mdb_dbi_open (env.tx (transaction), "pending", MDB_CREATE, &pending_v0) != 0; error_a |= mdb_dbi_open (env.tx (transaction), "pending_v1", MDB_CREATE, &pending_v1) != 0; - error_a |= mdb_dbi_open (env.tx (transaction), "blocks_info", MDB_CREATE, &blocks_info) != 0; error_a |= mdb_dbi_open (env.tx (transaction), "representation", MDB_CREATE, &representation) != 0; error_a |= mdb_dbi_open (env.tx (transaction), "unchecked", MDB_CREATE, &unchecked) != 0; error_a |= mdb_dbi_open (env.tx (transaction), "vote", MDB_CREATE, &vote) != 0; error_a |= mdb_dbi_open (env.tx (transaction), "meta", MDB_CREATE, &meta) != 0; + if (!full_sideband (transaction)) + { + error_a |= mdb_dbi_open (env.tx (transaction), "blocks_info", MDB_CREATE, &blocks_info) != 0; + } if (!error_a) { - do_upgrades (transaction); + do_upgrades (transaction, slow_upgrade); } } + if (slow_upgrade) + { + upgrades = std::thread ([this]() { + nano::thread_role::set (nano::thread_role::name::slow_db_upgrade); + do_slow_upgrades (); + }); + } +} + +nano::mdb_store::~mdb_store () +{ + stop (); +} + +void nano::mdb_store::stop () +{ + stopped = true; + if (upgrades.joinable ()) + { + upgrades.join (); + } } nano::transaction nano::mdb_store::tx_begin_write () @@ -783,7 +791,8 @@ void nano::mdb_store::initialize (nano::transaction const & transaction_a, nano: auto hash_l (genesis_a.hash ()); assert (latest_v0_begin (transaction_a) == latest_v0_end ()); assert (latest_v1_begin (transaction_a) == latest_v1_end ()); - block_put (transaction_a, hash_l, *genesis_a.open); + nano::block_sideband sideband (nano::block_type::open, nano::genesis_account, 0, nano::genesis_amount, 0, nano::seconds_since_epoch ()); + block_put (transaction_a, hash_l, *genesis_a.open, sideband); account_put (transaction_a, genesis_account, { hash_l, genesis_a.open->hash (), genesis_a.open->hash (), std::numeric_limits::max (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0 }); representation_put (transaction_a, genesis_account, std::numeric_limits::max ()); frontier_put (transaction_a, hash_l, genesis_account); @@ -795,6 +804,16 @@ void nano::mdb_store::version_put (nano::transaction const & transaction_a, int nano::uint256_union version_value (version_a); auto status (mdb_put (env.tx (transaction_a), meta, nano::mdb_val (version_key), nano::mdb_val (version_value), 0)); release_assert (status == 0); + if (blocks_info == 0 && !full_sideband (transaction_a)) + { + auto status (mdb_dbi_open (env.tx (transaction_a), "blocks_info", MDB_CREATE, &blocks_info)); + release_assert (status == MDB_SUCCESS); + } + if (blocks_info != 0 && full_sideband (transaction_a)) + { + auto status (mdb_drop (env.tx (transaction_a), blocks_info, 1)); + release_assert (status == MDB_SUCCESS); + } } int nano::mdb_store::version_get (nano::transaction const & transaction_a) @@ -840,7 +859,7 @@ void nano::mdb_store::delete_node_id (nano::transaction const & transaction_a) assert (!error || error == MDB_NOTFOUND); } -void nano::mdb_store::do_upgrades (nano::transaction const & transaction_a) +void nano::mdb_store::do_upgrades (nano::transaction const & transaction_a, bool & slow_upgrade) { switch (version_get (transaction_a)) { @@ -865,8 +884,13 @@ void nano::mdb_store::do_upgrades (nano::transaction const & transaction_a) case 10: upgrade_v10_to_v11 (transaction_a); case 11: + // Signal the start of sideband upgrade upgrade_v11_to_v12 (transaction_a); + // [[fallthrough]]; case 12: + slow_upgrade = true; + break; + case 13: break; default: assert (false); @@ -956,7 +980,23 @@ void nano::mdb_store::upgrade_v4_to_v5 (nano::transaction const & transaction_a) auto hash (block->hash ()); if (block_successor (transaction_a, hash).is_zero () && !successor.is_zero ()) { - block_put (transaction_a, hash, *block, successor); + std::vector vector; + { + nano::vectorstream stream (vector); + block->serialize (stream); + nano::write (stream, successor.bytes); + } + block_raw_put (transaction_a, block_database (block->type (), nano::epoch::epoch_0), hash, { vector.size (), vector.data () }); + if (!block->previous ().is_zero ()) + { + nano::block_type type; + auto value (block_raw_get (transaction_a, block->previous (), type)); + auto version (block_version (transaction_a, block->previous ())); + assert (value.mv_size != 0); + std::vector data (static_cast (value.mv_data), static_cast (value.mv_data) + value.mv_size); + std::copy (hash.bytes.begin (), hash.bytes.end (), data.end () - nano::block_sideband::size (type)); + block_raw_put (transaction_a, block_database (type, version), block->previous (), nano::mdb_val (data.size (), data.data ())); + } } successor = hash; block = block_get (transaction_a, block->previous ()); @@ -1032,32 +1072,6 @@ void nano::mdb_store::upgrade_v8_to_v9 (nano::transaction const & transaction_a) void nano::mdb_store::upgrade_v9_to_v10 (nano::transaction const & transaction_a) { - //std::cerr << boost::str (boost::format ("Performing database upgrade to version 10...\n")); - version_put (transaction_a, 10); - for (auto i (latest_v0_begin (transaction_a)), n (latest_v0_end ()); i != n; ++i) - { - nano::account_info info (i->second); - if (info.block_count >= block_info_max) - { - nano::account account (i->first); - //std::cerr << boost::str (boost::format ("Upgrading account %1%...\n") % account.to_account ()); - size_t block_count (1); - auto hash (info.open_block); - while (!hash.is_zero ()) - { - if ((block_count % block_info_max) == 0) - { - nano::block_info block_info; - block_info.account = account; - nano::amount balance (block_balance (transaction_a, hash)); - block_info.balance = balance; - block_info_put (transaction_a, hash, block_info); - } - hash = block_successor (transaction_a, hash); - ++block_count; - } - } - } } void nano::mdb_store::upgrade_v10_to_v11 (nano::transaction const & transaction_a) @@ -1068,6 +1082,39 @@ void nano::mdb_store::upgrade_v10_to_v11 (nano::transaction const & transaction_ mdb_drop (env.tx (transaction_a), unsynced, 1); } +void nano::mdb_store::do_slow_upgrades () +{ + int version; + { + nano::transaction transaction (tx_begin_read ()); + version = version_get (transaction); + } + switch (version) + { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + case 10: + case 11: + break; + case 12: + upgrade_v12_to_v13 (); + break; + case 13: + break; + default: + assert (false); + break; + } +} + void nano::mdb_store::upgrade_v11_to_v12 (nano::transaction const & transaction_a) { version_put (transaction_a, 12); @@ -1078,6 +1125,71 @@ void nano::mdb_store::upgrade_v11_to_v12 (nano::transaction const & transaction_ mdb_drop (env.tx (transaction_a), checksum, 1); } +void nano::mdb_store::upgrade_v12_to_v13 () +{ + size_t cost (0); + size_t const max (16384); + nano::account account (0); + auto transaction (tx_begin_write ()); + while (!stopped && account != nano::not_an_account) + { + nano::account first (0); + nano::account_info second; + { + auto current (latest_begin (transaction, account)); + if (current != latest_end ()) + { + first = current->first; + second = current->second; + } + } + if (!first.is_zero ()) + { + auto hash (second.open_block); + uint64_t height (0); + nano::block_sideband sideband; + while (!stopped && !hash.is_zero ()) + { + if (cost >= max) + { + BOOST_LOG (logging.log) << boost::str (boost::format ("Upgrading sideband information for account %1%... height %2%") % first.to_account ().substr (0, 24) % std::to_string (height)); + auto tx (boost::polymorphic_downcast (transaction.impl.get ())); + auto status0 (mdb_txn_commit (*tx)); + release_assert (status0 == MDB_SUCCESS); + std::this_thread::yield (); + auto status1 (mdb_txn_begin (env, nullptr, 0, &tx->handle)); + release_assert (status1 == MDB_SUCCESS); + cost = 0; + } + auto block (block_get (transaction, hash, &sideband)); + assert (block != nullptr); + if (sideband.height == std::numeric_limits::max ()) + { + sideband.height = height; + block_put (transaction, hash, *block, sideband, block_version (transaction, hash)); + cost += 16; + } + else + { + cost += 1; + } + hash = sideband.successor; + ++height; + } + account = first.number () + 1; + } + else + { + account = nano::not_an_account; + } + } + if (account == nano::not_an_account) + { + BOOST_LOG (logging.log) << boost::str (boost::format ("Completed sideband upgrade")); + version_put (transaction, 13); + } +} + void nano::mdb_store::clear (MDB_dbi db_a) { auto transaction (tx_begin_write ()); @@ -1087,6 +1199,33 @@ void nano::mdb_store::clear (MDB_dbi db_a) nano::uint128_t nano::mdb_store::block_balance (nano::transaction const & transaction_a, nano::block_hash const & hash_a) { + nano::block_sideband sideband; + auto block (block_get (transaction_a, hash_a, &sideband)); + nano::uint128_t result; + switch (block->type ()) + { + case nano::block_type::open: + case nano::block_type::receive: + case nano::block_type::change: + result = sideband.balance.number (); + break; + case nano::block_type::send: + result = boost::polymorphic_downcast (block.get ())->hashables.balance.number (); + break; + case nano::block_type::state: + result = boost::polymorphic_downcast (block.get ())->hashables.balance.number (); + break; + case nano::block_type::invalid: + case nano::block_type::not_a_block: + release_assert (false); + break; + } + return result; +} + +nano::uint128_t nano::mdb_store::block_balance_computed (nano::transaction const & transaction_a, nano::block_hash const & hash_a) +{ + assert (!full_sideband (transaction_a)); summation_visitor visitor (transaction_a, *this); return visitor.compute_balance (hash_a); } @@ -1159,14 +1298,15 @@ void nano::mdb_store::block_raw_put (nano::transaction const & transaction_a, MD release_assert (status2 == 0); } -void nano::mdb_store::block_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a, nano::block_hash const & successor_a, nano::epoch epoch_a) +void nano::mdb_store::block_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a, nano::block_sideband const & sideband_a, nano::epoch epoch_a) { - assert (successor_a.is_zero () || block_exists (transaction_a, successor_a)); + assert (block_a.type () == sideband_a.type); + assert (sideband_a.successor.is_zero () || block_exists (transaction_a, sideband_a.successor)); std::vector vector; { nano::vectorstream stream (vector); block_a.serialize (stream); - nano::write (stream, successor_a.bytes); + sideband_a.serialize (stream); } block_raw_put (transaction_a, block_database (block_a.type (), epoch_a), hash_a, { vector.size (), vector.data () }); nano::block_predecessor_set predecessor (transaction_a, *this); @@ -1299,12 +1439,29 @@ std::shared_ptr nano::mdb_store::block_random (nano::transaction co return result; } -size_t nano::mdb_store::block_successor_offset (nano::transaction const &, MDB_val entry_a, nano::block_type type_a) +bool nano::mdb_store::full_sideband (nano::transaction const & transaction_a) +{ + return version_get (transaction_a) > 12; +} + +bool nano::mdb_store::entry_has_sideband (MDB_val entry_a, nano::block_type type_a) +{ + return entry_a.mv_size == nano::block::size (type_a) + nano::block_sideband::size (type_a); +} + +size_t nano::mdb_store::block_successor_offset (nano::transaction const & transaction_a, MDB_val entry_a, nano::block_type type_a) { size_t result; - // Read old successor-only sideband - assert (entry_a.mv_size = nano::block::size (type_a) + sizeof (nano::uint256_union)); - result = entry_a.mv_size - sizeof (nano::uint256_union); + if (full_sideband (transaction_a) || entry_has_sideband (entry_a, type_a)) + { + result = entry_a.mv_size - nano::block_sideband::size (type_a); + } + else + { + // Read old successor-only sideband + assert (entry_a.mv_size = nano::block::size (type_a) + sizeof (nano::uint256_union)); + result = entry_a.mv_size - sizeof (nano::uint256_union); + } return result; } @@ -1329,12 +1486,16 @@ nano::block_hash nano::mdb_store::block_successor (nano::transaction const & tra void nano::mdb_store::block_successor_clear (nano::transaction const & transaction_a, nano::block_hash const & hash_a) { - auto block (block_get (transaction_a, hash_a)); + nano::block_type type; + auto value (block_raw_get (transaction_a, hash_a, type)); auto version (block_version (transaction_a, hash_a)); - block_put (transaction_a, hash_a, *block, 0, version); + assert (value.mv_size != 0); + std::vector data (static_cast (value.mv_data), static_cast (value.mv_data) + value.mv_size); + std::fill_n (data.begin () + block_successor_offset (transaction_a, value, type), sizeof (nano::uint256_union), 0); + block_raw_put (transaction_a, block_database (type, version), hash_a, nano::mdb_val (data.size (), data.data ())); } -std::shared_ptr nano::mdb_store::block_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a) +std::shared_ptr nano::mdb_store::block_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband * sideband_a) { nano::block_type type; auto value (block_raw_get (transaction_a, hash_a, type)); @@ -1344,6 +1505,24 @@ std::shared_ptr nano::mdb_store::block_get (nano::transaction const nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); result = nano::deserialize_block (stream, type); assert (result != nullptr); + if (sideband_a) + { + sideband_a->type = type; + if (full_sideband (transaction_a) || entry_has_sideband (value, type)) + { + auto error (sideband_a->deserialize (stream)); + assert (!error); + } + else + { + // Reconstruct sideband data for block. + sideband_a->account = block_account_computed (transaction_a, hash_a); + sideband_a->balance = block_balance_computed (transaction_a, hash_a); + sideband_a->successor = block_successor (transaction_a, hash_a); + sideband_a->height = std::numeric_limits::max (); + sideband_a->timestamp = std::numeric_limits::max (); + } + } } return result; } @@ -1482,6 +1661,62 @@ bool nano::mdb_store::root_exists (nano::transaction const & transaction_a, nano return block_exists (transaction_a, root_a) || account_exists (transaction_a, root_a); } +nano::account nano::mdb_store::block_account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) +{ + nano::block_sideband sideband; + auto block (block_get (transaction_a, hash_a, &sideband)); + nano::account result (block->account ()); + if (result.is_zero ()) + { + result = sideband.account; + } + assert (!result.is_zero ()); + return result; +} + +// Return account containing hash +nano::account nano::mdb_store::block_account_computed (nano::transaction const & transaction_a, nano::block_hash const & hash_a) +{ + assert (!full_sideband (transaction_a)); + nano::account result (0); + auto hash (hash_a); + while (result.is_zero ()) + { + auto block (block_get (transaction_a, hash)); + assert (block); + result = block->account (); + if (result.is_zero ()) + { + auto type (nano::block_type::invalid); + auto value (block_raw_get (transaction_a, block->previous (), type)); + if (entry_has_sideband (value, type)) + { + result = block_account (transaction_a, block->previous ()); + } + else + { + nano::block_info block_info; + if (!block_info_get (transaction_a, hash, block_info)) + { + result = block_info.account; + } + else + { + result = frontier_get (transaction_a, hash); + if (result.is_zero ()) + { + auto successor (block_successor (transaction_a, hash)); + assert (!successor.is_zero ()); + hash = successor; + } + } + } + } + } + assert (!result.is_zero ()); + return result; +} + void nano::mdb_store::account_del (nano::transaction const & transaction_a, nano::account const & account_a) { auto status1 (mdb_del (env.tx (transaction_a), accounts_v1, nano::mdb_val (account_a), nullptr)); @@ -1711,26 +1946,9 @@ nano::store_iterator nano::mdb_store::pen return result; } -void nano::mdb_store::block_info_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_info const & block_info_a) -{ - auto status (mdb_put (env.tx (transaction_a), blocks_info, nano::mdb_val (hash_a), nano::mdb_val (block_info_a), 0)); - release_assert (status == 0); -} - -void nano::mdb_store::block_info_del (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto status (mdb_del (env.tx (transaction_a), blocks_info, nano::mdb_val (hash_a), nullptr)); - release_assert (status == 0); -} - -bool nano::mdb_store::block_info_exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto iterator (block_info_begin (transaction_a, hash_a)); - return iterator != block_info_end () && nano::block_hash (iterator->first) == hash_a; -} - bool nano::mdb_store::block_info_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_info & block_info_a) { + assert (!full_sideband (transaction_a)); nano::mdb_val value; auto status (mdb_get (env.tx (transaction_a), blocks_info, nano::mdb_val (hash_a), value)); release_assert (status == 0 || status == MDB_NOTFOUND); diff --git a/nano/node/lmdb.hpp b/nano/node/lmdb.hpp index fdf90d65..1a114c59 100644 --- a/nano/node/lmdb.hpp +++ b/nano/node/lmdb.hpp @@ -9,6 +9,8 @@ #include #include +#include + namespace nano { class mdb_env; @@ -148,23 +150,25 @@ class mdb_store : public block_store public: mdb_store (bool &, nano::logging &, boost::filesystem::path const &, int lmdb_max_dbs = 128); + ~mdb_store (); nano::transaction tx_begin_write () override; nano::transaction tx_begin_read () override; nano::transaction tx_begin (bool write = false) override; void initialize (nano::transaction const &, nano::genesis const &) override; - void block_put (nano::transaction const &, nano::block_hash const &, nano::block const &, nano::block_hash const & = nano::block_hash (0), nano::epoch version = nano::epoch::epoch_0) override; + void block_put (nano::transaction const &, nano::block_hash const &, nano::block const &, nano::block_sideband const &, nano::epoch version = nano::epoch::epoch_0) override; size_t block_successor_offset (nano::transaction const &, MDB_val, nano::block_type); nano::block_hash block_successor (nano::transaction const &, nano::block_hash const &) override; void block_successor_clear (nano::transaction const &, nano::block_hash const &) override; - std::shared_ptr block_get (nano::transaction const &, nano::block_hash const &) override; + std::shared_ptr block_get (nano::transaction const &, nano::block_hash const &, nano::block_sideband * = nullptr) override; std::shared_ptr block_random (nano::transaction const &) override; void block_del (nano::transaction const &, nano::block_hash const &) override; bool block_exists (nano::transaction const &, nano::block_hash const &) override; bool block_exists (nano::transaction const &, nano::block_type, nano::block_hash const &) override; nano::block_counts block_count (nano::transaction const &) override; bool root_exists (nano::transaction const &, nano::uint256_union const &) override; + nano::account block_account (nano::transaction const &, nano::block_hash const &) override; void frontier_put (nano::transaction const &, nano::block_hash const &, nano::account const &) override; nano::account frontier_get (nano::transaction const &, nano::block_hash const &) override; @@ -199,13 +203,7 @@ public: nano::store_iterator pending_begin (nano::transaction const &) override; nano::store_iterator pending_end () override; - void block_info_put (nano::transaction const &, nano::block_hash const &, nano::block_info const &) override; - void block_info_del (nano::transaction const &, nano::block_hash const &) override; bool block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) override; - bool block_info_exists (nano::transaction const &, nano::block_hash const &) override; - nano::store_iterator block_info_begin (nano::transaction const &, nano::block_hash const &) override; - nano::store_iterator block_info_begin (nano::transaction const &) override; - nano::store_iterator block_info_end () override; nano::uint128_t block_balance (nano::transaction const &, nano::block_hash const &) override; nano::epoch block_version (nano::transaction const &, nano::block_hash const &) override; @@ -244,7 +242,7 @@ public: void version_put (nano::transaction const &, int) override; int version_get (nano::transaction const &) override; - void do_upgrades (nano::transaction const &); + void do_upgrades (nano::transaction const &, bool &); void upgrade_v1_to_v2 (nano::transaction const &); void upgrade_v2_to_v3 (nano::transaction const &); void upgrade_v3_to_v4 (nano::transaction const &); @@ -256,6 +254,9 @@ public: void upgrade_v9_to_v10 (nano::transaction const &); void upgrade_v10_to_v11 (nano::transaction const &); void upgrade_v11_to_v12 (nano::transaction const &); + void do_slow_upgrades (); + void upgrade_v12_to_v13 (); + bool full_sideband (nano::transaction const &); // Requires a write transaction nano::raw_key get_node_id (nano::transaction const &) override; @@ -263,6 +264,8 @@ public: /** Deletes the node ID from the store */ void delete_node_id (nano::transaction const &) override; + void stop (); + nano::logging & logging; nano::mdb_env env; @@ -364,12 +367,17 @@ public: MDB_dbi meta; private: + bool entry_has_sideband (MDB_val, nano::block_type); + nano::account block_account_computed (nano::transaction const &, nano::block_hash const &); + nano::uint128_t block_balance_computed (nano::transaction const &, nano::block_hash const &); MDB_dbi block_database (nano::block_type, nano::epoch); template std::shared_ptr block_random (nano::transaction const &, MDB_dbi); MDB_val block_raw_get (nano::transaction const &, nano::block_hash const &, nano::block_type &); void block_raw_put (nano::transaction const &, MDB_dbi, nano::block_hash const &, MDB_val); void clear (MDB_dbi); + bool stopped; + std::thread upgrades; }; class wallet_value { diff --git a/nano/node/rpc.cpp b/nano/node/rpc.cpp index 625a6260..bcf76639 100644 --- a/nano/node/rpc.cpp +++ b/nano/node/rpc.cpp @@ -818,15 +818,24 @@ void nano::rpc_handler::available_supply () response_errors (); } -void nano::rpc_handler::block () +void nano::rpc_handler::block_info () { auto hash (hash_impl ()); if (!ec) { + nano::block_sideband sideband; auto transaction (node.store.tx_begin_read ()); - auto block (node.store.block_get (transaction, hash)); + auto block (node.store.block_get (transaction, hash, &sideband)); if (block != nullptr) { + nano::account account (block->account ().is_zero () ? sideband.account : block->account ()); + response_l.put ("block_account", account.to_account ()); + auto amount (node.ledger.amount (transaction, hash)); + response_l.put ("amount", amount.convert_to ()); + auto balance (node.ledger.balance (transaction, hash)); + response_l.put ("balance", balance.convert_to ()); + response_l.put ("height", std::to_string (sideband.height)); + response_l.put ("local_timestamp", std::to_string (sideband.timestamp)); std::string contents; block->serialize_json (contents); response_l.put ("contents", contents); @@ -898,7 +907,6 @@ void nano::rpc_handler::blocks_info () { const bool pending = request.get ("pending", false); const bool source = request.get ("source", false); - const bool balance = request.get ("balance", false); std::vector hashes; boost::property_tree::ptree blocks; auto transaction (node.store.tx_begin_read ()); @@ -910,14 +918,19 @@ void nano::rpc_handler::blocks_info () nano::uint256_union hash; if (!hash.decode_hex (hash_text)) { - auto block (node.store.block_get (transaction, hash)); + nano::block_sideband sideband; + auto block (node.store.block_get (transaction, hash, &sideband)); if (block != nullptr) { boost::property_tree::ptree entry; - auto account (node.ledger.account (transaction, hash)); + nano::account account (block->account ().is_zero () ? sideband.account : block->account ()); entry.put ("block_account", account.to_account ()); auto amount (node.ledger.amount (transaction, hash)); entry.put ("amount", amount.convert_to ()); + auto balance (node.ledger.balance (transaction, hash)); + entry.put ("balance", balance.convert_to ()); + entry.put ("height", std::to_string (sideband.height)); + entry.put ("local_timestamp", std::to_string (sideband.timestamp)); std::string contents; block->serialize_json (contents); entry.put ("contents", contents); @@ -945,11 +958,6 @@ void nano::rpc_handler::blocks_info () entry.put ("source_account", "0"); } } - if (balance) - { - auto balance (node.ledger.balance (transaction, hash)); - entry.put ("balance", balance.convert_to ()); - } blocks.push_back (std::make_pair (hash_text, entry)); } else @@ -1868,7 +1876,8 @@ void nano::rpc_handler::account_history () { boost::property_tree::ptree history; response_l.put ("account", account.to_account ()); - auto block (node.store.block_get (transaction, hash)); + nano::block_sideband sideband; + auto block (node.store.block_get (transaction, hash, &sideband)); while (block != nullptr && count > 0) { if (offset > 0) @@ -1882,6 +1891,7 @@ void nano::rpc_handler::account_history () block->visit (visitor); if (!entry.empty ()) { + entry.put ("local_timestamp", std::to_string (sideband.timestamp)); entry.put ("hash", hash.to_string ()); if (output_raw) { @@ -1893,7 +1903,7 @@ void nano::rpc_handler::account_history () } } hash = block->previous (); - block = node.store.block_get (transaction, hash); + block = node.store.block_get (transaction, hash, &sideband); } response_l.add_child ("history", history); if (!hash.is_zero ()) @@ -4028,7 +4038,11 @@ void nano::rpc_handler::process_request () } else if (action == "block") { - block (); + block_info (); + } + else if (action == "block_info") + { + block_info (); } else if (action == "block_confirm") { diff --git a/nano/node/rpc.hpp b/nano/node/rpc.hpp index 09d9f66d..f2e8abe4 100644 --- a/nano/node/rpc.hpp +++ b/nano/node/rpc.hpp @@ -138,7 +138,7 @@ public: void accounts_frontiers (); void accounts_pending (); void available_supply (); - void block (); + void block_info (); void block_confirm (); void blocks (); void blocks_info (); diff --git a/nano/secure/blockstore.cpp b/nano/secure/blockstore.cpp index 14c80bd8..615a7850 100644 --- a/nano/secure/blockstore.cpp +++ b/nano/secure/blockstore.cpp @@ -4,6 +4,82 @@ #include +#include + +nano::block_sideband::block_sideband (nano::block_type type_a, nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t height_a, uint64_t timestamp_a) : +type (type_a), +successor (successor_a), +account (account_a), +balance (balance_a), +height (height_a), +timestamp (timestamp_a) +{ +} + +size_t nano::block_sideband::size (nano::block_type type_a) +{ + size_t result (0); + result += sizeof (successor); + if (type_a != nano::block_type::state && type_a != nano::block_type::open) + { + result += sizeof (account); + } + if (type_a != nano::block_type::open) + { + result += sizeof (height); + } + if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open) + { + result += sizeof (balance); + } + result += sizeof (timestamp); + return result; +} + +void nano::block_sideband::serialize (nano::stream & stream_a) const +{ + nano::write (stream_a, successor.bytes); + if (type != nano::block_type::state && type != nano::block_type::open) + { + nano::write (stream_a, account.bytes); + } + if (type != nano::block_type::open) + { + nano::write (stream_a, boost::endian::native_to_big (height)); + } + if (type == nano::block_type::receive || type == nano::block_type::change || type == nano::block_type::open) + { + nano::write (stream_a, balance.bytes); + } + nano::write (stream_a, boost::endian::native_to_big (timestamp)); +} + +bool nano::block_sideband::deserialize (nano::stream & stream_a) +{ + bool result (false); + result |= nano::read (stream_a, successor.bytes); + if (type != nano::block_type::state && type != nano::block_type::open) + { + result |= nano::read (stream_a, account.bytes); + } + if (type != nano::block_type::open) + { + result |= nano::read (stream_a, height); + boost::endian::big_to_native_inplace (height); + } + else + { + height = 0; + } + if (type == nano::block_type::receive || type == nano::block_type::change || type == nano::block_type::open) + { + nano::read (stream_a, balance.bytes); + } + result |= nano::read (stream_a, timestamp); + boost::endian::big_to_native_inplace (timestamp); + return result; +} + nano::summation_visitor::summation_visitor (nano::transaction const & transaction_a, nano::block_store & store_a) : transaction (transaction_a), store (store_a) diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index be424a69..731cd64a 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -5,6 +5,21 @@ namespace nano { +class block_sideband +{ +public: + block_sideband () = default; + block_sideband (nano::block_type, nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t); + void serialize (nano::stream &) const; + bool deserialize (nano::stream &); + static size_t size (nano::block_type); + nano::block_type type; + nano::block_hash successor; + nano::account account; + nano::amount balance; + uint64_t height; + uint64_t timestamp; +}; class transaction; class block_store; @@ -193,16 +208,17 @@ class block_store public: virtual ~block_store () = default; virtual void initialize (nano::transaction const &, nano::genesis const &) = 0; - virtual void block_put (nano::transaction const &, nano::block_hash const &, nano::block const &, nano::block_hash const & = nano::block_hash (0), nano::epoch version = nano::epoch::epoch_0) = 0; + virtual void block_put (nano::transaction const &, nano::block_hash const &, nano::block const &, nano::block_sideband const &, nano::epoch version = nano::epoch::epoch_0) = 0; virtual nano::block_hash block_successor (nano::transaction const &, nano::block_hash const &) = 0; virtual void block_successor_clear (nano::transaction const &, nano::block_hash const &) = 0; - virtual std::shared_ptr block_get (nano::transaction const &, nano::block_hash const &) = 0; + virtual std::shared_ptr block_get (nano::transaction const &, nano::block_hash const &, nano::block_sideband * = nullptr) = 0; virtual std::shared_ptr block_random (nano::transaction const &) = 0; virtual void block_del (nano::transaction const &, nano::block_hash const &) = 0; virtual bool block_exists (nano::transaction const &, nano::block_hash const &) = 0; virtual bool block_exists (nano::transaction const &, nano::block_type, nano::block_hash const &) = 0; virtual nano::block_counts block_count (nano::transaction const &) = 0; virtual bool root_exists (nano::transaction const &, nano::uint256_union const &) = 0; + virtual nano::account block_account (nano::transaction const &, nano::block_hash const &) = 0; virtual void frontier_put (nano::transaction const &, nano::block_hash const &, nano::account const &) = 0; virtual nano::account frontier_get (nano::transaction const &, nano::block_hash const &) = 0; @@ -237,16 +253,9 @@ public: virtual nano::store_iterator pending_begin (nano::transaction const &) = 0; virtual nano::store_iterator pending_end () = 0; - virtual void block_info_put (nano::transaction const &, nano::block_hash const &, nano::block_info const &) = 0; - virtual void block_info_del (nano::transaction const &, nano::block_hash const &) = 0; virtual bool block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) = 0; - virtual bool block_info_exists (nano::transaction const &, nano::block_hash const &) = 0; - virtual nano::store_iterator block_info_begin (nano::transaction const &, nano::block_hash const &) = 0; - virtual nano::store_iterator block_info_begin (nano::transaction const &) = 0; - virtual nano::store_iterator block_info_end () = 0; virtual nano::uint128_t block_balance (nano::transaction const &, nano::block_hash const &) = 0; virtual nano::epoch block_version (nano::transaction const &, nano::block_hash const &) = 0; - static size_t const block_info_max = 32; virtual nano::uint128_t representation_get (nano::transaction const &, nano::account const &) = 0; virtual void representation_put (nano::transaction const &, nano::account const &, nano::uint128_t const &) = 0; diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index 4bc5ee73..d670bf93 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -36,10 +36,6 @@ public: ledger.store.frontier_del (transaction, hash); ledger.store.frontier_put (transaction, block_a.hashables.previous, pending.source); ledger.store.block_successor_clear (transaction, block_a.hashables.previous); - if (!(info.block_count % ledger.store.block_info_max)) - { - ledger.store.block_info_del (transaction, hash); - } ledger.stats.inc (nano::stat::type::rollback, nano::stat::detail::send); } void receive_block (nano::receive_block const & block_a) override @@ -59,10 +55,6 @@ public: ledger.store.frontier_del (transaction, hash); ledger.store.frontier_put (transaction, block_a.hashables.previous, destination_account); ledger.store.block_successor_clear (transaction, block_a.hashables.previous); - if (!(info.block_count % ledger.store.block_info_max)) - { - ledger.store.block_info_del (transaction, hash); - } ledger.stats.inc (nano::stat::type::rollback, nano::stat::detail::receive); } void open_block (nano::open_block const & block_a) override @@ -94,10 +86,6 @@ public: ledger.store.frontier_del (transaction, hash); ledger.store.frontier_put (transaction, block_a.hashables.previous, account); ledger.store.block_successor_clear (transaction, block_a.hashables.previous); - if (!(info.block_count % ledger.store.block_info_max)) - { - ledger.store.block_info_del (transaction, hash); - } ledger.stats.inc (nano::stat::type::rollback, nano::stat::detail::change); } void state_block (nano::state_block const & block_a) override @@ -291,7 +279,8 @@ void ledger_processor::state_block_impl (nano::state_block const & block_a) { ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::state_block); result.state_is_send = is_send; - ledger.store.block_put (transaction, hash, block_a, 0, epoch); + nano::block_sideband sideband (nano::block_type::state, block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch ()); + ledger.store.block_put (transaction, hash, block_a, sideband, epoch); if (!info.rep_block.is_zero ()) { @@ -370,7 +359,8 @@ void ledger_processor::epoch_block_impl (nano::state_block const & block_a) ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::epoch_block); result.account = block_a.hashables.account; result.amount = 0; - ledger.store.block_put (transaction, hash, block_a, 0, nano::epoch::epoch_1); + nano::block_sideband sideband (nano::block_type::state, block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch ()); + ledger.store.block_put (transaction, hash, block_a, sideband, nano::epoch::epoch_1); ledger.change_latest (transaction, block_a.hashables.account, hash, hash, info.balance, info.block_count + 1, true, nano::epoch::epoch_1); if (!ledger.store.frontier_get (transaction, info.head).is_zero ()) { @@ -409,7 +399,8 @@ void ledger_processor::change_block (nano::change_block const & block_a) result.code = validate_message (account, hash, block_a.signature) ? nano::process_result::bad_signature : nano::process_result::progress; // Is this block signed correctly (Malformed) if (result.code == nano::process_result::progress) { - ledger.store.block_put (transaction, hash, block_a); + nano::block_sideband sideband (nano::block_type::change, account, 0, info.balance, info.block_count + 1, nano::seconds_since_epoch ()); + ledger.store.block_put (transaction, hash, block_a, sideband); auto balance (ledger.balance (transaction, block_a.hashables.previous)); ledger.store.representation_add (transaction, hash, balance); ledger.store.representation_add (transaction, info.rep_block, 0 - balance); @@ -456,7 +447,8 @@ void ledger_processor::send_block (nano::send_block const & block_a) { auto amount (info.balance.number () - block_a.hashables.balance.number ()); ledger.store.representation_add (transaction, info.rep_block, 0 - amount); - ledger.store.block_put (transaction, hash, block_a); + nano::block_sideband sideband (nano::block_type::send, account, 0, block_a.hashables.balance /* unused */, info.block_count + 1, nano::seconds_since_epoch ()); + ledger.store.block_put (transaction, hash, block_a, sideband); ledger.change_latest (transaction, account, hash, info.rep_block, block_a.hashables.balance, info.block_count + 1); ledger.store.pending_put (transaction, nano::pending_key (block_a.hashables.destination, hash), { account, amount, nano::epoch::epoch_0 }); ledger.store.frontier_del (transaction, block_a.hashables.previous); @@ -515,7 +507,8 @@ void ledger_processor::receive_block (nano::receive_block const & block_a) auto error (ledger.store.account_get (transaction, pending.source, source_info)); assert (!error); ledger.store.pending_del (transaction, key); - ledger.store.block_put (transaction, hash, block_a); + nano::block_sideband sideband (nano::block_type::receive, account, 0, new_balance, info.block_count + 1, nano::seconds_since_epoch ()); + ledger.store.block_put (transaction, hash, block_a, sideband); ledger.change_latest (transaction, account, hash, info.rep_block, new_balance, info.block_count + 1); ledger.store.representation_add (transaction, info.rep_block, pending.amount.number ()); ledger.store.frontier_del (transaction, block_a.hashables.previous); @@ -571,7 +564,8 @@ void ledger_processor::open_block (nano::open_block const & block_a) auto error (ledger.store.account_get (transaction, pending.source, source_info)); assert (!error); ledger.store.pending_del (transaction, key); - ledger.store.block_put (transaction, hash, block_a); + nano::block_sideband sideband (nano::block_type::open, block_a.hashables.account, 0, pending.amount, 0, nano::seconds_since_epoch ()); + ledger.store.block_put (transaction, hash, block_a, sideband); ledger.change_latest (transaction, block_a.hashables.account, hash, hash, pending.amount.number (), info.block_count + 1); ledger.store.representation_add (transaction, hash, pending.amount.number ()); ledger.store.frontier_put (transaction, hash, block_a.hashables.account); @@ -619,8 +613,7 @@ epoch_signer (epoch_signer_a) // Balance for account containing hash nano::uint128_t nano::ledger::balance (nano::transaction const & transaction_a, nano::block_hash const & hash_a) { - nano::summation_visitor visitor (transaction_a, store); - return visitor.compute_balance (hash_a); + return hash_a.is_zero () ? 0 : store.block_balance (transaction_a, hash_a); } // Balance for an account by account number @@ -796,43 +789,25 @@ void nano::ledger::rollback (nano::transaction const & transaction_a, nano::bloc // Return account containing hash nano::account nano::ledger::account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) { - nano::account result; - auto hash (hash_a); - nano::block_hash successor (1); - nano::block_info block_info; - auto block (store.block_get (transaction_a, hash)); - assert (block); - while (!successor.is_zero () && block->type () != nano::block_type::state && store.block_info_get (transaction_a, successor, block_info)) - { - successor = store.block_successor (transaction_a, hash); - if (!successor.is_zero ()) - { - hash = successor; - block = store.block_get (transaction_a, hash); - } - } - if (block->type () == nano::block_type::state) - { - auto state_block (dynamic_cast (block.get ())); - result = state_block->hashables.account; - } - else if (successor.is_zero ()) - { - result = store.frontier_get (transaction_a, hash); - } - else - { - result = block_info.account; - } - assert (!result.is_zero ()); - return result; + return store.block_account (transaction_a, hash_a); } // Return amount decrease or increase for block nano::uint128_t nano::ledger::amount (nano::transaction const & transaction_a, nano::block_hash const & hash_a) { - summation_visitor amount (transaction_a, store); - return amount.compute_amount (hash_a); + nano::uint128_t result; + if (hash_a != nano::genesis_account) + { + auto block (store.block_get (transaction_a, hash_a)); + auto block_balance (balance (transaction_a, hash_a)); + auto previous_balance (balance (transaction_a, block->previous ())); + result = block_balance > previous_balance ? block_balance - previous_balance : previous_balance - block_balance; + } + else + { + result = nano::genesis_amount; + } + return result; } // Return latest block for account @@ -947,13 +922,6 @@ void nano::ledger::change_latest (nano::transaction const & transaction_a, nano: } info.epoch = epoch_a; store.account_put (transaction_a, account_a, info); - if (!(block_count_a % store.block_info_max) && !is_state) - { - nano::block_info block_info; - block_info.account = account_a; - block_info.balance = balance_a; - store.block_info_put (transaction_a, hash_a, block_info); - } } else {