From dfef4ac6ce95cb2c73c02edde76012e052cc19c6 Mon Sep 17 00:00:00 2001 From: Guilherme Lawless Date: Mon, 10 Feb 2020 13:47:47 +0000 Subject: [PATCH] State blocks sideband upgrade adding is_send/receive/epoch (#2545) This changes the epoch byte in the sideband to also store these flags in the 3 most significant bits. These can be used to avoid grabbing the previous block to check its balance in some situations after ledger processing. The upgrade is done in-place, but takes a long time due to having to a random read to retrieve the previous block. A vacuum is done at the end. During the upgrade the ledger grows to almost 2x the ledger size, and up to 3x is required to vacuum as well. Due to using the most significant bits, the upgrade can safely be stopped and restarted (from the beginning), and no additional versioning was needed. --- nano/core_test/block_store.cpp | 301 +++++++++++++++++++++++++++-- nano/core_test/ledger.cpp | 83 +++++++- nano/core_test/versioning.cpp | 6 +- nano/nano_node/entry.cpp | 39 ++++ nano/node/lmdb/lmdb.cpp | 70 ++++++- nano/node/lmdb/lmdb.hpp | 1 + nano/secure/blockstore.cpp | 69 ++++++- nano/secure/blockstore.hpp | 89 ++++++--- nano/secure/blockstore_partial.hpp | 6 +- nano/secure/ledger.cpp | 15 +- 10 files changed, 612 insertions(+), 67 deletions(-) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index e51cc5a7..7cf0596a 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -32,6 +32,7 @@ void modify_genesis_account_info_to_v5 (nano::mdb_store & store, nano::transacti void modify_confirmation_height_to_v15 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height); void write_sideband_v12 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block & block_a, nano::block_hash const & successor_a, MDB_dbi db_a); void write_sideband_v14 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block const & block_a, MDB_dbi db_a); +void write_sideband_v15 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block const & block_a); } TEST (block_store, construction) @@ -41,6 +42,51 @@ TEST (block_store, construction) ASSERT_TRUE (!store->init_error ()); } +TEST (block_store, block_details) +{ + nano::block_details details_send (nano::epoch::epoch_0, true, false, false); + ASSERT_TRUE (details_send.is_send); + ASSERT_FALSE (details_send.is_receive); + ASSERT_FALSE (details_send.is_epoch); + ASSERT_EQ (nano::epoch::epoch_0, details_send.epoch); + + nano::block_details details_receive (nano::epoch::epoch_1, false, true, false); + ASSERT_FALSE (details_receive.is_send); + ASSERT_TRUE (details_receive.is_receive); + ASSERT_FALSE (details_receive.is_epoch); + ASSERT_EQ (nano::epoch::epoch_1, details_receive.epoch); + + nano::block_details details_epoch (nano::epoch::epoch_2, false, false, true); + ASSERT_FALSE (details_epoch.is_send); + ASSERT_FALSE (details_epoch.is_receive); + ASSERT_TRUE (details_epoch.is_epoch); + ASSERT_EQ (nano::epoch::epoch_2, details_epoch.epoch); + + nano::block_details details_none (nano::epoch::unspecified, false, false, false); + ASSERT_FALSE (details_none.is_send); + ASSERT_FALSE (details_none.is_receive); + ASSERT_FALSE (details_none.is_epoch); + ASSERT_EQ (nano::epoch::unspecified, details_none.epoch); +} + +TEST (block_store, block_details_serialization) +{ + nano::block_details details1; + details1.epoch = nano::epoch::epoch_2; + details1.is_epoch = false; + details1.is_receive = true; + details1.is_send = false; + std::vector vector; + { + nano::vectorstream stream1 (vector); + details1.serialize (stream1); + } + nano::bufferstream stream2 (vector.data (), vector.size ()); + nano::block_details details2; + ASSERT_FALSE (details2.deserialize (stream2)); + ASSERT_EQ (details1, details2); +} + TEST (block_store, sideband_serialization) { nano::block_sideband sideband1; @@ -77,7 +123,7 @@ TEST (block_store, add_item) auto latest1 (store->block_get (transaction, hash1)); ASSERT_EQ (nullptr, latest1); ASSERT_FALSE (store->block_exists (transaction, hash1)); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, hash1, block, sideband); auto latest2 (store->block_get (transaction, hash1)); ASSERT_NE (nullptr, latest2); @@ -96,7 +142,7 @@ TEST (block_store, clear_successor) ASSERT_TRUE (!store->init_error ()); nano::open_block block1 (0, 1, 0, nano::keypair ().prv, 0, 0); auto transaction (store->tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, block1.hash (), block1, sideband); nano::open_block block2 (0, 2, 0, nano::keypair ().prv, 0, 0); store->block_put (transaction, block2.hash (), block2, sideband); @@ -123,7 +169,7 @@ TEST (block_store, add_nonempty_block) auto transaction (store->tx_begin_write ()); auto latest1 (store->block_get (transaction, hash1)); ASSERT_EQ (nullptr, latest1); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, hash1, block, sideband); auto latest2 (store->block_get (transaction, hash1)); ASSERT_NE (nullptr, latest2); @@ -148,9 +194,9 @@ TEST (block_store, add_two_items) block2.signature = nano::sign_message (key1.prv, key1.pub, hash2); auto latest2 (store->block_get (transaction, hash2)); ASSERT_EQ (nullptr, latest2); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, hash1, block, sideband); - nano::block_sideband sideband2 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband2 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, hash2, block2, sideband2); auto latest3 (store->block_get (transaction, hash1)); ASSERT_NE (nullptr, latest3); @@ -170,13 +216,13 @@ TEST (block_store, add_receive) nano::keypair key2; nano::open_block block1 (0, 1, 0, nano::keypair ().prv, 0, 0); auto transaction (store->tx_begin_write ()); - nano::block_sideband sideband1 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband1 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, block1.hash (), block1, sideband1); nano::receive_block block (block1.hash (), 1, nano::keypair ().prv, 2, 3); nano::block_hash hash1 (block.hash ()); auto latest1 (store->block_get (transaction, hash1)); ASSERT_EQ (nullptr, latest1); - nano::block_sideband sideband (nano::block_type::receive, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::receive, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, hash1, block, sideband); auto latest2 (store->block_get (transaction, hash1)); ASSERT_NE (nullptr, latest2); @@ -418,7 +464,7 @@ TEST (block_store, one_block) ASSERT_TRUE (!store->init_error ()); nano::open_block block1 (0, 1, 0, nano::keypair ().prv, 0, 0); auto transaction (store->tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, block1.hash (), block1, sideband); ASSERT_TRUE (store->block_exists (transaction, block1.hash ())); } @@ -520,12 +566,12 @@ TEST (block_store, two_block) hashes.push_back (block1.hash ()); blocks.push_back (block1); auto transaction (store->tx_begin_write ()); - nano::block_sideband sideband1 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband1 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, hashes[0], block1, sideband1); nano::open_block block2 (0, 1, 2, nano::keypair ().prv, 0, 0); hashes.push_back (block2.hash ()); blocks.push_back (block2); - nano::block_sideband sideband2 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband2 (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, hashes[1], block2, sideband2); ASSERT_TRUE (store->block_exists (transaction, block1.hash ())); ASSERT_TRUE (store->block_exists (transaction, block2.hash ())); @@ -711,9 +757,9 @@ TEST (block_store, block_replace) nano::send_block send1 (0, 0, 0, nano::keypair ().prv, 0, 1); nano::send_block send2 (0, 0, 0, nano::keypair ().prv, 0, 2); auto transaction (store->tx_begin_write ()); - nano::block_sideband sideband1 (nano::block_type::send, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband1 (nano::block_type::send, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, 0, send1, sideband1); - nano::block_sideband sideband2 (nano::block_type::send, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband2 (nano::block_type::send, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, 0, send2, sideband2); auto block3 (store->block_get (transaction, 0)); ASSERT_NE (nullptr, block3); @@ -730,7 +776,7 @@ TEST (block_store, block_count) ASSERT_EQ (0, store->block_count (transaction).sum ()); nano::open_block block (0, 1, 0, nano::keypair ().prv, 0, 0); auto hash1 (block.hash ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, hash1, block, sideband); } auto transaction (store->tx_begin_read ()); @@ -1132,7 +1178,7 @@ TEST (block_store, state_block) auto transaction (store->tx_begin_write ()); store->initialize (transaction, genesis, ledger_cache); ASSERT_EQ (nano::block_type::state, block1.type ()); - nano::block_sideband sideband1 (nano::block_type::state, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband1 (nano::block_type::state, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (transaction, block1.hash (), block1, sideband1); ASSERT_TRUE (store->block_exists (transaction, block1.hash ())); auto block2 (store->block_get (transaction, block1.hash ())); @@ -1709,11 +1755,11 @@ TEST (mdb_block_store, upgrade_v14_v15) nano::block_sideband sideband; auto block = store.block_get (transaction, state_send.hash (), &sideband); ASSERT_NE (block, nullptr); - ASSERT_EQ (sideband.epoch, nano::epoch::epoch_1); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_1); block = store.block_get (transaction, send.hash (), &sideband); ASSERT_NE (block, nullptr); nano::block_sideband sideband1; - ASSERT_EQ (sideband1.epoch, nano::epoch::epoch_0); + ASSERT_EQ (sideband1.details.epoch, nano::epoch::epoch_0); ASSERT_EQ (info.epoch (), nano::epoch::epoch_1); nano::pending_info pending_info; store.pending_get (transaction, nano::pending_key (nano::test_genesis_key.pub, send.hash ()), pending_info); @@ -1818,6 +1864,203 @@ TEST (mdb_block_store, upgrade_v16_v17) code (4, block3.hash ()); } +TEST (mdb_block_store, upgrade_v17_v18) +{ + auto path (nano::unique_path ()); + nano::genesis genesis; + nano::keypair key1; + nano::keypair key2; + nano::keypair key3; + nano::network_params network_params; + nano::work_pool pool (std::numeric_limits::max ()); + nano::send_block send_zero (genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (genesis.hash ())); + nano::state_block state_receive_zero (nano::test_genesis_key.pub, send_zero.hash (), nano::test_genesis_key.pub, nano::genesis_amount, send_zero.hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (send_zero.hash ())); + nano::state_block epoch (nano::test_genesis_key.pub, state_receive_zero.hash (), nano::test_genesis_key.pub, nano::genesis_amount, network_params.ledger.epochs.link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (state_receive_zero.hash ())); + nano::state_block state_send (nano::test_genesis_key.pub, epoch.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (epoch.hash ())); + nano::state_block state_receive (nano::test_genesis_key.pub, state_send.hash (), nano::test_genesis_key.pub, nano::genesis_amount, state_send.hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (state_send.hash ())); + nano::state_block state_change (nano::test_genesis_key.pub, state_receive.hash (), nano::test_genesis_key.pub, nano::genesis_amount, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (state_receive.hash ())); + nano::state_block state_send_change (nano::test_genesis_key.pub, state_change.hash (), key1.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (state_change.hash ())); + nano::state_block epoch_first (key1.pub, 0, 0, 0, network_params.ledger.epochs.link (nano::epoch::epoch_2), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (key1.pub)); + nano::state_block state_receive2 (key1.pub, epoch_first.hash (), key1.pub, nano::Gxrb_ratio, state_send_change.hash (), key1.prv, key1.pub, *pool.generate (epoch_first.hash ())); + nano::state_block state_send2 (nano::test_genesis_key.pub, state_send_change.hash (), key1.pub, nano::genesis_amount - nano::Gxrb_ratio * 2, key2.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (state_send_change.hash ())); + nano::state_block state_open (key2.pub, 0, key2.pub, nano::Gxrb_ratio, state_send2.hash (), key2.prv, key2.pub, *pool.generate (key2.pub)); + nano::state_block state_send_epoch_link (key2.pub, state_open.hash (), key2.pub, 0, network_params.ledger.epochs.link (nano::epoch::epoch_2), key2.prv, key2.pub, *pool.generate (state_open.hash ())); + { + nano::logger_mt logger; + nano::mdb_store store (logger, path); + auto transaction (store.tx_begin_write ()); + nano::stat stats; + nano::ledger ledger (store, stats); + store.initialize (transaction, genesis, ledger.cache); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send_zero).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_receive_zero).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_send).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_receive).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_change).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_send_change).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch_first).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_receive2).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_send2).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_open).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_send_epoch_link).code); + + // Downgrade the store + store.version_put (transaction, 17); + + // Replace with the previous sideband version for state blocks + // The upgrade can resume after upgrading some blocks, test this by only downgrading some of them + write_sideband_v15 (store, transaction, state_receive_zero); + write_sideband_v15 (store, transaction, epoch); + write_sideband_v15 (store, transaction, state_send); + // DISABLED write_sideband_v15 (store, transaction, state_receive); + write_sideband_v15 (store, transaction, state_change); + write_sideband_v15 (store, transaction, state_send_change); + // DISABLED write_sideband_v15 (store, transaction, epoch_first); + write_sideband_v15 (store, transaction, state_receive2); + // DISABLED write_sideband_v15 (store, transaction, state_send2); + write_sideband_v15 (store, transaction, state_open); + // DISABLED write_sideband_v15 (store, transaction, state_send_epoch_link); + } + + // Now do the upgrade + nano::logger_mt logger; + auto error (false); + nano::mdb_store store (logger, path); + ASSERT_FALSE (error); + auto transaction (store.tx_begin_read ()); + + // Size of state block should equal that set in db (no change) + nano::mdb_val value; + ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.state_blocks, nano::mdb_val (state_send.hash ()), value)); + ASSERT_EQ (value.size (), nano::state_block::size + nano::block_sideband::size (nano::block_type::state)); + + // Check that sidebands are correctly populated + { + // Non-state unaffected + nano::block_sideband sideband; + auto block = store.block_get (transaction, send_zero.hash (), &sideband); + ASSERT_NE (block, nullptr); + // All defaults + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_0); + ASSERT_FALSE (sideband.details.is_epoch); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + } + { + // State receive from old zero send + nano::block_sideband sideband; + auto block = store.block_get (transaction, state_receive_zero.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_0); + ASSERT_FALSE (sideband.details.is_epoch); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_TRUE (sideband.details.is_receive); + } + { + // Epoch + nano::block_sideband sideband; + auto block = store.block_get (transaction, epoch.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_1); + ASSERT_TRUE (sideband.details.is_epoch); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + } + { + // State send + nano::block_sideband sideband; + auto block = store.block_get (transaction, state_send.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_1); + ASSERT_FALSE (sideband.details.is_epoch); + ASSERT_TRUE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + } + { + // State receive + nano::block_sideband sideband; + auto block = store.block_get (transaction, state_receive.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_1); + ASSERT_FALSE (sideband.details.is_epoch); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_TRUE (sideband.details.is_receive); + } + { + // State change + nano::block_sideband sideband; + auto block = store.block_get (transaction, state_change.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_1); + ASSERT_FALSE (sideband.details.is_epoch); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + } + { + // State send + change + nano::block_sideband sideband; + auto block = store.block_get (transaction, state_send_change.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_1); + ASSERT_FALSE (sideband.details.is_epoch); + ASSERT_TRUE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + } + { + // Epoch on unopened account + nano::block_sideband sideband; + auto block = store.block_get (transaction, epoch_first.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_2); + ASSERT_TRUE (sideband.details.is_epoch); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + } + { + // State open following epoch + nano::block_sideband sideband; + auto block = store.block_get (transaction, state_receive2.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_2); + ASSERT_FALSE (sideband.details.is_epoch); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_TRUE (sideband.details.is_receive); + } + { + // Another state send + nano::block_sideband sideband; + auto block = store.block_get (transaction, state_send2.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_1); + ASSERT_FALSE (sideband.details.is_epoch); + ASSERT_TRUE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + } + { + // State open + nano::block_sideband sideband; + auto block = store.block_get (transaction, state_open.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_1); + ASSERT_FALSE (sideband.details.is_epoch); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_TRUE (sideband.details.is_receive); + } + { + // State send to an epoch link + nano::block_sideband sideband; + auto block = store.block_get (transaction, state_send_epoch_link.hash (), &sideband); + ASSERT_NE (block, nullptr); + ASSERT_EQ (sideband.details.epoch, nano::epoch::epoch_1); + ASSERT_FALSE (sideband.details.is_epoch); + ASSERT_TRUE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + } + // Version should be correct + ASSERT_LT (17, store.version_get (transaction)); +} + TEST (mdb_block_store, upgrade_backup) { auto dir (nano::unique_path ()); @@ -1924,7 +2167,7 @@ TEST (mdb_block_store, upgrade_confirmation_height_many) { nano::account account (i); nano::open_block open (1, nano::genesis_account, 3, nullptr); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store.block_put (transaction, open.hash (), open, sideband); nano::account_info_v13 account_info_v13 (open.hash (), open.hash (), open.hash (), 3, 4, 1, nano::epoch::epoch_0); auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0)); @@ -1995,7 +2238,7 @@ TEST (block_store, reset_renew_existing_transaction) // Write the block { auto write_transaction (store->tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store->block_put (write_transaction, hash1, block, sideband); } @@ -2068,7 +2311,27 @@ void write_sideband_v14 (nano::mdb_store & store_a, nano::transaction & transact } MDB_val val{ data.size (), data.data () }; - ASSERT_FALSE (mdb_put (store_a.env.tx (transaction_a), sideband.epoch == nano::epoch::epoch_0 ? store_a.state_blocks_v0 : store_a.state_blocks_v1, nano::mdb_val (block_a.hash ()), &val, 0)); + ASSERT_FALSE (mdb_put (store_a.env.tx (transaction_a), sideband.details.epoch == nano::epoch::epoch_0 ? store_a.state_blocks_v0 : store_a.state_blocks_v1, nano::mdb_val (block_a.hash ()), &val, 0)); +} + +void write_sideband_v15 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block const & block_a) +{ + nano::block_sideband sideband; + auto block = store_a.block_get (transaction_a, block_a.hash (), &sideband); + ASSERT_NE (block, nullptr); + + assert (sideband.details.epoch <= nano::epoch::max); + // Simulated by writing 0 on every of the most significant bits, leaving out epoch only, as if pre-upgrade + nano::block_sideband sideband_v15 (sideband.type, sideband.account, sideband.successor, sideband.balance, sideband.timestamp, sideband.height, sideband.details.epoch, false, false, false); + std::vector data; + { + nano::vectorstream stream (data); + block_a.serialize (stream); + sideband_v15.serialize (stream); + } + + MDB_val val{ data.size (), data.data () }; + ASSERT_FALSE (mdb_put (store_a.env.tx (transaction_a), store_a.state_blocks, nano::mdb_val (block_a.hash ()), &val, 0)); } // These functions take the latest account_info and create a legacy one so that upgrade tests can be emulated more easily. diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index f0ed2e8a..3c94809e 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -1695,17 +1695,22 @@ TEST (ledger, state_send_receive) nano::state_block send1 (nano::genesis_account, genesis.hash (), nano::genesis_account, nano::genesis_amount - nano::Gxrb_ratio, nano::genesis_account, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (genesis.hash ())); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send1).code); ASSERT_TRUE (store->block_exists (transaction, send1.hash ())); - auto send2 (store->block_get (transaction, send1.hash ())); + nano::block_sideband sideband; + auto send2 (store->block_get (transaction, send1.hash (), &sideband)); ASSERT_NE (nullptr, send2); ASSERT_EQ (send1, *send2); ASSERT_EQ (nano::genesis_amount - nano::Gxrb_ratio, ledger.balance (transaction, send1.hash ())); ASSERT_EQ (nano::Gxrb_ratio, ledger.amount (transaction, send1.hash ())); ASSERT_EQ (nano::genesis_amount - nano::Gxrb_ratio, ledger.weight (nano::genesis_account)); ASSERT_TRUE (store->pending_exists (transaction, nano::pending_key (nano::genesis_account, send1.hash ()))); + ASSERT_TRUE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + ASSERT_FALSE (sideband.details.is_epoch); nano::state_block receive1 (nano::genesis_account, send1.hash (), nano::genesis_account, nano::genesis_amount, send1.hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (send1.hash ())); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, receive1).code); ASSERT_TRUE (store->block_exists (transaction, receive1.hash ())); - auto receive2 (store->block_get (transaction, receive1.hash ())); + nano::block_sideband sideband2; + auto receive2 (store->block_get (transaction, receive1.hash (), &sideband2)); ASSERT_NE (nullptr, receive2); ASSERT_EQ (receive1, *receive2); ASSERT_EQ (nano::genesis_amount, ledger.balance (transaction, receive1.hash ())); @@ -1713,6 +1718,9 @@ TEST (ledger, state_send_receive) ASSERT_EQ (nano::genesis_amount, ledger.weight (nano::genesis_account)); ASSERT_FALSE (store->pending_exists (transaction, nano::pending_key (nano::genesis_account, send1.hash ()))); ASSERT_EQ (store->account_count (transaction), ledger.cache.account_count); + ASSERT_FALSE (sideband2.details.is_send); + ASSERT_TRUE (sideband2.details.is_receive); + ASSERT_FALSE (sideband2.details.is_epoch); } TEST (ledger, state_receive) @@ -1738,12 +1746,16 @@ TEST (ledger, state_receive) nano::state_block receive1 (nano::genesis_account, send1.hash (), nano::genesis_account, nano::genesis_amount, send1.hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (send1.hash ())); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, receive1).code); ASSERT_TRUE (store->block_exists (transaction, receive1.hash ())); - auto receive2 (store->block_get (transaction, receive1.hash ())); + nano::block_sideband sideband; + auto receive2 (store->block_get (transaction, receive1.hash (), &sideband)); ASSERT_NE (nullptr, receive2); ASSERT_EQ (receive1, *receive2); ASSERT_EQ (nano::genesis_amount, ledger.balance (transaction, receive1.hash ())); ASSERT_EQ (nano::Gxrb_ratio, ledger.amount (transaction, receive1.hash ())); ASSERT_EQ (nano::genesis_amount, ledger.weight (nano::genesis_account)); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_TRUE (sideband.details.is_receive); + ASSERT_FALSE (sideband.details.is_epoch); } TEST (ledger, state_rep_change) @@ -1761,13 +1773,17 @@ TEST (ledger, state_rep_change) nano::state_block change1 (nano::genesis_account, genesis.hash (), rep.pub, nano::genesis_amount, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (genesis.hash ())); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, change1).code); ASSERT_TRUE (store->block_exists (transaction, change1.hash ())); - auto change2 (store->block_get (transaction, change1.hash ())); + nano::block_sideband sideband; + auto change2 (store->block_get (transaction, change1.hash (), &sideband)); ASSERT_NE (nullptr, change2); ASSERT_EQ (change1, *change2); ASSERT_EQ (nano::genesis_amount, ledger.balance (transaction, change1.hash ())); ASSERT_EQ (0, ledger.amount (transaction, change1.hash ())); ASSERT_EQ (0, ledger.weight (nano::genesis_account)); ASSERT_EQ (nano::genesis_amount, ledger.weight (rep.pub)); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + ASSERT_FALSE (sideband.details.is_epoch); } TEST (ledger, state_open) @@ -1796,13 +1812,17 @@ TEST (ledger, state_open) ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, open1).code); ASSERT_FALSE (store->pending_exists (transaction, nano::pending_key (destination.pub, send1.hash ()))); ASSERT_TRUE (store->block_exists (transaction, open1.hash ())); - auto open2 (store->block_get (transaction, open1.hash ())); + nano::block_sideband sideband; + auto open2 (store->block_get (transaction, open1.hash (), &sideband)); ASSERT_NE (nullptr, open2); ASSERT_EQ (open1, *open2); ASSERT_EQ (nano::Gxrb_ratio, ledger.balance (transaction, open1.hash ())); ASSERT_EQ (nano::Gxrb_ratio, ledger.amount (transaction, open1.hash ())); ASSERT_EQ (nano::genesis_amount, ledger.weight (nano::genesis_account)); ASSERT_EQ (ledger.cache.account_count, store->account_count (transaction)); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_TRUE (sideband.details.is_receive); + ASSERT_FALSE (sideband.details.is_epoch); } // Make sure old block types can't be inserted after a state block. @@ -2045,13 +2065,17 @@ TEST (ledger, state_send_change) nano::state_block send1 (nano::genesis_account, genesis.hash (), rep.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::genesis_account, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (genesis.hash ())); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send1).code); ASSERT_TRUE (store->block_exists (transaction, send1.hash ())); - auto send2 (store->block_get (transaction, send1.hash ())); + nano::block_sideband sideband; + auto send2 (store->block_get (transaction, send1.hash (), &sideband)); ASSERT_NE (nullptr, send2); ASSERT_EQ (send1, *send2); ASSERT_EQ (nano::genesis_amount - nano::Gxrb_ratio, ledger.balance (transaction, send1.hash ())); ASSERT_EQ (nano::Gxrb_ratio, ledger.amount (transaction, send1.hash ())); ASSERT_EQ (0, ledger.weight (nano::genesis_account)); ASSERT_EQ (nano::genesis_amount - nano::Gxrb_ratio, ledger.weight (rep.pub)); + ASSERT_TRUE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + ASSERT_FALSE (sideband.details.is_epoch); } TEST (ledger, state_receive_change) @@ -2078,13 +2102,17 @@ TEST (ledger, state_receive_change) nano::state_block receive1 (nano::genesis_account, send1.hash (), rep.pub, nano::genesis_amount, send1.hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (send1.hash ())); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, receive1).code); ASSERT_TRUE (store->block_exists (transaction, receive1.hash ())); - auto receive2 (store->block_get (transaction, receive1.hash ())); + nano::block_sideband sideband2; + auto receive2 (store->block_get (transaction, receive1.hash (), &sideband2)); ASSERT_NE (nullptr, receive2); ASSERT_EQ (receive1, *receive2); ASSERT_EQ (nano::genesis_amount, ledger.balance (transaction, receive1.hash ())); ASSERT_EQ (nano::Gxrb_ratio, ledger.amount (transaction, receive1.hash ())); ASSERT_EQ (0, ledger.weight (nano::genesis_account)); ASSERT_EQ (nano::genesis_amount, ledger.weight (rep.pub)); + ASSERT_FALSE (sideband2.details.is_send); + ASSERT_TRUE (sideband2.details.is_receive); + ASSERT_FALSE (sideband2.details.is_epoch); } TEST (ledger, state_open_old) @@ -2328,6 +2356,13 @@ TEST (ledger, epoch_blocks_v1_general) nano::keypair destination; nano::state_block epoch1 (nano::genesis_account, genesis.hash (), nano::genesis_account, nano::genesis_amount, ledger.epoch_link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (genesis.hash ())); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch1).code); + { + nano::block_sideband sideband; + (void)ledger.store.block_get (transaction, epoch1.hash (), &sideband); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + ASSERT_TRUE (sideband.details.is_epoch); + } nano::state_block epoch2 (nano::genesis_account, epoch1.hash (), nano::genesis_account, nano::genesis_amount, ledger.epoch_link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (epoch1.hash ())); ASSERT_EQ (nano::process_result::block_position, ledger.process (transaction, epoch2).code); nano::account_info genesis_info; @@ -2339,16 +2374,37 @@ TEST (ledger, epoch_blocks_v1_general) ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch1).code); ASSERT_FALSE (ledger.store.account_get (transaction, nano::genesis_account, genesis_info)); ASSERT_EQ (genesis_info.epoch (), nano::epoch::epoch_1); + { + nano::block_sideband sideband; + (void)ledger.store.block_get (transaction, epoch1.hash (), &sideband); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + ASSERT_TRUE (sideband.details.is_epoch); + } nano::change_block change1 (epoch1.hash (), nano::genesis_account, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (epoch1.hash ())); ASSERT_EQ (nano::process_result::block_position, ledger.process (transaction, change1).code); nano::state_block send1 (nano::genesis_account, epoch1.hash (), nano::genesis_account, nano::genesis_amount - nano::Gxrb_ratio, destination.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (epoch1.hash ())); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send1).code); + { + nano::block_sideband sideband; + (void)ledger.store.block_get (transaction, send1.hash (), &sideband); + ASSERT_TRUE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + ASSERT_FALSE (sideband.details.is_epoch); + } nano::open_block open1 (send1.hash (), nano::genesis_account, destination.pub, destination.prv, destination.pub, *pool.generate (destination.pub)); ASSERT_EQ (nano::process_result::unreceivable, ledger.process (transaction, open1).code); nano::state_block epoch3 (destination.pub, 0, nano::genesis_account, 0, ledger.epoch_link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (destination.pub)); ASSERT_EQ (nano::process_result::representative_mismatch, ledger.process (transaction, epoch3).code); nano::state_block epoch4 (destination.pub, 0, 0, 0, ledger.epoch_link (nano::epoch::epoch_1), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (destination.pub)); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch4).code); + { + nano::block_sideband sideband; + (void)ledger.store.block_get (transaction, epoch4.hash (), &sideband); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_receive); + ASSERT_TRUE (sideband.details.is_epoch); + } nano::receive_block receive1 (epoch4.hash (), send1.hash (), destination.prv, destination.pub, *pool.generate (epoch4.hash ())); ASSERT_EQ (nano::process_result::block_position, ledger.process (transaction, receive1).code); nano::state_block receive2 (destination.pub, epoch4.hash (), destination.pub, nano::Gxrb_ratio, send1.hash (), destination.prv, destination.pub, *pool.generate (epoch4.hash ())); @@ -2358,6 +2414,13 @@ TEST (ledger, epoch_blocks_v1_general) ASSERT_EQ (nano::Gxrb_ratio, ledger.amount (transaction, receive2.hash ())); ASSERT_EQ (nano::genesis_amount - nano::Gxrb_ratio, ledger.weight (nano::genesis_account)); ASSERT_EQ (nano::Gxrb_ratio, ledger.weight (destination.pub)); + { + nano::block_sideband sideband; + (void)ledger.store.block_get (transaction, receive2.hash (), &sideband); + ASSERT_FALSE (sideband.details.is_send); + ASSERT_TRUE (sideband.details.is_receive); + ASSERT_FALSE (sideband.details.is_epoch); + } } TEST (ledger, epoch_blocks_v2_general) @@ -2759,6 +2822,12 @@ TEST (ledger, unchecked_epoch_invalid) nano::account_info info; ASSERT_FALSE (node1.store.account_get (transaction, destination.pub, info)); ASSERT_NE (info.epoch (), nano::epoch::epoch_1); + nano::block_sideband sideband; + ASSERT_NE (nullptr, node1.store.block_get (transaction, epoch2->hash (), &sideband)); + ASSERT_EQ (nano::epoch::epoch_0, sideband.details.epoch); + ASSERT_TRUE (sideband.details.is_send); + ASSERT_FALSE (sideband.details.is_epoch); + ASSERT_FALSE (sideband.details.is_receive); } } diff --git a/nano/core_test/versioning.cpp b/nano/core_test/versioning.cpp index c7cd7dcb..7e5c8696 100644 --- a/nano/core_test/versioning.cpp +++ b/nano/core_test/versioning.cpp @@ -17,7 +17,7 @@ TEST (versioning, account_info_v1) nano::mdb_store store (logger, file); ASSERT_FALSE (store.init_error ()); auto transaction (store.tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store.block_put (transaction, open.hash (), open, sideband); auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (sizeof (v1), &v1), 0)); ASSERT_EQ (0, status); @@ -53,7 +53,7 @@ TEST (versioning, account_info_v5) nano::mdb_store store (logger, file); ASSERT_FALSE (store.init_error ()); auto transaction (store.tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store.block_put (transaction, open.hash (), open, sideband); auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (sizeof (v5), &v5), 0)); ASSERT_EQ (0, status); @@ -89,7 +89,7 @@ TEST (versioning, account_info_v13) nano::mdb_store store (logger, file); ASSERT_FALSE (store.init_error ()); auto transaction (store.tx_begin_write ()); - nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false); store.block_put (transaction, open.hash (), open, sideband); auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (v13), 0)); ASSERT_EQ (0, status); diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index 47cf42e7..34dbf065 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -1050,6 +1050,45 @@ int main (int argc, char * const * argv) std::cerr << boost::str (boost::format ("Invalid signature for block %1%\n") % hash.to_string ()); } } + // Validate block details set in the sideband + bool block_details_error = false; + if (block->type () != nano::block_type::state) + { + // Not state + block_details_error = sideband.details.is_send || sideband.details.is_receive || sideband.details.is_epoch; + } + else + { + auto prev_balance (node.node->ledger.balance (transaction, block->previous ())); + if (block->balance () < prev_balance) + { + // State send + block_details_error = !sideband.details.is_send || sideband.details.is_receive || sideband.details.is_epoch; + } + else + { + if (block->link ().is_zero ()) + { + // State change + block_details_error = sideband.details.is_send || sideband.details.is_receive || sideband.details.is_epoch; + } + else if (block->balance () == prev_balance && node.node->ledger.is_epoch_link (block->link ())) + { + // State epoch + block_details_error = !sideband.details.is_epoch || sideband.details.is_send || sideband.details.is_receive; + } + else + { + // State receive + block_details_error = !sideband.details.is_receive || sideband.details.is_send || sideband.details.is_epoch; + block_details_error |= !node.node->store.source_exists (transaction, block->link ()); + } + } + } + if (block_details_error) + { + std::cerr << boost::str (boost::format ("Incorrect sideband block details for block %1%\n") % hash.to_string ()); + } // Check if block work value is correct if (nano::work_validate (*block.get ())) { diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 0f0eab8a..2eb700b2 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -249,11 +249,14 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, bool upgrade_v14_to_v15 (transaction_a); needs_vacuuming = true; case 15: - // Upgrades to v16 & v17 are both part of the v21 node release + // Upgrades to v16, v17 & v18 are all part of the v21 node release upgrade_v15_to_v16 (transaction_a); case 16: upgrade_v16_to_v17 (transaction_a); case 17: + upgrade_v17_to_v18 (transaction_a); + needs_vacuuming = true; + case 18: break; default: logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); @@ -625,7 +628,7 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction & transaction_ nano::state_block_w_sideband_v14 state_block_w_sideband_v14 (i_state->second); auto & sideband_v14 = state_block_w_sideband_v14.sideband; - nano::block_sideband sideband{ sideband_v14.type, sideband_v14.account, sideband_v14.successor, sideband_v14.balance, sideband_v14.height, sideband_v14.timestamp, i_state.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 }; + nano::block_sideband sideband (sideband_v14.type, sideband_v14.account, sideband_v14.successor, sideband_v14.balance, sideband_v14.height, sideband_v14.timestamp, i_state.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1, false, false, false); // Write these out std::vector data; @@ -781,6 +784,69 @@ void nano::mdb_store::upgrade_v16_to_v17 (nano::write_transaction const & transa logger.always_log ("Finished upgrading confirmation height frontiers"); } +void nano::mdb_store::upgrade_v17_to_v18 (nano::write_transaction const & transaction_a) +{ + logger.always_log ("Preparing v17 to v18 database upgrade..."); + + auto count_pre (count (transaction_a, state_blocks)); + + nano::network_params network_params; + auto num = 0u; + for (nano::mdb_iterator state_i (transaction_a, state_blocks), state_n{}; state_i != state_n; ++state_i, ++num) + { + nano::state_block_w_sideband block_sideband (state_i->second); + auto & block (block_sideband.state_block); + auto & sideband (block_sideband.sideband); + + bool is_send{ false }; + bool is_receive{ false }; + bool is_epoch{ false }; + + nano::amount prev_balance (0); + if (!block->hashables.previous.is_zero ()) + { + prev_balance = block_balance (transaction_a, block->hashables.previous); + } + if (block->hashables.balance == prev_balance && network_params.ledger.epochs.is_epoch_link (block->hashables.link)) + { + is_epoch = true; + } + else if (block->hashables.balance < prev_balance) + { + is_send = true; + } + else if (!block->hashables.link.is_zero ()) + { + is_receive = true; + } + + nano::block_sideband new_sideband (sideband.type, sideband.account, sideband.successor, sideband.balance, sideband.height, sideband.timestamp, sideband.details.epoch, is_send, is_receive, is_epoch); + // Write these out + std::vector data; + { + nano::vectorstream stream (data); + block->serialize (stream); + new_sideband.serialize (stream); + } + nano::mdb_val value{ data.size (), (void *)data.data () }; + auto s = mdb_cursor_put (state_i.cursor, state_i->first, value, MDB_CURRENT); + release_assert (success (s)); + + // Every so often output to the log to indicate progress + constexpr auto output_cutoff = 1000000; + if (num > 0 && num % output_cutoff == 0) + { + logger.always_log (boost::str (boost::format ("Database sideband upgrade %1% million state blocks upgraded (out of %2%)") % (num / output_cutoff) % count_pre)); + } + } + + auto count_post (count (transaction_a, state_blocks)); + release_assert (count_pre == count_post); + + version_put (transaction_a, 18); + logger.always_log ("Finished upgrading the sideband"); +} + /** Takes a filepath, appends '_backup_' to the end (but before any extension) and saves that file in the same directory */ void nano::mdb_store::create_backup_file (nano::mdb_env & env_a, boost::filesystem::path const & filepath_a, nano::logger_mt & logger_a) { diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index 009deaf0..a0cc27f8 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -242,6 +242,7 @@ private: void upgrade_v14_to_v15 (nano::write_transaction &); void upgrade_v15_to_v16 (nano::write_transaction const &); void upgrade_v16_to_v17 (nano::write_transaction const &); + void upgrade_v17_to_v18 (nano::write_transaction const &); void open_databases (bool &, nano::transaction const &, unsigned); diff --git a/nano/secure/blockstore.cpp b/nano/secure/blockstore.cpp index c865952c..d82df471 100644 --- a/nano/secure/blockstore.cpp +++ b/nano/secure/blockstore.cpp @@ -3,14 +3,72 @@ #include -nano::block_sideband::block_sideband (nano::block_type type_a, nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t height_a, uint64_t timestamp_a, nano::epoch epoch_a) : +#include + +nano::block_details::block_details (nano::epoch const epoch_a, bool const is_send_a, bool const is_receive_a, bool const is_epoch_a) : +epoch (epoch_a), is_send (is_send_a), is_receive (is_receive_a), is_epoch (is_epoch_a) +{ +} + +constexpr size_t nano::block_details::size () +{ + return 1; +} + +bool nano::block_details::operator== (nano::block_details const & other_a) const +{ + return epoch == other_a.epoch && is_send == other_a.is_send && is_receive == other_a.is_receive && is_epoch == other_a.is_epoch; +} + +uint8_t nano::block_details::packed () const +{ + std::bitset<8> result (static_cast (epoch)); + result.set (7, is_send); + result.set (6, is_receive); + result.set (5, is_epoch); + return static_cast (result.to_ulong ()); +} + +void nano::block_details::unpack (uint8_t details_a) +{ + constexpr std::bitset<8> epoch_mask{ 0b00011111 }; + auto as_bitset = static_cast> (details_a); + is_send = as_bitset.test (7); + is_receive = as_bitset.test (6); + is_epoch = as_bitset.test (5); + epoch = static_cast ((as_bitset & epoch_mask).to_ulong ()); +} + +void nano::block_details::serialize (nano::stream & stream_a) const +{ + nano::write (stream_a, packed ()); +} + +bool nano::block_details::deserialize (nano::stream & stream_a) +{ + bool result (false); + try + { + uint8_t packed{ 0 }; + nano::read (stream_a, packed); + unpack (packed); + } + catch (std::runtime_error &) + { + result = true; + } + + return result; +} + +nano::block_sideband::block_sideband (nano::block_type type_a, nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t height_a, uint64_t timestamp_a, nano::epoch epoch_a, bool is_send, bool is_receive, bool is_epoch) : type (type_a), successor (successor_a), account (account_a), balance (balance_a), height (height_a), timestamp (timestamp_a), -epoch (epoch_a) +details (epoch_a, is_send, is_receive, is_epoch) { } @@ -33,7 +91,8 @@ size_t nano::block_sideband::size (nano::block_type type_a) result += sizeof (timestamp); if (type_a == nano::block_type::state) { - result += sizeof (epoch); + static_assert (sizeof (nano::epoch) == nano::block_details::size (), "block_details is larger than the epoch enum"); + result += nano::block_details::size (); } return result; } @@ -56,7 +115,7 @@ void nano::block_sideband::serialize (nano::stream & stream_a) const nano::write (stream_a, boost::endian::native_to_big (timestamp)); if (type == nano::block_type::state) { - nano::write (stream_a, epoch); + details.serialize (stream_a); } } @@ -87,7 +146,7 @@ bool nano::block_sideband::deserialize (nano::stream & stream_a) boost::endian::big_to_native_inplace (timestamp); if (type == nano::block_type::state) { - nano::read (stream_a, epoch); + result = details.deserialize (stream_a); } } catch (std::runtime_error &) diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index 0fcf2c8f..f17dd94f 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -16,6 +16,53 @@ namespace nano { +class block_details +{ + static_assert (std::is_same::type, uint8_t> (), "Epoch enum is not the proper type"); + static_assert (static_cast (nano::epoch::max) < (1 << 5), "Epoch max is too large for the sideband"); + +public: + block_details () = default; + block_details (nano::epoch const epoch_a, bool const is_send_a, bool const is_receive_a, bool const is_epoch_a); + static constexpr size_t size (); + bool operator== (block_details const & other_a) const; + void serialize (nano::stream &) const; + bool deserialize (nano::stream &); + nano::epoch epoch{ nano::epoch::epoch_0 }; + bool is_send{ false }; + bool is_receive{ false }; + bool is_epoch{ false }; + +private: + uint8_t packed () const; + void unpack (uint8_t); +}; + +class block_sideband final +{ +public: + block_sideband () = default; + block_sideband (nano::block_type, nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t, nano::epoch, bool is_send, bool is_receive, bool is_epoch); + void serialize (nano::stream &) const; + bool deserialize (nano::stream &); + static size_t size (nano::block_type); + nano::block_type type{ nano::block_type::invalid }; + nano::block_hash successor{ 0 }; + nano::account account{ 0 }; + nano::amount balance{ 0 }; + uint64_t height{ 0 }; + uint64_t timestamp{ 0 }; + nano::block_details details; +}; + +// Move to versioning with a specific version if required for a future upgrade +class state_block_w_sideband +{ +public: + std::shared_ptr state_block; + nano::block_sideband sideband; +}; + /** * Encapsulates database specific container */ @@ -264,19 +311,33 @@ public: return result; } - explicit operator nano::state_block_w_sideband_v14 () const +private: + // Common usage for versioning + template ::value || std::is_same::value>> + T as () const { nano::bufferstream stream (reinterpret_cast (data ()), size ()); auto error (false); - nano::state_block_w_sideband_v14 state_block_w_sideband_v14; - state_block_w_sideband_v14.state_block = std::make_shared (error, stream); + T block_w_sideband; + block_w_sideband.state_block = std::make_shared (error, stream); assert (!error); - state_block_w_sideband_v14.sideband.type = nano::block_type::state; - error = state_block_w_sideband_v14.sideband.deserialize (stream); + block_w_sideband.sideband.type = nano::block_type::state; + error = block_w_sideband.sideband.deserialize (stream); assert (!error); - return state_block_w_sideband_v14; + return block_w_sideband; + } + +public: + explicit operator state_block_w_sideband () const + { + return as (); + } + + explicit operator state_block_w_sideband_v14 () const + { + return as (); } explicit operator nano::no_value () const @@ -377,22 +438,6 @@ private: } }; -class block_sideband final -{ -public: - block_sideband () = default; - block_sideband (nano::block_type, nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t, nano::epoch); - void serialize (nano::stream &) const; - bool deserialize (nano::stream &); - static size_t size (nano::block_type); - nano::block_type type{ nano::block_type::invalid }; - nano::block_hash successor{ 0 }; - nano::account account{ 0 }; - nano::amount balance{ 0 }; - uint64_t height{ 0 }; - uint64_t timestamp{ 0 }; - nano::epoch epoch{ nano::epoch::epoch_0 }; -}; class transaction; class block_store; diff --git a/nano/secure/blockstore_partial.hpp b/nano/secure/blockstore_partial.hpp index da83a8fc..33f9b806 100644 --- a/nano/secure/blockstore_partial.hpp +++ b/nano/secure/blockstore_partial.hpp @@ -31,7 +31,7 @@ public: { auto hash_l (genesis_a.hash ()); assert (latest_begin (transaction_a) == latest_end ()); - nano::block_sideband sideband (nano::block_type::open, network_params.ledger.genesis_account, 0, network_params.ledger.genesis_amount, 1, nano::seconds_since_epoch (), nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, network_params.ledger.genesis_account, 0, network_params.ledger.genesis_amount, 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false, false, false); block_put (transaction_a, hash_l, *genesis_a.open, sideband); ++ledger_cache_a.block_count; confirmation_height_put (transaction_a, network_params.ledger.genesis_account, nano::confirmation_height_info{ 1, genesis_a.hash () }); @@ -411,7 +411,7 @@ public: auto block = block_get (transaction_a, hash_a, &sideband); if (sideband.type == nano::block_type::state) { - return sideband.epoch; + return sideband.details.epoch; } return nano::epoch::epoch_0; @@ -781,7 +781,7 @@ protected: nano::network_params network_params; std::unordered_map> vote_cache_l1; std::unordered_map> vote_cache_l2; - static int constexpr version{ 17 }; + static int constexpr version{ 18 }; template std::shared_ptr block_random (nano::transaction const & transaction_a, tables table_a) diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index 98a1906c..38dd1748 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -276,6 +276,7 @@ void ledger_processor::state_block_impl (nano::state_block const & block_a) nano::account_info info; result.amount = block_a.hashables.balance; auto is_send (false); + auto is_receive (false); auto account_error (ledger.store.account_get (transaction, block_a.hashables.account, info)); if (!account_error) { @@ -288,6 +289,7 @@ void ledger_processor::state_block_impl (nano::state_block const & block_a) if (result.code == nano::process_result::progress) { is_send = block_a.hashables.balance < info.balance; + is_receive = !is_send && !block_a.hashables.link.is_zero (); result.amount = is_send ? (info.balance.number () - result.amount.number ()) : (result.amount.number () - info.balance.number ()); result.code = block_a.hashables.previous == info.head ? nano::process_result::progress : nano::process_result::fork; // Is the previous block the account's head block? (Ambigious) } @@ -299,6 +301,7 @@ void ledger_processor::state_block_impl (nano::state_block const & block_a) result.code = block_a.previous ().is_zero () ? nano::process_result::progress : nano::process_result::gap_previous; // Does the first block in an account yield 0 for previous() ? (Unambigious) if (result.code == nano::process_result::progress) { + is_receive = true; result.code = !block_a.hashables.link.is_zero () ? nano::process_result::progress : nano::process_result::gap_source; // Is the first block receiving from a send ? (Unambigious) } } @@ -332,7 +335,7 @@ void ledger_processor::state_block_impl (nano::state_block const & block_a) { ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::state_block); result.state_is_send = is_send; - nano::block_sideband sideband (nano::block_type::state, block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch (), epoch); + nano::block_sideband sideband (nano::block_type::state, block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch (), epoch, is_send, is_receive, false); ledger.store.block_put (transaction, hash, block_a, sideband); if (!info.head.is_zero ()) @@ -420,7 +423,7 @@ void ledger_processor::epoch_block_impl (nano::state_block const & block_a) ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::epoch_block); result.account = block_a.hashables.account; result.amount = 0; - nano::block_sideband sideband (nano::block_type::state, block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch (), epoch); + nano::block_sideband sideband (nano::block_type::state, block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch (), epoch, false, false, true); ledger.store.block_put (transaction, hash, block_a, sideband); nano::account_info new_info (hash, block_a.representative (), info.open_block.is_zero () ? hash : info.open_block, info.balance, nano::seconds_since_epoch (), info.block_count + 1, epoch); ledger.change_latest (transaction, block_a.hashables.account, info, new_info); @@ -468,7 +471,7 @@ void ledger_processor::change_block (nano::change_block const & block_a) { assert (!validate_message (account, hash, block_a.signature)); result.verified = nano::signature_verification::valid; - nano::block_sideband sideband (nano::block_type::change, account, 0, info.balance, info.block_count + 1, nano::seconds_since_epoch (), nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::change, account, 0, info.balance, info.block_count + 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false, false, false); ledger.store.block_put (transaction, hash, block_a, sideband); auto balance (ledger.balance (transaction, block_a.hashables.previous)); ledger.cache.rep_weights.representation_add (block_a.representative (), balance); @@ -524,7 +527,7 @@ void ledger_processor::send_block (nano::send_block const & block_a) { auto amount (info.balance.number () - block_a.hashables.balance.number ()); ledger.cache.rep_weights.representation_add (info.representative, 0 - amount); - nano::block_sideband sideband (nano::block_type::send, account, 0, block_a.hashables.balance /* unused */, info.block_count + 1, nano::seconds_since_epoch (), nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::send, account, 0, block_a.hashables.balance /* unused */, info.block_count + 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false /* unused */, false /* unused */, false /* unused */); ledger.store.block_put (transaction, hash, block_a, sideband); nano::account_info new_info (hash, info.representative, info.open_block, block_a.hashables.balance, nano::seconds_since_epoch (), info.block_count + 1, nano::epoch::epoch_0); ledger.change_latest (transaction, account, info, new_info); @@ -592,7 +595,7 @@ void ledger_processor::receive_block (nano::receive_block const & block_a) (void)error; assert (!error); ledger.store.pending_del (transaction, key); - nano::block_sideband sideband (nano::block_type::receive, account, 0, new_balance, info.block_count + 1, nano::seconds_since_epoch (), nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::receive, account, 0, new_balance, info.block_count + 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false /* unused */, false /* unused */, false /* unused */); ledger.store.block_put (transaction, hash, block_a, sideband); nano::account_info new_info (hash, info.representative, info.open_block, new_balance, nano::seconds_since_epoch (), info.block_count + 1, nano::epoch::epoch_0); ledger.change_latest (transaction, account, info, new_info); @@ -656,7 +659,7 @@ void ledger_processor::open_block (nano::open_block const & block_a) (void)error; assert (!error); ledger.store.pending_del (transaction, key); - nano::block_sideband sideband (nano::block_type::open, block_a.hashables.account, 0, pending.amount, 1, nano::seconds_since_epoch (), nano::epoch::epoch_0); + nano::block_sideband sideband (nano::block_type::open, block_a.hashables.account, 0, pending.amount, 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false /* unused */, false /* unused */, false /* unused */); ledger.store.block_put (transaction, hash, block_a, sideband); nano::account_info new_info (hash, block_a.representative (), hash, pending.amount.number (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0); ledger.change_latest (transaction, block_a.hashables.account, info, new_info);