From 53384d734258f1316592bd80c2362f0f418a8c9b Mon Sep 17 00:00:00 2001 From: Wesley Shillingford Date: Fri, 12 Jul 2019 13:15:56 +0100 Subject: [PATCH] Refactor database code to allow easier reuse with different backends (#2136) --- nano/node/lmdb.cpp | 797 +++---------------------------------- nano/node/lmdb.hpp | 104 +---- nano/secure/blockstore.cpp | 24 -- nano/secure/blockstore.hpp | 764 ++++++++++++++++++++++++++++++++++- 4 files changed, 827 insertions(+), 862 deletions(-) diff --git a/nano/node/lmdb.cpp b/nano/node/lmdb.cpp index ae5aaad6..6774ec5d 100644 --- a/nano/node/lmdb.cpp +++ b/nano/node/lmdb.cpp @@ -9,6 +9,28 @@ #include +namespace nano +{ +template <> +void * mdb_val::data () const +{ + return value.mv_data; +} + +template <> +size_t mdb_val::size () const +{ + return value.mv_size; +} + +template <> +mdb_val::db_val (nano::DB_val const & value_a, nano::epoch epoch_a) : +value ({ value_a.size, value_a.data }), +epoch (epoch_a) +{ +} +} + nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a, int max_dbs_a, bool use_no_mem_init_a, size_t map_size_a) { boost::system::error_code error_mkdir, error_chmod; @@ -160,333 +182,6 @@ void * nano::write_mdb_txn::get_handle () const return handle; } -nano::mdb_val::mdb_val (nano::epoch epoch_a) : -value ({ 0, nullptr }), -epoch (epoch_a) -{ -} - -nano::mdb_val::mdb_val (MDB_val const & value_a, nano::epoch epoch_a) : -value (value_a), -epoch (epoch_a) -{ -} - -nano::mdb_val::mdb_val (size_t size_a, void * data_a) : -value ({ size_a, data_a }) -{ -} - -nano::mdb_val::mdb_val (nano::uint128_union const & val_a) : -mdb_val (sizeof (val_a), const_cast (&val_a)) -{ -} - -nano::mdb_val::mdb_val (nano::uint256_union const & val_a) : -mdb_val (sizeof (val_a), const_cast (&val_a)) -{ -} - -nano::mdb_val::mdb_val (nano::account_info const & val_a) : -mdb_val (val_a.db_size (), const_cast (&val_a)) -{ -} - -nano::mdb_val::mdb_val (nano::account_info_v13 const & val_a) : -mdb_val (val_a.db_size (), const_cast (&val_a)) -{ -} - -nano::mdb_val::mdb_val (nano::pending_info const & val_a) : -mdb_val (sizeof (val_a.source) + sizeof (val_a.amount), const_cast (&val_a)) -{ - static_assert (std::is_standard_layout::value, "Standard layout is required"); -} - -nano::mdb_val::mdb_val (nano::pending_key const & val_a) : -mdb_val (sizeof (val_a), const_cast (&val_a)) -{ - static_assert (std::is_standard_layout::value, "Standard layout is required"); -} - -nano::mdb_val::mdb_val (nano::unchecked_info const & val_a) : -buffer (std::make_shared> ()) -{ - { - nano::vectorstream stream (*buffer); - val_a.serialize (stream); - } - value = { buffer->size (), const_cast (buffer->data ()) }; -} - -nano::mdb_val::mdb_val (nano::block_info const & val_a) : -mdb_val (sizeof (val_a), const_cast (&val_a)) -{ - static_assert (std::is_standard_layout::value, "Standard layout is required"); -} - -nano::mdb_val::mdb_val (nano::endpoint_key const & val_a) : -mdb_val (sizeof (val_a), const_cast (&val_a)) -{ - static_assert (std::is_standard_layout::value, "Standard layout is required"); -} - -nano::mdb_val::mdb_val (std::shared_ptr const & val_a) : -buffer (std::make_shared> ()) -{ - { - nano::vectorstream stream (*buffer); - nano::serialize_block (stream, *val_a); - } - value = { buffer->size (), const_cast (buffer->data ()) }; -} - -nano::mdb_val::mdb_val (uint64_t val_a) : -buffer (std::make_shared> ()) -{ - { - boost::endian::native_to_big_inplace (val_a); - nano::vectorstream stream (*buffer); - nano::write (stream, val_a); - } - value = { buffer->size (), const_cast (buffer->data ()) }; -} - -void * nano::mdb_val::data () const -{ - return value.mv_data; -} - -size_t nano::mdb_val::size () const -{ - return value.mv_size; -} - -nano::mdb_val::operator nano::account_info () const -{ - nano::account_info result; - result.epoch = epoch; - assert (value.mv_size == result.db_size ()); - std::copy (reinterpret_cast (value.mv_data), reinterpret_cast (value.mv_data) + result.db_size (), reinterpret_cast (&result)); - return result; -} - -nano::mdb_val::operator nano::account_info_v13 () const -{ - nano::account_info_v13 result; - result.epoch = epoch; - assert (value.mv_size == result.db_size ()); - std::copy (reinterpret_cast (value.mv_data), reinterpret_cast (value.mv_data) + result.db_size (), reinterpret_cast (&result)); - return result; -} - -nano::mdb_val::operator nano::block_info () const -{ - nano::block_info result; - assert (value.mv_size == sizeof (result)); - static_assert (sizeof (nano::block_info::account) + sizeof (nano::block_info::balance) == sizeof (result), "Packed class"); - std::copy (reinterpret_cast (value.mv_data), reinterpret_cast (value.mv_data) + sizeof (result), reinterpret_cast (&result)); - return result; -} - -nano::mdb_val::operator nano::pending_info () const -{ - nano::pending_info result; - result.epoch = epoch; - std::copy (reinterpret_cast (value.mv_data), reinterpret_cast (value.mv_data) + sizeof (nano::pending_info::source) + sizeof (nano::pending_info::amount), reinterpret_cast (&result)); - return result; -} - -nano::mdb_val::operator nano::pending_key () const -{ - nano::pending_key result; - assert (value.mv_size == sizeof (result)); - static_assert (sizeof (nano::pending_key::account) + sizeof (nano::pending_key::hash) == sizeof (result), "Packed class"); - std::copy (reinterpret_cast (value.mv_data), reinterpret_cast (value.mv_data) + sizeof (result), reinterpret_cast (&result)); - return result; -} - -nano::mdb_val::operator nano::unchecked_info () const -{ - nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); - nano::unchecked_info result; - bool error (result.deserialize (stream)); - assert (!error); - return result; -} - -nano::mdb_val::operator nano::uint128_union () const -{ - nano::uint128_union result; - assert (size () == sizeof (result)); - std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), result.bytes.data ()); - return result; -} - -nano::mdb_val::operator nano::uint256_union () const -{ - nano::uint256_union result; - assert (size () == sizeof (result)); - std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), result.bytes.data ()); - return result; -} - -nano::mdb_val::operator std::array () const -{ - nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); - std::array result; - auto error = nano::try_read (stream, result); - assert (!error); - return result; -} - -nano::mdb_val::operator nano::endpoint_key () const -{ - nano::endpoint_key result; - std::copy (reinterpret_cast (value.mv_data), reinterpret_cast (value.mv_data) + sizeof (result), reinterpret_cast (&result)); - return result; -} - -nano::mdb_val::operator nano::no_value () const -{ - return no_value::dummy; -} - -nano::mdb_val::operator std::shared_ptr () const -{ - nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); - std::shared_ptr result (nano::deserialize_block (stream)); - return result; -} - -nano::mdb_val::operator std::shared_ptr () const -{ - nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); - auto error (false); - std::shared_ptr result (std::make_shared (error, stream)); - assert (!error); - return result; -} - -nano::mdb_val::operator std::shared_ptr () const -{ - nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); - auto error (false); - std::shared_ptr result (std::make_shared (error, stream)); - assert (!error); - return result; -} - -nano::mdb_val::operator std::shared_ptr () const -{ - nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); - auto error (false); - std::shared_ptr result (std::make_shared (error, stream)); - assert (!error); - return result; -} - -nano::mdb_val::operator std::shared_ptr () const -{ - nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); - auto error (false); - std::shared_ptr result (std::make_shared (error, stream)); - assert (!error); - return result; -} - -nano::mdb_val::operator std::shared_ptr () const -{ - nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); - auto error (false); - std::shared_ptr result (std::make_shared (error, stream)); - assert (!error); - return result; -} - -nano::mdb_val::operator std::shared_ptr () const -{ - nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); - auto error (false); - auto result (nano::make_shared (error, stream)); - assert (!error); - return result; -} - -nano::mdb_val::operator uint64_t () const -{ - uint64_t result; - nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); - auto error (nano::try_read (stream, result)); - assert (!error); - boost::endian::big_to_native_inplace (result); - return result; -} - -nano::mdb_val::operator MDB_val * () const -{ - // Allow passing a temporary to a non-c++ function which doesn't have constness - return const_cast (&value); -} - -nano::mdb_val::operator MDB_val const & () const -{ - return value; -} - -namespace nano -{ -/** - * Fill in our predecessors - */ -class block_predecessor_set : public nano::block_visitor -{ -public: - block_predecessor_set (nano::transaction const & transaction_a, nano::mdb_store & store_a) : - transaction (transaction_a), - store (store_a) - { - } - virtual ~block_predecessor_set () = default; - void fill_value (nano::block const & block_a) - { - auto hash (block_a.hash ()); - nano::block_type type; - auto value (store.block_raw_get (transaction, block_a.previous (), type)); - auto version (store.block_version (transaction, block_a.previous ())); - assert (value.mv_size != 0); - std::vector data (static_cast (value.mv_data), static_cast (value.mv_data) + value.mv_size); - std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + store.block_successor_offset (transaction, value, type)); - store.block_raw_put (transaction, store.block_database (type, version), block_a.previous (), nano::mdb_val (data.size (), data.data ())); - } - void send_block (nano::send_block const & block_a) override - { - fill_value (block_a); - } - void receive_block (nano::receive_block const & block_a) override - { - fill_value (block_a); - } - void open_block (nano::open_block const & block_a) override - { - // Open blocks don't have a predecessor - } - void change_block (nano::change_block const & block_a) override - { - fill_value (block_a); - } - void state_block (nano::state_block const & block_a) override - { - if (!block_a.previous ().is_zero ()) - { - fill_value (block_a); - } - } - nano::transaction const & transaction; - nano::mdb_store & store; -}; -} - template nano::mdb_iterator::mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a, nano::epoch epoch_a) : cursor (nullptr) @@ -912,22 +607,6 @@ nano::mdb_txn_callbacks nano::mdb_store::create_txn_callbacks () return mdb_txn_callbacks; } -/** - * This is only used with testing. If using a different store version than the latest then you may need - * to modify some of the objects in the store to be appropriate for the version before an upgrade. - */ -void nano::mdb_store::initialize (nano::transaction const & transaction_a, nano::genesis const & genesis_a) -{ - auto hash_l (genesis_a.hash ()); - assert (latest_v0_begin (transaction_a) == latest_v0_end ()); - assert (latest_v1_begin (transaction_a) == latest_v1_end ()); - nano::block_sideband sideband (nano::block_type::open, network_params.ledger.genesis_account, 0, network_params.ledger.genesis_amount, 1, nano::seconds_since_epoch ()); - block_put (transaction_a, hash_l, *genesis_a.open, sideband); - account_put (transaction_a, network_params.ledger.genesis_account, { hash_l, genesis_a.open->hash (), genesis_a.open->hash (), std::numeric_limits::max (), nano::seconds_since_epoch (), 1, 1, nano::epoch::epoch_0 }); - representation_put (transaction_a, network_params.ledger.genesis_account, std::numeric_limits::max ()); - frontier_put (transaction_a, hash_l, network_params.ledger.genesis_account); -} - void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & transaction_a, unsigned flags) { error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontiers) != 0; @@ -1010,10 +689,7 @@ bool nano::mdb_store::peer_exists (nano::transaction const & transaction_a, nano size_t nano::mdb_store::peer_count (nano::transaction const & transaction_a) const { - MDB_stat stats; - auto status (mdb_stat (env.tx (transaction_a), peers, &stats)); - release_assert (status == 0); - return stats.ms_entries; + return count (transaction_a, peers); } void nano::mdb_store::peer_clear (nano::transaction const & transaction_a) @@ -1165,16 +841,16 @@ void nano::mdb_store::upgrade_v4_to_v5 (nano::transaction const & transaction_a) block->serialize (stream); nano::write (stream, successor.bytes); } - block_raw_put (transaction_a, block_database (block->type (), nano::epoch::epoch_0), hash, { vector.size (), vector.data () }); + block_raw_put (transaction_a, vector, block->type (), nano::epoch::epoch_0, hash); if (!block->previous ().is_zero ()) { nano::block_type type; auto value (block_raw_get (transaction_a, block->previous (), type)); auto version (block_version (transaction_a, block->previous ())); - assert (value.mv_size != 0); - std::vector data (static_cast (value.mv_data), static_cast (value.mv_data) + value.mv_size); + assert (value.size () != 0); + std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); std::copy (hash.bytes.begin (), hash.bytes.end (), data.end () - nano::block_sideband::size (type)); - block_raw_put (transaction_a, block_database (type, version), block->previous (), nano::mdb_val (data.size (), data.data ())); + block_raw_put (transaction_a, data, type, version, block->previous ()); } } successor = hash; @@ -1372,21 +1048,6 @@ void nano::mdb_store::clear (MDB_dbi db_a) release_assert (status == 0); } -nano::uint128_t nano::mdb_store::block_balance (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - nano::block_sideband sideband; - auto block (block_get (transaction_a, hash_a, &sideband)); - nano::uint128_t result (block_balance_calculated (block, sideband)); - return result; -} - -nano::uint128_t nano::mdb_store::block_balance_computed (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - assert (!full_sideband (transaction_a)); - summation_visitor visitor (transaction_a, *this); - return visitor.compute_balance (hash_a); -} - nano::epoch nano::mdb_store::block_version (nano::transaction const & transaction_a, nano::block_hash const & hash_a) { nano::mdb_val value; @@ -1395,15 +1056,6 @@ nano::epoch nano::mdb_store::block_version (nano::transaction const & transactio return status == 0 ? nano::epoch::epoch_1 : nano::epoch::epoch_0; } -void nano::mdb_store::representation_add (nano::transaction const & transaction_a, nano::block_hash const & source_a, nano::uint128_t const & amount_a) -{ - auto source_block (block_get (transaction_a, source_a)); - assert (source_block != nullptr); - auto source_rep (source_block->representative ()); - auto source_previous (representation_get (transaction_a, source_rep)); - representation_put (transaction_a, source_rep, source_previous + amount_a); -} - MDB_dbi nano::mdb_store::block_database (nano::block_type type_a, nano::epoch epoch_a) { if (type_a == nano::block_type::state) @@ -1449,29 +1101,15 @@ MDB_dbi nano::mdb_store::block_database (nano::block_type type_a, nano::epoch ep return result; } -void nano::mdb_store::block_raw_put (nano::transaction const & transaction_a, MDB_dbi database_a, nano::block_hash const & hash_a, MDB_val value_a) +void nano::mdb_store::block_raw_put (nano::transaction const & transaction_a, std::vector const & data, nano::block_type block_type_a, nano::epoch epoch_a, nano::block_hash const & hash_a) { - auto status2 (mdb_put (env.tx (transaction_a), database_a, nano::mdb_val (hash_a), &value_a, 0)); + MDB_dbi database_a = block_database (block_type_a, epoch_a); + MDB_val value{ data.size (), (void *)data.data () }; + auto status2 (mdb_put (env.tx (transaction_a), database_a, nano::mdb_val (hash_a), &value, 0)); release_assert (status2 == 0); } -void nano::mdb_store::block_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a, nano::block_sideband const & sideband_a, nano::epoch epoch_a) -{ - assert (block_a.type () == sideband_a.type); - assert (sideband_a.successor.is_zero () || block_exists (transaction_a, sideband_a.successor)); - std::vector vector; - { - nano::vectorstream stream (vector); - block_a.serialize (stream); - sideband_a.serialize (stream); - } - block_raw_put (transaction_a, block_database (block_a.type (), epoch_a), hash_a, { vector.size (), vector.data () }); - nano::block_predecessor_set predecessor (transaction_a, *this); - block_a.visit (predecessor); - assert (block_a.previous ().is_zero () || block_successor (transaction_a, block_a.previous ()) == hash_a); -} - -boost::optional nano::mdb_store::block_raw_get_by_type (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const +boost::optional nano::mdb_store::block_raw_get_by_type (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const { nano::mdb_val value; auto status (MDB_NOTFOUND); @@ -1514,29 +1152,10 @@ boost::optional nano::mdb_store::block_raw_get_by_type (nano::transacti } release_assert (status == MDB_SUCCESS || status == MDB_NOTFOUND); - boost::optional result; + boost::optional result; if (status == MDB_SUCCESS) { - result = value; - } - - return result; -} - -MDB_val nano::mdb_store::block_raw_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const -{ - nano::mdb_val result; - // Table lookups are ordered by match probability - nano::block_type block_types[]{ nano::block_type::state, nano::block_type::send, nano::block_type::receive, nano::block_type::open, nano::block_type::change }; - for (auto current_type : block_types) - { - auto mdb_val (block_raw_get_by_type (transaction_a, hash_a, current_type)); - if (mdb_val.is_initialized ()) - { - type_a = current_type; - result = mdb_val.get (); - break; - } + result = nano::DB_val (value.size (), value.data ()); } return result; @@ -1607,103 +1226,6 @@ std::shared_ptr nano::mdb_store::block_random (nano::transaction co return result; } -bool nano::mdb_store::full_sideband (nano::transaction const & transaction_a) const -{ - return version_get (transaction_a) > 12; -} - -bool nano::mdb_store::entry_has_sideband (MDB_val entry_a, nano::block_type type_a) const -{ - return entry_a.mv_size == nano::block::size (type_a) + nano::block_sideband::size (type_a); -} - -size_t nano::mdb_store::block_successor_offset (nano::transaction const & transaction_a, MDB_val entry_a, nano::block_type type_a) const -{ - size_t result; - if (full_sideband (transaction_a) || entry_has_sideband (entry_a, type_a)) - { - result = entry_a.mv_size - nano::block_sideband::size (type_a); - } - else - { - // Read old successor-only sideband - assert (entry_a.mv_size == nano::block::size (type_a) + sizeof (nano::uint256_union)); - result = entry_a.mv_size - sizeof (nano::uint256_union); - } - return result; -} - -nano::block_hash nano::mdb_store::block_successor (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - nano::block_type type; - auto value (block_raw_get (transaction_a, hash_a, type)); - nano::block_hash result; - if (value.mv_size != 0) - { - assert (value.mv_size >= result.bytes.size ()); - nano::bufferstream stream (reinterpret_cast (value.mv_data) + block_successor_offset (transaction_a, value, type), result.bytes.size ()); - auto error (nano::try_read (stream, result.bytes)); - assert (!error); - } - else - { - result.clear (); - } - return result; -} - -void nano::mdb_store::block_successor_clear (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - nano::block_type type; - auto value (block_raw_get (transaction_a, hash_a, type)); - auto version (block_version (transaction_a, hash_a)); - assert (value.mv_size != 0); - std::vector data (static_cast (value.mv_data), static_cast (value.mv_data) + value.mv_size); - std::fill_n (data.begin () + block_successor_offset (transaction_a, value, type), sizeof (nano::uint256_union), 0); - block_raw_put (transaction_a, block_database (type, version), hash_a, nano::mdb_val (data.size (), data.data ())); -} - -// Converts a block hash to a block height -uint64_t nano::mdb_store::block_account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - nano::block_sideband sideband; - auto block = block_get (transaction_a, hash_a, &sideband); - assert (block != nullptr); - return sideband.height; -} - -std::shared_ptr nano::mdb_store::block_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband * sideband_a) const -{ - nano::block_type type; - auto value (block_raw_get (transaction_a, hash_a, type)); - std::shared_ptr result; - if (value.mv_size != 0) - { - nano::bufferstream stream (reinterpret_cast (value.mv_data), value.mv_size); - result = nano::deserialize_block (stream, type); - assert (result != nullptr); - if (sideband_a) - { - sideband_a->type = type; - if (full_sideband (transaction_a) || entry_has_sideband (value, type)) - { - auto error (sideband_a->deserialize (stream)); - assert (!error); - } - else - { - // Reconstruct sideband data for block. - sideband_a->account = block_account_computed (transaction_a, hash_a); - sideband_a->balance = block_balance_computed (transaction_a, hash_a); - sideband_a->successor = block_successor (transaction_a, hash_a); - sideband_a->height = 0; - sideband_a->timestamp = 0; - } - } - } - return result; -} - void nano::mdb_store::block_del (nano::transaction const & transaction_a, nano::block_hash const & hash_a) { auto status (mdb_del (env.tx (transaction_a), state_blocks_v1, nano::mdb_val (hash_a), nullptr)); @@ -1791,112 +1313,15 @@ bool nano::mdb_store::block_exists (nano::transaction const & transaction_a, nan return exists; } -bool nano::mdb_store::block_exists (nano::transaction const & tx_a, nano::block_hash const & hash_a) -{ - // Table lookups are ordered by match probability - // clang-format off - return - block_exists (tx_a, nano::block_type::state, hash_a) || - block_exists (tx_a, nano::block_type::send, hash_a) || - block_exists (tx_a, nano::block_type::receive, hash_a) || - block_exists (tx_a, nano::block_type::open, hash_a) || - block_exists (tx_a, nano::block_type::change, hash_a); - // clang-format on -} - nano::block_counts nano::mdb_store::block_count (nano::transaction const & transaction_a) { nano::block_counts result; - MDB_stat send_stats; - auto status1 (mdb_stat (env.tx (transaction_a), send_blocks, &send_stats)); - release_assert (status1 == 0); - MDB_stat receive_stats; - auto status2 (mdb_stat (env.tx (transaction_a), receive_blocks, &receive_stats)); - release_assert (status2 == 0); - MDB_stat open_stats; - auto status3 (mdb_stat (env.tx (transaction_a), open_blocks, &open_stats)); - release_assert (status3 == 0); - MDB_stat change_stats; - auto status4 (mdb_stat (env.tx (transaction_a), change_blocks, &change_stats)); - release_assert (status4 == 0); - MDB_stat state_v0_stats; - auto status5 (mdb_stat (env.tx (transaction_a), state_blocks_v0, &state_v0_stats)); - release_assert (status5 == 0); - MDB_stat state_v1_stats; - auto status6 (mdb_stat (env.tx (transaction_a), state_blocks_v1, &state_v1_stats)); - release_assert (status6 == 0); - result.send = send_stats.ms_entries; - result.receive = receive_stats.ms_entries; - result.open = open_stats.ms_entries; - result.change = change_stats.ms_entries; - result.state_v0 = state_v0_stats.ms_entries; - result.state_v1 = state_v1_stats.ms_entries; - return result; -} - -bool nano::mdb_store::root_exists (nano::transaction const & transaction_a, nano::uint256_union const & root_a) -{ - return block_exists (transaction_a, root_a) || account_exists (transaction_a, root_a); -} - -bool nano::mdb_store::source_exists (nano::transaction const & transaction_a, nano::block_hash const & source_a) -{ - return block_exists (transaction_a, nano::block_type::state, source_a) || block_exists (transaction_a, nano::block_type::send, source_a); -} - -nano::account nano::mdb_store::block_account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - nano::block_sideband sideband; - auto block (block_get (transaction_a, hash_a, &sideband)); - nano::account result (block->account ()); - if (result.is_zero ()) - { - result = sideband.account; - } - assert (!result.is_zero ()); - return result; -} - -// Return account containing hash -nano::account nano::mdb_store::block_account_computed (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - assert (!full_sideband (transaction_a)); - nano::account result (0); - auto hash (hash_a); - while (result.is_zero ()) - { - auto block (block_get (transaction_a, hash)); - assert (block); - result = block->account (); - if (result.is_zero ()) - { - auto type (nano::block_type::invalid); - auto value (block_raw_get (transaction_a, block->previous (), type)); - if (entry_has_sideband (value, type)) - { - result = block_account (transaction_a, block->previous ()); - } - else - { - nano::block_info block_info; - if (!block_info_get (transaction_a, hash, block_info)) - { - result = block_info.account; - } - else - { - result = frontier_get (transaction_a, hash); - if (result.is_zero ()) - { - auto successor (block_successor (transaction_a, hash)); - assert (!successor.is_zero ()); - hash = successor; - } - } - } - } - } - assert (!result.is_zero ()); + result.send = count (transaction_a, send_blocks); + result.receive = count (transaction_a, receive_blocks); + result.open = count (transaction_a, open_blocks); + result.change = count (transaction_a, change_blocks); + result.state_v0 = count (transaction_a, state_blocks_v0); + result.state_v1 = count (transaction_a, state_blocks_v1); return result; } @@ -1911,12 +1336,6 @@ void nano::mdb_store::account_del (nano::transaction const & transaction_a, nano } } -bool nano::mdb_store::account_exists (nano::transaction const & transaction_a, nano::account const & account_a) -{ - auto iterator (latest_begin (transaction_a, account_a)); - return iterator != latest_end () && nano::account (iterator->first) == account_a; -} - bool nano::mdb_store::account_get (nano::transaction const & transaction_a, nano::account const & account_a, nano::account_info & info_a) { nano::mdb_val value; @@ -1977,14 +1396,7 @@ void nano::mdb_store::frontier_del (nano::transaction const & transaction_a, nan size_t nano::mdb_store::account_count (nano::transaction const & transaction_a) { - MDB_stat stats1; - auto status1 (mdb_stat (env.tx (transaction_a), accounts_v0, &stats1)); - release_assert (status1 == 0); - MDB_stat stats2; - auto status2 (mdb_stat (env.tx (transaction_a), accounts_v1, &stats2)); - release_assert (status2 == 0); - auto result (stats1.ms_entries + stats2.ms_entries); - return result; + return count (transaction_a, { accounts_v0, accounts_v1 }); } MDB_dbi nano::mdb_store::get_account_db (nano::epoch epoch_a) const @@ -2029,35 +1441,6 @@ void nano::mdb_store::account_put (nano::transaction const & transaction_a, nano release_assert (status == 0); } -void nano::mdb_store::confirmation_height_clear (nano::transaction const & transaction_a, nano::account const & account, nano::account_info const & account_info) -{ - nano::account_info info_copy (account_info); - if (info_copy.confirmation_height > 0) - { - info_copy.confirmation_height = 0; - account_put (transaction_a, account, info_copy); - } -} - -void nano::mdb_store::confirmation_height_clear (nano::transaction const & transaction_a) -{ - for (auto i (latest_begin (transaction_a)), n (latest_end ()); i != n; ++i) - { - confirmation_height_clear (transaction_a, i->first, i->second); - } -} - -uint64_t nano::mdb_store::cemented_count (nano::transaction const & transaction_a) -{ - uint64_t sum = 0; - for (auto i (latest_begin (transaction_a)), n (latest_end ()); i != n; ++i) - { - nano::account_info const & info (i->second); - sum += info.confirmation_height; - } - return sum; -} - void nano::mdb_store::pending_put (nano::transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_a) { auto status (mdb_put (env.tx (transaction_a), get_pending_db (pending_a.epoch), nano::mdb_val (key_a), nano::mdb_val (pending_a), 0)); @@ -2075,12 +1458,6 @@ void nano::mdb_store::pending_del (nano::transaction const & transaction_a, nano } } -bool nano::mdb_store::pending_exists (nano::transaction const & transaction_a, nano::pending_key const & key_a) -{ - auto iterator (pending_begin (transaction_a, key_a)); - return iterator != pending_end () && nano::pending_key (iterator->first) == key_a; -} - bool nano::mdb_store::pending_get (nano::transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info & pending_a) { nano::mdb_val value; @@ -2224,13 +1601,6 @@ void nano::mdb_store::unchecked_put (nano::transaction const & transaction_a, na release_assert (status == 0); } -void nano::mdb_store::unchecked_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, std::shared_ptr const & block_a) -{ - nano::unchecked_key key (hash_a, block_a->hash ()); - nano::unchecked_info info (block_a, block_a->account (), nano::seconds_since_epoch (), nano::signature_verification::unknown); - unchecked_put (transaction_a, key, info); -} - std::shared_ptr nano::mdb_store::vote_get (nano::transaction const & transaction_a, nano::account const & account_a) { nano::mdb_val value; @@ -2245,17 +1615,6 @@ std::shared_ptr nano::mdb_store::vote_get (nano::transaction const & return nullptr; } -std::vector nano::mdb_store::unchecked_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - std::vector result; - for (auto i (unchecked_begin (transaction_a, nano::unchecked_key (hash_a, 0))), n (unchecked_end ()); i != n && nano::block_hash (i->first.key ()) == hash_a; ++i) - { - nano::unchecked_info const & unchecked_info (i->second); - result.push_back (unchecked_info); - } - return result; -} - void nano::mdb_store::unchecked_del (nano::transaction const & transaction_a, nano::unchecked_key const & key_a) { auto status (mdb_del (env.tx (transaction_a), unchecked, nano::mdb_val (key_a), nullptr)); @@ -2264,11 +1623,15 @@ void nano::mdb_store::unchecked_del (nano::transaction const & transaction_a, na size_t nano::mdb_store::unchecked_count (nano::transaction const & transaction_a) { - MDB_stat unchecked_stats; - auto status (mdb_stat (env.tx (transaction_a), unchecked, &unchecked_stats)); + return count (transaction_a, unchecked); +} + +size_t nano::mdb_store::count (nano::transaction const & transaction_a, MDB_dbi db_a) const +{ + MDB_stat stats; + auto status (mdb_stat (env.tx (transaction_a), db_a, &stats)); release_assert (status == 0); - auto result (unchecked_stats.ms_entries); - return result; + return (stats.ms_entries); } void nano::mdb_store::online_weight_put (nano::transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) @@ -2295,10 +1658,7 @@ nano::store_iterator nano::mdb_store::online_weight_end size_t nano::mdb_store::online_weight_count (nano::transaction const & transaction_a) const { - MDB_stat online_weight_stats; - auto status1 (mdb_stat (env.tx (transaction_a), online_weight, &online_weight_stats)); - release_assert (status1 == 0); - return online_weight_stats.ms_entries; + return count (transaction_a, online_weight); } void nano::mdb_store::online_weight_clear (nano::transaction const & transaction_a) @@ -2325,62 +1685,15 @@ void nano::mdb_store::flush (nano::transaction const & transaction_a) release_assert (status1 == 0); } } -std::shared_ptr nano::mdb_store::vote_current (nano::transaction const & transaction_a, nano::account const & account_a) -{ - assert (!cache_mutex.try_lock ()); - std::shared_ptr result; - auto existing (vote_cache_l1.find (account_a)); - auto have_existing (true); - if (existing == vote_cache_l1.end ()) - { - existing = vote_cache_l2.find (account_a); - if (existing == vote_cache_l2.end ()) - { - have_existing = false; - } - } - if (have_existing) - { - result = existing->second; - } - else - { - result = vote_get (transaction_a, account_a); - } - return result; -} -std::shared_ptr nano::mdb_store::vote_generate (nano::transaction const & transaction_a, nano::account const & account_a, nano::raw_key const & key_a, std::shared_ptr block_a) +size_t nano::mdb_store::count (nano::transaction const & transaction_a, std::initializer_list dbs_a) const { - std::lock_guard lock (cache_mutex); - auto result (vote_current (transaction_a, account_a)); - uint64_t sequence ((result ? result->sequence : 0) + 1); - result = std::make_shared (account_a, key_a, sequence, block_a); - vote_cache_l1[account_a] = result; - return result; -} - -std::shared_ptr nano::mdb_store::vote_generate (nano::transaction const & transaction_a, nano::account const & account_a, nano::raw_key const & key_a, std::vector blocks_a) -{ - std::lock_guard lock (cache_mutex); - auto result (vote_current (transaction_a, account_a)); - uint64_t sequence ((result ? result->sequence : 0) + 1); - result = std::make_shared (account_a, key_a, sequence, blocks_a); - vote_cache_l1[account_a] = result; - return result; -} - -std::shared_ptr nano::mdb_store::vote_max (nano::transaction const & transaction_a, std::shared_ptr vote_a) -{ - std::lock_guard lock (cache_mutex); - auto current (vote_current (transaction_a, vote_a->account)); - auto result (vote_a); - if (current != nullptr && current->sequence > result->sequence) + size_t total_count = 0; + for (auto db : dbs_a) { - result = current; + total_count += count (transaction_a, db); } - vote_cache_l1[vote_a->account] = result; - return result; + return total_count; } nano::store_iterator nano::mdb_store::latest_begin (nano::transaction const & transaction_a, nano::account const & account_a) diff --git a/nano/node/lmdb.hpp b/nano/node/lmdb.hpp index f682348c..f62d3153 100644 --- a/nano/node/lmdb.hpp +++ b/nano/node/lmdb.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -18,7 +19,6 @@ namespace nano { class mdb_env; -class account_info_v13; class mdb_txn_callbacks { @@ -70,55 +70,7 @@ public: MDB_env * environment; }; -/** - * Encapsulates MDB_val and provides uint256_union conversion of the data. - */ -class mdb_val -{ -public: - mdb_val (nano::epoch = nano::epoch::unspecified); - mdb_val (nano::account_info const &); - mdb_val (nano::account_info_v13 const &); - mdb_val (nano::block_info const &); - mdb_val (MDB_val const &, nano::epoch = nano::epoch::unspecified); - mdb_val (nano::pending_info const &); - mdb_val (nano::pending_key const &); - mdb_val (nano::unchecked_info const &); - mdb_val (size_t, void *); - mdb_val (nano::uint128_union const &); - mdb_val (nano::uint256_union const &); - mdb_val (nano::endpoint_key const &); - mdb_val (std::shared_ptr const &); - mdb_val (std::shared_ptr const &); - mdb_val (uint64_t); - void * data () const; - size_t size () const; - explicit operator nano::account_info () const; - explicit operator nano::account_info_v13 () const; - explicit operator nano::block_info () const; - explicit operator nano::pending_info () const; - explicit operator nano::pending_key () const; - explicit operator nano::unchecked_info () const; - explicit operator nano::uint128_union () const; - explicit operator nano::uint256_union () const; - explicit operator std::array () const; - explicit operator nano::endpoint_key () const; - explicit operator nano::no_value () const; - explicit operator std::shared_ptr () const; - explicit operator std::shared_ptr () const; - explicit operator std::shared_ptr () const; - explicit operator std::shared_ptr () const; - explicit operator std::shared_ptr () const; - explicit operator std::shared_ptr () const; - explicit operator std::shared_ptr () const; - explicit operator uint64_t () const; - operator MDB_val * () const; - operator MDB_val const & () const; - MDB_val value; - std::shared_ptr> buffer; - nano::epoch epoch{ nano::epoch::unspecified }; -}; -class block_store; +using mdb_val = db_val; template class mdb_iterator : public store_iterator_impl @@ -177,28 +129,20 @@ class logging_mt; /** * mdb implementation of the block store */ -class mdb_store : public block_store +class mdb_store : public block_store_partial { - friend class nano::block_predecessor_set; - public: + using block_store_partial::block_exists; + using block_store_partial::unchecked_put; + mdb_store (bool &, nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512); nano::write_transaction tx_begin_write () override; nano::read_transaction tx_begin_read () override; - void initialize (nano::transaction const &, nano::genesis const &) override; - void block_put (nano::transaction const &, nano::block_hash const &, nano::block const &, nano::block_sideband const &, nano::epoch version = nano::epoch::epoch_0) override; - nano::block_hash block_successor (nano::transaction const &, nano::block_hash const &) const override; - void block_successor_clear (nano::transaction const &, nano::block_hash const &) override; - std::shared_ptr block_get (nano::transaction const &, nano::block_hash const &, nano::block_sideband * = nullptr) const override; std::shared_ptr block_random (nano::transaction const &) override; void block_del (nano::transaction const &, nano::block_hash const &) override; - bool block_exists (nano::transaction const &, nano::block_hash const &) override; bool block_exists (nano::transaction const &, nano::block_type, nano::block_hash const &) override; nano::block_counts block_count (nano::transaction const &) override; - bool root_exists (nano::transaction const &, nano::uint256_union const &) override; - bool source_exists (nano::transaction const &, nano::block_hash const &) override; - nano::account block_account (nano::transaction const &, nano::block_hash const &) const override; void frontier_put (nano::transaction const &, nano::block_hash const &, nano::account const &) override; nano::account frontier_get (nano::transaction const &, nano::block_hash const &) const override; @@ -207,11 +151,7 @@ public: void account_put (nano::transaction const &, nano::account const &, nano::account_info const &) override; bool account_get (nano::transaction const &, nano::account const &, nano::account_info &) override; void account_del (nano::transaction const &, nano::account const &) override; - bool account_exists (nano::transaction const &, nano::account const &) override; size_t account_count (nano::transaction const &) override; - void confirmation_height_clear (nano::transaction const &, nano::account const & account, nano::account_info const & account_info) override; - void confirmation_height_clear (nano::transaction const &) override; - uint64_t cemented_count (nano::transaction const &) override; nano::store_iterator latest_v0_begin (nano::transaction const &, nano::account const &) override; nano::store_iterator latest_v0_begin (nano::transaction const &) override; nano::store_iterator latest_v0_end () override; @@ -225,7 +165,6 @@ public: void pending_put (nano::transaction const &, nano::pending_key const &, nano::pending_info const &) override; void pending_del (nano::transaction const &, nano::pending_key const &) override; bool pending_get (nano::transaction const &, nano::pending_key const &, nano::pending_info &) override; - bool pending_exists (nano::transaction const &, nano::pending_key const &) override; nano::store_iterator pending_v0_begin (nano::transaction const &, nano::pending_key const &) override; nano::store_iterator pending_v0_begin (nano::transaction const &) override; nano::store_iterator pending_v0_end () override; @@ -237,19 +176,15 @@ public: nano::store_iterator pending_end () override; bool block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) const override; - nano::uint128_t block_balance (nano::transaction const &, nano::block_hash const &) override; nano::epoch block_version (nano::transaction const &, nano::block_hash const &) override; nano::uint128_t representation_get (nano::transaction const &, nano::account const &) override; void representation_put (nano::transaction const &, nano::account const &, nano::uint128_t const &) override; - void representation_add (nano::transaction const &, nano::account const &, nano::uint128_t const &) override; nano::store_iterator representation_begin (nano::transaction const &) override; nano::store_iterator representation_end () override; void unchecked_clear (nano::transaction const &) override; void unchecked_put (nano::transaction const &, nano::unchecked_key const &, nano::unchecked_info const &) override; - void unchecked_put (nano::transaction const &, nano::block_hash const &, std::shared_ptr const &) override; - std::vector unchecked_get (nano::transaction const &, nano::block_hash const &) override; void unchecked_del (nano::transaction const &, nano::unchecked_key const &) override; nano::store_iterator unchecked_begin (nano::transaction const &) override; nano::store_iterator unchecked_begin (nano::transaction const &, nano::unchecked_key const &) override; @@ -258,13 +193,6 @@ public: // Return latest vote for an account from store std::shared_ptr vote_get (nano::transaction const &, nano::account const &) override; - // Populate vote with the next sequence number - std::shared_ptr vote_generate (nano::transaction const &, nano::account const &, nano::raw_key const &, std::shared_ptr) override; - std::shared_ptr vote_generate (nano::transaction const &, nano::account const &, nano::raw_key const &, std::vector) override; - // Return either vote or the stored vote with a higher sequence number - std::shared_ptr vote_max (nano::transaction const &, std::shared_ptr) override; - // Return latest vote for an account considering the vote cache - std::shared_ptr vote_current (nano::transaction const &, nano::account const &) override; void flush (nano::transaction const &) override; nano::store_iterator> vote_begin (nano::transaction const &) override; nano::store_iterator> vote_end () override; @@ -276,10 +204,6 @@ public: size_t online_weight_count (nano::transaction const &) const override; void online_weight_clear (nano::transaction const &) override; - std::mutex cache_mutex; - std::unordered_map> vote_cache_l1; - std::unordered_map> vote_cache_l2; - void version_put (nano::transaction const &, int) override; int version_get (nano::transaction const &) const override; @@ -292,11 +216,7 @@ public: nano::store_iterator peers_begin (nano::transaction const & transaction_a) override; nano::store_iterator peers_end () override; - uint64_t block_account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - - bool full_sideband (nano::transaction const &) const; MDB_dbi get_account_db (nano::epoch epoch_a) const; - size_t block_successor_offset (nano::transaction const &, MDB_val, nano::block_type) const; void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) override; nano::logger_mt & logger; @@ -412,16 +332,11 @@ public: MDB_dbi peers{ 0 }; private: - nano::network_params network_params; - bool entry_has_sideband (MDB_val, nano::block_type) const; - nano::account block_account_computed (nano::transaction const &, nano::block_hash const &) const; - nano::uint128_t block_balance_computed (nano::transaction const &, nano::block_hash const &) const; MDB_dbi block_database (nano::block_type, nano::epoch); template std::shared_ptr block_random (nano::transaction const &, MDB_dbi); - MDB_val block_raw_get (nano::transaction const &, nano::block_hash const &, nano::block_type &) const; - boost::optional block_raw_get_by_type (nano::transaction const &, nano::block_hash const &, nano::block_type &) const; - void block_raw_put (nano::transaction const &, MDB_dbi, nano::block_hash const &, MDB_val); + boost::optional block_raw_get_by_type (nano::transaction const &, nano::block_hash const &, nano::block_type &) const override; + void block_raw_put (nano::transaction const & transaction_a, std::vector const & data, nano::block_type block_type_a, nano::epoch epoch_a, nano::block_hash const & hash_a) override; void clear (MDB_dbi); bool do_upgrades (nano::write_transaction &, size_t); void upgrade_v1_to_v2 (nano::transaction const &); @@ -443,6 +358,9 @@ private: nano::mdb_txn_callbacks create_txn_callbacks (); bool txn_tracking_enabled; static int constexpr version{ 14 }; + + size_t count (nano::transaction const &, MDB_dbi) const; + size_t count (nano::transaction const &, std::initializer_list) const; }; class wallet_value { diff --git a/nano/secure/blockstore.cpp b/nano/secure/blockstore.cpp index ec3dbdce..54ac9f75 100644 --- a/nano/secure/blockstore.cpp +++ b/nano/secure/blockstore.cpp @@ -365,30 +365,6 @@ void nano::representative_visitor::state_block (nano::state_block const & block_ result = block_a.hash (); } -nano::uint128_t nano::block_store::block_balance_calculated (std::shared_ptr block_a, nano::block_sideband const & sideband_a) const -{ - nano::uint128_t result; - switch (block_a->type ()) - { - case nano::block_type::open: - case nano::block_type::receive: - case nano::block_type::change: - result = sideband_a.balance.number (); - break; - case nano::block_type::send: - result = boost::polymorphic_downcast (block_a.get ())->hashables.balance.number (); - break; - case nano::block_type::state: - result = boost::polymorphic_downcast (block_a.get ())->hashables.balance.number (); - break; - case nano::block_type::invalid: - case nano::block_type::not_a_block: - release_assert (false); - break; - } - return result; -} - nano::read_transaction::read_transaction (std::unique_ptr read_transaction_impl) : impl (std::move (read_transaction_impl)) { diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index 6eda7428..86276b71 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -1,12 +1,304 @@ #pragma once #include +#include #include +#include + +#include +#include #include namespace nano { +// Generic container to be used when templated db types cannot +class DB_val +{ +public: + DB_val () = default; + DB_val (size_t size_a, void * data_a) : + size (size_a), + data (data_a) + { + } + + size_t size; + void * data; +}; + +/** + * Encapsulates database specific container and provides uint256_union conversion of the data. + */ +template +class db_val +{ +public: + db_val (nano::epoch epoch_a = nano::epoch::unspecified) : + value ({ 0, nullptr }), + epoch (epoch_a) + { + } + + db_val (Val const & value_a, nano::epoch epoch_a = nano::epoch::unspecified) : + value (value_a), + epoch (epoch_a) + { + } + + db_val (DB_val const & value_a, nano::epoch epoch_a = nano::epoch::unspecified); + + db_val (size_t size_a, void * data_a) : + value ({ size_a, data_a }) + { + } + + db_val (nano::uint128_union const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + } + + db_val (nano::uint256_union const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + } + + db_val (nano::account_info const & val_a) : + db_val (val_a.db_size (), const_cast (&val_a)) + { + } + + db_val (nano::account_info_v13 const & val_a) : + db_val (val_a.db_size (), const_cast (&val_a)) + { + } + + db_val (nano::pending_info const & val_a) : + db_val (sizeof (val_a.source) + sizeof (val_a.amount), const_cast (&val_a)) + { + static_assert (std::is_standard_layout::value, "Standard layout is required"); + } + + db_val (nano::pending_key const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + static_assert (std::is_standard_layout::value, "Standard layout is required"); + } + + db_val (nano::unchecked_info const & val_a) : + buffer (std::make_shared> ()) + { + { + nano::vectorstream stream (*buffer); + val_a.serialize (stream); + } + value = { buffer->size (), const_cast (buffer->data ()) }; + } + + db_val (nano::block_info const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + static_assert (std::is_standard_layout::value, "Standard layout is required"); + } + + db_val (nano::endpoint_key const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + static_assert (std::is_standard_layout::value, "Standard layout is required"); + } + + db_val (std::shared_ptr const & val_a) : + buffer (std::make_shared> ()) + { + { + nano::vectorstream stream (*buffer); + nano::serialize_block (stream, *val_a); + } + value = { buffer->size (), const_cast (buffer->data ()) }; + } + + db_val (uint64_t val_a) : + buffer (std::make_shared> ()) + { + { + boost::endian::native_to_big_inplace (val_a); + nano::vectorstream stream (*buffer); + nano::write (stream, val_a); + } + value = { buffer->size (), const_cast (buffer->data ()) }; + } + + explicit operator nano::account_info () const + { + nano::account_info result; + result.epoch = epoch; + assert (size () == result.db_size ()); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); + return result; + } + + explicit operator nano::account_info_v13 () const + { + nano::account_info_v13 result; + result.epoch = epoch; + assert (size () == result.db_size ()); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); + return result; + } + + explicit operator nano::block_info () const + { + nano::block_info result; + assert (size () == sizeof (result)); + static_assert (sizeof (nano::block_info::account) + sizeof (nano::block_info::balance) == sizeof (result), "Packed class"); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), reinterpret_cast (&result)); + return result; + } + + explicit operator nano::pending_info () const + { + nano::pending_info result; + result.epoch = epoch; + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (nano::pending_info::source) + sizeof (nano::pending_info::amount), reinterpret_cast (&result)); + return result; + } + + explicit operator nano::pending_key () const + { + nano::pending_key result; + assert (size () == sizeof (result)); + static_assert (sizeof (nano::pending_key::account) + sizeof (nano::pending_key::hash) == sizeof (result), "Packed class"); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), reinterpret_cast (&result)); + return result; + } + + explicit operator nano::unchecked_info () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + nano::unchecked_info result; + bool error (result.deserialize (stream)); + assert (!error); + return result; + } + + explicit operator nano::uint128_union () const + { + nano::uint128_union result; + assert (size () == sizeof (result)); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), result.bytes.data ()); + return result; + } + + explicit operator nano::uint256_union () const + { + nano::uint256_union result; + assert (size () == sizeof (result)); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), result.bytes.data ()); + return result; + } + + explicit operator std::array () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + std::array result; + auto error = nano::try_read (stream, result); + assert (!error); + return result; + } + + explicit operator nano::endpoint_key () const + { + nano::endpoint_key result; + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), reinterpret_cast (&result)); + return result; + } + + explicit operator nano::no_value () const + { + return no_value::dummy; + } + + explicit operator std::shared_ptr () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + std::shared_ptr result (nano::deserialize_block (stream)); + return result; + } + + template + std::shared_ptr convert_to_block () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + auto error (false); + auto result (std::make_shared (error, stream)); + assert (!error); + return result; + } + + explicit operator std::shared_ptr () const + { + return convert_to_block (); + } + + explicit operator std::shared_ptr () const + { + return convert_to_block (); + } + + explicit operator std::shared_ptr () const + { + return convert_to_block (); + } + + explicit operator std::shared_ptr () const + { + return convert_to_block (); + } + + explicit operator std::shared_ptr () const + { + return convert_to_block (); + } + + explicit operator std::shared_ptr () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + auto error (false); + auto result (nano::make_shared (error, stream)); + assert (!error); + return result; + } + + explicit operator uint64_t () const + { + uint64_t result; + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + auto error (nano::try_read (stream, result)); + assert (!error); + boost::endian::big_to_native_inplace (result); + return result; + } + + operator Val * () const + { + // Allow passing a temporary to a non-c++ function which doesn't have constness + return const_cast (&value); + } + + operator Val const & () const + { + return value; + } + + /** Must be specialized in the sub-class */ + void * data () const; + size_t size () const; + + Val value; + std::shared_ptr> buffer; + nano::epoch epoch{ nano::epoch::unspecified }; +}; + class block_sideband final { public: @@ -186,8 +478,6 @@ private: std::unique_ptr> impl; }; -class block_predecessor_set; - class transaction_impl { public: @@ -308,7 +598,7 @@ public: virtual bool block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) const = 0; virtual nano::uint128_t block_balance (nano::transaction const &, nano::block_hash const &) = 0; - nano::uint128_t block_balance_calculated (std::shared_ptr, nano::block_sideband const &) const; + virtual nano::uint128_t block_balance_calculated (std::shared_ptr, nano::block_sideband const &) const = 0; virtual nano::epoch block_version (nano::transaction const &, nano::block_hash const &) = 0; virtual nano::uint128_t representation_get (nano::transaction const &, nano::account const &) = 0; @@ -367,4 +657,472 @@ public: /** Start read-only transaction */ virtual nano::read_transaction tx_begin_read () = 0; }; + +template +class block_predecessor_set; + +/** This base class implements the block_store interface functions which have DB agnostic functionality */ +template +class block_store_partial : public block_store +{ +public: + using block_store::block_exists; + using block_store::unchecked_put; + + friend class nano::block_predecessor_set; + + std::mutex cache_mutex; + + /** + * If using a different store version than the latest then you may need + * to modify some of the objects in the store to be appropriate for the version before an upgrade. + */ + void initialize (nano::transaction const & transaction_a, nano::genesis const & genesis_a) override + { + auto hash_l (genesis_a.hash ()); + assert (latest_v0_begin (transaction_a) == latest_v0_end ()); + assert (latest_v1_begin (transaction_a) == latest_v1_end ()); + nano::block_sideband sideband (nano::block_type::open, network_params.ledger.genesis_account, 0, network_params.ledger.genesis_amount, 1, nano::seconds_since_epoch ()); + block_put (transaction_a, hash_l, *genesis_a.open, sideband); + account_put (transaction_a, network_params.ledger.genesis_account, { hash_l, genesis_a.open->hash (), genesis_a.open->hash (), std::numeric_limits::max (), nano::seconds_since_epoch (), 1, 1, nano::epoch::epoch_0 }); + representation_put (transaction_a, network_params.ledger.genesis_account, std::numeric_limits::max ()); + frontier_put (transaction_a, hash_l, network_params.ledger.genesis_account); + } + + nano::uint128_t block_balance (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override + { + nano::block_sideband sideband; + auto block (block_get (transaction_a, hash_a, &sideband)); + nano::uint128_t result (block_balance_calculated (block, sideband)); + return result; + } + + void representation_add (nano::transaction const & transaction_a, nano::block_hash const & source_a, nano::uint128_t const & amount_a) override + { + auto source_block (block_get (transaction_a, source_a)); + assert (source_block != nullptr); + auto source_rep (source_block->representative ()); + auto source_previous (representation_get (transaction_a, source_rep)); + representation_put (transaction_a, source_rep, source_previous + amount_a); + } + + bool account_exists (nano::transaction const & transaction_a, nano::account const & account_a) override + { + auto iterator (latest_begin (transaction_a, account_a)); + return iterator != latest_end () && nano::account (iterator->first) == account_a; + } + + void confirmation_height_clear (nano::transaction const & transaction_a, nano::account const & account, nano::account_info const & account_info) override + { + nano::account_info info_copy (account_info); + if (info_copy.confirmation_height > 0) + { + info_copy.confirmation_height = 0; + account_put (transaction_a, account, info_copy); + } + } + + void confirmation_height_clear (nano::transaction const & transaction_a) override + { + for (auto i (latest_begin (transaction_a)), n (latest_end ()); i != n; ++i) + { + confirmation_height_clear (transaction_a, i->first, i->second); + } + } + + bool pending_exists (nano::transaction const & transaction_a, nano::pending_key const & key_a) override + { + auto iterator (pending_begin (transaction_a, key_a)); + return iterator != pending_end () && nano::pending_key (iterator->first) == key_a; + } + + std::vector unchecked_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override + { + std::vector result; + for (auto i (unchecked_begin (transaction_a, nano::unchecked_key (hash_a, 0))), n (unchecked_end ()); i != n && nano::block_hash (i->first.key ()) == hash_a; ++i) + { + nano::unchecked_info const & unchecked_info (i->second); + result.push_back (unchecked_info); + } + return result; + } + + void block_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a, nano::block_sideband const & sideband_a, nano::epoch epoch_a = nano::epoch::epoch_0) override + { + assert (block_a.type () == sideband_a.type); + assert (sideband_a.successor.is_zero () || block_exists (transaction_a, sideband_a.successor)); + std::vector vector; + { + nano::vectorstream stream (vector); + block_a.serialize (stream); + sideband_a.serialize (stream); + } + block_raw_put (transaction_a, vector, block_a.type (), epoch_a, hash_a); + nano::block_predecessor_set predecessor (transaction_a, *this); + block_a.visit (predecessor); + assert (block_a.previous ().is_zero () || block_successor (transaction_a, block_a.previous ()) == hash_a); + } + + // Converts a block hash to a block height + uint64_t block_account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override + { + nano::block_sideband sideband; + auto block = block_get (transaction_a, hash_a, &sideband); + assert (block != nullptr); + return sideband.height; + } + + std::shared_ptr block_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband * sideband_a = nullptr) const override + { + nano::block_type type; + auto value (block_raw_get (transaction_a, hash_a, type)); + std::shared_ptr result; + if (value.size () != 0) + { + nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + result = nano::deserialize_block (stream, type); + assert (result != nullptr); + if (sideband_a) + { + sideband_a->type = type; + if (full_sideband (transaction_a) || entry_has_sideband (value.size (), type)) + { + auto error (sideband_a->deserialize (stream)); + assert (!error); + } + else + { + // Reconstruct sideband data for block. + sideband_a->account = block_account_computed (transaction_a, hash_a); + sideband_a->balance = block_balance_computed (transaction_a, hash_a); + sideband_a->successor = block_successor (transaction_a, hash_a); + sideband_a->height = 0; + sideband_a->timestamp = 0; + } + } + } + return result; + } + + bool block_exists (nano::transaction const & tx_a, nano::block_hash const & hash_a) override + { + // Table lookups are ordered by match probability + // clang-format off + return + block_exists (tx_a, nano::block_type::state, hash_a) || + block_exists (tx_a, nano::block_type::send, hash_a) || + block_exists (tx_a, nano::block_type::receive, hash_a) || + block_exists (tx_a, nano::block_type::open, hash_a) || + block_exists (tx_a, nano::block_type::change, hash_a); + // clang-format on + } + + bool root_exists (nano::transaction const & transaction_a, nano::uint256_union const & root_a) override + { + return block_exists (transaction_a, root_a) || account_exists (transaction_a, root_a); + } + + bool source_exists (nano::transaction const & transaction_a, nano::block_hash const & source_a) override + { + return block_exists (transaction_a, nano::block_type::state, source_a) || block_exists (transaction_a, nano::block_type::send, source_a); + } + + nano::account block_account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override + { + nano::block_sideband sideband; + auto block (block_get (transaction_a, hash_a, &sideband)); + nano::account result (block->account ()); + if (result.is_zero ()) + { + result = sideband.account; + } + assert (!result.is_zero ()); + return result; + } + + nano::uint128_t block_balance_calculated (std::shared_ptr block_a, nano::block_sideband const & sideband_a) const override + { + nano::uint128_t result; + switch (block_a->type ()) + { + case nano::block_type::open: + case nano::block_type::receive: + case nano::block_type::change: + result = sideband_a.balance.number (); + break; + case nano::block_type::send: + result = boost::polymorphic_downcast (block_a.get ())->hashables.balance.number (); + break; + case nano::block_type::state: + result = boost::polymorphic_downcast (block_a.get ())->hashables.balance.number (); + break; + case nano::block_type::invalid: + case nano::block_type::not_a_block: + release_assert (false); + break; + } + return result; + } + + nano::block_hash block_successor (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override + { + nano::block_type type; + auto value (block_raw_get (transaction_a, hash_a, type)); + nano::block_hash result; + if (value.size () != 0) + { + assert (value.size () >= result.bytes.size ()); + nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset (transaction_a, value.size (), type), result.bytes.size ()); + auto error (nano::try_read (stream, result.bytes)); + assert (!error); + } + else + { + result.clear (); + } + return result; + } + + bool full_sideband (nano::transaction const & transaction_a) const + { + return version_get (transaction_a) > 12; + } + + void block_successor_clear (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override + { + nano::block_type type; + auto value (block_raw_get (transaction_a, hash_a, type)); + auto version (block_version (transaction_a, hash_a)); + assert (value.size () != 0); + std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); + std::fill_n (data.begin () + block_successor_offset (transaction_a, value.size (), type), sizeof (nano::uint256_union), 0); + block_raw_put (transaction_a, data, type, version, hash_a); + } + + uint64_t cemented_count (nano::transaction const & transaction_a) override + { + uint64_t sum = 0; + for (auto i (latest_begin (transaction_a)), n (latest_end ()); i != n; ++i) + { + nano::account_info const & info (i->second); + sum += info.confirmation_height; + } + return sum; + } + + void unchecked_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, std::shared_ptr const & block_a) override + { + nano::unchecked_key key (hash_a, block_a->hash ()); + nano::unchecked_info info (block_a, block_a->account (), nano::seconds_since_epoch (), nano::signature_verification::unknown); + unchecked_put (transaction_a, key, info); + } + + std::shared_ptr vote_current (nano::transaction const & transaction_a, nano::account const & account_a) override + { + assert (!cache_mutex.try_lock ()); + std::shared_ptr result; + auto existing (vote_cache_l1.find (account_a)); + auto have_existing (true); + if (existing == vote_cache_l1.end ()) + { + existing = vote_cache_l2.find (account_a); + if (existing == vote_cache_l2.end ()) + { + have_existing = false; + } + } + if (have_existing) + { + result = existing->second; + } + else + { + result = vote_get (transaction_a, account_a); + } + return result; + } + + std::shared_ptr vote_generate (nano::transaction const & transaction_a, nano::account const & account_a, nano::raw_key const & key_a, std::shared_ptr block_a) override + { + std::lock_guard lock (cache_mutex); + auto result (vote_current (transaction_a, account_a)); + uint64_t sequence ((result ? result->sequence : 0) + 1); + result = std::make_shared (account_a, key_a, sequence, block_a); + vote_cache_l1[account_a] = result; + return result; + } + + std::shared_ptr vote_generate (nano::transaction const & transaction_a, nano::account const & account_a, nano::raw_key const & key_a, std::vector blocks_a) override + { + std::lock_guard lock (cache_mutex); + auto result (vote_current (transaction_a, account_a)); + uint64_t sequence ((result ? result->sequence : 0) + 1); + result = std::make_shared (account_a, key_a, sequence, blocks_a); + vote_cache_l1[account_a] = result; + return result; + } + + std::shared_ptr vote_max (nano::transaction const & transaction_a, std::shared_ptr vote_a) override + { + std::lock_guard lock (cache_mutex); + auto current (vote_current (transaction_a, vote_a->account)); + auto result (vote_a); + if (current != nullptr && current->sequence > result->sequence) + { + result = current; + } + vote_cache_l1[vote_a->account] = result; + return result; + } + + virtual void block_raw_put (nano::transaction const & transaction_a, std::vector const & data, nano::block_type block_type_a, nano::epoch epoch_a, nano::block_hash const & hash_a) = 0; + +protected: + nano::network_params network_params; + std::unordered_map> vote_cache_l1; + std::unordered_map> vote_cache_l2; + + bool entry_has_sideband (size_t entry_size_a, nano::block_type type_a) const + { + return entry_size_a == nano::block::size (type_a) + nano::block_sideband::size (type_a); + } + + nano::db_val block_raw_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const + { + nano::db_val result; + // Table lookups are ordered by match probability + nano::block_type block_types[]{ nano::block_type::state, nano::block_type::send, nano::block_type::receive, nano::block_type::open, nano::block_type::change }; + for (auto current_type : block_types) + { + auto db_val (block_raw_get_by_type (transaction_a, hash_a, current_type)); + if (db_val.is_initialized ()) + { + type_a = current_type; + result = db_val.get (); + break; + } + } + + return result; + } + + // Return account containing hash + nano::account block_account_computed (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const + { + assert (!full_sideband (transaction_a)); + nano::account result (0); + auto hash (hash_a); + while (result.is_zero ()) + { + auto block (block_get (transaction_a, hash)); + assert (block); + result = block->account (); + if (result.is_zero ()) + { + auto type (nano::block_type::invalid); + auto value (block_raw_get (transaction_a, block->previous (), type)); + if (entry_has_sideband (value.size (), type)) + { + result = block_account (transaction_a, block->previous ()); + } + else + { + nano::block_info block_info; + if (!block_info_get (transaction_a, hash, block_info)) + { + result = block_info.account; + } + else + { + result = frontier_get (transaction_a, hash); + if (result.is_zero ()) + { + auto successor (block_successor (transaction_a, hash)); + assert (!successor.is_zero ()); + hash = successor; + } + } + } + } + } + assert (!result.is_zero ()); + return result; + } + + nano::uint128_t block_balance_computed (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const + { + assert (!full_sideband (transaction_a)); + summation_visitor visitor (transaction_a, *this); + return visitor.compute_balance (hash_a); + } + + size_t block_successor_offset (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const + { + size_t result; + if (full_sideband (transaction_a) || entry_has_sideband (entry_size_a, type_a)) + { + result = entry_size_a - nano::block_sideband::size (type_a); + } + else + { + // Read old successor-only sideband + assert (entry_size_a == nano::block::size (type_a) + sizeof (nano::uint256_union)); + result = entry_size_a - sizeof (nano::uint256_union); + } + return result; + } + + virtual boost::optional block_raw_get_by_type (nano::transaction const &, nano::block_hash const &, nano::block_type &) const = 0; +}; + +/** + * Fill in our predecessors + */ +template +class block_predecessor_set : public nano::block_visitor +{ +public: + block_predecessor_set (nano::transaction const & transaction_a, nano::block_store_partial & store_a) : + transaction (transaction_a), + store (store_a) + { + } + virtual ~block_predecessor_set () = default; + void fill_value (nano::block const & block_a) + { + auto hash (block_a.hash ()); + nano::block_type type; + auto value (store.block_raw_get (transaction, block_a.previous (), type)); + auto version (store.block_version (transaction, block_a.previous ())); + assert (value.size () != 0); + std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); + std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + store.block_successor_offset (transaction, value.size (), type)); + store.block_raw_put (transaction, data, type, version, block_a.previous ()); + } + void send_block (nano::send_block const & block_a) override + { + fill_value (block_a); + } + void receive_block (nano::receive_block const & block_a) override + { + fill_value (block_a); + } + void open_block (nano::open_block const & block_a) override + { + // Open blocks don't have a predecessor + } + void change_block (nano::change_block const & block_a) override + { + fill_value (block_a); + } + void state_block (nano::state_block const & block_a) override + { + if (!block_a.previous ().is_zero ()) + { + fill_value (block_a); + } + } + nano::transaction const & transaction; + nano::block_store_partial & store; +}; }