Minecon724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 1

Preparation for new db backends (#2177)

Browse source
This commit is contained in:
Wesley Shillingford 2019-07-29 16:35:35 +01:00 • committed by GitHub
commit 27371d43f8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 3724 additions and 1299 deletions
Download patch file Download diff file Expand all files Collapse all files

682
nano/core_test/block_store.cpp
View file

File diff suppressed because it is too large Load diff

884
nano/core_test/ledger.cpp
View file

File diff suppressed because it is too large Load diff

20
nano/core_test/node.cpp
View file

@ -2041,7 +2041,7 @@ TEST (node, vote_replay)
}
{
auto transaction (system.nodes[0]->store.tx_begin_read ());
std::lock_guard<std::mutex> lock (boost::polymorphic_downcast<nano::mdb_store *> (system.nodes[0]->store_impl.get ())->cache_mutex);
std::lock_guard<std::mutex> lock (system.nodes[0]->store.get_cache_mutex ());
auto vote (system.nodes[0]->store.vote_current (transaction, nano::test_genesis_key.pub));
ASSERT_EQ (nullptr, vote);
}
@ -2054,7 +2054,7 @@ TEST (node, vote_replay)
{
auto ec = system.poll ();
auto transaction (system.nodes[0]->store.tx_begin_read ());
std::lock_guard<std::mutex> lock (boost::polymorphic_downcast<nano::mdb_store *> (system.nodes[0]->store_impl.get ())->cache_mutex);
std::lock_guard<std::mutex> lock (system.nodes[0]->store.get_cache_mutex ());
auto vote (system.nodes[0]->store.vote_current (transaction, nano::test_genesis_key.pub));
done = vote && (vote->sequence >= 10000);
ASSERT_NO_ERROR (ec);
@ -2284,7 +2284,7 @@ TEST (node, local_votes_cache)
node.network.process_message (message2, channel);
}
{
std::lock_guard<std::mutex> lock (boost::polymorphic_downcast<nano::mdb_store *> (node.store_impl.get ())->cache_mutex);
std::lock_guard<std::mutex> lock (node.store.get_cache_mutex ());
auto transaction (node.store.tx_begin_read ());
auto current_vote (node.store.vote_current (transaction, nano::test_genesis_key.pub));
ASSERT_EQ (current_vote->sequence, 2);
@ -2300,7 +2300,7 @@ TEST (node, local_votes_cache)
node.network.process_message (message3, channel);
}
{
std::lock_guard<std::mutex> lock (boost::polymorphic_downcast<nano::mdb_store *> (node.store_impl.get ())->cache_mutex);
std::lock_guard<std::mutex> lock (node.store.get_cache_mutex ());
auto transaction (node.store.tx_begin_read ());
auto current_vote (node.store.vote_current (transaction, nano::test_genesis_key.pub));
ASSERT_EQ (current_vote->sequence, 3);
@ -2334,7 +2334,7 @@ TEST (node, local_votes_cache_generate_new_vote)
ASSERT_EQ (1, votes1[0]->blocks.size ());
ASSERT_EQ (send1->hash (), boost::get<nano::block_hash> (votes1[0]->blocks[0]));
{
std::lock_guard<std::mutex> lock (boost::polymorphic_downcast<nano::mdb_store *> (node.store_impl.get ())->cache_mutex);
std::lock_guard<std::mutex> lock (node.store.get_cache_mutex ());
auto transaction (node.store.tx_begin_read ());
auto current_vote (node.store.vote_current (transaction, nano::test_genesis_key.pub));
ASSERT_EQ (current_vote->sequence, 1);
@ -2355,7 +2355,7 @@ TEST (node, local_votes_cache_generate_new_vote)
ASSERT_EQ (1, votes2.size ());
ASSERT_EQ (2, votes2[0]->blocks.size ());
{
std::lock_guard<std::mutex> lock (boost::polymorphic_downcast<nano::mdb_store *> (node.store_impl.get ())->cache_mutex);
std::lock_guard<std::mutex> lock (node.store.get_cache_mutex ());
auto transaction (node.store.tx_begin_read ());
auto current_vote (node.store.vote_current (transaction, nano::test_genesis_key.pub));
ASSERT_EQ (current_vote->sequence, 2);
@ -2869,10 +2869,14 @@ TEST (node, peer_cache_restart)
// Restart node
{
nano::node_init init;
auto node (std::make_shared<nano::node> (init, system.io_ctx, 24002, path, system.alarm, system.logging, system.work));
nano::node_flags node_flags;
node_flags.read_only = true;
auto node (std::make_shared<nano::node> (init, system.io_ctx, 24002, path, system.alarm, system.logging, system.work, node_flags));
system.nodes.push_back (node);
// Check cached peers after restart
node->start ();
node->network.start ();
node->add_initial_peers ();
auto & store = node->store;
{
auto transaction (store.tx_begin_read ());

22
nano/core_test/processor_service.cpp
View file

@ -12,16 +12,16 @@ TEST (processor_service, bad_send_signature)
{
nano::logger_mt logger;
bool init (false);
nano::mdb_store store (init, logger, nano::unique_path ());
auto store = nano::make_store (init, logger, nano::unique_path ());
ASSERT_FALSE (init);
nano::stat stats;
nano::ledger ledger (store, stats);
nano::ledger ledger (*store, stats);
nano::genesis genesis;
auto transaction (store.tx_begin_write ());
store.initialize (transaction, genesis);
auto transaction (store->tx_begin_write ());
store->initialize (transaction, genesis);
nano::work_pool pool (std::numeric_limits<unsigned>::max ());
nano::account_info info1;
ASSERT_FALSE (store.account_get (transaction, nano::test_genesis_key.pub, info1));
ASSERT_FALSE (store->account_get (transaction, nano::test_genesis_key.pub, info1));
nano::keypair key2;
nano::send_block send (info1.head, nano::test_genesis_key.pub, 50, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (info1.head));
send.signature.bytes[32] ^= 0x1;
@ -32,21 +32,21 @@ TEST (processor_service, bad_receive_signature)
{
nano::logger_mt logger;
bool init (false);
nano::mdb_store store (init, logger, nano::unique_path ());
auto store = nano::make_store (init, logger, nano::unique_path ());
ASSERT_FALSE (init);
nano::stat stats;
nano::ledger ledger (store, stats);
nano::ledger ledger (*store, stats);
nano::genesis genesis;
auto transaction (store.tx_begin_write ());
store.initialize (transaction, genesis);
auto transaction (store->tx_begin_write ());
store->initialize (transaction, genesis);
nano::work_pool pool (std::numeric_limits<unsigned>::max ());
nano::account_info info1;
ASSERT_FALSE (store.account_get (transaction, nano::test_genesis_key.pub, info1));
ASSERT_FALSE (store->account_get (transaction, nano::test_genesis_key.pub, info1));
nano::send_block send (info1.head, nano::test_genesis_key.pub, 50, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (info1.head));
nano::block_hash hash1 (send.hash ());
ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send).code);
nano::account_info info2;
ASSERT_FALSE (store.account_get (transaction, nano::test_genesis_key.pub, info2));
ASSERT_FALSE (store->account_get (transaction, nano::test_genesis_key.pub, info2));
nano::receive_block receive (hash1, hash1, nano::test_genesis_key.prv, nano::test_genesis_key.pub, pool.generate (hash1));
receive.signature.bytes[32] ^= 0x1;
ASSERT_EQ (nano::process_result::bad_signature, ledger.process (transaction, receive).code);

2
nano/core_test/versioning.cpp
View file

@ -1,5 +1,5 @@
#include <nano/lib/logger_mt.hpp>
#include <nano/node/lmdb.hpp>
#include <nano/node/lmdb/lmdb.hpp>
#include <nano/secure/blockstore.hpp>
#include <nano/secure/versioning.hpp>

1
nano/core_test/wallet.cpp
View file

@ -1,5 +1,6 @@
#include <nano/core_test/testutil.hpp>
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/node/lmdb/wallet_value.hpp>
#include <nano/node/testing.hpp>
#include <gtest/gtest.h>

2
nano/core_test/wallets.cpp
View file

@ -99,7 +99,7 @@ TEST (wallets, upgrade)
nano::account_info info;
ASSERT_FALSE (mdb_store.account_get (transaction_destination, nano::genesis_account, info));
nano::account_info_v13 account_info_v13 (info.head, info.rep_block, info.open_block, info.balance, info.modified, info.block_count, info.epoch);
auto status (mdb_put (mdb_store.env.tx (transaction_destination), mdb_store.get_account_db (info.epoch), nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (account_info_v13), 0));
auto status (mdb_put (mdb_store.env.tx (transaction_destination), mdb_store.get_account_db (info.epoch) == nano::block_store_partial<MDB_val, nano::mdb_store>::tables::accounts_v0 ? mdb_store.accounts_v0 : mdb_store.accounts_v1, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (account_info_v13), 0));
(void)status;
assert (status == 0);
}

13
nano/node/CMakeLists.txt
View file

@ -49,10 +49,15 @@ add_library (node
json_handler.cpp
json_payment_observer.hpp
json_payment_observer.cpp
lmdb.hpp
lmdb.cpp
lmdb_txn_tracker.hpp
lmdb_txn_tracker.cpp
lmdb/lmdb.hpp
lmdb/lmdb.cpp
lmdb/lmdb_env.hpp
lmdb/lmdb_env.cpp
lmdb/lmdb_iterator.hpp
lmdb/lmdb_txn.hpp
lmdb/lmdb_txn.cpp
lmdb/wallet_value.hpp
lmdb/wallet_value.cpp
logging.hpp
logging.cpp
network.hpp

617
nano/node/lmdb/lmdb.cpp Normal file
View file

@ -0,0 +1,617 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/utility.hpp>
#include <nano/node/common.hpp>
#include <nano/node/lmdb/lmdb.hpp>
#include <nano/node/lmdb/lmdb_iterator.hpp>
#include <nano/node/lmdb/wallet_value.hpp>
#include <nano/secure/versioning.hpp>
#include <boost/endian/conversion.hpp>
#include <boost/polymorphic_cast.hpp>
#include <queue>
namespace nano
{
template <>
void * mdb_val::data () const
{
return value.mv_data;
}
template <>
size_t mdb_val::size () const
{
return value.mv_size;
}
template <>
mdb_val::db_val (size_t size_a, void * data_a, nano::epoch epoch_a) :
value ({ size_a, data_a }),
epoch (epoch_a)
{
}
template <>
void mdb_val::convert_buffer_to_value ()
{
value = { buffer->size (), const_cast<uint8_t *> (buffer->data ()) };
}
}
nano::mdb_store::mdb_store (bool & error_a, nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, bool drop_unchecked, size_t const batch_size) :
logger (logger_a),
env (error_a, path_a, lmdb_max_dbs, true),
mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a),
txn_tracking_enabled (txn_tracking_config_a.enable)
{
if (!error_a)
{
auto is_fully_upgraded (false);
{
auto transaction (tx_begin_read ());
auto err = mdb_dbi_open (env.tx (transaction), "meta", 0, &meta);
if (err == MDB_SUCCESS)
{
is_fully_upgraded = (version_get (transaction) == version);
mdb_dbi_close (env, meta);
}
}
// Only open a write lock when upgrades are needed. This is because CLI commands
// open inactive nodes which can otherwise be locked here if there is a long write
// (can be a few minutes with the --fastbootstrap flag for instance)
if (!is_fully_upgraded)
{
auto transaction (tx_begin_write ());
open_databases (error_a, transaction, MDB_CREATE);
if (!error_a)
{
error_a |= do_upgrades (transaction, batch_size);
}
}
else
{
auto transaction (tx_begin_read ());
open_databases (error_a, transaction, 0);
}
if (!error_a && drop_unchecked)
{
auto transaction (tx_begin_write ());
unchecked_clear (transaction);
}
}
}
void nano::mdb_store::serialize_mdb_tracker (boost::property_tree::ptree & json, std::chrono::milliseconds min_read_time, std::chrono::milliseconds min_write_time)
{
mdb_txn_tracker.serialize_json (json, min_read_time, min_write_time);
}
nano::write_transaction nano::mdb_store::tx_begin_write ()
{
return env.tx_begin_write (create_txn_callbacks ());
}
nano::read_transaction nano::mdb_store::tx_begin_read ()
{
return env.tx_begin_read (create_txn_callbacks ());
}
nano::mdb_txn_callbacks nano::mdb_store::create_txn_callbacks ()
{
nano::mdb_txn_callbacks mdb_txn_callbacks;
if (txn_tracking_enabled)
{
// clang-format off
mdb_txn_callbacks.txn_start = ([&mdb_txn_tracker = mdb_txn_tracker](const nano::transaction_impl * transaction_impl) {
mdb_txn_tracker.add (transaction_impl);
});
mdb_txn_callbacks.txn_end = ([&mdb_txn_tracker = mdb_txn_tracker](const nano::transaction_impl * transaction_impl) {
mdb_txn_tracker.erase (transaction_impl);
});
// clang-format on
}
return mdb_txn_callbacks;
}
void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & transaction_a, unsigned flags)
{
error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontiers) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts", flags, &accounts_v0) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts_v1", flags, &accounts_v1) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "send", flags, &send_blocks) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "receive", flags, &receive_blocks) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "open", flags, &open_blocks) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "change", flags, &change_blocks) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "state", flags, &state_blocks_v0) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "state_v1", flags, &state_blocks_v1) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "pending", flags, &pending_v0) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "pending_v1", flags, &pending_v1) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "representation", flags, &representation) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "unchecked", flags, &unchecked) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "vote", flags, &vote) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &meta) != 0;
error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peers) != 0;
if (!full_sideband (transaction_a))
{
error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks_info", flags, &blocks_info) != 0;
}
}
bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, size_t batch_size)
{
auto error (false);
auto version_l = version_get (transaction_a);
switch (version_l)
{
case 1:
upgrade_v1_to_v2 (transaction_a);
case 2:
upgrade_v2_to_v3 (transaction_a);
case 3:
upgrade_v3_to_v4 (transaction_a);
case 4:
upgrade_v4_to_v5 (transaction_a);
case 5:
upgrade_v5_to_v6 (transaction_a);
case 6:
upgrade_v6_to_v7 (transaction_a);
case 7:
upgrade_v7_to_v8 (transaction_a);
case 8:
upgrade_v8_to_v9 (transaction_a);
case 9:
case 10:
upgrade_v10_to_v11 (transaction_a);
case 11:
upgrade_v11_to_v12 (transaction_a);
case 12:
upgrade_v12_to_v13 (transaction_a, batch_size);
case 13:
upgrade_v13_to_v14 (transaction_a);
case 14:
break;
default:
logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l));
error = true;
break;
}
return error;
}
void nano::mdb_store::upgrade_v1_to_v2 (nano::transaction const & transaction_a)
{
version_put (transaction_a, 2);
nano::account account (1);
while (!account.is_zero ())
{
nano::mdb_iterator<nano::uint256_union, nano::account_info_v1> i (transaction_a, accounts_v0, nano::mdb_val (account));
std::cerr << std::hex;
if (i != nano::mdb_iterator<nano::uint256_union, nano::account_info_v1> (nullptr))
{
account = nano::uint256_union (i->first);
nano::account_info_v1 v1 (i->second);
nano::account_info_v5 v2;
v2.balance = v1.balance;
v2.head = v1.head;
v2.modified = v1.modified;
v2.rep_block = v1.rep_block;
auto block (block_get (transaction_a, v1.head));
while (!block->previous ().is_zero ())
{
block = block_get (transaction_a, block->previous ());
}
v2.open_block = block->hash ();
auto status (mdb_put (env.tx (transaction_a), accounts_v0, nano::mdb_val (account), nano::mdb_val (sizeof (v2), &v2), 0));
release_assert (status == 0);
account = account.number () + 1;
}
else
{
account.clear ();
}
}
}
void nano::mdb_store::upgrade_v2_to_v3 (nano::transaction const & transaction_a)
{
version_put (transaction_a, 3);
mdb_drop (env.tx (transaction_a), representation, 0);
for (auto i (std::make_unique<nano::mdb_iterator<nano::account, nano::account_info_v5>> (transaction_a, accounts_v0)), n (std::make_unique<nano::mdb_iterator<nano::account, nano::account_info_v5>> (nullptr)); *i != *n; ++(*i))
{
nano::account account_l ((*i)->first);
nano::account_info_v5 info ((*i)->second);
representative_visitor visitor (transaction_a, *this);
visitor.compute (info.head);
assert (!visitor.result.is_zero ());
info.rep_block = visitor.result;
auto impl (boost::polymorphic_downcast<nano::mdb_iterator<nano::account, nano::account_info_v5> *> (i.get ()));
mdb_cursor_put (impl->cursor, nano::mdb_val (account_l), nano::mdb_val (sizeof (info), &info), MDB_CURRENT);
representation_add (transaction_a, visitor.result, info.balance.number ());
}
}
void nano::mdb_store::upgrade_v3_to_v4 (nano::transaction const & transaction_a)
{
version_put (transaction_a, 4);
std::queue<std::pair<nano::pending_key, nano::pending_info>> items;
for (auto i (nano::store_iterator<nano::block_hash, nano::pending_info_v3> (std::make_unique<nano::mdb_iterator<nano::block_hash, nano::pending_info_v3>> (transaction_a, pending_v0))), n (nano::store_iterator<nano::block_hash, nano::pending_info_v3> (nullptr)); i != n; ++i)
{
nano::block_hash const & hash (i->first);
nano::pending_info_v3 const & info (i->second);
items.push (std::make_pair (nano::pending_key (info.destination, hash), nano::pending_info (info.source, info.amount, nano::epoch::epoch_0)));
}
mdb_drop (env.tx (transaction_a), pending_v0, 0);
while (!items.empty ())
{
pending_put (transaction_a, items.front ().first, items.front ().second);
items.pop ();
}
}
void nano::mdb_store::upgrade_v4_to_v5 (nano::transaction const & transaction_a)
{
version_put (transaction_a, 5);
for (auto i (nano::store_iterator<nano::account, nano::account_info_v5> (std::make_unique<nano::mdb_iterator<nano::account, nano::account_info_v5>> (transaction_a, accounts_v0))), n (nano::store_iterator<nano::account, nano::account_info_v5> (nullptr)); i != n; ++i)
{
nano::account_info_v5 const & info (i->second);
nano::block_hash successor (0);
auto block (block_get (transaction_a, info.head));
while (block != nullptr)
{
auto hash (block->hash ());
if (block_successor (transaction_a, hash).is_zero () && !successor.is_zero ())
{
std::vector<uint8_t> vector;
{
nano::vectorstream stream (vector);
block->serialize (stream);
nano::write (stream, successor.bytes);
}
block_raw_put (transaction_a, vector, block->type (), nano::epoch::epoch_0, hash);
if (!block->previous ().is_zero ())
{
nano::block_type type;
auto value (block_raw_get (transaction_a, block->previous (), type));
auto version (block_version (transaction_a, block->previous ()));
assert (value.size () != 0);
std::vector<uint8_t> data (static_cast<uint8_t *> (value.data ()), static_cast<uint8_t *> (value.data ()) + value.size ());
std::copy (hash.bytes.begin (), hash.bytes.end (), data.end () - nano::block_sideband::size (type));
block_raw_put (transaction_a, data, type, version, block->previous ());
}
}
successor = hash;
block = block_get (transaction_a, block->previous ());
}
}
}
void nano::mdb_store::upgrade_v5_to_v6 (nano::transaction const & transaction_a)
{
version_put (transaction_a, 6);
std::deque<std::pair<nano::account, nano::account_info_v13>> headers;
for (auto i (nano::store_iterator<nano::account, nano::account_info_v5> (std::make_unique<nano::mdb_iterator<nano::account, nano::account_info_v5>> (transaction_a, accounts_v0))), n (nano::store_iterator<nano::account, nano::account_info_v5> (nullptr)); i != n; ++i)
{
nano::account const & account (i->first);
nano::account_info_v5 info_old (i->second);
uint64_t block_count (0);
auto hash (info_old.head);
while (!hash.is_zero ())
{
++block_count;
auto block (block_get (transaction_a, hash));
assert (block != nullptr);
hash = block->previous ();
}
headers.emplace_back (account, nano::account_info_v13{ info_old.head, info_old.rep_block, info_old.open_block, info_old.balance, info_old.modified, block_count, nano::epoch::epoch_0 });
}
for (auto i (headers.begin ()), n (headers.end ()); i != n; ++i)
{
auto status (mdb_put (env.tx (transaction_a), accounts_v0, nano::mdb_val (i->first), nano::mdb_val (i->second), 0));
release_assert (status == 0);
}
}
void nano::mdb_store::upgrade_v6_to_v7 (nano::transaction const & transaction_a)
{
version_put (transaction_a, 7);
mdb_drop (env.tx (transaction_a), unchecked, 0);
}
void nano::mdb_store::upgrade_v7_to_v8 (nano::transaction const & transaction_a)
{
version_put (transaction_a, 8);
mdb_drop (env.tx (transaction_a), unchecked, 1);
mdb_dbi_open (env.tx (transaction_a), "unchecked", MDB_CREATE | MDB_DUPSORT, &unchecked);
}
void nano::mdb_store::upgrade_v8_to_v9 (nano::transaction const & transaction_a)
{
version_put (transaction_a, 9);
MDB_dbi sequence;
mdb_dbi_open (env.tx (transaction_a), "sequence", MDB_CREATE | MDB_DUPSORT, &sequence);
nano::genesis genesis;
std::shared_ptr<nano::block> block (std::move (genesis.open));
nano::keypair junk;
for (nano::mdb_iterator<nano::account, uint64_t> i (transaction_a, sequence), n (nano::mdb_iterator<nano::account, uint64_t> (nullptr)); i != n; ++i)
{
nano::bufferstream stream (reinterpret_cast<uint8_t const *> (i->second.data ()), i->second.size ());
uint64_t sequence;
auto error (nano::try_read (stream, sequence));
(void)error;
// Create a dummy vote with the same sequence number for easy upgrading. This won't have a valid signature.
nano::vote dummy (nano::account (i->first), junk.prv, sequence, block);
std::vector<uint8_t> vector;
{
nano::vectorstream stream (vector);
dummy.serialize (stream);
}
auto status1 (mdb_put (env.tx (transaction_a), vote, nano::mdb_val (i->first), nano::mdb_val (vector.size (), vector.data ()), 0));
release_assert (status1 == 0);
assert (!error);
}
mdb_drop (env.tx (transaction_a), sequence, 1);
}
void nano::mdb_store::upgrade_v10_to_v11 (nano::transaction const & transaction_a)
{
version_put (transaction_a, 11);
MDB_dbi unsynced;
mdb_dbi_open (env.tx (transaction_a), "unsynced", MDB_CREATE | MDB_DUPSORT, &unsynced);
mdb_drop (env.tx (transaction_a), unsynced, 1);
}
void nano::mdb_store::upgrade_v11_to_v12 (nano::transaction const & transaction_a)
{
version_put (transaction_a, 12);
mdb_drop (env.tx (transaction_a), unchecked, 1);
mdb_dbi_open (env.tx (transaction_a), "unchecked", MDB_CREATE, &unchecked);
MDB_dbi checksum;
mdb_dbi_open (env.tx (transaction_a), "checksum", MDB_CREATE, &checksum);
mdb_drop (env.tx (transaction_a), checksum, 1);
}
void nano::mdb_store::upgrade_v12_to_v13 (nano::write_transaction & transaction_a, size_t const batch_size)
{
size_t cost (0);
nano::account account (0);
auto const & not_an_account (network_params.random.not_an_account);
while (account != not_an_account)
{
nano::account first (0);
nano::account_info_v13 second;
{
nano::store_iterator<nano::account, nano::account_info_v13> current (std::make_unique<nano::mdb_merge_iterator<nano::account, nano::account_info_v13>> (transaction_a, accounts_v0, accounts_v1, nano::mdb_val (account)));
nano::store_iterator<nano::account, nano::account_info_v13> end (nullptr);
if (current != end)
{
first = current->first;
second = current->second;
}
}
if (!first.is_zero ())
{
auto hash (second.open_block);
uint64_t height (1);
nano::block_sideband sideband;
while (!hash.is_zero ())
{
if (cost >= batch_size)
{
logger.always_log (boost::str (boost::format ("Upgrading sideband information for account %1%... height %2%") % first.to_account ().substr (0, 24) % std::to_string (height)));
transaction_a.commit ();
std::this_thread::yield ();
transaction_a.renew ();
cost = 0;
}
auto block (block_get (transaction_a, hash, &sideband));
assert (block != nullptr);
if (sideband.height == 0)
{
sideband.height = height;
block_put (transaction_a, hash, *block, sideband, block_version (transaction_a, hash));
cost += 16;
}
else
{
cost += 1;
}
hash = sideband.successor;
++height;
}
account = first.number () + 1;
}
else
{
account = not_an_account;
}
}
if (account == not_an_account)
{
logger.always_log ("Completed sideband upgrade");
version_put (transaction_a, 13);
}
}
void nano::mdb_store::upgrade_v13_to_v14 (nano::transaction const & transaction_a)
{
// Upgrade all accounts to have a confirmation of 0 (except genesis which should have 1)
version_put (transaction_a, 14);
nano::store_iterator<nano::account, nano::account_info_v13> i (std::make_unique<nano::mdb_merge_iterator<nano::account, nano::account_info_v13>> (transaction_a, accounts_v0, accounts_v1));
nano::store_iterator<nano::account, nano::account_info_v13> n (nullptr);
std::vector<std::pair<nano::account, nano::account_info>> account_infos;
account_infos.reserve (account_count (transaction_a));
for (; i != n; ++i)
{
nano::account_info_v13 const & account_info_v13 (i->second);
uint64_t confirmation_height = 0;
if (i->first == network_params.ledger.genesis_account)
{
confirmation_height = 1;
}
account_infos.emplace_back (i->first, nano::account_info{ account_info_v13.head, account_info_v13.rep_block, account_info_v13.open_block, account_info_v13.balance, account_info_v13.modified, account_info_v13.block_count, confirmation_height, account_info_v13.epoch });
}
for (auto const & account_info : account_infos)
{
account_put (transaction_a, account_info.first, account_info.second);
}
logger.always_log ("Completed confirmation height upgrade");
nano::uint256_union node_id_mdb_key (3);
auto error (mdb_del (env.tx (transaction_a), meta, nano::mdb_val (node_id_mdb_key), nullptr));
release_assert (!error || error == MDB_NOTFOUND);
}
void nano::mdb_store::version_put (nano::transaction const & transaction_a, int version_a)
{
nano::uint256_union version_key (1);
nano::uint256_union version_value (version_a);
auto status (mdb_put (env.tx (transaction_a), meta, nano::mdb_val (version_key), nano::mdb_val (version_value), 0));
release_assert (status == 0);
if (blocks_info == 0 && !full_sideband (transaction_a))
{
auto status (mdb_dbi_open (env.tx (transaction_a), "blocks_info", MDB_CREATE, &blocks_info));
release_assert (status == MDB_SUCCESS);
}
if (blocks_info != 0 && full_sideband (transaction_a))
{
auto status (mdb_drop (env.tx (transaction_a), blocks_info, 1));
release_assert (status == MDB_SUCCESS);
blocks_info = 0;
}
}
bool nano::mdb_store::block_info_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_info & block_info_a) const
{
assert (!full_sideband (transaction_a));
nano::mdb_val value;
auto status (mdb_get (env.tx (transaction_a), blocks_info, nano::mdb_val (hash_a), value));
release_assert (status == 0 || status == MDB_NOTFOUND);
bool result (true);
if (status != MDB_NOTFOUND)
{
result = false;
assert (value.size () == sizeof (block_info_a.account.bytes) + sizeof (block_info_a.balance.bytes));
nano::bufferstream stream (reinterpret_cast<uint8_t const *> (value.data ()), value.size ());
auto error1 (nano::try_read (stream, block_info_a.account));
(void)error1;
assert (!error1);
auto error2 (nano::try_read (stream, block_info_a.balance));
(void)error2;
assert (!error2);
}
return result;
}
bool nano::mdb_store::exists (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const
{
nano::mdb_val junk;
auto status = get (transaction_a, table_a, key_a, junk);
release_assert (status == MDB_SUCCESS || status == MDB_NOTFOUND);
return (status == MDB_SUCCESS);
}
int nano::mdb_store::get (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, nano::mdb_val & value_a) const
{
return mdb_get (env.tx (transaction_a), table_to_dbi (table_a), key_a, value_a);
}
int nano::mdb_store::put (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, const nano::mdb_val & value_a) const
{
return (mdb_put (env.tx (transaction_a), table_to_dbi (table_a), key_a, value_a, 0));
}
int nano::mdb_store::del (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const
{
return (mdb_del (env.tx (transaction_a), table_to_dbi (table_a), key_a, nullptr));
}
int nano::mdb_store::drop (nano::transaction const & transaction_a, tables table_a)
{
return clear (transaction_a, table_to_dbi (table_a));
}
int nano::mdb_store::clear (nano::transaction const & transaction_a, MDB_dbi handle_a)
{
return mdb_drop (env.tx (transaction_a), handle_a, 0);
}
size_t nano::mdb_store::count (nano::transaction const & transaction_a, tables table_a) const
{
return count (transaction_a, table_to_dbi (table_a));
}
size_t nano::mdb_store::count (nano::transaction const & transaction_a, MDB_dbi db_a) const
{
MDB_stat stats;
auto status (mdb_stat (env.tx (transaction_a), db_a, &stats));
release_assert (status == 0);
return (stats.ms_entries);
}
MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const
{
switch (table_a)
{
case tables::frontiers:
return frontiers;
case tables::accounts_v0:
return accounts_v0;
case tables::accounts_v1:
return accounts_v1;
case tables::send_blocks:
return send_blocks;
case tables::receive_blocks:
return receive_blocks;
case tables::open_blocks:
return open_blocks;
case tables::change_blocks:
return change_blocks;
case tables::state_blocks_v0:
return state_blocks_v0;
case tables::state_blocks_v1:
return state_blocks_v1;
case tables::pending_v0:
return pending_v0;
case tables::pending_v1:
return pending_v1;
case tables::blocks_info:
return blocks_info;
case tables::representation:
return representation;
case tables::unchecked:
return unchecked;
case tables::vote:
return vote;
case tables::online_weight:
return online_weight;
case tables::meta:
return meta;
case tables::peers:
return peers;
default:
release_assert (false);
return peers;
}
}
bool nano::mdb_store::not_found (int status) const
{
return (MDB_NOTFOUND == status);
}
bool nano::mdb_store::success (int status) const
{
return (MDB_SUCCESS == status);
}
int nano::mdb_store::status_code_not_found () const
{
return MDB_NOTFOUND;
}

228
nano/node/lmdb/lmdb.hpp Normal file
View file

@ -0,0 +1,228 @@
#pragma once
#include <nano/lib/config.hpp>
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/node/diagnosticsconfig.hpp>
#include <nano/node/lmdb/lmdb_env.hpp>
#include <nano/node/lmdb/lmdb_iterator.hpp>
#include <nano/node/lmdb/lmdb_txn.hpp>
#include <nano/secure/blockstore_partial.hpp>
#include <nano/secure/common.hpp>
#include <nano/secure/versioning.hpp>
#include <boost/filesystem.hpp>
#include <boost/optional.hpp>
#include <thread>
#include <lmdb/libraries/liblmdb/lmdb.h>
namespace nano
{
using mdb_val = db_val<MDB_val>;
class logging_mt;
/**
* mdb implementation of the block store
*/
class mdb_store : public block_store_partial<MDB_val, mdb_store>
{
public:
using block_store_partial::block_exists;
using block_store_partial::unchecked_put;
mdb_store (bool &, nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512);
nano::write_transaction tx_begin_write () override;
nano::read_transaction tx_begin_read () override;
bool block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) const override;
void version_put (nano::transaction const &, int) override;
void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) override;
nano::logger_mt & logger;
nano::mdb_env env;
/**
* Maps head block to owning account
* nano::block_hash -> nano::account
*/
MDB_dbi frontiers{ 0 };
/**
* Maps account v1 to account information, head, rep, open, balance, timestamp and block count.
* nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t
*/
MDB_dbi accounts_v0{ 0 };
/**
* Maps account v0 to account information, head, rep, open, balance, timestamp and block count.
* nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t
*/
MDB_dbi accounts_v1{ 0 };
/**
* Maps block hash to send block.
* nano::block_hash -> nano::send_block
*/
MDB_dbi send_blocks{ 0 };
/**
* Maps block hash to receive block.
* nano::block_hash -> nano::receive_block
*/
MDB_dbi receive_blocks{ 0 };
/**
* Maps block hash to open block.
* nano::block_hash -> nano::open_block
*/
MDB_dbi open_blocks{ 0 };
/**
* Maps block hash to change block.
* nano::block_hash -> nano::change_block
*/
MDB_dbi change_blocks{ 0 };
/**
* Maps block hash to v0 state block.
* nano::block_hash -> nano::state_block
*/
MDB_dbi state_blocks_v0{ 0 };
/**
* Maps block hash to v1 state block.
* nano::block_hash -> nano::state_block
*/
MDB_dbi state_blocks_v1{ 0 };
/**
* Maps min_version 0 (destination account, pending block) to (source account, amount).
* nano::account, nano::block_hash -> nano::account, nano::amount
*/
MDB_dbi pending_v0{ 0 };
/**
* Maps min_version 1 (destination account, pending block) to (source account, amount).
* nano::account, nano::block_hash -> nano::account, nano::amount
*/
MDB_dbi pending_v1{ 0 };
/**
* Maps block hash to account and balance.
* block_hash -> nano::account, nano::amount
*/
MDB_dbi blocks_info{ 0 };
/**
* Representative weights.
* nano::account -> nano::uint128_t
*/
MDB_dbi representation{ 0 };
/**
* Unchecked bootstrap blocks info.
* nano::block_hash -> nano::unchecked_info
*/
MDB_dbi unchecked{ 0 };
/**
* Highest vote observed for account.
* nano::account -> uint64_t
*/
MDB_dbi vote{ 0 };
/**
* Samples of online vote weight
* uint64_t -> nano::amount
*/
MDB_dbi online_weight{ 0 };
/**
* Meta information about block store, such as versions.
* nano::uint256_union (arbitrary key) -> blob
*/
MDB_dbi meta{ 0 };
/*
* Endpoints for peers
* nano::endpoint_key -> no_value
*/
MDB_dbi peers{ 0 };
bool exists (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const;
int get (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, nano::mdb_val & value_a) const;
int put (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, const nano::mdb_val & value_a) const;
int del (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const;
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a)
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::mdb_iterator<Key, Value>> (transaction_a, table_to_dbi (table_a)));
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key)
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::mdb_iterator<Key, Value>> (transaction_a, table_to_dbi (table_a), key));
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, nano::mdb_val const & key)
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::mdb_merge_iterator<Key, Value>> (transaction_a, table_to_dbi (table1_a), table_to_dbi (table2_a), key));
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a)
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::mdb_merge_iterator<Key, Value>> (transaction_a, table_to_dbi (table1_a), table_to_dbi (table2_a)));
}
private:
bool do_upgrades (nano::write_transaction &, size_t);
void upgrade_v1_to_v2 (nano::transaction const &);
void upgrade_v2_to_v3 (nano::transaction const &);
void upgrade_v3_to_v4 (nano::transaction const &);
void upgrade_v4_to_v5 (nano::transaction const &);
void upgrade_v5_to_v6 (nano::transaction const &);
void upgrade_v6_to_v7 (nano::transaction const &);
void upgrade_v7_to_v8 (nano::transaction const &);
void upgrade_v8_to_v9 (nano::transaction const &);
void upgrade_v10_to_v11 (nano::transaction const &);
void upgrade_v11_to_v12 (nano::transaction const &);
void upgrade_v12_to_v13 (nano::write_transaction &, size_t);
void upgrade_v13_to_v14 (nano::transaction const &);
void open_databases (bool &, nano::transaction const &, unsigned);
int drop (nano::transaction const & transaction_a, tables table_a) override;
int clear (nano::transaction const & transaction_a, MDB_dbi handle_a);
bool not_found (int status) const override;
bool success (int status) const override;
int status_code_not_found () const override;
MDB_dbi table_to_dbi (tables table_a) const;
nano::mdb_txn_tracker mdb_txn_tracker;
nano::mdb_txn_callbacks create_txn_callbacks ();
bool txn_tracking_enabled;
size_t count (nano::transaction const & transaction_a, tables table_a) const override;
size_t count (nano::transaction const &, MDB_dbi) const;
};
template <>
void * mdb_val::data () const;
template <>
size_t mdb_val::size () const;
template <>
mdb_val::db_val (size_t size_a, void * data_a, nano::epoch epoch_a);
template <>
void mdb_val::convert_buffer_to_value ();
}

87
nano/node/lmdb/lmdb_env.cpp Normal file
View file

@ -0,0 +1,87 @@
#include <nano/node/lmdb/lmdb_env.hpp>
nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a, int max_dbs_a, bool use_no_mem_init_a, size_t map_size_a)
{
boost::system::error_code error_mkdir, error_chmod;
if (path_a.has_parent_path ())
{
boost::filesystem::create_directories (path_a.parent_path (), error_mkdir);
nano::set_secure_perm_directory (path_a.parent_path (), error_chmod);
if (!error_mkdir)
{
auto status1 (mdb_env_create (&environment));
release_assert (status1 == 0);
auto status2 (mdb_env_set_maxdbs (environment, max_dbs_a));
release_assert (status2 == 0);
auto map_size = map_size_a;
auto max_valgrind_map_size = 16 * 1024 * 1024;
if (running_within_valgrind () && map_size_a > max_valgrind_map_size)
{
// In order to run LMDB under Valgrind, the maximum map size must be smaller than half your available RAM
map_size = max_valgrind_map_size;
}
auto status3 (mdb_env_set_mapsize (environment, map_size));
release_assert (status3 == 0);
// It seems if there's ever more threads than mdb_env_set_maxreaders has read slots available, we get failures on transaction creation unless MDB_NOTLS is specified
// This can happen if something like 256 io_threads are specified in the node config
// MDB_NORDAHEAD will allow platforms that support it to load the DB in memory as needed.
// MDB_NOMEMINIT prevents zeroing malloc'ed pages. Can provide improvement for non-sensitive data but may make memory checkers noisy (e.g valgrind).
auto environment_flags = MDB_NOSUBDIR | MDB_NOTLS | MDB_NORDAHEAD;
if (!running_within_valgrind () && use_no_mem_init_a)
{
environment_flags |= MDB_NOMEMINIT;
}
auto status4 (mdb_env_open (environment, path_a.string ().c_str (), environment_flags, 00600));
if (status4 != 0)
{
std::cerr << "Could not open lmdb environment: " << status4;
char * error_str (mdb_strerror (status4));
if (error_str)
{
std::cerr << ", " << error_str;
}
std::cerr << std::endl;
}
release_assert (status4 == 0);
error_a = status4 != 0;
}
else
{
error_a = true;
environment = nullptr;
}
}
else
{
error_a = true;
environment = nullptr;
}
}
nano::mdb_env::~mdb_env ()
{
if (environment != nullptr)
{
mdb_env_close (environment);
}
}
nano::mdb_env::operator MDB_env * () const
{
return environment;
}
nano::read_transaction nano::mdb_env::tx_begin_read (mdb_txn_callbacks mdb_txn_callbacks) const
{
return nano::read_transaction{ std::make_unique<nano::read_mdb_txn> (*this, mdb_txn_callbacks) };
}
nano::write_transaction nano::mdb_env::tx_begin_write (mdb_txn_callbacks mdb_txn_callbacks) const
{
return nano::write_transaction{ std::make_unique<nano::write_mdb_txn> (*this, mdb_txn_callbacks) };
}
MDB_txn * nano::mdb_env::tx (nano::transaction const & transaction_a) const
{
return static_cast<MDB_txn *> (transaction_a.get_handle ());
}

24
nano/node/lmdb/lmdb_env.hpp Normal file
View file

@ -0,0 +1,24 @@
#pragma once
#include <nano/node/lmdb/lmdb_txn.hpp>
#include <nano/secure/blockstore.hpp>
namespace nano
{
/**
* RAII wrapper for MDB_env
*/
class mdb_env final
{
public:
mdb_env (bool &, boost::filesystem::path const &, int max_dbs = 128, bool use_no_mem_init = false, size_t map_size = 128ULL * 1024 * 1024 * 1024);
~mdb_env ();
operator MDB_env * () const;
// clang-format off
nano::read_transaction tx_begin_read (mdb_txn_callbacks txn_callbacks = mdb_txn_callbacks{}) const;
nano::write_transaction tx_begin_write (mdb_txn_callbacks txn_callbacks = mdb_txn_callbacks{}) const;
MDB_txn * tx (nano::transaction const & transaction_a) const;
// clang-format on
MDB_env * environment;
};
}

286
nano/node/lmdb/lmdb_iterator.hpp Normal file
View file

@ -0,0 +1,286 @@
#pragma once
#include <nano/secure/blockstore.hpp>
#include <lmdb/libraries/liblmdb/lmdb.h>
namespace nano
{
template <typename T, typename U>
class mdb_iterator : public store_iterator_impl<T, U>
{
public:
mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a, nano::epoch epoch_a = nano::epoch::unspecified) :
cursor (nullptr)
{
current.first.epoch = epoch_a;
current.second.epoch = epoch_a;
auto status (mdb_cursor_open (tx (transaction_a), db_a, &cursor));
release_assert (status == 0);
auto status2 (mdb_cursor_get (cursor, &current.first.value, &current.second.value, MDB_FIRST));
release_assert (status2 == 0 || status2 == MDB_NOTFOUND);
if (status2 != MDB_NOTFOUND)
{
auto status3 (mdb_cursor_get (cursor, &current.first.value, &current.second.value, MDB_GET_CURRENT));
release_assert (status3 == 0 || status3 == MDB_NOTFOUND);
if (current.first.size () != sizeof (T))
{
clear ();
}
}
else
{
clear ();
}
}
mdb_iterator (std::nullptr_t, nano::epoch epoch_a = nano::epoch::unspecified) :
cursor (nullptr)
{
current.first.epoch = epoch_a;
current.second.epoch = epoch_a;
}
mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a, MDB_val const & val_a, nano::epoch epoch_a = nano::epoch::unspecified) :
cursor (nullptr)
{
current.first.epoch = epoch_a;
current.second.epoch = epoch_a;
auto status (mdb_cursor_open (tx (transaction_a), db_a, &cursor));
release_assert (status == 0);
current.first = val_a;
auto status2 (mdb_cursor_get (cursor, &current.first.value, &current.second.value, MDB_SET_RANGE));
release_assert (status2 == 0 || status2 == MDB_NOTFOUND);
if (status2 != MDB_NOTFOUND)
{
auto status3 (mdb_cursor_get (cursor, &current.first.value, &current.second.value, MDB_GET_CURRENT));
release_assert (status3 == 0 || status3 == MDB_NOTFOUND);
if (current.first.size () != sizeof (T))
{
clear ();
}
}
else
{
clear ();
}
}
mdb_iterator (nano::mdb_iterator<T, U> && other_a)
{
cursor = other_a.cursor;
other_a.cursor = nullptr;
current = other_a.current;
}
mdb_iterator (nano::mdb_iterator<T, U> const &) = delete;
~mdb_iterator ()
{
if (cursor != nullptr)
{
mdb_cursor_close (cursor);
}
}
nano::store_iterator_impl<T, U> & operator++ () override
{
assert (cursor != nullptr);
auto status (mdb_cursor_get (cursor, &current.first.value, &current.second.value, MDB_NEXT));
release_assert (status == 0 || status == MDB_NOTFOUND);
if (status == MDB_NOTFOUND)
{
clear ();
}
if (current.first.size () != sizeof (T))
{
clear ();
}
return *this;
}
std::pair<nano::db_val<MDB_val>, nano::db_val<MDB_val>> * operator-> ()
{
return &current;
}
bool operator== (nano::store_iterator_impl<T, U> const & base_a) const override
{
auto const other_a (boost::polymorphic_downcast<nano::mdb_iterator<T, U> const *> (&base_a));
auto result (current.first.data () == other_a->current.first.data ());
assert (!result || (current.first.size () == other_a->current.first.size ()));
assert (!result || (current.second.data () == other_a->current.second.data ()));
assert (!result || (current.second.size () == other_a->current.second.size ()));
return result;
}
bool is_end_sentinal () const override
{
return current.first.size () == 0;
}
void fill (std::pair<T, U> & value_a) const override
{
if (current.first.size () != 0)
{
value_a.first = static_cast<T> (current.first);
}
else
{
value_a.first = T ();
}
if (current.second.size () != 0)
{
value_a.second = static_cast<U> (current.second);
}
else
{
value_a.second = U ();
}
}
void clear ()
{
current.first = nano::db_val<MDB_val> (current.first.epoch);
current.second = nano::db_val<MDB_val> (current.second.epoch);
assert (is_end_sentinal ());
}
nano::mdb_iterator<T, U> & operator= (nano::mdb_iterator<T, U> && other_a)
{
if (cursor != nullptr)
{
mdb_cursor_close (cursor);
}
cursor = other_a.cursor;
other_a.cursor = nullptr;
current = other_a.current;
other_a.clear ();
return *this;
}
nano::store_iterator_impl<T, U> & operator= (nano::store_iterator_impl<T, U> const &) = delete;
MDB_cursor * cursor;
std::pair<nano::db_val<MDB_val>, nano::db_val<MDB_val>> current;
private:
MDB_txn * tx (nano::transaction const & transaction_a) const
{
return static_cast<MDB_txn *> (transaction_a.get_handle ());
}
};
/**
* Iterates the key/value pairs of two stores merged together
*/
template <typename T, typename U>
class mdb_merge_iterator : public store_iterator_impl<T, U>
{
public:
mdb_merge_iterator (nano::transaction const & transaction_a, MDB_dbi db1_a, MDB_dbi db2_a) :
impl1 (std::make_unique<nano::mdb_iterator<T, U>> (transaction_a, db1_a, nano::epoch::epoch_0)),
impl2 (std::make_unique<nano::mdb_iterator<T, U>> (transaction_a, db2_a, nano::epoch::epoch_1))
{
}
mdb_merge_iterator (std::nullptr_t) :
impl1 (std::make_unique<nano::mdb_iterator<T, U>> (nullptr, nano::epoch::epoch_0)),
impl2 (std::make_unique<nano::mdb_iterator<T, U>> (nullptr, nano::epoch::epoch_1))
{
}
mdb_merge_iterator (nano::transaction const & transaction_a, MDB_dbi db1_a, MDB_dbi db2_a, MDB_val const & val_a) :
impl1 (std::make_unique<nano::mdb_iterator<T, U>> (transaction_a, db1_a, val_a, nano::epoch::epoch_0)),
impl2 (std::make_unique<nano::mdb_iterator<T, U>> (transaction_a, db2_a, val_a, nano::epoch::epoch_1))
{
}
mdb_merge_iterator (nano::mdb_merge_iterator<T, U> && other_a)
{
impl1 = std::move (other_a.impl1);
impl2 = std::move (other_a.impl2);
}
mdb_merge_iterator (nano::mdb_merge_iterator<T, U> const &) = delete;
nano::store_iterator_impl<T, U> & operator++ () override
{
++least_iterator ();
return *this;
}
std::pair<nano::db_val<MDB_val>, nano::db_val<MDB_val>> * operator-> ()
{
return least_iterator ().operator-> ();
}
bool operator== (nano::store_iterator_impl<T, U> const & base_a) const override
{
assert ((dynamic_cast<nano::mdb_merge_iterator<T, U> const *> (&base_a) != nullptr) && "Incompatible iterator comparison");
auto & other (static_cast<nano::mdb_merge_iterator<T, U> const &> (base_a));
return *impl1 == *other.impl1 && *impl2 == *other.impl2;
}
bool is_end_sentinal () const override
{
return least_iterator ().is_end_sentinal ();
}
void fill (std::pair<T, U> & value_a) const override
{
auto & current (least_iterator ());
if (current->first.size () != 0)
{
value_a.first = static_cast<T> (current->first);
}
else
{
value_a.first = T ();
}
if (current->second.size () != 0)
{
value_a.second = static_cast<U> (current->second);
}
else
{
value_a.second = U ();
}
}
nano::mdb_merge_iterator<T, U> & operator= (nano::mdb_merge_iterator<T, U> &&) = default;
nano::mdb_merge_iterator<T, U> & operator= (nano::mdb_merge_iterator<T, U> const &) = delete;
private:
nano::mdb_iterator<T, U> & least_iterator () const
{
nano::mdb_iterator<T, U> * result;
if (impl1->is_end_sentinal ())
{
result = impl2.get ();
}
else if (impl2->is_end_sentinal ())
{
result = impl1.get ();
}
else
{
auto key_cmp (mdb_cmp (mdb_cursor_txn (impl1->cursor), mdb_cursor_dbi (impl1->cursor), impl1->current.first, impl2->current.first));
if (key_cmp < 0)
{
result = impl1.get ();
}
else if (key_cmp > 0)
{
result = impl2.get ();
}
else
{
auto val_cmp (mdb_cmp (mdb_cursor_txn (impl1->cursor), mdb_cursor_dbi (impl1->cursor), impl1->current.second, impl2->current.second));
result = val_cmp < 0 ? impl1.get () : impl2.get ();
}
}
return *result;
}
std::unique_ptr<nano::mdb_iterator<T, U>> impl1;
std::unique_ptr<nano::mdb_iterator<T, U>> impl2;
};
}

227
nano/node/lmdb/lmdb_txn.cpp Normal file
View file

@ -0,0 +1,227 @@
#include <nano/lib/jsonconfig.hpp>
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/utility.hpp>
#include <nano/node/lmdb/lmdb_env.hpp>
#include <nano/node/lmdb/lmdb_txn.hpp>
#include <nano/secure/blockstore.hpp>
#include <boost/polymorphic_cast.hpp>
// Some builds (mac) fail due to "Boost.Stacktrace requires `_Unwind_Backtrace` function".
#ifndef _WIN32
#ifndef _GNU_SOURCE
#define BEFORE_GNU_SOURCE 0
#define _GNU_SOURCE
#else
#define BEFORE_GNU_SOURCE 1
#endif
#endif
// On Windows this include defines min/max macros, so keep below other includes
// to reduce conflicts with other std functions
#include <boost/stacktrace.hpp>
#ifndef _WIN32
#if !BEFORE_GNU_SOURCE
#undef _GNU_SOURCE
#endif
#endif
namespace
{
class matches_txn
{
public:
matches_txn (const nano::transaction_impl * transaction_impl_a) :
transaction_impl (transaction_impl_a)
{
}
bool operator() (nano::mdb_txn_stats const & mdb_txn_stats)
{
return (mdb_txn_stats.transaction_impl == transaction_impl);
}
private:
const nano::transaction_impl * transaction_impl;
};
}
nano::read_mdb_txn::read_mdb_txn (nano::mdb_env const & environment_a, nano::mdb_txn_callbacks txn_callbacks_a) :
txn_callbacks (txn_callbacks_a)
{
auto status (mdb_txn_begin (environment_a, nullptr, MDB_RDONLY, &handle));
release_assert (status == 0);
txn_callbacks.txn_start (this);
}
nano::read_mdb_txn::~read_mdb_txn ()
{
// This uses commit rather than abort, as it is needed when opening databases with a read only transaction
auto status (mdb_txn_commit (handle));
release_assert (status == MDB_SUCCESS);
txn_callbacks.txn_end (this);
}
void nano::read_mdb_txn::reset ()
{
mdb_txn_reset (handle);
txn_callbacks.txn_end (this);
}
void nano::read_mdb_txn::renew ()
{
auto status (mdb_txn_renew (handle));
release_assert (status == 0);
txn_callbacks.txn_start (this);
}
void * nano::read_mdb_txn::get_handle () const
{
return handle;
}
nano::write_mdb_txn::write_mdb_txn (nano::mdb_env const & environment_a, nano::mdb_txn_callbacks txn_callbacks_a) :
env (environment_a),
txn_callbacks (txn_callbacks_a)
{
renew ();
}
nano::write_mdb_txn::~write_mdb_txn ()
{
commit ();
}
void nano::write_mdb_txn::commit () const
{
auto status (mdb_txn_commit (handle));
release_assert (status == MDB_SUCCESS);
txn_callbacks.txn_end (this);
}
void nano::write_mdb_txn::renew ()
{
auto status (mdb_txn_begin (env, nullptr, 0, &handle));
release_assert (status == MDB_SUCCESS);
txn_callbacks.txn_start (this);
}
void * nano::write_mdb_txn::get_handle () const
{
return handle;
}
nano::mdb_txn_tracker::mdb_txn_tracker (nano::logger_mt & logger_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a) :
logger (logger_a),
txn_tracking_config (txn_tracking_config_a),
block_processor_batch_max_time (block_processor_batch_max_time_a)
{
}
void nano::mdb_txn_tracker::serialize_json (boost::property_tree::ptree & json, std::chrono::milliseconds min_read_time, std::chrono::milliseconds min_write_time)
{
// Copying is cheap compared to generating the stack trace strings, so reduce time holding the mutex
std::vector<mdb_txn_stats> copy_stats;
{
std::lock_guard<std::mutex> guard (mutex);
copy_stats = stats;
}
// Get the time difference now as creating stacktraces (Debug/Windows for instance) can take a while so results won't be as accurate
std::vector<std::chrono::milliseconds> times_since_start;
times_since_start.reserve (copy_stats.size ());
// clang-format off
std::transform (copy_stats.cbegin (), copy_stats.cend (), std::back_inserter (times_since_start), [] (const auto & stat) {
return stat.timer.since_start ();
});
// clang-format on
assert (times_since_start.size () == copy_stats.size ());
for (size_t i = 0; i < times_since_start.size (); ++i)
{
auto const & stat = copy_stats[i];
auto time_held_open = times_since_start[i];
if ((stat.is_write () && time_held_open >= min_write_time) || (!stat.is_write () && time_held_open >= min_read_time))
{
nano::jsonconfig mdb_lock_config;
mdb_lock_config.put ("thread", stat.thread_name);
mdb_lock_config.put ("time_held_open", time_held_open.count ());
mdb_lock_config.put ("write", stat.is_write ());
boost::property_tree::ptree stacktrace_config;
for (auto frame : *stat.stacktrace)
{
nano::jsonconfig frame_json;
frame_json.put ("name", frame.name ());
frame_json.put ("address", frame.address ());
frame_json.put ("source_file", frame.source_file ());
frame_json.put ("source_line", frame.source_line ());
stacktrace_config.push_back (std::make_pair ("", frame_json.get_tree ()));
}
nano::jsonconfig stack (stacktrace_config);
mdb_lock_config.put_child ("stacktrace", stack);
json.push_back (std::make_pair ("", mdb_lock_config.get_tree ()));
}
}
}
void nano::mdb_txn_tracker::output_finished (nano::mdb_txn_stats const & mdb_txn_stats) const
{
// Only output them if transactions were held for longer than a certain period of time
auto is_write = mdb_txn_stats.is_write ();
auto time_open = mdb_txn_stats.timer.since_start ();
auto should_ignore = false;
// Reduce noise in log files by removing any entries from the block processor (if enabled) which are less than the max batch time (+ a few second buffer) because these are expected writes during bootstrapping.
auto is_below_max_time = time_open <= (block_processor_batch_max_time + std::chrono::seconds (3));
bool is_blk_processing_thread = mdb_txn_stats.thread_name == nano::thread_role::get_string (nano::thread_role::name::block_processing);
if (txn_tracking_config.ignore_writes_below_block_processor_max_time && is_blk_processing_thread && is_write && is_below_max_time)
{
should_ignore = true;
}
if (!should_ignore && ((is_write && time_open >= txn_tracking_config.min_write_txn_time) || (!is_write && time_open >= txn_tracking_config.min_read_txn_time)))
{
assert (mdb_txn_stats.stacktrace);
logger.always_log (boost::str (boost::format ("%1%ms %2% held on thread %3%\n%4%") % mdb_txn_stats.timer.since_start ().count () % (is_write ? "write lock" : "read") % mdb_txn_stats.thread_name % *mdb_txn_stats.stacktrace));
}
}
void nano::mdb_txn_tracker::add (const nano::transaction_impl * transaction_impl)
{
std::lock_guard<std::mutex> guard (mutex);
// clang-format off
assert (std::find_if (stats.cbegin (), stats.cend (), matches_txn (transaction_impl)) == stats.cend ());
// clang-format on
stats.emplace_back (transaction_impl);
}
/** Can be called without error if transaction does not exist */
void nano::mdb_txn_tracker::erase (const nano::transaction_impl * transaction_impl)
{
std::lock_guard<std::mutex> guard (mutex);
// clang-format off
auto it = std::find_if (stats.begin (), stats.end (), matches_txn (transaction_impl));
// clang-format on
if (it != stats.end ())
{
output_finished (*it);
it->timer.stop ();
stats.erase (it);
}
}
nano::mdb_txn_stats::mdb_txn_stats (const nano::transaction_impl * transaction_impl) :
transaction_impl (transaction_impl),
thread_name (nano::thread_role::get_string ()),
stacktrace (std::make_shared<boost::stacktrace::stacktrace> ())
{
timer.start ();
}
bool nano::mdb_txn_stats::is_write () const
{
return (dynamic_cast<const nano::write_transaction_impl *> (transaction_impl) != nullptr);
}

84
nano/node/lmdb/lmdb_txn.hpp Normal file
View file

@ -0,0 +1,84 @@
#pragma once
#include <nano/lib/timer.hpp>
#include <nano/node/diagnosticsconfig.hpp>
#include <nano/secure/blockstore.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/stacktrace/stacktrace_fwd.hpp>
#include <mutex>
#include <lmdb/libraries/liblmdb/lmdb.h>
namespace nano
{
class transaction_impl;
class logger_mt;
class mdb_env;
class mdb_txn_callbacks
{
public:
// clang-format off
std::function<void (const nano::transaction_impl *)> txn_start{ [] (const nano::transaction_impl *) {} };
std::function<void (const nano::transaction_impl *)> txn_end{ [] (const nano::transaction_impl *) {} };
// clang-format on
};
class read_mdb_txn final : public read_transaction_impl
{
public:
read_mdb_txn (nano::mdb_env const &, mdb_txn_callbacks mdb_txn_callbacks);
~read_mdb_txn ();
void reset () override;
void renew () override;
void * get_handle () const override;
MDB_txn * handle;
mdb_txn_callbacks txn_callbacks;
};
class write_mdb_txn final : public write_transaction_impl
{
public:
write_mdb_txn (nano::mdb_env const &, mdb_txn_callbacks mdb_txn_callbacks);
~write_mdb_txn ();
void commit () const override;
void renew () override;
void * get_handle () const override;
MDB_txn * handle;
nano::mdb_env const & env;
mdb_txn_callbacks txn_callbacks;
};
class mdb_txn_stats
{
public:
mdb_txn_stats (const nano::transaction_impl * transaction_impl_a);
bool is_write () const;
nano::timer<std::chrono::milliseconds> timer;
const nano::transaction_impl * transaction_impl;
std::string thread_name;
// Smart pointer so that we don't need the full definition which causes min/max issues on Windows
std::shared_ptr<boost::stacktrace::stacktrace> stacktrace;
};
class mdb_txn_tracker
{
public:
mdb_txn_tracker (nano::logger_mt & logger_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a);
void serialize_json (boost::property_tree::ptree & json, std::chrono::milliseconds min_read_time, std::chrono::milliseconds min_write_time);
void add (const nano::transaction_impl * transaction_impl);
void erase (const nano::transaction_impl * transaction_impl);
private:
std::mutex mutex;
std::vector<mdb_txn_stats> stats;
nano::logger_mt & logger;
nano::txn_tracking_config txn_tracking_config;
std::chrono::milliseconds block_processor_batch_max_time;
void output_finished (nano::mdb_txn_stats const & mdb_txn_stats) const;
};
}

20
nano/node/lmdb/wallet_value.cpp Normal file
View file

@ -0,0 +1,20 @@
#include <nano/node/lmdb/wallet_value.hpp>
nano::wallet_value::wallet_value (nano::db_val<MDB_val> const & val_a)
{
assert (val_a.size () == sizeof (*this));
std::copy (reinterpret_cast<uint8_t const *> (val_a.data ()), reinterpret_cast<uint8_t const *> (val_a.data ()) + sizeof (key), key.chars.begin ());
std::copy (reinterpret_cast<uint8_t const *> (val_a.data ()) + sizeof (key), reinterpret_cast<uint8_t const *> (val_a.data ()) + sizeof (key) + sizeof (work), reinterpret_cast<char *> (&work));
}
nano::wallet_value::wallet_value (nano::uint256_union const & key_a, uint64_t work_a) :
key (key_a),
work (work_a)
{
}
nano::db_val<MDB_val> nano::wallet_value::val () const
{
static_assert (sizeof (*this) == sizeof (key) + sizeof (work), "Class not packed");
return nano::db_val<MDB_val> (sizeof (*this), const_cast<nano::wallet_value *> (this));
}

20
nano/node/lmdb/wallet_value.hpp Normal file
View file

@ -0,0 +1,20 @@
#pragma once
#include <nano/lib/numbers.hpp>
#include <nano/secure/blockstore.hpp>
#include <lmdb/libraries/liblmdb/lmdb.h>
namespace nano
{
class wallet_value
{
public:
wallet_value () = default;
wallet_value (nano::db_val<MDB_val> const &);
wallet_value (nano::uint256_union const &, uint64_t);
nano::db_val<MDB_val> val () const;
nano::private_key key;
uint64_t work;
};
}

9
nano/node/node.cpp
View file

@ -109,8 +109,8 @@ std::unique_ptr<seq_con_info_component> collect_seq_con_info (block_processor &
}
}
nano::node::node (nano::node_init & init_a, boost::asio::io_context & io_ctx_a, uint16_t peering_port_a, boost::filesystem::path const & application_path_a, nano::alarm & alarm_a, nano::logging const & logging_a, nano::work_pool & work_a) :
node (init_a, io_ctx_a, application_path_a, alarm_a, nano::node_config (peering_port_a, logging_a), work_a)
nano::node::node (nano::node_init & init_a, boost::asio::io_context & io_ctx_a, uint16_t peering_port_a, boost::filesystem::path const & application_path_a, nano::alarm & alarm_a, nano::logging const & logging_a, nano::work_pool & work_a, nano::node_flags flags_a) :
node (init_a, io_ctx_a, application_path_a, alarm_a, nano::node_config (peering_port_a, logging_a), work_a, flags_a)
{
}
@ -1562,3 +1562,8 @@ nano::inactive_node::~inactive_node ()
{
node->stop ();
}
std::unique_ptr<nano::block_store> nano::make_store (bool & init, nano::logger_mt & logger, boost::filesystem::path const & path)
{
return std::make_unique<nano::mdb_store> (init, logger, path);
}

2
nano/node/node.hpp
View file

@ -86,7 +86,7 @@ std::unique_ptr<seq_con_info_component> collect_seq_con_info (block_processor &
class node final : public std::enable_shared_from_this<nano::node>
{
public:
node (nano::node_init &, boost::asio::io_context &, uint16_t, boost::filesystem::path const &, nano::alarm &, nano::logging const &, nano::work_pool &);
node (nano::node_init &, boost::asio::io_context &, uint16_t, boost::filesystem::path const &, nano::alarm &, nano::logging const &, nano::work_pool &, nano::node_flags = nano::node_flags ());
node (nano::node_init &, boost::asio::io_context &, boost::filesystem::path const &, nano::alarm &, nano::node_config const &, nano::work_pool &, nano::node_flags = nano::node_flags ());
~node ();
template <typename T>

1
nano/node/nodeconfig.hpp
View file

@ -97,6 +97,7 @@ public:
bool disable_unchecked_drop{ true };
bool fast_bootstrap{ false };
bool delay_frontier_confirmation_height_updating{ false };
bool read_only{ false };
size_t sideband_batch_size{ 512 };
size_t block_processor_batch_size{ 0 };
size_t block_processor_full_size{ 65536 };

1
nano/node/wallet.cpp
View file

@ -1,5 +1,6 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/utility.hpp>
#include <nano/node/lmdb/lmdb_iterator.hpp>
#include <nano/node/node.hpp>
#include <nano/node/wallet.hpp>
#include <nano/node/xorshift.hpp>

5
nano/node/wallet.hpp
View file

@ -1,7 +1,8 @@
#pragma once
#include <nano/lib/config.hpp>
#include <nano/node/lmdb.hpp>
#include <nano/node/lmdb/lmdb.hpp>
#include <nano/node/lmdb/wallet_value.hpp>
#include <nano/node/openclwork.hpp>
#include <nano/secure/blockstore.hpp>
#include <nano/secure/common.hpp>
@ -122,7 +123,6 @@ public:
std::shared_ptr<nano::block> change_action (nano::account const &, nano::account const &, uint64_t = 0, bool = true);
std::shared_ptr<nano::block> receive_action (nano::block const &, nano::account const &, nano::uint128_union const &, uint64_t = 0, bool = true);
std::shared_ptr<nano::block> send_action (nano::account const &, nano::account const &, nano::uint128_t const &, uint64_t = 0, bool = true, boost::optional<std::string> = {});
std::shared_ptr<nano::block> regenerate_action (nano::qualified_root const &, std::shared_ptr<nano::block_builder>);
wallet (bool &, nano::transaction &, nano::wallets &, std::string const &);
wallet (bool &, nano::transaction &, nano::wallets &, std::string const &, std::string const &);
void enter_initial_password ();
@ -142,7 +142,6 @@ public:
void receive_async (std::shared_ptr<nano::block>, nano::account const &, nano::uint128_t const &, std::function<void(std::shared_ptr<nano::block>)> const &, uint64_t = 0, bool = true);
nano::block_hash send_sync (nano::account const &, nano::account const &, nano::uint128_t const &);
void send_async (nano::account const &, nano::account const &, nano::uint128_t const &, std::function<void(std::shared_ptr<nano::block>)> const &, uint64_t = 0, bool = true, boost::optional<std::string> = {});
void work_apply (nano::account const &, std::function<void(uint64_t)>);
void work_cache_blocking (nano::account const &, nano::block_hash const &);
void work_update (nano::transaction const &, nano::account const &, nano::block_hash const &, uint64_t);
void work_ensure (nano::account const &, nano::block_hash const &);

1
nano/secure/CMakeLists.txt
View file

@ -36,6 +36,7 @@ add_library (secure
common.hpp
common.cpp
blockstore.hpp
blockstore_partial.hpp
blockstore.cpp
ledger.hpp
ledger.cpp

538
nano/secure/blockstore.hpp
View file

@ -1,6 +1,8 @@
#pragma once
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/config.hpp>
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/memory.hpp>
#include <nano/secure/common.hpp>
#include <nano/secure/versioning.hpp>
@ -12,21 +14,6 @@
namespace nano
{
// Generic container to be used when templated db types cannot
class DB_val
{
public:
DB_val () = default;
DB_val (size_t size_a, void * data_a) :
size (size_a),
data (data_a)
{
}
size_t size;
void * data;
};
/**
* Encapsulates database specific container and provides uint256_union conversion of the data.
*/
@ -34,22 +21,14 @@ template <typename Val>
class db_val
{
public:
db_val (nano::epoch epoch_a = nano::epoch::unspecified) :
value ({ 0, nullptr }),
epoch (epoch_a)
{
}
db_val (Val const & value_a, nano::epoch epoch_a = nano::epoch::unspecified) :
value (value_a),
epoch (epoch_a)
{
}
db_val (DB_val const & value_a, nano::epoch epoch_a = nano::epoch::unspecified);
db_val (size_t size_a, void * data_a) :
value ({ size_a, data_a })
db_val (nano::epoch epoch_a = nano::epoch::unspecified) :
db_val (0, nullptr, epoch_a)
{
}
@ -92,7 +71,7 @@ public:
nano::vectorstream stream (*buffer);
val_a.serialize (stream);
}
value = { buffer->size (), const_cast<uint8_t *> (buffer->data ()) };
convert_buffer_to_value ();
}
db_val (nano::block_info const & val_a) :
@ -114,7 +93,7 @@ public:
nano::vectorstream stream (*buffer);
nano::serialize_block (stream, *val_a);
}
value = { buffer->size (), const_cast<uint8_t *> (buffer->data ()) };
convert_buffer_to_value ();
}
db_val (uint64_t val_a) :
@ -125,7 +104,7 @@ public:
nano::vectorstream stream (*buffer);
nano::write (stream, val_a);
}
value = { buffer->size (), const_cast<uint8_t *> (buffer->data ()) };
convert_buffer_to_value ();
}
explicit operator nano::account_info () const
@ -293,9 +272,11 @@ public:
return value;
}
/** Must be specialized in the sub-class */
// Must be specialized
void * data () const;
size_t size () const;
db_val (size_t size_a, void * data_a, nano::epoch epoch_a = nano::epoch::unspecified);
void convert_buffer_to_value ();
Val value;
std::shared_ptr<std::vector<uint8_t>> buffer;
@ -491,8 +472,8 @@ public:
class read_transaction_impl : public transaction_impl
{
public:
virtual void reset () const = 0;
virtual void renew () const = 0;
virtual void reset () = 0;
virtual void renew () = 0;
};
class write_transaction_impl : public transaction_impl
@ -548,6 +529,28 @@ private:
class block_store
{
public:
enum class tables
{
frontiers,
accounts_v0,
accounts_v1,
send_blocks,
receive_blocks,
open_blocks,
change_blocks,
state_blocks_v0,
state_blocks_v1,
pending_v0,
pending_v1,
blocks_info,
representation,
unchecked,
vote,
online_weight,
meta,
peers
};
virtual ~block_store () = default;
virtual void initialize (nano::transaction const &, nano::genesis const &) = 0;
virtual void block_put (nano::transaction const &, nano::block_hash const &, nano::block const &, nano::block_sideband const &, nano::epoch version = nano::epoch::epoch_0) = 0;
@ -650,8 +653,9 @@ public:
virtual void peer_clear (nano::transaction const & transaction_a) = 0;
virtual nano::store_iterator<nano::endpoint_key, nano::no_value> peers_begin (nano::transaction const & transaction_a) = 0;
virtual nano::store_iterator<nano::endpoint_key, nano::no_value> peers_end () = 0;
virtual uint64_t block_account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0;
virtual std::mutex & get_cache_mutex () = 0;
virtual void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) = 0;
/** Start read-write transaction */
@ -661,473 +665,5 @@ public:
virtual nano::read_transaction tx_begin_read () = 0;
};
template <typename Val>
class block_predecessor_set;
/** This base class implements the block_store interface functions which have DB agnostic functionality */
template <class Val>
class block_store_partial : public block_store
{
public:
using block_store::block_exists;
using block_store::unchecked_put;
friend class nano::block_predecessor_set<Val>;
std::mutex cache_mutex;
/**
* If using a different store version than the latest then you may need
* to modify some of the objects in the store to be appropriate for the version before an upgrade.
*/
void initialize (nano::transaction const & transaction_a, nano::genesis const & genesis_a) override
{
auto hash_l (genesis_a.hash ());
assert (latest_v0_begin (transaction_a) == latest_v0_end ());
assert (latest_v1_begin (transaction_a) == latest_v1_end ());
nano::block_sideband sideband (nano::block_type::open, network_params.ledger.genesis_account, 0, network_params.ledger.genesis_amount, 1, nano::seconds_since_epoch ());
block_put (transaction_a, hash_l, *genesis_a.open, sideband);
account_put (transaction_a, network_params.ledger.genesis_account, { hash_l, genesis_a.open->hash (), genesis_a.open->hash (), std::numeric_limits<nano::uint128_t>::max (), nano::seconds_since_epoch (), 1, 1, nano::epoch::epoch_0 });
representation_put (transaction_a, network_params.ledger.genesis_account, std::numeric_limits<nano::uint128_t>::max ());
frontier_put (transaction_a, hash_l, network_params.ledger.genesis_account);
}
nano::uint128_t block_balance (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override
{
nano::block_sideband sideband;
auto block (block_get (transaction_a, hash_a, &sideband));
nano::uint128_t result (block_balance_calculated (block, sideband));
return result;
}
void representation_add (nano::transaction const & transaction_a, nano::block_hash const & source_a, nano::uint128_t const & amount_a) override
{
auto source_block (block_get (transaction_a, source_a));
assert (source_block != nullptr);
auto source_rep (source_block->representative ());
auto source_previous (representation_get (transaction_a, source_rep));
representation_put (transaction_a, source_rep, source_previous + amount_a);
}
bool account_exists (nano::transaction const & transaction_a, nano::account const & account_a) override
{
auto iterator (latest_begin (transaction_a, account_a));
return iterator != latest_end () && nano::account (iterator->first) == account_a;
}
void confirmation_height_clear (nano::transaction const & transaction_a, nano::account const & account, nano::account_info const & account_info) override
{
nano::account_info info_copy (account_info);
if (info_copy.confirmation_height > 0)
{
info_copy.confirmation_height = 0;
account_put (transaction_a, account, info_copy);
}
}
void confirmation_height_clear (nano::transaction const & transaction_a) override
{
for (auto i (latest_begin (transaction_a)), n (latest_end ()); i != n; ++i)
{
confirmation_height_clear (transaction_a, i->first, i->second);
}
}
bool pending_exists (nano::transaction const & transaction_a, nano::pending_key const & key_a) override
{
auto iterator (pending_begin (transaction_a, key_a));
return iterator != pending_end () && nano::pending_key (iterator->first) == key_a;
}
std::vector<nano::unchecked_info> unchecked_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override
{
std::vector<nano::unchecked_info> result;
for (auto i (unchecked_begin (transaction_a, nano::unchecked_key (hash_a, 0))), n (unchecked_end ()); i != n && nano::block_hash (i->first.key ()) == hash_a; ++i)
{
nano::unchecked_info const & unchecked_info (i->second);
result.push_back (unchecked_info);
}
return result;
}
void block_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a, nano::block_sideband const & sideband_a, nano::epoch epoch_a = nano::epoch::epoch_0) override
{
assert (block_a.type () == sideband_a.type);
assert (sideband_a.successor.is_zero () || block_exists (transaction_a, sideband_a.successor));
std::vector<uint8_t> vector;
{
nano::vectorstream stream (vector);
block_a.serialize (stream);
sideband_a.serialize (stream);
}
block_raw_put (transaction_a, vector, block_a.type (), epoch_a, hash_a);
nano::block_predecessor_set<Val> predecessor (transaction_a, *this);
block_a.visit (predecessor);
assert (block_a.previous ().is_zero () || block_successor (transaction_a, block_a.previous ()) == hash_a);
}
// Converts a block hash to a block height
uint64_t block_account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override
{
nano::block_sideband sideband;
auto block = block_get (transaction_a, hash_a, &sideband);
assert (block != nullptr);
return sideband.height;
}
std::shared_ptr<nano::block> block_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband * sideband_a = nullptr) const override
{
nano::block_type type;
auto value (block_raw_get (transaction_a, hash_a, type));
std::shared_ptr<nano::block> result;
if (value.size () != 0)
{
nano::bufferstream stream (reinterpret_cast<uint8_t const *> (value.data ()), value.size ());
result = nano::deserialize_block (stream, type);
assert (result != nullptr);
if (sideband_a)
{
sideband_a->type = type;
if (full_sideband (transaction_a) || entry_has_sideband (value.size (), type))
{
auto error (sideband_a->deserialize (stream));
(void)error;
assert (!error);
}
else
{
// Reconstruct sideband data for block.
sideband_a->account = block_account_computed (transaction_a, hash_a);
sideband_a->balance = block_balance_computed (transaction_a, hash_a);
sideband_a->successor = block_successor (transaction_a, hash_a);
sideband_a->height = 0;
sideband_a->timestamp = 0;
}
}
}
return result;
}
bool block_exists (nano::transaction const & tx_a, nano::block_hash const & hash_a) override
{
// Table lookups are ordered by match probability
// clang-format off
return
block_exists (tx_a, nano::block_type::state, hash_a) ||
block_exists (tx_a, nano::block_type::send, hash_a) ||
block_exists (tx_a, nano::block_type::receive, hash_a) ||
block_exists (tx_a, nano::block_type::open, hash_a) ||
block_exists (tx_a, nano::block_type::change, hash_a);
// clang-format on
}
bool root_exists (nano::transaction const & transaction_a, nano::uint256_union const & root_a) override
{
return block_exists (transaction_a, root_a) || account_exists (transaction_a, root_a);
}
bool source_exists (nano::transaction const & transaction_a, nano::block_hash const & source_a) override
{
return block_exists (transaction_a, nano::block_type::state, source_a) || block_exists (transaction_a, nano::block_type::send, source_a);
}
nano::account block_account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override
{
nano::block_sideband sideband;
auto block (block_get (transaction_a, hash_a, &sideband));
nano::account result (block->account ());
if (result.is_zero ())
{
result = sideband.account;
}
assert (!result.is_zero ());
return result;
}
nano::uint128_t block_balance_calculated (std::shared_ptr<nano::block> block_a, nano::block_sideband const & sideband_a) const override
{
nano::uint128_t result;
switch (block_a->type ())
{
case nano::block_type::open:
case nano::block_type::receive:
case nano::block_type::change:
result = sideband_a.balance.number ();
break;
case nano::block_type::send:
result = boost::polymorphic_downcast<nano::send_block *> (block_a.get ())->hashables.balance.number ();
break;
case nano::block_type::state:
result = boost::polymorphic_downcast<nano::state_block *> (block_a.get ())->hashables.balance.number ();
break;
case nano::block_type::invalid:
case nano::block_type::not_a_block:
release_assert (false);
break;
}
return result;
}
nano::block_hash block_successor (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override
{
nano::block_type type;
auto value (block_raw_get (transaction_a, hash_a, type));
nano::block_hash result;
if (value.size () != 0)
{
assert (value.size () >= result.bytes.size ());
nano::bufferstream stream (reinterpret_cast<uint8_t const *> (value.data ()) + block_successor_offset (transaction_a, value.size (), type), result.bytes.size ());
auto error (nano::try_read (stream, result.bytes));
(void)error;
assert (!error);
}
else
{
result.clear ();
}
return result;
}
bool full_sideband (nano::transaction const & transaction_a) const
{
return version_get (transaction_a) > 12;
}
void block_successor_clear (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override
{
nano::block_type type;
auto value (block_raw_get (transaction_a, hash_a, type));
auto version (block_version (transaction_a, hash_a));
assert (value.size () != 0);
std::vector<uint8_t> data (static_cast<uint8_t *> (value.data ()), static_cast<uint8_t *> (value.data ()) + value.size ());
std::fill_n (data.begin () + block_successor_offset (transaction_a, value.size (), type), sizeof (nano::uint256_union), uint8_t{ 0 });
block_raw_put (transaction_a, data, type, version, hash_a);
}
uint64_t cemented_count (nano::transaction const & transaction_a) override
{
uint64_t sum = 0;
for (auto i (latest_begin (transaction_a)), n (latest_end ()); i != n; ++i)
{
nano::account_info const & info (i->second);
sum += info.confirmation_height;
}
return sum;
}
void unchecked_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, std::shared_ptr<nano::block> const & block_a) override
{
nano::unchecked_key key (hash_a, block_a->hash ());
nano::unchecked_info info (block_a, block_a->account (), nano::seconds_since_epoch (), nano::signature_verification::unknown);
unchecked_put (transaction_a, key, info);
}
std::shared_ptr<nano::vote> vote_current (nano::transaction const & transaction_a, nano::account const & account_a) override
{
assert (!cache_mutex.try_lock ());
std::shared_ptr<nano::vote> result;
auto existing (vote_cache_l1.find (account_a));
auto have_existing (true);
if (existing == vote_cache_l1.end ())
{
existing = vote_cache_l2.find (account_a);
if (existing == vote_cache_l2.end ())
{
have_existing = false;
}
}
if (have_existing)
{
result = existing->second;
}
else
{
result = vote_get (transaction_a, account_a);
}
return result;
}
std::shared_ptr<nano::vote> vote_generate (nano::transaction const & transaction_a, nano::account const & account_a, nano::raw_key const & key_a, std::shared_ptr<nano::block> block_a) override
{
std::lock_guard<std::mutex> lock (cache_mutex);
auto result (vote_current (transaction_a, account_a));
uint64_t sequence ((result ? result->sequence : 0) + 1);
result = std::make_shared<nano::vote> (account_a, key_a, sequence, block_a);
vote_cache_l1[account_a] = result;
return result;
}
std::shared_ptr<nano::vote> vote_generate (nano::transaction const & transaction_a, nano::account const & account_a, nano::raw_key const & key_a, std::vector<nano::block_hash> blocks_a) override
{
std::lock_guard<std::mutex> lock (cache_mutex);
auto result (vote_current (transaction_a, account_a));
uint64_t sequence ((result ? result->sequence : 0) + 1);
result = std::make_shared<nano::vote> (account_a, key_a, sequence, blocks_a);
vote_cache_l1[account_a] = result;
return result;
}
std::shared_ptr<nano::vote> vote_max (nano::transaction const & transaction_a, std::shared_ptr<nano::vote> vote_a) override
{
std::lock_guard<std::mutex> lock (cache_mutex);
auto current (vote_current (transaction_a, vote_a->account));
auto result (vote_a);
if (current != nullptr && current->sequence > result->sequence)
{
result = current;
}
vote_cache_l1[vote_a->account] = result;
return result;
}
virtual void block_raw_put (nano::transaction const & transaction_a, std::vector<uint8_t> const & data, nano::block_type block_type_a, nano::epoch epoch_a, nano::block_hash const & hash_a) = 0;
protected:
nano::network_params network_params;
std::unordered_map<nano::account, std::shared_ptr<nano::vote>> vote_cache_l1;
std::unordered_map<nano::account, std::shared_ptr<nano::vote>> vote_cache_l2;
bool entry_has_sideband (size_t entry_size_a, nano::block_type type_a) const
{
return entry_size_a == nano::block::size (type_a) + nano::block_sideband::size (type_a);
}
nano::db_val<Val> block_raw_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const
{
nano::db_val<Val> result;
// Table lookups are ordered by match probability
nano::block_type block_types[]{ nano::block_type::state, nano::block_type::send, nano::block_type::receive, nano::block_type::open, nano::block_type::change };
for (auto current_type : block_types)
{
auto db_val (block_raw_get_by_type (transaction_a, hash_a, current_type));
if (db_val.is_initialized ())
{
type_a = current_type;
result = db_val.get ();
break;
}
}
return result;
}
// Return account containing hash
nano::account block_account_computed (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const
{
assert (!full_sideband (transaction_a));
nano::account result (0);
auto hash (hash_a);
while (result.is_zero ())
{
auto block (block_get (transaction_a, hash));
assert (block);
result = block->account ();
if (result.is_zero ())
{
auto type (nano::block_type::invalid);
auto value (block_raw_get (transaction_a, block->previous (), type));
if (entry_has_sideband (value.size (), type))
{
result = block_account (transaction_a, block->previous ());
}
else
{
nano::block_info block_info;
if (!block_info_get (transaction_a, hash, block_info))
{
result = block_info.account;
}
else
{
result = frontier_get (transaction_a, hash);
if (result.is_zero ())
{
auto successor (block_successor (transaction_a, hash));
assert (!successor.is_zero ());
hash = successor;
}
}
}
}
}
assert (!result.is_zero ());
return result;
}
nano::uint128_t block_balance_computed (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const
{
assert (!full_sideband (transaction_a));
summation_visitor visitor (transaction_a, *this);
return visitor.compute_balance (hash_a);
}
size_t block_successor_offset (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const
{
size_t result;
if (full_sideband (transaction_a) || entry_has_sideband (entry_size_a, type_a))
{
result = entry_size_a - nano::block_sideband::size (type_a);
}
else
{
// Read old successor-only sideband
assert (entry_size_a == nano::block::size (type_a) + sizeof (nano::uint256_union));
result = entry_size_a - sizeof (nano::uint256_union);
}
return result;
}
virtual boost::optional<DB_val> block_raw_get_by_type (nano::transaction const &, nano::block_hash const &, nano::block_type &) const = 0;
};
/**
* Fill in our predecessors
*/
template <class Val>
class block_predecessor_set : public nano::block_visitor
{
public:
block_predecessor_set (nano::transaction const & transaction_a, nano::block_store_partial<Val> & store_a) :
transaction (transaction_a),
store (store_a)
{
}
virtual ~block_predecessor_set () = default;
void fill_value (nano::block const & block_a)
{
auto hash (block_a.hash ());
nano::block_type type;
auto value (store.block_raw_get (transaction, block_a.previous (), type));
auto version (store.block_version (transaction, block_a.previous ()));
assert (value.size () != 0);
std::vector<uint8_t> data (static_cast<uint8_t *> (value.data ()), static_cast<uint8_t *> (value.data ()) + value.size ());
std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + store.block_successor_offset (transaction, value.size (), type));
store.block_raw_put (transaction, data, type, version, block_a.previous ());
}
void send_block (nano::send_block const & block_a) override
{
fill_value (block_a);
}
void receive_block (nano::receive_block const & block_a) override
{
fill_value (block_a);
}
void open_block (nano::open_block const & block_a) override
{
// Open blocks don't have a predecessor
}
void change_block (nano::change_block const & block_a) override
{
fill_value (block_a);
}
void state_block (nano::state_block const & block_a) override
{
if (!block_a.previous ().is_zero ())
{
fill_value (block_a);
}
}
nano::transaction const & transaction;
nano::block_store_partial<Val> & store;
};
std::unique_ptr<nano::block_store> make_store (bool & init, nano::logger_mt & logger, boost::filesystem::path const & path);
}

1233
nano/secure/blockstore_partial.hpp Normal file
View file

File diff suppressed because it is too large Load diff

14
nano/slow_test/node.cpp
View file

@ -95,13 +95,13 @@ TEST (ledger, deep_account_compute)
{
nano::logger_mt logger;
bool init (false);
nano::mdb_store store (init, logger, nano::unique_path ());
auto store = nano::make_store (init, logger, nano::unique_path ());
ASSERT_FALSE (init);
nano::stat stats;
nano::ledger ledger (store, stats);
nano::ledger ledger (*store, stats);
nano::genesis genesis;
auto transaction (store.tx_begin_write ());
store.initialize (transaction, genesis);
auto transaction (store->tx_begin_write ());
store->initialize (transaction, genesis);
nano::work_pool pool (std::numeric_limits<unsigned>::max ());
nano::keypair key;
auto balance (nano::genesis_amount - 1);
@ -124,10 +124,8 @@ TEST (ledger, deep_account_compute)
{
std::cerr << i << ' ';
}
auto account (ledger.account (transaction, sprevious));
(void)account;
auto balance (ledger.balance (transaction, rprevious));
(void)balance;
ledger.account (transaction, sprevious);
ledger.balance (transaction, rprevious);
}
}