Minecon724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 1

Adds RocksDB support (#2197)

Browse source 
* Add RocksDB support

* Remove unused variable

* node.confirm_locked test is using the wrong store

* Fix Tests, ASAN issues and other improvements

* Delete column family after dropping it

* (Unrelated) Output to cerr if there is a problem reading genesis block

* Vacuum fails if the backup directory does not exist

* Copy WAL log files over when doing snapshot/vacuuming and open the database to explicitly flush them

* Don't atomic_flush yet since WAL are enabled by default
This commit is contained in:
Wesley Shillingford 2019-08-28 08:08:43 +01:00 • committed by GitHub
commit bcb7f39be4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
49 changed files with 1731 additions and 373 deletions
Download patch file Download diff file Expand all files Collapse all files

3
CMakeLists.txt
View file

@ -32,6 +32,9 @@ set (NANO_SECURE_RPC OFF CACHE BOOL "")
set (NANO_ROCKSDB OFF CACHE BOOL "")
set (NANO_WARN_TO_ERR OFF CACHE BOOL "")
if (NANO_ROCKSDB)
add_definitions (-DNANO_ROCKSDB=1)
endif ()
option(NANO_ASAN_INT "Enable ASan+UBSan+Integer overflow" OFF)
option(NANO_ASAN "Enable ASan+UBSan" OFF)
option(NANO_TSAN "Enable TSan" OFF)

2
ci/build-travis.sh
View file

@ -27,6 +27,8 @@ else
SANITIZERS=""
fi
ulimit -S -n 8192
cmake \
-G'Unix Makefiles' \
-DACTIVE_NETWORK=nano_test_network \

2
ci/test.sh
View file

@ -1,7 +1,7 @@
#!/usr/bin/env bash
build_dir=${1-${PWD}}
TIMEOUT_DEFAULT=180
TIMEOUT_DEFAULT=360
BUSYBOX_BASH=${BUSYBOX_BASH-0}

7
docker/ci/Dockerfile-clang
View file

@ -1,7 +1,12 @@
FROM nanocurrency/nano-env:base
RUN apt-get update -qq && apt-get install -yqq \
clang-3.9 lldb-3.9 librocksdb-dev
clang-3.9 lldb-3.9 git
RUN git clone https://github.com/facebook/rocksdb.git && \
cd rocksdb && \
make static_lib && \
make install
ENV CXX=/usr/bin/clang++
ENV CC=/usr/bin/clang

7
docker/ci/Dockerfile-gcc
View file

@ -1,6 +1,11 @@
FROM nanocurrency/nano-env:base
RUN apt-get install -yqq librocksdb-dev
RUN apt-get install -yqq git
RUN git clone https://github.com/facebook/rocksdb.git && \
cd rocksdb && \
make static_lib && \
make install
ENV BOOST_ROOT=/usr/local
ADD util/build_prep/bootstrap_boost.sh bootstrap_boost.sh

8
nano/core_test/CMakeLists.txt
View file

@ -1,8 +1,3 @@
if (NANO_ROCKSDB)
set (rocksdb_test rocksdb_test.cpp)
set (rocksdb_libs ${ROCKSDB_LIBRARIES} ${ZLIB_LIBRARIES})
endif ()
add_executable (core_test
core_test_main.cc
testutil.hpp
@ -23,7 +18,6 @@ add_executable (core_test
memory_pool.cpp
processor_service.cpp
peer_container.cpp
${rocksdb_test}
signing.cpp
socket.cpp
toml.cpp
@ -41,4 +35,4 @@ target_compile_definitions(core_test
-DTAG_VERSION_STRING=${TAG_VERSION_STRING}
-DGIT_COMMIT_HASH=${GIT_COMMIT_HASH}
-DBOOST_PROCESS_SUPPORTED=${BOOST_PROCESS_SUPPORTED})
target_link_libraries (core_test node secure gtest libminiupnpc-static Boost::boost ${rocksdb_libs})
target_link_libraries (core_test node secure gtest libminiupnpc-static Boost::boost)

14
nano/core_test/block_store.cpp
View file

@ -614,6 +614,7 @@ TEST (block_store, latest_find)
ASSERT_EQ (second, find3);
}
#if !NANO_ROCKSDB
TEST (block_store, bad_path)
{
nano::logger_mt logger;
@ -621,6 +622,7 @@ TEST (block_store, bad_path)
auto store = nano::make_store (init, logger, boost::filesystem::path ("///"));
ASSERT_TRUE (init);
}
#endif
TEST (block_store, DISABLED_already_open) // File can be shared
{
@ -1849,14 +1851,18 @@ TEST (block_store, incompatible_version)
// Put version to an unreachable number so that it should always be incompatible
auto transaction (store->tx_begin_write ());
store->version_put (transaction, std::numeric_limits<unsigned>::max ());
store->version_put (transaction, std::numeric_limits<int>::max ());
}
// Now try and read it, should give an error
{
auto error (false);
auto store = nano::make_store (error, logger, path);
auto store = nano::make_store (error, logger, path, true);
ASSERT_TRUE (error);
auto transaction = store->tx_begin_read ();
auto version_l = store->version_get (transaction);
ASSERT_EQ (version_l, std::numeric_limits<int>::max ());
}
}
@ -1901,7 +1907,7 @@ void modify_account_info_to_v13 (nano::mdb_store & store, nano::transaction cons
nano::account_info info;
ASSERT_FALSE (store.account_get (transaction_a, account, info));
nano::account_info_v13 account_info_v13 (info.head, info.rep_block, info.open_block, info.balance, info.modified, info.block_count, info.epoch);
auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::block_store_partial<MDB_val, nano::mdb_store>::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0));
auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0));
(void)status;
assert (status == 0);
}
@ -1911,7 +1917,7 @@ void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction cons
nano::account_info info;
ASSERT_FALSE (store.account_get (transaction_a, account, info));
nano::account_info_v14 account_info_v14 (info.head, info.rep_block, info.open_block, info.balance, info.modified, info.block_count, confirmation_height, info.epoch);
auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::block_store_partial<MDB_val, nano::mdb_store>::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v14), 0));
auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v14), 0));
(void)status;
assert (status == 0);
}

4
nano/core_test/ledger.cpp
View file

@ -8,16 +8,18 @@
using namespace std::chrono_literals;
#if !NANO_ROCKSDB
// Init returns an error if it can't open files at the path
TEST (ledger, store_error)
{
nano::logger_mt logger;
bool init (false);
auto store = nano::make_store (init, logger, boost::filesystem::path ("///"));
ASSERT_FALSE (!init);
ASSERT_TRUE (init);
nano::stat stats;
nano::ledger ledger (*store, stats);
}
#endif
// Ledger can be initialized and returns a basic query for an empty account
TEST (ledger, empty)

2
nano/core_test/network.cpp
View file

@ -1544,7 +1544,7 @@ TEST (confirmation_height, single)
ASSERT_NO_ERROR (system.poll ());
}
auto transaction = node->store.tx_begin_read ();
auto transaction = node->store.tx_begin_write ();
ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height));
ASSERT_EQ (2, confirmation_height);

11
nano/core_test/node.cpp
View file

@ -242,6 +242,7 @@ TEST (node, auto_bootstrap)
node1->network.send_keepalive (channel);
node1->start ();
system.nodes.push_back (node1);
system.deadline_set (10s);
while (!node1->bootstrap_initiator.in_progress ())
{
ASSERT_NO_ERROR (system.poll ());
@ -497,7 +498,7 @@ TEST (node, confirm_locked)
{
nano::system system (24000, 1);
system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv);
auto transaction (system.nodes[0]->store.tx_begin_read ());
auto transaction (system.wallet (0)->wallets.tx_begin_read ());
system.wallet (0)->enter_password (transaction, "1");
auto block (std::make_shared<nano::send_block> (0, 0, 0, nano::keypair ().prv, 0, 0));
system.nodes[0]->network.flood_block (block);
@ -3059,15 +3060,15 @@ TEST (node, dont_write_lock_node)
std::thread ([&path, &write_lock_held_promise, &finished_promise]() {
nano::logger_mt logger;
bool init (false);
nano::mdb_store store (init, logger, path / "data.ldb");
auto store = nano::make_store (init, logger, path, false, true);
nano::genesis genesis;
{
auto transaction (store.tx_begin_write ());
store.initialize (transaction, genesis);
auto transaction (store->tx_begin_write ());
store->initialize (transaction, genesis);
}
// Hold write lock open until main thread is done needing it
auto transaction (store.tx_begin_write ());
auto transaction (store->tx_begin_write ());
write_lock_held_promise.set_value ();
finished_promise.get_future ().wait ();
})

31
nano/core_test/rocksdb_test.cpp
View file

@ -1,31 +0,0 @@
#include <nano/secure/utility.hpp>
#include <gtest/gtest.h>
#include <thread>
#include <rocksdb/db.h>
#include <rocksdb/options.h>
#include <rocksdb/utilities/optimistic_transaction_db.h>
#include <rocksdb/utilities/transaction.h>
#include <rocksdb/utilities/transaction_db.h>
TEST (rocksdb, build_test)
{
auto path = nano::unique_path ();
std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
column_families.push_back (rocksdb::ColumnFamilyDescriptor (
rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions ()));
rocksdb::Options options;
options.create_if_missing = true;
options.IncreaseParallelism (std::thread::hardware_concurrency ());
options.OptimizeLevelStyleCompaction ();
options.OptimizeUniversalStyleCompaction ();
std::vector<rocksdb::ColumnFamilyHandle *> handles;
rocksdb::OptimisticTransactionDB * db;
auto s = rocksdb::OptimisticTransactionDB::Open (options, path.string (), column_families, &handles, &db);
ASSERT_TRUE (s.ok ());
}

2
nano/core_test/socket.cpp
View file

@ -10,7 +10,7 @@ using namespace std::chrono_literals;
TEST (socket, concurrent_writes)
{
nano::inactive_node inactivenode;
nano::inactive_node inactivenode (nano::working_path (), 24000, false);
auto node = inactivenode.node;
// This gives more realistic execution than using system#poll, allowing writes to

50
nano/core_test/utility.cpp
View file

@ -32,3 +32,53 @@ TEST (thread, worker)
}
ASSERT_TRUE (passed_sleep);
}
TEST (filesystem, remove_all_files)
{
auto path = nano::unique_path ();
auto dummy_directory = path / "tmp";
boost::filesystem::create_directories (dummy_directory);
auto dummy_file1 = path / "my_file1.txt";
auto dummy_file2 = path / "my_file2.txt";
std::ofstream (dummy_file1.string ());
std::ofstream (dummy_file2.string ());
// Check all exist
ASSERT_TRUE (boost::filesystem::exists (dummy_directory));
ASSERT_TRUE (boost::filesystem::exists (dummy_file1));
ASSERT_TRUE (boost::filesystem::exists (dummy_file2));
// Should remove only the files
nano::remove_all_files_in_dir (path);
ASSERT_TRUE (boost::filesystem::exists (dummy_directory));
ASSERT_FALSE (boost::filesystem::exists (dummy_file1));
ASSERT_FALSE (boost::filesystem::exists (dummy_file2));
}
TEST (filesystem, move_all_files)
{
auto path = nano::unique_path ();
auto dummy_directory = path / "tmp";
boost::filesystem::create_directories (dummy_directory);
auto dummy_file1 = dummy_directory / "my_file1.txt";
auto dummy_file2 = dummy_directory / "my_file2.txt";
std::ofstream (dummy_file1.string ());
std::ofstream (dummy_file2.string ());
// Check all exist
ASSERT_TRUE (boost::filesystem::exists (dummy_directory));
ASSERT_TRUE (boost::filesystem::exists (dummy_file1));
ASSERT_TRUE (boost::filesystem::exists (dummy_file2));
// Should move only the files
nano::move_all_files_to_dir (dummy_directory, path);
ASSERT_TRUE (boost::filesystem::exists (dummy_directory));
ASSERT_TRUE (boost::filesystem::exists (path / "my_file1.txt"));
ASSERT_TRUE (boost::filesystem::exists (path / "my_file2.txt"));
ASSERT_FALSE (boost::filesystem::exists (dummy_file1));
ASSERT_FALSE (boost::filesystem::exists (dummy_file2));
}

4
nano/core_test/wallets.cpp
View file

@ -78,6 +78,7 @@ TEST (wallets, remove)
}
}
#if !NANO_ROCKSDB
TEST (wallets, upgrade)
{
nano::system system (24000, 1);
@ -101,7 +102,7 @@ TEST (wallets, upgrade)
nano::account_info info;
ASSERT_FALSE (mdb_store.account_get (transaction_destination, nano::genesis_account, info));
nano::account_info_v13 account_info_v13 (info.head, info.rep_block, info.open_block, info.balance, info.modified, info.block_count, info.epoch);
auto status (mdb_put (mdb_store.env.tx (transaction_destination), mdb_store.get_account_db (info.epoch) == nano::block_store_partial<MDB_val, nano::mdb_store>::tables::accounts_v0 ? mdb_store.accounts_v0 : mdb_store.accounts_v1, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (account_info_v13), 0));
auto status (mdb_put (mdb_store.env.tx (transaction_destination), mdb_store.get_account_db (info.epoch) == nano::tables::accounts_v0 ? mdb_store.accounts_v0 : mdb_store.accounts_v1, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (account_info_v13), 0));
(void)status;
assert (status == 0);
}
@ -118,6 +119,7 @@ TEST (wallets, upgrade)
MDB_dbi new_handle;
ASSERT_EQ (0, mdb_dbi_open (tx_new, id.pub.to_string ().c_str (), 0, &new_handle));
}
#endif
// Keeps breaking whenever we add new DBs
TEST (wallets, DISABLED_wallet_create_max)

2
nano/lib/CMakeLists.txt
View file

@ -21,6 +21,8 @@ add_library (nano_lib
config.hpp
config.cpp
configbase.hpp
diagnosticsconfig.hpp
diagnosticsconfig.cpp
errors.hpp
errors.cpp
ipc.hpp

2
nano/node/diagnosticsconfig.cpp → nano/lib/diagnosticsconfig.cpp
View file

@ -1,6 +1,6 @@
#include <nano/lib/diagnosticsconfig.hpp>
#include <nano/lib/jsonconfig.hpp>
#include <nano/lib/tomlconfig.hpp>
#include <nano/node/diagnosticsconfig.hpp>
nano::error nano::diagnostics_config::serialize_json (nano::jsonconfig & json) const
{

0
nano/node/diagnosticsconfig.hpp → nano/lib/diagnosticsconfig.hpp
View file

24
nano/lib/utility.cpp
View file

@ -298,6 +298,30 @@ std::unique_ptr<nano::seq_con_info_component> nano::collect_seq_con_info (nano::
return composite;
}
void nano::remove_all_files_in_dir (boost::filesystem::path const & dir)
{
for (auto & p : boost::filesystem::directory_iterator (dir))
{
auto path = p.path ();
if (boost::filesystem::is_regular_file (path))
{
boost::filesystem::remove (path);
}
}
}
void nano::move_all_files_to_dir (boost::filesystem::path const & from, boost::filesystem::path const & to)
{
for (auto & p : boost::filesystem::directory_iterator (from))
{
auto path = p.path ();
if (boost::filesystem::is_regular_file (path))
{
boost::filesystem::rename (path, to / path.filename ());
}
}
}
/*
* Backing code for "release_assert", which is itself a macro
*/

3
nano/lib/utility.hpp
View file

@ -217,6 +217,9 @@ inline std::unique_ptr<seq_con_info_component> collect_seq_con_info (observer_se
composite->add_component (std::make_unique<seq_con_info_leaf> (seq_con_info{ "observers", count, sizeof_element }));
return composite;
}
void remove_all_files_in_dir (boost::filesystem::path const & dir);
void move_all_files_to_dir (boost::filesystem::path const & from, boost::filesystem::path const & to);
}
void release_assert_internal (bool check, const char * check_expr, const char * file, unsigned int line);

4
nano/node/CMakeLists.txt
View file

@ -1,5 +1,6 @@
if (NANO_ROCKSDB)
set (rocksdb_libs ${ROCKSDB_LIBRARIES} ${ZLIB_LIBRARIES})
set (rocksdb_sources rocksdb/rocksdb.hpp rocksdb/rocksdb.cpp rocksdb/rocksdb_iterator.hpp rocksdb/rocksdb_txn.hpp rocksdb/rocksdb_txn.cpp)
endif ()
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
@ -20,6 +21,7 @@ endif ()
add_library (node
${platform_sources}
${rocksdb_sources}
active_transactions.hpp
active_transactions.cpp
blockprocessor.cpp
@ -35,8 +37,6 @@ add_library (node
confirmation_height_processor.cpp
daemonconfig.hpp
daemonconfig.cpp
diagnosticsconfig.hpp
diagnosticsconfig.cpp
election.hpp
election.cpp
gap_cache.hpp

26
nano/node/active_transactions.cpp
View file

@ -76,21 +76,23 @@ void nano::active_transactions::confirm_frontiers (nano::transaction const & tra
lk.unlock ();
nano::account_info info;
auto error = node.store.account_get (transaction_a, cementable_account.account, info);
release_assert (!error);
uint64_t confirmation_height;
error = node.store.confirmation_height_get (transaction_a, cementable_account.account, confirmation_height);
release_assert (!error);
if (info.block_count > confirmation_height && !this->node.pending_confirmation_height.is_processing_block (info.head))
if (!error)
{
auto block (this->node.store.block_get (transaction_a, info.head));
if (!this->start (block))
uint64_t confirmation_height;
error = node.store.confirmation_height_get (transaction_a, cementable_account.account, confirmation_height);
release_assert (!error);
if (info.block_count > confirmation_height && !this->node.pending_confirmation_height.is_processing_block (info.head))
{
++elections_count;
// Calculate votes for local representatives
if (representative)
auto block (this->node.store.block_get (transaction_a, info.head));
if (!this->start (block))
{
this->node.block_processor.generator.add (block->hash ());
++elections_count;
// Calculate votes for local representatives
if (representative)
{
this->node.block_processor.generator.add (block->hash ());
}
}
}
}

11
nano/node/blockprocessor.cpp
View file

@ -262,7 +262,7 @@ void nano::block_processor::process_batch (std::unique_lock<std::mutex> & lock_a
}
lock_a.unlock ();
auto scoped_write_guard = write_database_queue.wait (nano::writer::process_batch);
auto transaction (node.store.tx_begin_write ());
auto transaction (node.store.tx_begin_write ({ nano::tables::accounts_v0, nano::tables::accounts_v1, nano::tables::cached_counts, nano::tables::change_blocks, nano::tables::frontiers, nano::tables::open_blocks, nano::tables::pending_v0, nano::tables::pending_v1, nano::tables::receive_blocks, nano::tables::representation, nano::tables::send_blocks, nano::tables::state_blocks_v0, nano::tables::state_blocks_v1, nano::tables::unchecked }, { nano::tables::confirmation_height }));
timer_l.restart ();
lock_a.lock ();
// Processing blocks
@ -348,8 +348,7 @@ void nano::block_processor::process_batch (std::unique_lock<std::mutex> & lock_a
}
}
number_of_blocks_processed++;
auto process_result (process_one (transaction, info));
(void)process_result;
process_one (transaction, info);
lock_a.lock ();
/* Verify more state blocks if blocks deque is empty
Because verification is long process, avoid large deque verification inside of write transaction */
@ -407,7 +406,7 @@ void nano::block_processor::process_live (nano::block_hash const & hash_a, std::
});
}
nano::process_return nano::block_processor::process_one (nano::transaction const & transaction_a, nano::unchecked_info info_a, const bool watch_work_a)
nano::process_return nano::block_processor::process_one (nano::write_transaction const & transaction_a, nano::unchecked_info info_a, const bool watch_work_a)
{
nano::process_return result;
auto hash (info_a.block->hash ());
@ -540,14 +539,14 @@ nano::process_return nano::block_processor::process_one (nano::transaction const
return result;
}
nano::process_return nano::block_processor::process_one (nano::transaction const & transaction_a, std::shared_ptr<nano::block> block_a, const bool watch_work_a)
nano::process_return nano::block_processor::process_one (nano::write_transaction const & transaction_a, std::shared_ptr<nano::block> block_a, const bool watch_work_a)
{
nano::unchecked_info info (block_a, block_a->account (), 0, nano::signature_verification::unknown);
auto result (process_one (transaction_a, info, watch_work_a));
return result;
}
void nano::block_processor::queue_unchecked (nano::transaction const & transaction_a, nano::block_hash const & hash_a)
void nano::block_processor::queue_unchecked (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a)
{
auto unchecked_blocks (node.store.unchecked_get (transaction_a, hash_a));
for (auto & info : unchecked_blocks)

7
nano/node/blockprocessor.hpp
View file

@ -18,6 +18,7 @@ namespace nano
{
class node;
class transaction;
class write_transaction;
class write_database_queue;
class rolled_hash
@ -47,14 +48,14 @@ public:
bool should_log (bool);
bool have_blocks ();
void process_blocks ();
nano::process_return process_one (nano::transaction const &, nano::unchecked_info, const bool = false);
nano::process_return process_one (nano::transaction const &, std::shared_ptr<nano::block>, const bool = false);
nano::process_return process_one (nano::write_transaction const &, nano::unchecked_info, const bool = false);
nano::process_return process_one (nano::write_transaction const &, std::shared_ptr<nano::block>, const bool = false);
nano::vote_generator generator;
// Delay required for average network propagartion before requesting confirmation
static std::chrono::milliseconds constexpr confirmation_request_delay{ 1500 };
private:
void queue_unchecked (nano::transaction const &, nano::block_hash const &);
void queue_unchecked (nano::write_transaction const &, nano::block_hash const &);
void verify_state_blocks (nano::transaction const & transaction_a, std::unique_lock<std::mutex> &, size_t = std::numeric_limits<size_t>::max ());
void process_batch (std::unique_lock<std::mutex> &);
void process_live (nano::block_hash const &, std::shared_ptr<nano::block>, const bool = false);

265
nano/node/cli.cpp
View file

@ -22,6 +22,8 @@ std::string nano::error_cli_messages::message (int ev) const
return "Invalid arguments";
case nano::error_cli::unknown_command:
return "Unknown command";
case nano::error_cli::database_write_error:
return "Database write error";
}
return "Invalid error code";
@ -68,6 +70,57 @@ void nano::add_node_options (boost::program_options::options_description & descr
// clang-format on
}
namespace
{
void database_write_lock_error (std::error_code & ec)
{
std::cerr << "Write database error, this cannot be run while the node is already running\n";
ec = nano::error_cli::database_write_error;
}
bool copy_database (boost::filesystem::path const & data_path, boost::program_options::variables_map & vm, boost::filesystem::path const & output_path, std::error_code & ec)
{
bool success = false;
bool needs_to_write = vm.count ("unchecked_clear") || vm.count ("clear_send_ids") || vm.count ("online_weight_clear") || vm.count ("peer_clear") || vm.count ("confirmation_height_clear");
nano::inactive_node node (data_path, 24000, !needs_to_write);
if (!node.init.error ())
{
if (vm.count ("unchecked_clear"))
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.unchecked_clear (transaction);
}
if (vm.count ("clear_send_ids"))
{
auto transaction (node.node->wallets.tx_begin_write ());
node.node->wallets.clear_send_ids (transaction);
}
if (vm.count ("online_weight_clear"))
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.online_weight_clear (transaction);
}
if (vm.count ("peer_clear"))
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.peer_clear (transaction);
}
if (vm.count ("confirmation_height_clear"))
{
reset_confirmation_heights (node.node->store);
}
success = node.node->copy_with_compaction (output_path);
}
else
{
database_write_lock_error (ec);
}
return success;
}
}
std::error_code nano::handle_node_options (boost::program_options::variables_map & vm)
{
std::error_code ec;
@ -151,53 +204,45 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
{
try
{
auto vacuum_path = data_path / "vacuumed.ldb";
auto source_path = data_path / "data.ldb";
auto backup_path = data_path / "backup.vacuum.ldb";
std::cout << "Vacuuming database copy in " << data_path << std::endl;
std::cout << "This may take a while..." << std::endl;
// Scope the node so the mdb environment gets cleaned up properly before
// the original file is replaced with the vacuumed file.
bool success = false;
std::cout << "Vacuuming database copy in ";
#if NANO_ROCKSDB
auto source_path = data_path / "rocksdb";
auto backup_path = source_path / "backup";
auto vacuum_path = backup_path / "vacuumed";
if (!boost::filesystem::exists (vacuum_path))
{
inactive_node node (data_path);
if (vm.count ("unchecked_clear"))
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.unchecked_clear (transaction);
}
if (vm.count ("clear_send_ids"))
{
auto transaction (node.node->wallets.tx_begin_write ());
node.node->wallets.clear_send_ids (transaction);
}
if (vm.count ("online_weight_clear"))
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.online_weight_clear (transaction);
}
if (vm.count ("peer_clear"))
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.peer_clear (transaction);
}
success = node.node->copy_with_compaction (vacuum_path);
boost::filesystem::create_directories (vacuum_path);
}
std::cout << source_path << "\n";
#else
auto source_path = data_path / "data.ldb";
auto backup_path = data_path / "backup.vacuum.ldb";
auto vacuum_path = data_path / "vacuumed.ldb";
std::cout << data_path << "\n";
#endif
std::cout << "This may take a while..." << std::endl;
bool success = copy_database (data_path, vm, vacuum_path, ec);
if (success)
{
// Note that these throw on failure
std::cout << "Finalizing" << std::endl;
#ifdef NANO_ROCKSDB
nano::remove_all_files_in_dir (backup_path);
nano::move_all_files_to_dir (source_path, backup_path);
nano::move_all_files_to_dir (vacuum_path, source_path);
boost::filesystem::remove_all (vacuum_path);
#else
boost::filesystem::remove (backup_path);
boost::filesystem::rename (source_path, backup_path);
boost::filesystem::rename (vacuum_path, source_path);
#endif
std::cout << "Vacuum completed" << std::endl;
}
else
{
std::cerr << "Vacuum failed (copy_with_compaction returned false)" << std::endl;
std::cerr << "Vacuum failed (copying returned false)" << std::endl;
}
}
catch (const boost::filesystem::filesystem_error & ex)
@ -213,51 +258,24 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
{
try
{
boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as<std::string> ()) : nano::working_path ();
#if NANO_ROCKSDB
auto source_path = data_path / "rocksdb";
auto snapshot_path = source_path / "backup";
#else
auto source_path = data_path / "data.ldb";
auto snapshot_path = data_path / "snapshot.ldb";
#endif
std::cout << "Database snapshot of " << source_path << " to " << snapshot_path << " in progress" << std::endl;
std::cout << "This may take a while..." << std::endl;
bool success = false;
{
inactive_node node (data_path);
if (vm.count ("unchecked_clear"))
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.unchecked_clear (transaction);
}
if (vm.count ("clear_send_ids"))
{
auto transaction (node.node->wallets.tx_begin_write ());
node.node->wallets.clear_send_ids (transaction);
}
if (vm.count ("online_weight_clear"))
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.online_weight_clear (transaction);
}
if (vm.count ("peer_clear"))
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.peer_clear (transaction);
}
if (vm.count ("confirmation_height_clear"))
{
reset_confirmation_heights (node.node->store);
}
success = node.node->copy_with_compaction (snapshot_path);
}
bool success = copy_database (data_path, vm, snapshot_path, ec);
if (success)
{
std::cout << "Snapshot completed, This can be found at " << snapshot_path << std::endl;
}
else
{
std::cerr << "Snapshot Failed (copy_with_compaction returned false)" << std::endl;
std::cerr << "Snapshot Failed (copying returned false)" << std::endl;
}
}
catch (const boost::filesystem::filesystem_error & ex)
@ -272,80 +290,115 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
else if (vm.count ("unchecked_clear"))
{
boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as<std::string> ()) : nano::working_path ();
inactive_node node (data_path);
auto transaction (node.node->store.tx_begin_write ());
node.node->store.unchecked_clear (transaction);
std::cout << "Unchecked blocks deleted" << std::endl;
inactive_node node (data_path, 24000, false);
if (!node.init.error ())
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.unchecked_clear (transaction);
std::cout << "Unchecked blocks deleted" << std::endl;
}
else
{
database_write_lock_error (ec);
}
}
else if (vm.count ("clear_send_ids"))
{
boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as<std::string> ()) : nano::working_path ();
inactive_node node (data_path);
auto transaction (node.node->wallets.tx_begin_write ());
node.node->wallets.clear_send_ids (transaction);
std::cout << "Send IDs deleted" << std::endl;
inactive_node node (data_path, 24000, false);
if (!node.init.error ())
{
auto transaction (node.node->wallets.tx_begin_write ());
node.node->wallets.clear_send_ids (transaction);
std::cout << "Send IDs deleted" << std::endl;
}
else
{
database_write_lock_error (ec);
}
}
else if (vm.count ("online_weight_clear"))
{
boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as<std::string> ()) : nano::working_path ();
inactive_node node (data_path);
auto transaction (node.node->store.tx_begin_write ());
node.node->store.online_weight_clear (transaction);
std::cout << "Onine weight records are removed" << std::endl;
inactive_node node (data_path, 24000, false);
if (!node.init.error ())
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.online_weight_clear (transaction);
std::cout << "Onine weight records are removed" << std::endl;
}
else
{
database_write_lock_error (ec);
}
}
else if (vm.count ("peer_clear"))
{
boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as<std::string> ()) : nano::working_path ();
inactive_node node (data_path);
auto transaction (node.node->store.tx_begin_write ());
node.node->store.peer_clear (transaction);
std::cout << "Database peers are removed" << std::endl;
inactive_node node (data_path, 24000, false);
if (!node.init.error ())
{
auto transaction (node.node->store.tx_begin_write ());
node.node->store.peer_clear (transaction);
std::cout << "Database peers are removed" << std::endl;
}
else
{
database_write_lock_error (ec);
}
}
else if (vm.count ("confirmation_height_clear"))
{
boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as<std::string> ()) : nano::working_path ();
inactive_node node (data_path);
auto account_it = vm.find ("account");
if (account_it != vm.cend ())
inactive_node node (data_path, 24000, false);
if (!node.init.error ())
{
auto account_str = account_it->second.as<std::string> ();
nano::account account;
if (!account.decode_account (account_str))
auto account_it = vm.find ("account");
if (account_it != vm.cend ())
{
uint64_t confirmation_height;
auto transaction (node.node->store.tx_begin_read ());
if (!node.node->store.confirmation_height_get (transaction, account, confirmation_height))
auto account_str = account_it->second.as<std::string> ();
nano::account account;
if (!account.decode_account (account_str))
{
auto transaction (node.node->store.tx_begin_write ());
auto conf_height_reset_num = 0;
if (account == node.node->network_params.ledger.genesis_account)
uint64_t confirmation_height;
auto transaction (node.node->store.tx_begin_read ());
if (!node.node->store.confirmation_height_get (transaction, account, confirmation_height))
{
conf_height_reset_num = 1;
node.node->store.confirmation_height_put (transaction, account, confirmation_height);
auto transaction (node.node->store.tx_begin_write ());
auto conf_height_reset_num = 0;
if (account == node.node->network_params.ledger.genesis_account)
{
conf_height_reset_num = 1;
node.node->store.confirmation_height_put (transaction, account, confirmation_height);
}
else
{
node.node->store.confirmation_height_clear (transaction, account, confirmation_height);
}
std::cout << "Confirmation height of account " << account_str << " is set to " << conf_height_reset_num << std::endl;
}
else
{
node.node->store.confirmation_height_clear (transaction, account, confirmation_height);
std::cerr << "Could not find account" << std::endl;
ec = nano::error_cli::generic;
}
std::cout << "Confirmation height of account " << account_str << " is set to " << conf_height_reset_num << std::endl;
}
else
{
std::cerr << "Could not find account" << std::endl;
ec = nano::error_cli::generic;
std::cerr << "Invalid account id\n";
ec = nano::error_cli::invalid_arguments;
}
}
else
{
std::cerr << "Invalid account id\n";
ec = nano::error_cli::invalid_arguments;
reset_confirmation_heights (node.node->store);
std::cout << "Confirmation heights of all accounts (except genesis which is set to 1) are set to 0" << std::endl;
}
}
else
{
reset_confirmation_heights (node.node->store);
std::cout << "Confirmation heights of all accounts (except genesis) are set to 0" << std::endl;
database_write_lock_error (ec);
}
}
else if (vm.count ("generate_config"))

3
nano/node/cli.hpp
View file

@ -12,7 +12,8 @@ enum class error_cli
generic = 1,
parse_error = 2,
invalid_arguments = 3,
unknown_command = 4
unknown_command = 4,
database_write_error = 5
};
void add_node_options (boost::program_options::options_description &);

3
nano/node/confirmation_height_processor.cpp
View file

@ -269,7 +269,8 @@ bool nano::confirmation_height_processor::write_pending (std::deque<conf_height_
while (total_pending_write_block_count > 0)
{
uint64_t num_accounts_processed = 0;
auto transaction (store.tx_begin_write ());
auto transaction (store.tx_begin_write ({}, { nano::tables::confirmation_height }));
while (!all_pending_a.empty ())
{
const auto & pending = all_pending_a.front ();

6
nano/node/json_handler.cpp
View file

@ -4439,8 +4439,7 @@ void nano::json_handler::work_generate ()
{
auto hash (hash_impl ());
auto difficulty (difficulty_optional_impl ());
auto multiplier (multiplier_optional_impl (difficulty));
(void)multiplier;
multiplier_optional_impl (difficulty);
if (!ec && (difficulty > node.config.max_work_generate_difficulty || difficulty < node.network_params.network.publish_threshold))
{
ec = nano::error_rpc::difficulty_limit;
@ -4540,8 +4539,7 @@ void nano::json_handler::work_validate ()
auto hash (hash_impl ());
auto work (work_optional_impl ());
auto difficulty (difficulty_optional_impl ());
auto multiplier (multiplier_optional_impl (difficulty));
(void)multiplier;
multiplier_optional_impl (difficulty);
if (!ec)
{
uint64_t result_difficulty (0);

45
nano/node/lmdb/lmdb.cpp
View file

@ -82,7 +82,7 @@ txn_tracking_enabled (txn_tracking_config_a.enable)
if (!error_a && drop_unchecked)
{
auto transaction (tx_begin_write ());
auto transaction (tx_begin_write ({ nano::tables::cached_counts, tables::unchecked }));
unchecked_clear (transaction);
}
}
@ -93,7 +93,7 @@ void nano::mdb_store::serialize_mdb_tracker (boost::property_tree::ptree & json,
mdb_txn_tracker.serialize_json (json, min_read_time, min_write_time);
}
nano::write_transaction nano::mdb_store::tx_begin_write ()
nano::write_transaction nano::mdb_store::tx_begin_write (std::vector<nano::tables> const &, std::vector<nano::tables> const &)
{
return env.tx_begin_write (create_txn_callbacks ());
}
@ -189,7 +189,7 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, size
return error;
}
void nano::mdb_store::upgrade_v1_to_v2 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v1_to_v2 (nano::write_transaction const & transaction_a)
{
version_put (transaction_a, 2);
nano::account account (1);
@ -223,7 +223,7 @@ void nano::mdb_store::upgrade_v1_to_v2 (nano::transaction const & transaction_a)
}
}
void nano::mdb_store::upgrade_v2_to_v3 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v2_to_v3 (nano::write_transaction const & transaction_a)
{
version_put (transaction_a, 3);
mdb_drop (env.tx (transaction_a), representation, 0);
@ -241,7 +241,7 @@ void nano::mdb_store::upgrade_v2_to_v3 (nano::transaction const & transaction_a)
}
}
void nano::mdb_store::upgrade_v3_to_v4 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v3_to_v4 (nano::write_transaction const & transaction_a)
{
version_put (transaction_a, 4);
std::queue<std::pair<nano::pending_key, nano::pending_info>> items;
@ -259,7 +259,7 @@ void nano::mdb_store::upgrade_v3_to_v4 (nano::transaction const & transaction_a)
}
}
void nano::mdb_store::upgrade_v4_to_v5 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v4_to_v5 (nano::write_transaction const & transaction_a)
{
version_put (transaction_a, 5);
for (auto i (nano::store_iterator<nano::account, nano::account_info_v5> (std::make_unique<nano::mdb_iterator<nano::account, nano::account_info_v5>> (transaction_a, accounts_v0))), n (nano::store_iterator<nano::account, nano::account_info_v5> (nullptr)); i != n; ++i)
@ -296,7 +296,7 @@ void nano::mdb_store::upgrade_v4_to_v5 (nano::transaction const & transaction_a)
}
}
void nano::mdb_store::upgrade_v5_to_v6 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v5_to_v6 (nano::write_transaction const & transaction_a)
{
version_put (transaction_a, 6);
std::deque<std::pair<nano::account, nano::account_info_v13>> headers;
@ -322,20 +322,20 @@ void nano::mdb_store::upgrade_v5_to_v6 (nano::transaction const & transaction_a)
}
}
void nano::mdb_store::upgrade_v6_to_v7 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v6_to_v7 (nano::write_transaction const & transaction_a)
{
version_put (transaction_a, 7);
mdb_drop (env.tx (transaction_a), unchecked, 0);
}
void nano::mdb_store::upgrade_v7_to_v8 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v7_to_v8 (nano::write_transaction const & transaction_a)
{
version_put (transaction_a, 8);
mdb_drop (env.tx (transaction_a), unchecked, 1);
mdb_dbi_open (env.tx (transaction_a), "unchecked", MDB_CREATE | MDB_DUPSORT, &unchecked);
}
void nano::mdb_store::upgrade_v8_to_v9 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v8_to_v9 (nano::write_transaction const & transaction_a)
{
version_put (transaction_a, 9);
MDB_dbi sequence;
@ -363,7 +363,7 @@ void nano::mdb_store::upgrade_v8_to_v9 (nano::transaction const & transaction_a)
mdb_drop (env.tx (transaction_a), sequence, 1);
}
void nano::mdb_store::upgrade_v10_to_v11 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v10_to_v11 (nano::write_transaction const & transaction_a)
{
version_put (transaction_a, 11);
MDB_dbi unsynced;
@ -371,7 +371,7 @@ void nano::mdb_store::upgrade_v10_to_v11 (nano::transaction const & transaction_
mdb_drop (env.tx (transaction_a), unsynced, 1);
}
void nano::mdb_store::upgrade_v11_to_v12 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v11_to_v12 (nano::write_transaction const & transaction_a)
{
version_put (transaction_a, 12);
mdb_drop (env.tx (transaction_a), unchecked, 1);
@ -443,7 +443,7 @@ void nano::mdb_store::upgrade_v12_to_v13 (nano::write_transaction & transaction_
}
}
void nano::mdb_store::upgrade_v13_to_v14 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v13_to_v14 (nano::write_transaction const & transaction_a)
{
// Upgrade all accounts to have a confirmation of 0 (except genesis which should have 1)
version_put (transaction_a, 14);
@ -476,7 +476,7 @@ void nano::mdb_store::upgrade_v13_to_v14 (nano::transaction const & transaction_
release_assert (!error || error == MDB_NOTFOUND);
}
void nano::mdb_store::upgrade_v14_to_v15 (nano::transaction const & transaction_a)
void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transaction_a)
{
version_put (transaction_a, 15);
@ -530,7 +530,7 @@ void nano::mdb_store::create_backup_file (nano::mdb_env & env_a, boost::filesyst
}
}
void nano::mdb_store::version_put (nano::transaction const & transaction_a, int version_a)
void nano::mdb_store::version_put (nano::write_transaction const & transaction_a, int version_a)
{
nano::uint256_union version_key (1);
nano::uint256_union version_value (version_a);
@ -584,22 +584,22 @@ int nano::mdb_store::get (nano::transaction const & transaction_a, tables table_
return mdb_get (env.tx (transaction_a), table_to_dbi (table_a), key_a, value_a);
}
int nano::mdb_store::put (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, const nano::mdb_val & value_a) const
int nano::mdb_store::put (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, const nano::mdb_val & value_a) const
{
return (mdb_put (env.tx (transaction_a), table_to_dbi (table_a), key_a, value_a, 0));
}
int nano::mdb_store::del (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const
int nano::mdb_store::del (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const
{
return (mdb_del (env.tx (transaction_a), table_to_dbi (table_a), key_a, nullptr));
}
int nano::mdb_store::drop (nano::transaction const & transaction_a, tables table_a)
int nano::mdb_store::drop (nano::write_transaction const & transaction_a, tables table_a)
{
return clear (transaction_a, table_to_dbi (table_a));
}
int nano::mdb_store::clear (nano::transaction const & transaction_a, MDB_dbi handle_a)
int nano::mdb_store::clear (nano::write_transaction const & transaction_a, MDB_dbi handle_a)
{
return mdb_drop (env.tx (transaction_a), handle_a, 0);
}
@ -667,7 +667,7 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const
bool nano::mdb_store::not_found (int status) const
{
return (MDB_NOTFOUND == status);
return (status_code_not_found () == status);
}
bool nano::mdb_store::success (int status) const
@ -679,3 +679,8 @@ int nano::mdb_store::status_code_not_found () const
{
return MDB_NOTFOUND;
}
bool nano::mdb_store::copy_db (boost::filesystem::path const & destination_file)
{
return !mdb_env_copy2 (env.environment, destination_file.string ().c_str (), MDB_CP_COMPACT);
}

48
nano/node/lmdb/lmdb.hpp
View file

@ -1,9 +1,9 @@
#pragma once
#include <nano/lib/config.hpp>
#include <nano/lib/diagnosticsconfig.hpp>
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/node/diagnosticsconfig.hpp>
#include <nano/node/lmdb/lmdb_env.hpp>
#include <nano/node/lmdb/lmdb_iterator.hpp>
#include <nano/node/lmdb/lmdb_txn.hpp>
@ -33,12 +33,12 @@ public:
using block_store_partial::unchecked_put;
mdb_store (bool &, nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512, bool backup_before_upgrade = false);
nano::write_transaction tx_begin_write () override;
nano::write_transaction tx_begin_write (std::vector<nano::tables> const & tables_requiring_lock = {}, std::vector<nano::tables> const & tables_no_lock = {}) override;
nano::read_transaction tx_begin_read () override;
bool block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) const override;
void version_put (nano::transaction const &, int) override;
void version_put (nano::write_transaction const &, int) override;
void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) override;
@ -165,52 +165,54 @@ public:
bool exists (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const;
int get (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, nano::mdb_val & value_a) const;
int put (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, const nano::mdb_val & value_a) const;
int del (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const;
int put (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, const nano::mdb_val & value_a) const;
int del (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const;
bool copy_db (boost::filesystem::path const & destination_file) override;
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a)
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a) const
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::mdb_iterator<Key, Value>> (transaction_a, table_to_dbi (table_a)));
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key)
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key) const
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::mdb_iterator<Key, Value>> (transaction_a, table_to_dbi (table_a), key));
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, nano::mdb_val const & key)
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, nano::mdb_val const & key) const
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::mdb_merge_iterator<Key, Value>> (transaction_a, table_to_dbi (table1_a), table_to_dbi (table2_a), key));
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a)
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a) const
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::mdb_merge_iterator<Key, Value>> (transaction_a, table_to_dbi (table1_a), table_to_dbi (table2_a)));
}
private:
bool do_upgrades (nano::write_transaction &, size_t);
void upgrade_v1_to_v2 (nano::transaction const &);
void upgrade_v2_to_v3 (nano::transaction const &);
void upgrade_v3_to_v4 (nano::transaction const &);
void upgrade_v4_to_v5 (nano::transaction const &);
void upgrade_v5_to_v6 (nano::transaction const &);
void upgrade_v6_to_v7 (nano::transaction const &);
void upgrade_v7_to_v8 (nano::transaction const &);
void upgrade_v8_to_v9 (nano::transaction const &);
void upgrade_v10_to_v11 (nano::transaction const &);
void upgrade_v11_to_v12 (nano::transaction const &);
void upgrade_v1_to_v2 (nano::write_transaction const &);
void upgrade_v2_to_v3 (nano::write_transaction const &);
void upgrade_v3_to_v4 (nano::write_transaction const &);
void upgrade_v4_to_v5 (nano::write_transaction const &);
void upgrade_v5_to_v6 (nano::write_transaction const &);
void upgrade_v6_to_v7 (nano::write_transaction const &);
void upgrade_v7_to_v8 (nano::write_transaction const &);
void upgrade_v8_to_v9 (nano::write_transaction const &);
void upgrade_v10_to_v11 (nano::write_transaction const &);
void upgrade_v11_to_v12 (nano::write_transaction const &);
void upgrade_v12_to_v13 (nano::write_transaction &, size_t);
void upgrade_v13_to_v14 (nano::transaction const &);
void upgrade_v14_to_v15 (nano::transaction const &);
void upgrade_v13_to_v14 (nano::write_transaction const &);
void upgrade_v14_to_v15 (nano::write_transaction const &);
void open_databases (bool &, nano::transaction const &, unsigned);
int drop (nano::transaction const & transaction_a, tables table_a) override;
int clear (nano::transaction const & transaction_a, MDB_dbi handle_a);
int drop (nano::write_transaction const & transaction_a, tables table_a) override;
int clear (nano::write_transaction const & transaction_a, MDB_dbi handle_a);
bool not_found (int status) const override;
bool success (int status) const override;

6
nano/node/lmdb/lmdb_txn.cpp
View file

@ -110,6 +110,12 @@ void * nano::write_mdb_txn::get_handle () const
return handle;
}
bool nano::write_mdb_txn::contains (nano::tables table_a) const
{
// LMDB locks on every write
return true;
}
nano::mdb_txn_tracker::mdb_txn_tracker (nano::logger_mt & logger_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a) :
logger (logger_a),
txn_tracking_config (txn_tracking_config_a),

3
nano/node/lmdb/lmdb_txn.hpp
View file

@ -1,7 +1,7 @@
#pragma once
#include <nano/lib/diagnosticsconfig.hpp>
#include <nano/lib/timer.hpp>
#include <nano/node/diagnosticsconfig.hpp>
#include <nano/secure/blockstore.hpp>
#include <boost/property_tree/ptree.hpp>
@ -46,6 +46,7 @@ public:
void commit () const override;
void renew () override;
void * get_handle () const override;
bool contains (nano::tables table_a) const override;
MDB_txn * handle;
nano::mdb_env const & env;
mdb_txn_callbacks txn_callbacks;

42
nano/node/node.cpp
View file

@ -5,6 +5,10 @@
#include <nano/node/node.hpp>
#include <nano/rpc/rpc.hpp>
#if NANO_ROCKSDB
#include <nano/node/rocksdb/rocksdb.hpp>
#endif
#include <boost/polymorphic_cast.hpp>
#include <boost/property_tree/json_parser.hpp>
@ -123,7 +127,7 @@ flags (flags_a),
alarm (alarm_a),
work (work_a),
logger (config_a.logging.min_time_between_log_output),
store_impl (std::make_unique<nano::mdb_store> (init_a.block_store_init, logger, application_path_a / "data.ldb", config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_max_dbs, !flags.disable_unchecked_drop, flags.sideband_batch_size, config_a.backup_before_upgrade)),
store_impl (nano::make_store (init_a.block_store_init, logger, application_path_a, flags.read_only, true, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_max_dbs, !flags.disable_unchecked_drop, flags.sideband_batch_size, config_a.backup_before_upgrade)),
store (*store_impl),
wallets_store_impl (std::make_unique<nano::mdb_wallets_store> (init_a.wallets_store_init, application_path_a / "wallets.ldb", config_a.lmdb_max_dbs)),
wallets_store (*wallets_store_impl),
@ -143,7 +147,7 @@ block_processor_thread ([this]() {
nano::thread_role::set (nano::thread_role::name::block_processing);
this->block_processor.process_blocks ();
}),
online_reps (*this, config.online_weight_minimum.number ()),
online_reps (*this, config.online_weight_minimum.number (), init_a.block_store_init),
vote_uniquer (block_uniquer),
active (*this),
confirmation_height_processor (pending_confirmation_height, store, ledger.stats, active, ledger.epoch_link, write_database_queue, config.conf_height_processor_batch_min_time, logger),
@ -389,6 +393,7 @@ startup_time (std::chrono::steady_clock::now ())
nano::genesis genesis;
if (!is_initialized)
{
release_assert (!flags.read_only);
auto transaction (store.tx_begin_write ());
// Store was empty meaning we just created it, add the genesis block
store.initialize (transaction, genesis);
@ -397,7 +402,16 @@ startup_time (std::chrono::steady_clock::now ())
auto transaction (store.tx_begin_read ());
if (!store.block_exists (transaction, genesis.hash ()))
{
logger.always_log ("Genesis block not found. Make sure the node network ID is correct.");
std::stringstream ss;
ss << "Genesis block not found. Make sure the node network ID is correct.";
if (network_params.network.is_beta_network ())
{
ss << " Beta network may have reset, try clearing database files";
}
auto str = ss.str ();
logger.always_log (str);
std::cerr << str << std::endl;
std::exit (1);
}
@ -520,9 +534,9 @@ void nano::node::do_rpc_callback (boost::asio::ip::tcp::resolver::iterator i_a,
}
}
bool nano::node::copy_with_compaction (boost::filesystem::path const & destination_file)
bool nano::node::copy_with_compaction (boost::filesystem::path const & destination)
{
return !mdb_env_copy2 (boost::polymorphic_downcast<nano::mdb_store *> (store_impl.get ())->env.environment, destination_file.string ().c_str (), MDB_CP_COMPACT);
return store.copy_db (destination);
}
void nano::node::process_fork (nano::transaction const & transaction_a, std::shared_ptr<nano::block> block_a)
@ -602,7 +616,7 @@ void nano::node::process_active (std::shared_ptr<nano::block> incoming)
nano::process_return nano::node::process (nano::block const & block_a)
{
auto transaction (store.tx_begin_write ());
auto transaction (store.tx_begin_write ({ tables::accounts_v0, tables::accounts_v1, tables::cached_counts, tables::change_blocks, tables::frontiers, tables::open_blocks, tables::pending_v0, tables::pending_v1, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks_v0, tables::state_blocks_v1 }, { tables::confirmation_height }));
auto result (ledger.process (transaction, block_a));
return result;
}
@ -791,7 +805,7 @@ void nano::node::ongoing_bootstrap ()
void nano::node::ongoing_store_flush ()
{
{
auto transaction (store.tx_begin_write ());
auto transaction (store.tx_begin_write ({ tables::vote }));
store.flush (transaction);
}
std::weak_ptr<nano::node> node_w (shared_from_this ());
@ -1625,7 +1639,7 @@ int nano::node::store_version ()
return store.version_get (transaction);
}
nano::inactive_node::inactive_node (boost::filesystem::path const & path_a, uint16_t peering_port_a) :
nano::inactive_node::inactive_node (boost::filesystem::path const & path_a, uint16_t peering_port_a, bool read_only_a) :
path (path_a),
io_context (std::make_shared<boost::asio::io_context> ()),
alarm (*io_context),
@ -1641,7 +1655,9 @@ peering_port (peering_port_a)
nano::set_secure_perm_directory (path, error_chmod);
logging.max_size = std::numeric_limits<std::uintmax_t>::max ();
logging.init (path);
node = std::make_shared<nano::node> (init, *io_context, peering_port, path, alarm, logging, work);
auto node_flags = nano::node_flags ();
node_flags.read_only = read_only_a;
node = std::make_shared<nano::node> (init, *io_context, peering_port, path, alarm, logging, work, node_flags);
node->active.stop ();
}
@ -1650,7 +1666,11 @@ nano::inactive_node::~inactive_node ()
node->stop ();
}
std::unique_ptr<nano::block_store> nano::make_store (bool & init, nano::logger_mt & logger, boost::filesystem::path const & path)
std::unique_ptr<nano::block_store> nano::make_store (bool & init, nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, bool drop_unchecked, size_t batch_size, bool backup_before_upgrade)
{
return std::make_unique<nano::mdb_store> (init, logger, path);
#if NANO_ROCKSDB
return std::make_unique<nano::rocksdb_store> (init, logger, add_db_postfix ? path / "rocksdb" : path, drop_unchecked, read_only);
#else
return std::make_unique<nano::mdb_store> (init, logger, add_db_postfix ? path / "data.ldb" : path, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_max_dbs, drop_unchecked, batch_size, backup_before_upgrade);
#endif
}

2
nano/node/node.hpp
View file

@ -195,7 +195,7 @@ std::unique_ptr<seq_con_info_component> collect_seq_con_info (node & node, const
class inactive_node final
{
public:
inactive_node (boost::filesystem::path const & path = nano::working_path (), uint16_t = 24000);
inactive_node (boost::filesystem::path const & path = nano::working_path (), uint16_t = 24000, bool read_only_a = true);
~inactive_node ();
boost::filesystem::path path;
std::shared_ptr<boost::asio::io_context> io_context;

2
nano/node/nodeconfig.hpp
View file

@ -1,11 +1,11 @@
#pragma once
#include <nano/lib/config.hpp>
#include <nano/lib/diagnosticsconfig.hpp>
#include <nano/lib/errors.hpp>
#include <nano/lib/jsonconfig.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/lib/stats.hpp>
#include <nano/node/diagnosticsconfig.hpp>
#include <nano/node/ipcconfig.hpp>
#include <nano/node/logging.hpp>
#include <nano/node/websocketconfig.hpp>

9
nano/node/online_reps.cpp
View file

@ -3,12 +3,15 @@
#include <cassert>
nano::online_reps::online_reps (nano::node & node_a, nano::uint128_t minimum_a) :
nano::online_reps::online_reps (nano::node & node_a, nano::uint128_t minimum_a, bool block_store_error_a) :
node (node_a),
minimum (minimum_a)
{
auto transaction (node.ledger.store.tx_begin_read ());
online = trend (transaction);
if (!block_store_error_a)
{
auto transaction (node.ledger.store.tx_begin_read ());
online = trend (transaction);
}
}
void nano::online_reps::observe (nano::account const & rep_a)

2
nano/node/online_reps.hpp
View file

@ -16,7 +16,7 @@ class transaction;
class online_reps final
{
public:
online_reps (nano::node &, nano::uint128_t);
online_reps (nano::node &, nano::uint128_t, bool);
/** Add voting account \p rep_account to the set of online representatives */
void observe (nano::account const & rep_account);

616
nano/node/rocksdb/rocksdb.cpp Normal file
View file

@ -0,0 +1,616 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/node/rocksdb/rocksdb.hpp>
#include <nano/node/rocksdb/rocksdb_iterator.hpp>
#include <nano/node/rocksdb/rocksdb_txn.hpp>
#include <boost/endian/conversion.hpp>
#include <boost/polymorphic_cast.hpp>
#include <rocksdb/merge_operator.h>
#include <rocksdb/slice.h>
#include <rocksdb/utilities/backupable_db.h>
#include <rocksdb/utilities/transaction.h>
#include <rocksdb/utilities/transaction_db.h>
namespace nano
{
template <>
void * rocksdb_val::data () const
{
return (void *)value.data ();
}
template <>
size_t rocksdb_val::size () const
{
return value.size ();
}
template <>
rocksdb_val::db_val (size_t size_a, void * data_a, nano::epoch epoch_a) :
value (static_cast<const char *> (data_a), size_a),
epoch (epoch_a)
{
}
template <>
void rocksdb_val::convert_buffer_to_value ()
{
value = rocksdb::Slice (reinterpret_cast<const char *> (buffer->data ()), buffer->size ());
}
}
nano::rocksdb_store::rocksdb_store (bool & error_a, nano::logger_mt & logger_a, boost::filesystem::path const & path_a, bool drop_unchecked_a, bool open_read_only_a) :
logger (logger_a)
{
boost::system::error_code error_mkdir, error_chmod;
boost::filesystem::create_directories (path_a, error_mkdir);
nano::set_secure_perm_directory (path_a, error_chmod);
error_a |= static_cast<bool> (error_mkdir);
if (!error_a)
{
auto table_options = get_table_options ();
table_factory.reset (rocksdb::NewBlockBasedTableFactory (table_options));
if (!open_read_only_a)
{
construct_column_family_mutexes ();
}
open (error_a, path_a, open_read_only_a);
}
if (!error_a && !open_read_only_a && drop_unchecked_a)
{
auto transaction (tx_begin_write ({ nano::tables::cached_counts, tables::unchecked }));
unchecked_clear (transaction);
}
}
nano::rocksdb_store::~rocksdb_store ()
{
for (auto handle : handles)
{
delete handle;
}
delete db;
}
void nano::rocksdb_store::open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a)
{
std::initializer_list<const char *> names{ rocksdb::kDefaultColumnFamilyName.c_str (), "frontiers", "accounts", "accounts_v1", "send", "receive", "open", "change", "state", "state_v1", "pending", "pending_v1", "representation", "unchecked", "vote", "online_weight", "meta", "peers", "cached_counts", "confirmation_height" };
std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
for (const auto & cf_name : names)
{
column_families.emplace_back (cf_name, get_cf_options ());
}
auto options = get_db_options ();
rocksdb::Status s;
if (open_read_only_a)
{
s = rocksdb::DB::OpenForReadOnly (options, path_a.string (), column_families, &handles, &db);
}
else
{
s = rocksdb::OptimisticTransactionDB::Open (options, path_a.string (), column_families, &handles, &optimistic_db);
if (optimistic_db)
{
db = optimistic_db;
}
}
// Assign handles to supplied
error_a |= !s.ok ();
if (!error_a)
{
auto transaction = tx_begin_read ();
auto version_l = version_get (transaction);
if (version_l > version)
{
error_a = true;
logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l));
}
}
}
nano::write_transaction nano::rocksdb_store::tx_begin_write (std::vector<nano::tables> const & tables_requiring_locks_a, std::vector<nano::tables> const & tables_no_locks_a)
{
std::unique_ptr<nano::write_rocksdb_txn> txn;
release_assert (optimistic_db != nullptr);
if (tables_requiring_locks_a.empty () && tables_no_locks_a.empty ())
{
// Use all tables if none are specified
txn = std::make_unique<nano::write_rocksdb_txn> (optimistic_db, all_tables (), tables_no_locks_a, write_lock_mutexes);
}
else
{
txn = std::make_unique<nano::write_rocksdb_txn> (optimistic_db, tables_requiring_locks_a, tables_no_locks_a, write_lock_mutexes);
}
// Tables must be kept in alphabetical order. These can be used for mutex locking, so order is important to prevent deadlocking
assert (std::is_sorted (tables_requiring_locks_a.begin (), tables_requiring_locks_a.end ()));
return nano::write_transaction{ std::move (txn) };
}
nano::read_transaction nano::rocksdb_store::tx_begin_read ()
{
return nano::read_transaction{ std::make_unique<nano::read_rocksdb_txn> (db) };
}
rocksdb::ColumnFamilyHandle * nano::rocksdb_store::table_to_column_family (tables table_a) const
{
auto & handles_l = handles;
auto get_handle = [&handles_l](const char * name) {
auto iter = std::find_if (handles_l.begin (), handles_l.end (), [name](auto handle) {
return (handle->GetName () == name);
});
assert (iter != handles_l.end ());
return *iter;
};
switch (table_a)
{
case tables::frontiers:
return get_handle ("frontiers");
case tables::accounts_v0:
return get_handle ("accounts");
case tables::accounts_v1:
return get_handle ("accounts_v1");
case tables::send_blocks:
return get_handle ("send");
case tables::receive_blocks:
return get_handle ("receive");
case tables::open_blocks:
return get_handle ("open");
case tables::change_blocks:
return get_handle ("change");
case tables::state_blocks_v0:
return get_handle ("state");
case tables::state_blocks_v1:
return get_handle ("state_v1");
case tables::pending_v0:
return get_handle ("pending");
case tables::pending_v1:
return get_handle ("pending_v1");
case tables::blocks_info:
assert (false);
case tables::representation:
return get_handle ("representation");
case tables::unchecked:
return get_handle ("unchecked");
case tables::vote:
return get_handle ("vote");
case tables::online_weight:
return get_handle ("online_weight");
case tables::meta:
return get_handle ("meta");
case tables::peers:
return get_handle ("peers");
case tables::cached_counts:
return get_handle ("cached_counts");
case tables::confirmation_height:
return get_handle ("confirmation_height");
default:
release_assert (false);
return get_handle ("peers");
}
}
bool nano::rocksdb_store::exists (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a) const
{
rocksdb::PinnableSlice slice;
rocksdb::Status status;
if (is_read (transaction_a))
{
status = db->Get (snapshot_options (transaction_a), table_to_column_family (table_a), key_a, &slice);
}
else
{
rocksdb::ReadOptions options;
status = tx (transaction_a)->Get (options, table_to_column_family (table_a), key_a, &slice);
}
return (status.ok ());
}
int nano::rocksdb_store::del (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a)
{
assert (transaction_a.contains (table_a));
if (!exists (transaction_a, table_a, key_a))
{
return status_code_not_found ();
}
else
{
// Adding a new entry so counts need adjusting (use RMW otherwise known as merge)
if (is_caching_counts (table_a))
{
decrement (transaction_a, tables::cached_counts, rocksdb_val (rocksdb::Slice (table_to_column_family (table_a)->GetName ())), 1);
}
}
return tx (transaction_a)->Delete (table_to_column_family (table_a), key_a).code ();
}
bool nano::rocksdb_store::block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) const
{
// Should not be called
assert (false);
return true;
}
void nano::rocksdb_store::version_put (nano::write_transaction const & transaction_a, int version_a)
{
assert (transaction_a.contains (tables::meta));
nano::uint256_union version_key (1);
nano::uint256_union version_value (version_a);
auto status (put (transaction_a, tables::meta, version_key, nano::rocksdb_val (version_value)));
release_assert (success (status));
}
rocksdb::Transaction * nano::rocksdb_store::tx (nano::transaction const & transaction_a) const
{
assert (!is_read (transaction_a));
return static_cast<rocksdb::Transaction *> (transaction_a.get_handle ());
}
int nano::rocksdb_store::get (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val & value_a) const
{
rocksdb::ReadOptions options;
rocksdb::PinnableSlice slice;
auto handle = table_to_column_family (table_a);
rocksdb::Status status;
if (is_read (transaction_a))
{
status = db->Get (snapshot_options (transaction_a), handle, key_a, &slice);
}
else
{
status = tx (transaction_a)->Get (options, handle, key_a, &slice);
}
if (status.ok ())
{
value_a.buffer = std::make_shared<std::vector<uint8_t>> (slice.size ());
std::memcpy (value_a.buffer->data (), slice.data (), slice.size ());
value_a.convert_buffer_to_value ();
}
return status.code ();
}
/** The column families which need to have their counts cached for later querying */
bool nano::rocksdb_store::is_caching_counts (nano::tables table_a) const
{
switch (table_a)
{
case tables::accounts_v0:
case tables::accounts_v1:
case tables::unchecked:
case tables::send_blocks:
case tables::receive_blocks:
case tables::open_blocks:
case tables::change_blocks:
case tables::state_blocks_v0:
case tables::state_blocks_v1:
return true;
default:
return false;
}
}
int nano::rocksdb_store::increment (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, uint64_t amount_a)
{
release_assert (transaction_a.contains (table_a));
uint64_t base;
nano::rocksdb_val value;
if (!success (get (transaction_a, table_a, key_a, value)))
{
base = 0;
}
else
{
base = static_cast<uint64_t> (value);
}
return put (transaction_a, table_a, key_a, nano::rocksdb_val (base + amount_a));
}
int nano::rocksdb_store::decrement (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, uint64_t amount_a)
{
release_assert (transaction_a.contains (table_a));
nano::rocksdb_val value;
auto status = get (transaction_a, table_a, key_a, value);
release_assert (success (status));
auto base = static_cast<uint64_t> (value);
return put (transaction_a, table_a, key_a, nano::rocksdb_val (base - amount_a));
}
int nano::rocksdb_store::put (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val const & value_a)
{
assert (transaction_a.contains (table_a));
auto txn = tx (transaction_a);
if (is_caching_counts (table_a))
{
if (!exists (transaction_a, table_a, key_a))
{
// Adding a new entry so counts need adjusting
increment (transaction_a, tables::cached_counts, rocksdb_val (rocksdb::Slice (table_to_column_family (table_a)->GetName ())), 1);
}
}
return txn->Put (table_to_column_family (table_a), key_a, value_a).code ();
}
bool nano::rocksdb_store::not_found (int status) const
{
return (status_code_not_found () == status);
}
bool nano::rocksdb_store::success (int status) const
{
return (static_cast<int> (rocksdb::Status::Code::kOk) == status);
}
int nano::rocksdb_store::status_code_not_found () const
{
return static_cast<int> (rocksdb::Status::Code::kNotFound);
}
uint64_t nano::rocksdb_store::count (nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle) const
{
uint64_t count = 0;
nano::rocksdb_val val;
auto const & key = handle->GetName ();
auto status = get (transaction_a, tables::cached_counts, nano::rocksdb_val (key.size (), (void *)key.data ()), val);
if (success (status))
{
count = static_cast<uint64_t> (val);
}
release_assert (success (status) || not_found (status));
return count;
}
size_t nano::rocksdb_store::count (nano::transaction const & transaction_a, tables table_a) const
{
size_t sum = 0;
// Some column families are small enough that they can just be iterated, rather than doing extra io caching counts
if (table_a == tables::peers)
{
for (auto i (peers_begin (transaction_a)), n (peers_end ()); i != n; ++i)
{
++sum;
}
}
else if (table_a == tables::online_weight)
{
for (auto i (online_weight_begin (transaction_a)), n (online_weight_end ()); i != n; ++i)
{
++sum;
}
}
else
{
return count (transaction_a, table_to_column_family (table_a));
}
return sum;
}
int nano::rocksdb_store::drop (nano::write_transaction const & transaction_a, tables table_a)
{
assert (transaction_a.contains (table_a));
auto col = table_to_column_family (table_a);
int status = static_cast<int> (rocksdb::Status::Code::kOk);
if (is_caching_counts (table_a))
{
// Reset counter to 0
status = put (transaction_a, tables::cached_counts, nano::rocksdb_val (rocksdb::Slice (col->GetName ())), nano::rocksdb_val (uint64_t{ 0 }));
}
if (success (status))
{
// Dropping/Creating families like in node::ongoing_peer_clear can cause write stalls, just delete them manually.
if (table_a == tables::peers)
{
int status = 0;
for (auto i = peers_begin (transaction_a), n = peers_end (); i != n; ++i)
{
status = del (transaction_a, tables::peers, nano::rocksdb_val (i->first));
release_assert (success (status));
}
return status;
}
else
{
return clear (col);
}
}
return status;
}
int nano::rocksdb_store::clear (rocksdb::ColumnFamilyHandle * column_family)
{
// Dropping completely removes the column
auto name = column_family->GetName ();
auto status = db->DropColumnFamily (column_family);
release_assert (status.ok ());
delete column_family;
// Need to add it back as we just want to clear the contents
auto handle_it = std::find (handles.begin (), handles.end (), column_family);
assert (handle_it != handles.cend ());
status = db->CreateColumnFamily (get_cf_options (), name, &column_family);
release_assert (status.ok ());
*handle_it = column_family;
return status.code ();
}
void nano::rocksdb_store::construct_column_family_mutexes ()
{
for (auto table : all_tables ())
{
write_lock_mutexes.emplace (std::piecewise_construct, std::forward_as_tuple (table), std::forward_as_tuple ());
}
}
rocksdb::Options nano::rocksdb_store::get_db_options () const
{
rocksdb::Options db_options;
db_options.create_if_missing = true;
db_options.create_missing_column_families = true;
// Sets the compaction priority
db_options.compaction_pri = rocksdb::CompactionPri::kMinOverlappingRatio;
// Start agressively flushing WAL files when they reach over 1GB
db_options.max_total_wal_size = 1 * 1024 * 1024 * 1024LL;
if (!low_end_system ())
{
// Adds a separate write queue for memtable/WAL
db_options.enable_pipelined_write = true;
// Optimize RocksDB. This is the easiest way to get RocksDB to perform well
db_options.IncreaseParallelism (std::thread::hardware_concurrency ());
db_options.OptimizeLevelStyleCompaction ();
}
return db_options;
}
/** As options are currently hardcoded, a heuristic is taken based off the number of cores to modify config options */
bool nano::rocksdb_store::low_end_system () const
{
return (std::thread::hardware_concurrency () < 2);
}
rocksdb::BlockBasedTableOptions nano::rocksdb_store::get_table_options () const
{
rocksdb::BlockBasedTableOptions table_options;
if (!low_end_system ())
{
// 512MB block cache
table_options.block_cache = rocksdb::NewLRUCache (512 * 1024 * 1024LL);
// Bloom filter to help with point reads
table_options.filter_policy.reset (rocksdb::NewBloomFilterPolicy (10, false));
table_options.block_size = 16 * 1024;
table_options.cache_index_and_filter_blocks = true;
table_options.pin_l0_filter_and_index_blocks_in_cache = true;
}
return table_options;
}
rocksdb::ColumnFamilyOptions nano::rocksdb_store::get_cf_options () const
{
rocksdb::ColumnFamilyOptions cf_options;
cf_options.table_factory = table_factory;
if (!low_end_system ())
{
cf_options.level_compaction_dynamic_level_bytes = true;
// Number of files in level which triggers compaction. Size of L0 and L1 should be kept similar as this is the only compaction which is single threaded
cf_options.level0_file_num_compaction_trigger = 4;
// L1 size, compaction is triggered for L0 at this size (512MB)
cf_options.max_bytes_for_level_base = 512 * 1024 * 1024LL;
// Each level is a multiple of the above. L1 will be 512MB. Le will be 512 * 8 = 2GB. L3 will be 2GB * 8 = 16GB, and so on...
cf_options.max_bytes_for_level_multiplier = 8;
// Size of level 1 sst files (128MB)
cf_options.target_file_size_base = 128 * 1024 * 1024LL;
// Size of each memtable (128MB)
cf_options.write_buffer_size = 128 * 1024 * 1024LL;
// Number of memtables to keep in memory (1 active, rest inactive/immutable)
cf_options.max_write_buffer_number = 3;
// Files older than this (1 day) will be scheduled for compaction when there is no other background work. This can lead to more writes however.
cf_options.ttl = 1 * 24 * 60 * 60;
}
return cf_options;
}
std::vector<nano::tables> nano::rocksdb_store::all_tables () const
{
return std::vector<nano::tables>{ tables::accounts_v0, tables::accounts_v1, tables::cached_counts, tables::change_blocks, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::open_blocks, tables::peers, tables::pending_v0, tables::pending_v1, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks_v0, tables::state_blocks_v1, tables::unchecked, tables::vote };
}
bool nano::rocksdb_store::copy_db (boost::filesystem::path const & destination_path)
{
std::unique_ptr<rocksdb::BackupEngine> backup_engine;
{
rocksdb::BackupEngine * backup_engine_raw;
rocksdb::BackupableDBOptions backup_options (destination_path.string ());
// Use incremental backups (default)
backup_options.share_table_files = true;
// Increase number of threads used for copying
backup_options.max_background_operations = std::thread::hardware_concurrency ();
auto status = rocksdb::BackupEngine::Open (rocksdb::Env::Default (), backup_options, &backup_engine_raw);
backup_engine.reset (backup_engine_raw);
if (!status.ok ())
{
return false;
}
}
auto status = backup_engine->CreateNewBackup (db);
if (!status.ok ())
{
return false;
}
std::vector<rocksdb::BackupInfo> backup_infos;
backup_engine->GetBackupInfo (&backup_infos);
for (auto const & backup_info : backup_infos)
{
status = backup_engine->VerifyBackup (backup_info.backup_id);
if (!status.ok ())
{
return false;
}
}
rocksdb::BackupEngineReadOnly * backup_engine_read;
status = rocksdb::BackupEngineReadOnly::Open (rocksdb::Env::Default (), rocksdb::BackupableDBOptions (destination_path.string ()), &backup_engine_read);
if (!status.ok ())
{
delete backup_engine_read;
return false;
}
// First remove all files (not directories) in the destination
for (boost::filesystem::directory_iterator end_dir_it, it (destination_path); it != end_dir_it; ++it)
{
auto path = it->path ();
if (boost::filesystem::is_regular_file (path))
{
boost::filesystem::remove (it->path ());
}
}
// Now generate the relevant files from the backup
status = backup_engine->RestoreDBFromLatestBackup (destination_path.string (), destination_path.string ());
delete backup_engine_read;
// Open it so that it flushes all WAL files
if (status.ok ())
{
auto error{ false };
nano::rocksdb_store rocksdb_store (error, logger, destination_path.string (), false, false);
return !error;
}
return false;
}

104
nano/node/rocksdb/rocksdb.hpp Normal file
View file

@ -0,0 +1,104 @@
#pragma once
#include <nano/lib/config.hpp>
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/numbers.hpp>
#include <nano/node/rocksdb/rocksdb_iterator.hpp>
#include <nano/secure/blockstore_partial.hpp>
#include <nano/secure/common.hpp>
#include <rocksdb/db.h>
#include <rocksdb/filter_policy.h>
#include <rocksdb/options.h>
#include <rocksdb/slice.h>
#include <rocksdb/table.h>
#include <rocksdb/utilities/optimistic_transaction_db.h>
#include <rocksdb/utilities/transaction.h>
namespace nano
{
class logging_mt;
/**
* rocksdb implementation of the block store
*/
class rocksdb_store : public block_store_partial<rocksdb::Slice, rocksdb_store>
{
public:
rocksdb_store (bool &, nano::logger_mt &, boost::filesystem::path const &, bool drop_unchecked = false, bool open_read_only = false);
~rocksdb_store ();
nano::write_transaction tx_begin_write (std::vector<nano::tables> const & tables_requiring_lock = {}, std::vector<nano::tables> const & tables_no_lock = {}) override;
nano::read_transaction tx_begin_read () override;
bool block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) const override;
size_t count (nano::transaction const & transaction_a, tables table_a) const override;
void version_put (nano::write_transaction const &, int) override;
bool exists (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a) const;
int get (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val & value_a) const;
int put (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val const & value_a);
int del (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a);
void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) override
{
// Do nothing
}
bool copy_db (boost::filesystem::path const & destination) override;
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a) const
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::rocksdb_iterator<Key, Value>> (db, transaction_a, table_to_column_family (table_a)));
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key) const
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::rocksdb_iterator<Key, Value>> (db, transaction_a, table_to_column_family (table_a), key));
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, rocksdb_val const & key) const
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::rocksdb_merge_iterator<Key, Value>> (db, transaction_a, table_to_column_family (table1_a), table_to_column_family (table2_a), key));
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a) const
{
return nano::store_iterator<Key, Value> (std::make_unique<nano::rocksdb_merge_iterator<Key, Value>> (db, transaction_a, table_to_column_family (table1_a), table_to_column_family (table2_a)));
}
private:
nano::logger_mt & logger;
std::vector<rocksdb::ColumnFamilyHandle *> handles;
// Optimistic transactions are used in write mode
rocksdb::OptimisticTransactionDB * optimistic_db = nullptr;
rocksdb::DB * db = nullptr;
std::shared_ptr<rocksdb::TableFactory> table_factory;
std::unordered_map<nano::tables, std::mutex> write_lock_mutexes;
rocksdb::Transaction * tx (nano::transaction const & transaction_a) const;
std::vector<nano::tables> all_tables () const;
bool not_found (int status) const override;
bool success (int status) const override;
int status_code_not_found () const override;
int drop (nano::write_transaction const &, tables) override;
rocksdb::ColumnFamilyHandle * table_to_column_family (tables table_a) const;
int clear (rocksdb::ColumnFamilyHandle * column_family);
void open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a);
uint64_t count (nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle) const;
bool is_caching_counts (nano::tables table_a) const;
int increment (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, uint64_t amount_a);
int decrement (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, uint64_t amount_a);
bool low_end_system () const;
rocksdb::ColumnFamilyOptions get_cf_options () const;
void construct_column_family_mutexes ();
rocksdb::Options get_db_options () const;
rocksdb::BlockBasedTableOptions get_table_options () const;
};
}

313
nano/node/rocksdb/rocksdb_iterator.hpp Normal file
View file

@ -0,0 +1,313 @@
#pragma once
#include <nano/secure/blockstore.hpp>
#include <rocksdb/db.h>
#include <rocksdb/filter_policy.h>
#include <rocksdb/options.h>
#include <rocksdb/slice.h>
#include <rocksdb/utilities/optimistic_transaction_db.h>
#include <rocksdb/utilities/transaction.h>
namespace
{
inline bool is_read (nano::transaction const & transaction_a)
{
return (dynamic_cast<const nano::read_transaction *> (&transaction_a) != nullptr);
}
inline rocksdb::ReadOptions const & snapshot_options (nano::transaction const & transaction_a)
{
assert (is_read (transaction_a));
return *static_cast<const rocksdb::ReadOptions *> (transaction_a.get_handle ());
}
}
namespace nano
{
using rocksdb_val = db_val<rocksdb::Slice>;
template <typename T, typename U>
class rocksdb_iterator : public store_iterator_impl<T, U>
{
public:
rocksdb_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle_a, nano::epoch epoch_a = nano::epoch::unspecified) :
cursor (nullptr)
{
current.first.epoch = epoch_a;
current.second.epoch = epoch_a;
rocksdb::Iterator * iter;
if (is_read (transaction_a))
{
iter = db->NewIterator (snapshot_options (transaction_a), handle_a);
}
else
{
rocksdb::ReadOptions ropts;
ropts.fill_cache = false;
iter = tx (transaction_a)->GetIterator (ropts, handle_a);
}
cursor.reset (iter);
cursor->SeekToFirst ();
if (cursor->Valid ())
{
current.first.value = cursor->key ();
current.second.value = cursor->value ();
}
else
{
clear ();
}
}
rocksdb_iterator (std::nullptr_t, nano::epoch epoch_a = nano::epoch::unspecified) :
cursor (nullptr)
{
current.first.epoch = epoch_a;
current.second.epoch = epoch_a;
}
rocksdb_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle_a, rocksdb_val const & val_a, nano::epoch epoch_a = nano::epoch::unspecified) :
cursor (nullptr)
{
current.first.epoch = epoch_a;
current.second.epoch = epoch_a;
rocksdb::Iterator * iter;
if (is_read (transaction_a))
{
iter = db->NewIterator (snapshot_options (transaction_a), handle_a);
}
else
{
iter = tx (transaction_a)->GetIterator (rocksdb::ReadOptions (), handle_a);
}
cursor.reset (iter);
cursor->Seek (val_a);
if (cursor->Valid ())
{
current.first = cursor->key ();
current.second = cursor->value ();
}
else
{
clear ();
}
}
rocksdb_iterator (nano::rocksdb_iterator<T, U> && other_a)
{
cursor = other_a.cursor;
other_a.cursor = nullptr;
current = other_a.current;
}
rocksdb_iterator (nano::rocksdb_iterator<T, U> const &) = delete;
nano::store_iterator_impl<T, U> & operator++ () override
{
cursor->Next ();
if (cursor->Valid ())
{
current.first = cursor->key ();
current.second = cursor->value ();
if (current.first.size () != sizeof (T))
{
clear ();
}
}
else
{
clear ();
}
return *this;
}
std::pair<nano::rocksdb_val, nano::rocksdb_val> * operator-> ()
{
return &current;
}
bool operator== (nano::store_iterator_impl<T, U> const & base_a) const override
{
auto const other_a (boost::polymorphic_downcast<nano::rocksdb_iterator<T, U> const *> (&base_a));
if (!current.first.data () && !other_a->current.first.data ())
{
return true;
}
else if (!current.first.data () || !other_a->current.first.data ())
{
return false;
}
auto result (std::memcmp (current.first.data (), other_a->current.first.data (), current.first.size ()) == 0);
assert (!result || (current.first.size () == other_a->current.first.size ()));
assert (!result || (current.second.data () == other_a->current.second.data ()));
assert (!result || (current.second.size () == other_a->current.second.size ()));
return result;
}
bool is_end_sentinal () const override
{
return current.first.size () == 0;
}
void fill (std::pair<T, U> & value_a) const override
{
{
if (current.first.size () != 0)
{
value_a.first = static_cast<T> (current.first);
}
else
{
value_a.first = T ();
}
if (current.second.size () != 0)
{
value_a.second = static_cast<U> (current.second);
}
else
{
value_a.second = U ();
}
}
}
void clear ()
{
current.first = nano::rocksdb_val (current.first.epoch);
current.second = nano::rocksdb_val (current.second.epoch);
assert (is_end_sentinal ());
}
nano::rocksdb_iterator<T, U> & operator= (nano::rocksdb_iterator<T, U> && other_a)
{
cursor = std::move (other_a.cursor);
current = other_a.current;
return *this;
}
nano::store_iterator_impl<T, U> & operator= (nano::store_iterator_impl<T, U> const &) = delete;
std::unique_ptr<rocksdb::Iterator> cursor;
std::pair<nano::rocksdb_val, nano::rocksdb_val> current;
private:
rocksdb::Transaction * tx (nano::transaction const & transaction_a) const
{
return static_cast<rocksdb::Transaction *> (transaction_a.get_handle ());
}
};
/**
* Iterates the key/value pairs of two stores merged together
*/
template <typename T, typename U>
class rocksdb_merge_iterator : public store_iterator_impl<T, U>
{
public:
rocksdb_merge_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * db1_a, rocksdb::ColumnFamilyHandle * db2_a) :
impl1 (std::make_unique<nano::rocksdb_iterator<T, U>> (db, transaction_a, db1_a, nano::epoch::epoch_0)),
impl2 (std::make_unique<nano::rocksdb_iterator<T, U>> (db, transaction_a, db2_a, nano::epoch::epoch_1))
{
}
rocksdb_merge_iterator (std::nullptr_t) :
impl1 (std::make_unique<nano::rocksdb_iterator<T, U>> (nullptr, nano::epoch::epoch_0)),
impl2 (std::make_unique<nano::rocksdb_iterator<T, U>> (nullptr, nano::epoch::epoch_1))
{
}
rocksdb_merge_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * db1_a, rocksdb::ColumnFamilyHandle * db2_a, rocksdb::Slice const & val_a) :
impl1 (std::make_unique<nano::rocksdb_iterator<T, U>> (db, transaction_a, db1_a, val_a, nano::epoch::epoch_0)),
impl2 (std::make_unique<nano::rocksdb_iterator<T, U>> (db, transaction_a, db2_a, val_a, nano::epoch::epoch_1))
{
}
rocksdb_merge_iterator (nano::rocksdb_merge_iterator<T, U> && other_a)
{
impl1 = std::move (other_a.impl1);
impl2 = std::move (other_a.impl2);
}
rocksdb_merge_iterator (nano::rocksdb_merge_iterator<T, U> const &) = delete;
nano::store_iterator_impl<T, U> & operator++ () override
{
++least_iterator ();
return *this;
}
std::pair<nano::rocksdb_val, nano::rocksdb_val> * operator-> ();
bool operator== (nano::store_iterator_impl<T, U> const & base_a) const override
{
assert ((dynamic_cast<nano::rocksdb_merge_iterator<T, U> const *> (&base_a) != nullptr) && "Incompatible iterator comparison");
auto & other (static_cast<nano::rocksdb_merge_iterator<T, U> const &> (base_a));
return *impl1 == *other.impl1 && *impl2 == *other.impl2;
}
bool is_end_sentinal () const override
{
return least_iterator ().is_end_sentinal ();
}
void fill (std::pair<T, U> & value_a) const override
{
auto & current (least_iterator ());
if (current->first.size () != 0)
{
value_a.first = static_cast<T> (current->first);
}
else
{
value_a.first = T ();
}
if (current->second.size () != 0)
{
value_a.second = static_cast<U> (current->second);
}
else
{
value_a.second = U ();
}
}
nano::rocksdb_merge_iterator<T, U> & operator= (nano::rocksdb_merge_iterator<T, U> &&) = default;
nano::rocksdb_merge_iterator<T, U> & operator= (nano::rocksdb_merge_iterator<T, U> const &) = delete;
private:
nano::rocksdb_iterator<T, U> & least_iterator () const
{
nano::rocksdb_iterator<T, U> * result;
if (impl1->is_end_sentinal ())
{
result = impl2.get ();
}
else if (impl2->is_end_sentinal ())
{
result = impl1.get ();
}
else
{
if (impl1->current.first.value.compare (impl2->current.first.value) < 0)
{
result = impl1.get ();
}
else if (impl1->current.first.value.compare (impl2->current.first.value) > 0)
{
result = impl2.get ();
}
else
{
result = impl1->current.second.value.compare (impl2->current.second.value) < 0 ? impl1.get () : impl2.get ();
}
}
return *result;
}
std::unique_ptr<nano::rocksdb_iterator<T, U>> impl1;
std::unique_ptr<nano::rocksdb_iterator<T, U>> impl2;
};
}

95
nano/node/rocksdb/rocksdb_txn.cpp Normal file
View file

@ -0,0 +1,95 @@
#include <nano/node/rocksdb/rocksdb_txn.hpp>
nano::read_rocksdb_txn::read_rocksdb_txn (rocksdb::DB * db_a) :
db (db_a)
{
options.snapshot = db_a->GetSnapshot ();
}
nano::read_rocksdb_txn::~read_rocksdb_txn ()
{
reset ();
}
void nano::read_rocksdb_txn::reset ()
{
db->ReleaseSnapshot (options.snapshot);
}
void nano::read_rocksdb_txn::renew ()
{
options.snapshot = db->GetSnapshot ();
}
void * nano::read_rocksdb_txn::get_handle () const
{
return (void *)&options;
}
nano::write_rocksdb_txn::write_rocksdb_txn (rocksdb::OptimisticTransactionDB * db_a, std::vector<nano::tables> const & tables_requiring_locks_a, std::vector<nano::tables> const & tables_no_locks_a, std::unordered_map<nano::tables, std::mutex> & mutexes_a) :
db (db_a),
tables_requiring_locks (tables_requiring_locks_a),
tables_no_locks (tables_no_locks_a),
mutexes (mutexes_a)
{
lock ();
rocksdb::OptimisticTransactionOptions txn_options;
txn_options.set_snapshot = true;
txn = db->BeginTransaction (rocksdb::WriteOptions (), txn_options);
}
nano::write_rocksdb_txn::~write_rocksdb_txn ()
{
commit ();
delete txn;
unlock ();
}
void nano::write_rocksdb_txn::commit () const
{
auto status = txn->Commit ();
// If there are no available memtables try again a few more times
constexpr auto num_attempts = 10;
auto attempt_num = 0;
while (status.IsTryAgain () && attempt_num < num_attempts)
{
status = txn->Commit ();
++attempt_num;
}
release_assert (status.ok ());
}
void nano::write_rocksdb_txn::renew ()
{
rocksdb::OptimisticTransactionOptions txn_options;
txn_options.set_snapshot = true;
db->BeginTransaction (rocksdb::WriteOptions (), txn_options, txn);
}
void * nano::write_rocksdb_txn::get_handle () const
{
return txn;
}
void nano::write_rocksdb_txn::lock ()
{
for (auto table : tables_requiring_locks)
{
mutexes.at (table).lock ();
}
}
void nano::write_rocksdb_txn::unlock ()
{
for (auto table : tables_requiring_locks)
{
mutexes.at (table).unlock ();
}
}
bool nano::write_rocksdb_txn::contains (nano::tables table_a) const
{
return (std::find (tables_requiring_locks.begin (), tables_requiring_locks.end (), table_a) != tables_requiring_locks.end ()) || (std::find (tables_no_locks.begin (), tables_no_locks.end (), table_a) != tables_no_locks.end ());
}

48
nano/node/rocksdb/rocksdb_txn.hpp Normal file
View file

@ -0,0 +1,48 @@
#pragma once
#include <rocksdb/db.h>
#include <rocksdb/filter_policy.h>
#include <rocksdb/options.h>
#include <rocksdb/slice.h>
#include <rocksdb/utilities/optimistic_transaction_db.h>
#include <rocksdb/utilities/transaction.h>
#include <nano/secure/blockstore.hpp>
namespace nano
{
class read_rocksdb_txn final : public read_transaction_impl
{
public:
read_rocksdb_txn (rocksdb::DB * db);
~read_rocksdb_txn ();
void reset () override;
void renew () override;
void * get_handle () const override;
private:
rocksdb::DB * db;
rocksdb::ReadOptions options;
};
class write_rocksdb_txn final : public write_transaction_impl
{
public:
write_rocksdb_txn (rocksdb::OptimisticTransactionDB * db_a, std::vector<nano::tables> const & tables_requiring_locks_a, std::vector<nano::tables> const & tables_no_locks_a, std::unordered_map<nano::tables, std::mutex> & mutexes_a);
~write_rocksdb_txn ();
void commit () const override;
void renew () override;
void * get_handle () const override;
bool contains (nano::tables table_a) const override;
private:
rocksdb::Transaction * txn;
rocksdb::OptimisticTransactionDB * db;
std::vector<nano::tables> tables_requiring_locks;
std::vector<nano::tables> tables_no_locks;
std::unordered_map<nano::tables, std::mutex> & mutexes;
void lock ();
void unlock ();
};
}

2
nano/node/transport/tcp.cpp
View file

@ -191,7 +191,7 @@ bool nano::transport::tcp_channels::store_all (bool clear_peers)
if (!endpoints.empty ())
{
// Clear all peers then refresh with the current list of peers
auto transaction (node.store.tx_begin_write ());
auto transaction (node.store.tx_begin_write ({ tables::peers }));
if (clear_peers)
{
node.store.peer_clear (transaction);

2
nano/node/transport/udp.cpp
View file

@ -188,7 +188,7 @@ bool nano::transport::udp_channels::store_all (bool clear_peers)
if (!endpoints.empty ())
{
// Clear all peers then refresh with the current list of peers
auto transaction (node.store.tx_begin_write ());
auto transaction (node.store.tx_begin_write ({ tables::peers }));
if (clear_peers)
{
node.store.peer_clear (transaction);

7
nano/rpc_test/rpc.cpp
View file

@ -6646,11 +6646,6 @@ TEST (rpc, block_confirmed)
auto send = std::make_shared<nano::send_block> (latest, key.pub, 10, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (latest));
node->process_active (send);
node->block_processor.flush ();
system.deadline_set (10s);
while (!node->pending_confirmation_height.is_processing_block (send->hash ()))
{
ASSERT_NO_ERROR (system.poll ());
}
// Wait until the confirmation height has been set
system.deadline_set (10s);
@ -6681,6 +6676,7 @@ TEST (rpc, block_confirmed)
ASSERT_TRUE (response3.json.get<bool> ("confirmed"));
}
#if !NANO_ROCKSDB
TEST (rpc, database_txn_tracker)
{
// First try when database tracking is disabled
@ -6808,6 +6804,7 @@ TEST (rpc, database_txn_tracker)
ASSERT_TRUE (!std::get<3> (json_l.front ()).empty ());
thread.join ();
}
#endif
TEST (rpc, active_difficulty)
{

5
nano/secure/blockstore.cpp
View file

@ -414,3 +414,8 @@ void nano::write_transaction::renew ()
{
impl->renew ();
}
bool nano::write_transaction::contains (nano::tables table_a) const
{
return impl->contains (table_a);
}

134
nano/secure/blockstore.hpp
View file

@ -2,6 +2,7 @@
#include <nano/crypto_lib/random_pool.hpp>
#include <nano/lib/config.hpp>
#include <nano/lib/diagnosticsconfig.hpp>
#include <nano/lib/logger_mt.hpp>
#include <nano/lib/memory.hpp>
#include <nano/secure/common.hpp>
@ -476,6 +477,31 @@ private:
std::unique_ptr<nano::store_iterator_impl<T, U>> impl;
};
// Keep this in alphabetical order
enum class tables
{
accounts_v0,
accounts_v1,
blocks_info, // LMDB only
cached_counts, // RocksDB only
change_blocks,
confirmation_height,
frontiers,
meta,
online_weight,
open_blocks,
peers,
pending_v0,
pending_v1,
receive_blocks,
representation,
send_blocks,
state_blocks_v0,
state_blocks_v1,
unchecked,
vote
};
class transaction_impl
{
public:
@ -495,6 +521,7 @@ class write_transaction_impl : public transaction_impl
public:
virtual void commit () const = 0;
virtual void renew () = 0;
virtual bool contains (nano::tables table_a) const = 0;
};
class transaction
@ -532,6 +559,7 @@ public:
void * get_handle () const override;
void commit () const;
void renew ();
bool contains (nano::tables table_a) const;
private:
std::unique_ptr<nano::write_transaction_impl> impl;
@ -543,37 +571,14 @@ private:
class block_store
{
public:
enum class tables
{
frontiers,
accounts_v0,
accounts_v1,
send_blocks,
receive_blocks,
open_blocks,
change_blocks,
state_blocks_v0,
state_blocks_v1,
pending_v0,
pending_v1,
blocks_info,
representation,
unchecked,
vote,
online_weight,
meta,
peers,
confirmation_height
};
virtual ~block_store () = default;
virtual void initialize (nano::transaction const &, nano::genesis const &) = 0;
virtual void block_put (nano::transaction const &, nano::block_hash const &, nano::block const &, nano::block_sideband const &, nano::epoch version = nano::epoch::epoch_0) = 0;
virtual void initialize (nano::write_transaction const &, nano::genesis const &) = 0;
virtual void block_put (nano::write_transaction const &, nano::block_hash const &, nano::block const &, nano::block_sideband const &, nano::epoch version = nano::epoch::epoch_0) = 0;
virtual nano::block_hash block_successor (nano::transaction const &, nano::block_hash const &) const = 0;
virtual void block_successor_clear (nano::transaction const &, nano::block_hash const &) = 0;
virtual void block_successor_clear (nano::write_transaction const &, nano::block_hash const &) = 0;
virtual std::shared_ptr<nano::block> block_get (nano::transaction const &, nano::block_hash const &, nano::block_sideband * = nullptr) const = 0;
virtual std::shared_ptr<nano::block> block_random (nano::transaction const &) = 0;
virtual void block_del (nano::transaction const &, nano::block_hash const &) = 0;
virtual void block_del (nano::write_transaction const &, nano::block_hash const &) = 0;
virtual bool block_exists (nano::transaction const &, nano::block_hash const &) = 0;
virtual bool block_exists (nano::transaction const &, nano::block_type, nano::block_hash const &) = 0;
virtual nano::block_counts block_count (nano::transaction const &) = 0;
@ -581,17 +586,17 @@ public:
virtual bool source_exists (nano::transaction const &, nano::block_hash const &) = 0;
virtual nano::account block_account (nano::transaction const &, nano::block_hash const &) const = 0;
virtual void frontier_put (nano::transaction const &, nano::block_hash const &, nano::account const &) = 0;
virtual void frontier_put (nano::write_transaction const &, nano::block_hash const &, nano::account const &) = 0;
virtual nano::account frontier_get (nano::transaction const &, nano::block_hash const &) const = 0;
virtual void frontier_del (nano::transaction const &, nano::block_hash const &) = 0;
virtual void frontier_del (nano::write_transaction const &, nano::block_hash const &) = 0;
virtual void account_put (nano::transaction const &, nano::account const &, nano::account_info const &) = 0;
virtual void account_put (nano::write_transaction const &, nano::account const &, nano::account_info const &) = 0;
virtual bool account_get (nano::transaction const &, nano::account const &, nano::account_info &) = 0;
virtual void account_del (nano::transaction const &, nano::account const &) = 0;
virtual void account_del (nano::write_transaction const &, nano::account const &) = 0;
virtual bool account_exists (nano::transaction const &, nano::account const &) = 0;
virtual size_t account_count (nano::transaction const &) = 0;
virtual void confirmation_height_clear (nano::transaction const &, nano::account const & account, uint64_t existing_confirmation_height) = 0;
virtual void confirmation_height_clear (nano::transaction const &) = 0;
virtual void confirmation_height_clear (nano::write_transaction const &, nano::account const & account, uint64_t existing_confirmation_height) = 0;
virtual void confirmation_height_clear (nano::write_transaction const &) = 0;
virtual uint64_t cemented_count (nano::transaction const &) = 0;
virtual nano::store_iterator<nano::account, nano::account_info> latest_v0_begin (nano::transaction const &, nano::account const &) = 0;
virtual nano::store_iterator<nano::account, nano::account_info> latest_v0_begin (nano::transaction const &) = 0;
@ -603,8 +608,8 @@ public:
virtual nano::store_iterator<nano::account, nano::account_info> latest_begin (nano::transaction const &) = 0;
virtual nano::store_iterator<nano::account, nano::account_info> latest_end () = 0;
virtual void pending_put (nano::transaction const &, nano::pending_key const &, nano::pending_info const &) = 0;
virtual void pending_del (nano::transaction const &, nano::pending_key const &) = 0;
virtual void pending_put (nano::write_transaction const &, nano::pending_key const &, nano::pending_info const &) = 0;
virtual void pending_del (nano::write_transaction const &, nano::pending_key const &) = 0;
virtual bool pending_get (nano::transaction const &, nano::pending_key const &, nano::pending_info &) = 0;
virtual bool pending_exists (nano::transaction const &, nano::pending_key const &) = 0;
virtual nano::store_iterator<nano::pending_key, nano::pending_info> pending_v0_begin (nano::transaction const &, nano::pending_key const &) = 0;
@ -623,16 +628,16 @@ public:
virtual nano::epoch block_version (nano::transaction const &, nano::block_hash const &) = 0;
virtual nano::uint128_t representation_get (nano::transaction const &, nano::account const &) = 0;
virtual void representation_put (nano::transaction const &, nano::account const &, nano::uint128_union const &) = 0;
virtual void representation_add (nano::transaction const &, nano::account const &, nano::uint128_t const &) = 0;
virtual void representation_put (nano::write_transaction const &, nano::account const &, nano::uint128_union const &) = 0;
virtual void representation_add (nano::write_transaction const &, nano::account const &, nano::uint128_t const &) = 0;
virtual nano::store_iterator<nano::account, nano::uint128_union> representation_begin (nano::transaction const &) = 0;
virtual nano::store_iterator<nano::account, nano::uint128_union> representation_end () = 0;
virtual void unchecked_clear (nano::transaction const &) = 0;
virtual void unchecked_put (nano::transaction const &, nano::unchecked_key const &, nano::unchecked_info const &) = 0;
virtual void unchecked_put (nano::transaction const &, nano::block_hash const &, std::shared_ptr<nano::block> const &) = 0;
virtual void unchecked_clear (nano::write_transaction const &) = 0;
virtual void unchecked_put (nano::write_transaction const &, nano::unchecked_key const &, nano::unchecked_info const &) = 0;
virtual void unchecked_put (nano::write_transaction const &, nano::block_hash const &, std::shared_ptr<nano::block> const &) = 0;
virtual std::vector<nano::unchecked_info> unchecked_get (nano::transaction const &, nano::block_hash const &) = 0;
virtual void unchecked_del (nano::transaction const &, nano::unchecked_key const &) = 0;
virtual void unchecked_del (nano::write_transaction const &, nano::unchecked_key const &) = 0;
virtual nano::store_iterator<nano::unchecked_key, nano::unchecked_info> unchecked_begin (nano::transaction const &) = 0;
virtual nano::store_iterator<nano::unchecked_key, nano::unchecked_info> unchecked_begin (nano::transaction const &, nano::unchecked_key const &) = 0;
virtual nano::store_iterator<nano::unchecked_key, nano::unchecked_info> unchecked_end () = 0;
@ -647,32 +652,32 @@ public:
virtual std::shared_ptr<nano::vote> vote_max (nano::transaction const &, std::shared_ptr<nano::vote>) = 0;
// Return latest vote for an account considering the vote cache
virtual std::shared_ptr<nano::vote> vote_current (nano::transaction const &, nano::account const &) = 0;
virtual void flush (nano::transaction const &) = 0;
virtual void flush (nano::write_transaction const &) = 0;
virtual nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> vote_begin (nano::transaction const &) = 0;
virtual nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> vote_end () = 0;
virtual void online_weight_put (nano::transaction const &, uint64_t, nano::amount const &) = 0;
virtual void online_weight_del (nano::transaction const &, uint64_t) = 0;
virtual nano::store_iterator<uint64_t, nano::amount> online_weight_begin (nano::transaction const &) = 0;
virtual nano::store_iterator<uint64_t, nano::amount> online_weight_end () = 0;
virtual void online_weight_put (nano::write_transaction const &, uint64_t, nano::amount const &) = 0;
virtual void online_weight_del (nano::write_transaction const &, uint64_t) = 0;
virtual nano::store_iterator<uint64_t, nano::amount> online_weight_begin (nano::transaction const &) const = 0;
virtual nano::store_iterator<uint64_t, nano::amount> online_weight_end () const = 0;
virtual size_t online_weight_count (nano::transaction const &) const = 0;
virtual void online_weight_clear (nano::transaction const &) = 0;
virtual void online_weight_clear (nano::write_transaction const &) = 0;
virtual void version_put (nano::transaction const &, int) = 0;
virtual void version_put (nano::write_transaction const &, int) = 0;
virtual int version_get (nano::transaction const &) const = 0;
virtual void peer_put (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0;
virtual void peer_del (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0;
virtual void peer_put (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0;
virtual void peer_del (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0;
virtual bool peer_exists (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const = 0;
virtual size_t peer_count (nano::transaction const & transaction_a) const = 0;
virtual void peer_clear (nano::transaction const & transaction_a) = 0;
virtual nano::store_iterator<nano::endpoint_key, nano::no_value> peers_begin (nano::transaction const & transaction_a) = 0;
virtual nano::store_iterator<nano::endpoint_key, nano::no_value> peers_end () = 0;
virtual void peer_clear (nano::write_transaction const & transaction_a) = 0;
virtual nano::store_iterator<nano::endpoint_key, nano::no_value> peers_begin (nano::transaction const & transaction_a) const = 0;
virtual nano::store_iterator<nano::endpoint_key, nano::no_value> peers_end () const = 0;
virtual void confirmation_height_put (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) = 0;
virtual void confirmation_height_put (nano::write_transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) = 0;
virtual bool confirmation_height_get (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t & confirmation_height_a) = 0;
virtual bool confirmation_height_exists (nano::transaction const & transaction_a, nano::account const & account_a) const = 0;
virtual void confirmation_height_del (nano::transaction const & transaction_a, nano::account const & account_a) = 0;
virtual void confirmation_height_del (nano::write_transaction const & transaction_a, nano::account const & account_a) = 0;
virtual uint64_t confirmation_height_count (nano::transaction const & transaction_a) = 0;
virtual nano::store_iterator<nano::account, uint64_t> confirmation_height_begin (nano::transaction const & transaction_a, nano::account const & account_a) = 0;
virtual nano::store_iterator<nano::account, uint64_t> confirmation_height_begin (nano::transaction const & transaction_a) = 0;
@ -681,14 +686,29 @@ public:
virtual uint64_t block_account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0;
virtual std::mutex & get_cache_mutex () = 0;
virtual bool copy_db (boost::filesystem::path const & destination) = 0;
/** Not applicable to all sub-classes */
virtual void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) = 0;
/** Start read-write transaction */
virtual nano::write_transaction tx_begin_write () = 0;
virtual nano::write_transaction tx_begin_write (std::vector<nano::tables> const & tables_to_lock = {}, std::vector<nano::tables> const & tables_no_lock = {}) = 0;
/** Start read-only transaction */
virtual nano::read_transaction tx_begin_read () = 0;
};
std::unique_ptr<nano::block_store> make_store (bool & init, nano::logger_mt & logger, boost::filesystem::path const & path);
std::unique_ptr<nano::block_store> make_store (bool & init, nano::logger_mt & logger, boost::filesystem::path const & path, bool open_read_only = false, bool add_db_postfix = false, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512, bool backup_before_upgrade = false);
}
namespace std
{
template <>
struct hash<::nano::tables>
{
size_t operator() (::nano::tables const & table_a) const
{
return static_cast<size_t> (table_a);
}
};
}

90
nano/secure/blockstore_partial.hpp
View file

@ -23,7 +23,7 @@ public:
* If using a different store version than the latest then you may need
* to modify some of the objects in the store to be appropriate for the version before an upgrade.
*/
void initialize (nano::transaction const & transaction_a, nano::genesis const & genesis_a) override
void initialize (nano::write_transaction const & transaction_a, nano::genesis const & genesis_a) override
{
auto hash_l (genesis_a.hash ());
assert (latest_v0_begin (transaction_a) == latest_v0_end ());
@ -44,7 +44,7 @@ public:
return result;
}
void representation_add (nano::transaction const & transaction_a, nano::block_hash const & source_a, nano::uint128_t const & amount_a) override
void representation_add (nano::write_transaction const & transaction_a, nano::block_hash const & source_a, nano::uint128_t const & amount_a) override
{
auto source_block (block_get (transaction_a, source_a));
assert (source_block != nullptr);
@ -59,7 +59,7 @@ public:
return iterator != latest_end () && nano::account (iterator->first) == account_a;
}
void confirmation_height_clear (nano::transaction const & transaction_a, nano::account const & account, uint64_t existing_confirmation_height) override
void confirmation_height_clear (nano::write_transaction const & transaction_a, nano::account const & account, uint64_t existing_confirmation_height) override
{
if (existing_confirmation_height > 0)
{
@ -67,7 +67,7 @@ public:
}
}
void confirmation_height_clear (nano::transaction const & transaction_a) override
void confirmation_height_clear (nano::write_transaction const & transaction_a) override
{
for (auto i (confirmation_height_begin (transaction_a)), n (confirmation_height_end ()); i != n; ++i)
{
@ -92,7 +92,7 @@ public:
return result;
}
void block_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a, nano::block_sideband const & sideband_a, nano::epoch epoch_a = nano::epoch::epoch_0) override
void block_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a, nano::block_sideband const & sideband_a, nano::epoch epoch_a = nano::epoch::epoch_0) override
{
assert (block_a.type () == sideband_a.type);
assert (sideband_a.successor.is_zero () || block_exists (transaction_a, sideband_a.successor));
@ -241,7 +241,7 @@ public:
return version_get (transaction_a) > 12;
}
void block_successor_clear (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override
void block_successor_clear (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override
{
nano::block_type type;
auto value (block_raw_get (transaction_a, hash_a, type));
@ -262,7 +262,7 @@ public:
return sum;
}
void unchecked_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, std::shared_ptr<nano::block> const & block_a) override
void unchecked_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, std::shared_ptr<nano::block> const & block_a) override
{
nano::unchecked_key key (hash_a, block_a->hash ());
nano::unchecked_info info (block_a, block_a->account (), nano::seconds_since_epoch (), nano::signature_verification::unknown);
@ -342,7 +342,7 @@ public:
return nano::store_iterator<nano::account, std::shared_ptr<nano::vote>> (nullptr);
}
nano::store_iterator<nano::endpoint_key, nano::no_value> peers_end () override
nano::store_iterator<nano::endpoint_key, nano::no_value> peers_end () const override
{
return nano::store_iterator<nano::endpoint_key, nano::no_value> (nullptr);
}
@ -362,7 +362,7 @@ public:
return nano::store_iterator<nano::pending_key, nano::pending_info> (nullptr);
}
nano::store_iterator<uint64_t, nano::amount> online_weight_end () override
nano::store_iterator<uint64_t, nano::amount> online_weight_end () const override
{
return nano::store_iterator<uint64_t, nano::amount> (nullptr);
}
@ -392,7 +392,7 @@ public:
return cache_mutex;
}
void block_del (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override
void block_del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override
{
auto status = del (transaction_a, tables::state_blocks_v1, hash_a);
release_assert (success (status) || not_found (status));
@ -446,7 +446,7 @@ public:
return status == 0 ? nano::epoch::epoch_1 : nano::epoch::epoch_0;
}
void block_raw_put (nano::transaction const & transaction_a, std::vector<uint8_t> const & data, nano::block_type block_type_a, nano::epoch epoch_a, nano::block_hash const & hash_a)
void block_raw_put (nano::write_transaction const & transaction_a, std::vector<uint8_t> const & data, nano::block_type block_type_a, nano::epoch epoch_a, nano::block_hash const & hash_a)
{
auto database_a = block_database (block_type_a, epoch_a);
nano::db_val<Val> value{ data.size (), (void *)data.data () };
@ -454,14 +454,14 @@ public:
release_assert (success (status));
}
void pending_put (nano::transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_a) override
void pending_put (nano::write_transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_a) override
{
nano::db_val<Val> pending (pending_a);
auto status = put (transaction_a, get_pending_db (pending_a.epoch), key_a, pending);
release_assert (success (status));
}
void pending_del (nano::transaction const & transaction_a, nano::pending_key const & key_a) override
void pending_del (nano::write_transaction const & transaction_a, nano::pending_key const & key_a) override
{
auto status1 = del (transaction_a, tables::pending_v1, key_a);
if (!success (status1))
@ -506,7 +506,7 @@ public:
return result;
}
void frontier_put (nano::transaction const & transaction_a, nano::block_hash const & block_a, nano::account const & account_a) override
void frontier_put (nano::write_transaction const & transaction_a, nano::block_hash const & block_a, nano::account const & account_a) override
{
nano::db_val<Val> account (account_a);
auto status (put (transaction_a, tables::frontiers, block_a, account));
@ -526,7 +526,7 @@ public:
return result;
}
void frontier_del (nano::transaction const & transaction_a, nano::block_hash const & block_a) override
void frontier_del (nano::write_transaction const & transaction_a, nano::block_hash const & block_a) override
{
auto status (del (transaction_a, tables::frontiers, block_a));
release_assert (success (status));
@ -550,21 +550,21 @@ public:
return result;
}
void representation_put (nano::transaction const & transaction_a, nano::account const & account_a, nano::uint128_union const & representation_a) override
void representation_put (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::uint128_union const & representation_a) override
{
nano::db_val<Val> rep (representation_a);
auto status (put (transaction_a, tables::representation, account_a, rep));
release_assert (success (status));
}
void unchecked_put (nano::transaction const & transaction_a, nano::unchecked_key const & key_a, nano::unchecked_info const & info_a) override
void unchecked_put (nano::write_transaction const & transaction_a, nano::unchecked_key const & key_a, nano::unchecked_info const & info_a) override
{
nano::db_val<Val> info (info_a);
auto status (put (transaction_a, tables::unchecked, key_a, info));
release_assert (success (status));
}
void unchecked_del (nano::transaction const & transaction_a, nano::unchecked_key const & key_a) override
void unchecked_del (nano::write_transaction const & transaction_a, nano::unchecked_key const & key_a) override
{
auto status (del (transaction_a, tables::unchecked, key_a));
release_assert (success (status) || not_found (status));
@ -584,7 +584,7 @@ public:
return nullptr;
}
void flush (nano::transaction const & transaction_a) override
void flush (nano::write_transaction const & transaction_a) override
{
{
std::lock_guard<std::mutex> lock (cache_mutex);
@ -604,14 +604,14 @@ public:
}
}
void online_weight_put (nano::transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override
void online_weight_put (nano::write_transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override
{
nano::db_val<Val> value (amount_a);
auto status (put (transaction_a, tables::online_weight, time_a, value));
release_assert (success (status));
}
void online_weight_del (nano::transaction const & transaction_a, uint64_t time_a) override
void online_weight_del (nano::write_transaction const & transaction_a, uint64_t time_a) override
{
auto status (del (transaction_a, tables::online_weight, time_a));
release_assert (success (status));
@ -635,7 +635,7 @@ public:
return db;
}
void account_put (nano::transaction const & transaction_a, nano::account const & account_a, nano::account_info const & info_a) override
void account_put (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::account_info const & info_a) override
{
// Check we are still in sync with other tables
assert (confirmation_height_exists (transaction_a, account_a));
@ -644,7 +644,7 @@ public:
release_assert (success (status));
}
void account_del (nano::transaction const & transaction_a, nano::account const & account_a) override
void account_del (nano::write_transaction const & transaction_a, nano::account const & account_a) override
{
auto status1 = del (transaction_a, tables::accounts_v1, account_a);
if (!success (status1))
@ -689,7 +689,7 @@ public:
return result;
}
void unchecked_clear (nano::transaction const & transaction_a) override
void unchecked_clear (nano::write_transaction const & transaction_a) override
{
auto status = drop (transaction_a, tables::unchecked);
release_assert (success (status));
@ -700,20 +700,20 @@ public:
return count (transaction_a, tables::online_weight);
}
void online_weight_clear (nano::transaction const & transaction_a) override
void online_weight_clear (nano::write_transaction const & transaction_a) override
{
auto status (drop (transaction_a, tables::online_weight));
release_assert (success (status));
}
void peer_put (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override
void peer_put (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override
{
nano::db_val<Val> zero (static_cast<uint64_t> (0));
auto status = put (transaction_a, tables::peers, endpoint_a, zero);
release_assert (success (status));
}
void peer_del (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override
void peer_del (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override
{
auto status (del (transaction_a, tables::peers, endpoint_a));
release_assert (success (status));
@ -729,7 +729,7 @@ public:
return count (transaction_a, tables::peers);
}
void peer_clear (nano::transaction const & transaction_a) override
void peer_clear (nano::write_transaction const & transaction_a) override
{
auto status = drop (transaction_a, tables::peers);
release_assert (success (status));
@ -813,7 +813,7 @@ public:
return count (transaction_a, tables::confirmation_height);
}
void confirmation_height_put (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) override
void confirmation_height_put (nano::write_transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) override
{
nano::db_val<Val> confirmation_height (confirmation_height_a);
auto status = put (transaction_a, tables::confirmation_height, account_a, confirmation_height);
@ -833,7 +833,7 @@ public:
return (!success (status));
}
void confirmation_height_del (nano::transaction const & transaction_a, nano::account const & account_a) override
void confirmation_height_del (nano::write_transaction const & transaction_a, nano::account const & account_a) override
{
auto status (del (transaction_a, tables::confirmation_height, nano::db_val<Val> (account_a)));
release_assert (success (status));
@ -924,12 +924,12 @@ public:
return make_iterator<nano::account, std::shared_ptr<nano::vote>> (transaction_a, tables::vote);
}
nano::store_iterator<uint64_t, nano::amount> online_weight_begin (nano::transaction const & transaction_a) override
nano::store_iterator<uint64_t, nano::amount> online_weight_begin (nano::transaction const & transaction_a) const override
{
return make_iterator<uint64_t, nano::amount> (transaction_a, tables::online_weight);
}
nano::store_iterator<nano::endpoint_key, nano::no_value> peers_begin (nano::transaction const & transaction_a) override
nano::store_iterator<nano::endpoint_key, nano::no_value> peers_begin (nano::transaction const & transaction_a) const override
{
return make_iterator<nano::endpoint_key, nano::no_value> (transaction_a, tables::peers);
}
@ -971,27 +971,27 @@ protected:
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a)
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a) const
{
return static_cast<Derived_Store &> (*this).template make_iterator<Key, Value> (transaction_a, table_a);
return static_cast<Derived_Store const &> (*this).template make_iterator<Key, Value> (transaction_a, table_a);
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a, nano::db_val<Val> const & key)
nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a, nano::db_val<Val> const & key) const
{
return static_cast<Derived_Store &> (*this).template make_iterator<Key, Value> (transaction_a, table_a, key);
return static_cast<Derived_Store const &> (*this).template make_iterator<Key, Value> (transaction_a, table_a, key);
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a)
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a) const
{
return static_cast<Derived_Store &> (*this).template make_merge_iterator<Key, Value> (transaction_a, table1_a, table2_a);
return static_cast<Derived_Store const &> (*this).template make_merge_iterator<Key, Value> (transaction_a, table1_a, table2_a);
}
template <typename Key, typename Value>
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, nano::db_val<Val> const & key)
nano::store_iterator<Key, Value> make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, nano::db_val<Val> const & key) const
{
return static_cast<Derived_Store &> (*this).template make_merge_iterator<Key, Value> (transaction_a, table1_a, table2_a, key);
return static_cast<Derived_Store const &> (*this).template make_merge_iterator<Key, Value> (transaction_a, table1_a, table2_a, key);
}
bool entry_has_sideband (size_t entry_size_a, nano::block_type type_a) const
@ -1214,18 +1214,18 @@ protected:
return static_cast<Derived_Store const &> (*this).get (transaction_a, table_a, key_a, value_a);
}
int put (nano::transaction const & transaction_a, tables table_a, nano::db_val<Val> const & key_a, nano::db_val<Val> const & value_a)
int put (nano::write_transaction const & transaction_a, tables table_a, nano::db_val<Val> const & key_a, nano::db_val<Val> const & value_a)
{
return static_cast<Derived_Store &> (*this).put (transaction_a, table_a, key_a, value_a);
}
int del (nano::transaction const & transaction_a, tables table_a, nano::db_val<Val> const & key_a)
int del (nano::write_transaction const & transaction_a, tables table_a, nano::db_val<Val> const & key_a)
{
return static_cast<Derived_Store &> (*this).del (transaction_a, table_a, key_a);
}
virtual size_t count (nano::transaction const & transaction_a, tables table_a) const = 0;
virtual int drop (nano::transaction const & transaction_a, tables table_a) = 0;
virtual int drop (nano::write_transaction const & transaction_a, tables table_a) = 0;
virtual bool not_found (int status) const = 0;
virtual bool success (int status) const = 0;
virtual int status_code_not_found () const = 0;
@ -1238,7 +1238,7 @@ template <typename Val, typename Derived_Store>
class block_predecessor_set : public nano::block_visitor
{
public:
block_predecessor_set (nano::transaction const & transaction_a, nano::block_store_partial<Val, Derived_Store> & store_a) :
block_predecessor_set (nano::write_transaction const & transaction_a, nano::block_store_partial<Val, Derived_Store> & store_a) :
transaction (transaction_a),
store (store_a)
{
@ -1278,7 +1278,7 @@ public:
fill_value (block_a);
}
}
nano::transaction const & transaction;
nano::write_transaction const & transaction;
nano::block_store_partial<Val, Derived_Store> & store;
};
}

18
nano/secure/ledger.cpp
View file

@ -12,7 +12,7 @@ namespace
class rollback_visitor : public nano::block_visitor
{
public:
rollback_visitor (nano::transaction const & transaction_a, nano::ledger & ledger_a, std::vector<std::shared_ptr<nano::block>> & list_a) :
rollback_visitor (nano::write_transaction const & transaction_a, nano::ledger & ledger_a, std::vector<std::shared_ptr<nano::block>> & list_a) :
transaction (transaction_a),
ledger (ledger_a),
list (list_a)
@ -154,7 +154,7 @@ public:
}
ledger.store.block_del (transaction, hash);
}
nano::transaction const & transaction;
nano::write_transaction const & transaction;
nano::ledger & ledger;
std::vector<std::shared_ptr<nano::block>> & list;
bool error{ false };
@ -163,7 +163,7 @@ public:
class ledger_processor : public nano::block_visitor
{
public:
ledger_processor (nano::ledger &, nano::transaction const &, nano::signature_verification = nano::signature_verification::unknown);
ledger_processor (nano::ledger &, nano::write_transaction const &, nano::signature_verification = nano::signature_verification::unknown);
virtual ~ledger_processor () = default;
void send_block (nano::send_block const &) override;
void receive_block (nano::receive_block const &) override;
@ -173,7 +173,7 @@ public:
void state_block_impl (nano::state_block const &);
void epoch_block_impl (nano::state_block const &);
nano::ledger & ledger;
nano::transaction const & transaction;
nano::write_transaction const & transaction;
nano::signature_verification verification;
nano::process_return result;
};
@ -647,7 +647,7 @@ void ledger_processor::open_block (nano::open_block const & block_a)
}
}
ledger_processor::ledger_processor (nano::ledger & ledger_a, nano::transaction const & transaction_a, nano::signature_verification verification_a) :
ledger_processor::ledger_processor (nano::ledger & ledger_a, nano::write_transaction const & transaction_a, nano::signature_verification verification_a) :
ledger (ledger_a),
transaction (transaction_a),
verification (verification_a)
@ -713,7 +713,7 @@ nano::uint128_t nano::ledger::account_pending (nano::transaction const & transac
return result;
}
nano::process_return nano::ledger::process (nano::transaction const & transaction_a, nano::block const & block_a, nano::signature_verification verification)
nano::process_return nano::ledger::process (nano::write_transaction const & transaction_a, nano::block const & block_a, nano::signature_verification verification)
{
assert (!nano::work_validate (block_a));
ledger_processor processor (*this, transaction_a, verification);
@ -839,7 +839,7 @@ nano::uint128_t nano::ledger::weight (nano::transaction const & transaction_a, n
}
// Rollback blocks until `block_a' doesn't exist or it tries to penetrate the confirmation height
bool nano::ledger::rollback (nano::transaction const & transaction_a, nano::block_hash const & block_a, std::vector<std::shared_ptr<nano::block>> & list_a)
bool nano::ledger::rollback (nano::write_transaction const & transaction_a, nano::block_hash const & block_a, std::vector<std::shared_ptr<nano::block>> & list_a)
{
assert (store.block_exists (transaction_a, block_a));
auto account_l (account (transaction_a, block_a));
@ -870,7 +870,7 @@ bool nano::ledger::rollback (nano::transaction const & transaction_a, nano::bloc
return error;
}
bool nano::ledger::rollback (nano::transaction const & transaction_a, nano::block_hash const & block_a)
bool nano::ledger::rollback (nano::write_transaction const & transaction_a, nano::block_hash const & block_a)
{
std::vector<std::shared_ptr<nano::block>> rollback_list;
return rollback (transaction_a, block_a, rollback_list);
@ -989,7 +989,7 @@ bool nano::ledger::is_epoch_link (nano::uint256_union const & link_a)
return link_a == epoch_link;
}
void nano::ledger::change_latest (nano::transaction const & transaction_a, nano::account const & account_a, nano::block_hash const & hash_a, nano::block_hash const & rep_block_a, nano::amount const & balance_a, uint64_t block_count_a, bool is_state, nano::epoch epoch_a)
void nano::ledger::change_latest (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::block_hash const & hash_a, nano::block_hash const & rep_block_a, nano::amount const & balance_a, uint64_t block_count_a, bool is_state, nano::epoch epoch_a)
{
nano::account_info info;
auto exists (!store.account_get (transaction_a, account_a, info));

8
nano/secure/ledger.hpp
View file

@ -40,10 +40,10 @@ public:
bool is_send (nano::transaction const &, nano::state_block const &) const;
nano::block_hash block_destination (nano::transaction const &, nano::block const &);
nano::block_hash block_source (nano::transaction const &, nano::block const &);
nano::process_return process (nano::transaction const &, nano::block const &, nano::signature_verification = nano::signature_verification::unknown);
bool rollback (nano::transaction const &, nano::block_hash const &, std::vector<std::shared_ptr<nano::block>> &);
bool rollback (nano::transaction const &, nano::block_hash const &);
void change_latest (nano::transaction const &, nano::account const &, nano::block_hash const &, nano::account const &, nano::uint128_union const &, uint64_t, bool = false, nano::epoch = nano::epoch::epoch_0);
nano::process_return process (nano::write_transaction const &, nano::block const &, nano::signature_verification = nano::signature_verification::unknown);
bool rollback (nano::write_transaction const &, nano::block_hash const &, std::vector<std::shared_ptr<nano::block>> &);
bool rollback (nano::write_transaction const &, nano::block_hash const &);
void change_latest (nano::write_transaction const &, nano::account const &, nano::block_hash const &, nano::account const &, nano::uint128_union const &, uint64_t, bool = false, nano::epoch = nano::epoch::epoch_0);
void dump_account_chain (nano::account const &);
bool could_fit (nano::transaction const &, nano::block const &);
bool is_epoch_link (nano::uint256_union const &);