diff --git a/CMakeLists.txt b/CMakeLists.txt index fb71a03a..10abe20f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,9 @@ set (NANO_SECURE_RPC OFF CACHE BOOL "") set (NANO_ROCKSDB OFF CACHE BOOL "") set (NANO_WARN_TO_ERR OFF CACHE BOOL "") +if (NANO_ROCKSDB) + add_definitions (-DNANO_ROCKSDB=1) +endif () option(NANO_ASAN_INT "Enable ASan+UBSan+Integer overflow" OFF) option(NANO_ASAN "Enable ASan+UBSan" OFF) option(NANO_TSAN "Enable TSan" OFF) diff --git a/ci/build-travis.sh b/ci/build-travis.sh index a0b2f3e8..5a21b49d 100755 --- a/ci/build-travis.sh +++ b/ci/build-travis.sh @@ -27,6 +27,8 @@ else SANITIZERS="" fi +ulimit -S -n 8192 + cmake \ -G'Unix Makefiles' \ -DACTIVE_NETWORK=nano_test_network \ diff --git a/ci/test.sh b/ci/test.sh index 9cb1b6bb..97cad4f8 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash build_dir=${1-${PWD}} -TIMEOUT_DEFAULT=180 +TIMEOUT_DEFAULT=360 BUSYBOX_BASH=${BUSYBOX_BASH-0} diff --git a/docker/ci/Dockerfile-clang b/docker/ci/Dockerfile-clang index 2ff79eda..6fbd365d 100644 --- a/docker/ci/Dockerfile-clang +++ b/docker/ci/Dockerfile-clang @@ -1,7 +1,12 @@ FROM nanocurrency/nano-env:base RUN apt-get update -qq && apt-get install -yqq \ - clang-3.9 lldb-3.9 librocksdb-dev + clang-3.9 lldb-3.9 git + +RUN git clone https://github.com/facebook/rocksdb.git && \ + cd rocksdb && \ + make static_lib && \ + make install ENV CXX=/usr/bin/clang++ ENV CC=/usr/bin/clang diff --git a/docker/ci/Dockerfile-gcc b/docker/ci/Dockerfile-gcc index adf956f3..23cb201a 100644 --- a/docker/ci/Dockerfile-gcc +++ b/docker/ci/Dockerfile-gcc @@ -1,6 +1,11 @@ FROM nanocurrency/nano-env:base -RUN apt-get install -yqq librocksdb-dev +RUN apt-get install -yqq git + +RUN git clone https://github.com/facebook/rocksdb.git && \ + cd rocksdb && \ + make static_lib && \ + make install ENV BOOST_ROOT=/usr/local ADD util/build_prep/bootstrap_boost.sh bootstrap_boost.sh diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index 0b6b2e1e..41a0da0d 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -1,8 +1,3 @@ -if (NANO_ROCKSDB) - set (rocksdb_test rocksdb_test.cpp) - set (rocksdb_libs ${ROCKSDB_LIBRARIES} ${ZLIB_LIBRARIES}) -endif () - add_executable (core_test core_test_main.cc testutil.hpp @@ -23,7 +18,6 @@ add_executable (core_test memory_pool.cpp processor_service.cpp peer_container.cpp - ${rocksdb_test} signing.cpp socket.cpp toml.cpp @@ -41,4 +35,4 @@ target_compile_definitions(core_test -DTAG_VERSION_STRING=${TAG_VERSION_STRING} -DGIT_COMMIT_HASH=${GIT_COMMIT_HASH} -DBOOST_PROCESS_SUPPORTED=${BOOST_PROCESS_SUPPORTED}) -target_link_libraries (core_test node secure gtest libminiupnpc-static Boost::boost ${rocksdb_libs}) +target_link_libraries (core_test node secure gtest libminiupnpc-static Boost::boost) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 59659077..239772d1 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -614,6 +614,7 @@ TEST (block_store, latest_find) ASSERT_EQ (second, find3); } +#if !NANO_ROCKSDB TEST (block_store, bad_path) { nano::logger_mt logger; @@ -621,6 +622,7 @@ TEST (block_store, bad_path) auto store = nano::make_store (init, logger, boost::filesystem::path ("///")); ASSERT_TRUE (init); } +#endif TEST (block_store, DISABLED_already_open) // File can be shared { @@ -1849,14 +1851,18 @@ TEST (block_store, incompatible_version) // Put version to an unreachable number so that it should always be incompatible auto transaction (store->tx_begin_write ()); - store->version_put (transaction, std::numeric_limits::max ()); + store->version_put (transaction, std::numeric_limits::max ()); } // Now try and read it, should give an error { auto error (false); - auto store = nano::make_store (error, logger, path); + auto store = nano::make_store (error, logger, path, true); ASSERT_TRUE (error); + + auto transaction = store->tx_begin_read (); + auto version_l = store->version_get (transaction); + ASSERT_EQ (version_l, std::numeric_limits::max ()); } } @@ -1901,7 +1907,7 @@ void modify_account_info_to_v13 (nano::mdb_store & store, nano::transaction cons nano::account_info info; ASSERT_FALSE (store.account_get (transaction_a, account, info)); nano::account_info_v13 account_info_v13 (info.head, info.rep_block, info.open_block, info.balance, info.modified, info.block_count, info.epoch); - auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::block_store_partial::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0)); + auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0)); (void)status; assert (status == 0); } @@ -1911,7 +1917,7 @@ void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction cons nano::account_info info; ASSERT_FALSE (store.account_get (transaction_a, account, info)); nano::account_info_v14 account_info_v14 (info.head, info.rep_block, info.open_block, info.balance, info.modified, info.block_count, confirmation_height, info.epoch); - auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::block_store_partial::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v14), 0)); + auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v14), 0)); (void)status; assert (status == 0); } diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index e1a7dbf5..0e920842 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -8,16 +8,18 @@ using namespace std::chrono_literals; +#if !NANO_ROCKSDB // Init returns an error if it can't open files at the path TEST (ledger, store_error) { nano::logger_mt logger; bool init (false); auto store = nano::make_store (init, logger, boost::filesystem::path ("///")); - ASSERT_FALSE (!init); + ASSERT_TRUE (init); nano::stat stats; nano::ledger ledger (*store, stats); } +#endif // Ledger can be initialized and returns a basic query for an empty account TEST (ledger, empty) diff --git a/nano/core_test/network.cpp b/nano/core_test/network.cpp index bcf09911..c4809fa2 100644 --- a/nano/core_test/network.cpp +++ b/nano/core_test/network.cpp @@ -1544,7 +1544,7 @@ TEST (confirmation_height, single) ASSERT_NO_ERROR (system.poll ()); } - auto transaction = node->store.tx_begin_read (); + auto transaction = node->store.tx_begin_write (); ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); ASSERT_EQ (2, confirmation_height); diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index e2ab79e2..37810dc3 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -242,6 +242,7 @@ TEST (node, auto_bootstrap) node1->network.send_keepalive (channel); node1->start (); system.nodes.push_back (node1); + system.deadline_set (10s); while (!node1->bootstrap_initiator.in_progress ()) { ASSERT_NO_ERROR (system.poll ()); @@ -497,7 +498,7 @@ TEST (node, confirm_locked) { nano::system system (24000, 1); system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - auto transaction (system.nodes[0]->store.tx_begin_read ()); + auto transaction (system.wallet (0)->wallets.tx_begin_read ()); system.wallet (0)->enter_password (transaction, "1"); auto block (std::make_shared (0, 0, 0, nano::keypair ().prv, 0, 0)); system.nodes[0]->network.flood_block (block); @@ -3059,15 +3060,15 @@ TEST (node, dont_write_lock_node) std::thread ([&path, &write_lock_held_promise, &finished_promise]() { nano::logger_mt logger; bool init (false); - nano::mdb_store store (init, logger, path / "data.ldb"); + auto store = nano::make_store (init, logger, path, false, true); nano::genesis genesis; { - auto transaction (store.tx_begin_write ()); - store.initialize (transaction, genesis); + auto transaction (store->tx_begin_write ()); + store->initialize (transaction, genesis); } // Hold write lock open until main thread is done needing it - auto transaction (store.tx_begin_write ()); + auto transaction (store->tx_begin_write ()); write_lock_held_promise.set_value (); finished_promise.get_future ().wait (); }) diff --git a/nano/core_test/rocksdb_test.cpp b/nano/core_test/rocksdb_test.cpp deleted file mode 100644 index 0c578398..00000000 --- a/nano/core_test/rocksdb_test.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include - -#include - -#include - -#include -#include -#include -#include -#include - -TEST (rocksdb, build_test) -{ - auto path = nano::unique_path (); - - std::vector column_families; - column_families.push_back (rocksdb::ColumnFamilyDescriptor ( - rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions ())); - - rocksdb::Options options; - options.create_if_missing = true; - options.IncreaseParallelism (std::thread::hardware_concurrency ()); - options.OptimizeLevelStyleCompaction (); - options.OptimizeUniversalStyleCompaction (); - - std::vector handles; - rocksdb::OptimisticTransactionDB * db; - auto s = rocksdb::OptimisticTransactionDB::Open (options, path.string (), column_families, &handles, &db); - ASSERT_TRUE (s.ok ()); -} diff --git a/nano/core_test/socket.cpp b/nano/core_test/socket.cpp index 28a3c11e..bcfcb056 100644 --- a/nano/core_test/socket.cpp +++ b/nano/core_test/socket.cpp @@ -10,7 +10,7 @@ using namespace std::chrono_literals; TEST (socket, concurrent_writes) { - nano::inactive_node inactivenode; + nano::inactive_node inactivenode (nano::working_path (), 24000, false); auto node = inactivenode.node; // This gives more realistic execution than using system#poll, allowing writes to diff --git a/nano/core_test/utility.cpp b/nano/core_test/utility.cpp index 386bb90b..2304ceb5 100644 --- a/nano/core_test/utility.cpp +++ b/nano/core_test/utility.cpp @@ -32,3 +32,53 @@ TEST (thread, worker) } ASSERT_TRUE (passed_sleep); } + +TEST (filesystem, remove_all_files) +{ + auto path = nano::unique_path (); + auto dummy_directory = path / "tmp"; + boost::filesystem::create_directories (dummy_directory); + + auto dummy_file1 = path / "my_file1.txt"; + auto dummy_file2 = path / "my_file2.txt"; + std::ofstream (dummy_file1.string ()); + std::ofstream (dummy_file2.string ()); + + // Check all exist + ASSERT_TRUE (boost::filesystem::exists (dummy_directory)); + ASSERT_TRUE (boost::filesystem::exists (dummy_file1)); + ASSERT_TRUE (boost::filesystem::exists (dummy_file2)); + + // Should remove only the files + nano::remove_all_files_in_dir (path); + + ASSERT_TRUE (boost::filesystem::exists (dummy_directory)); + ASSERT_FALSE (boost::filesystem::exists (dummy_file1)); + ASSERT_FALSE (boost::filesystem::exists (dummy_file2)); +} + +TEST (filesystem, move_all_files) +{ + auto path = nano::unique_path (); + auto dummy_directory = path / "tmp"; + boost::filesystem::create_directories (dummy_directory); + + auto dummy_file1 = dummy_directory / "my_file1.txt"; + auto dummy_file2 = dummy_directory / "my_file2.txt"; + std::ofstream (dummy_file1.string ()); + std::ofstream (dummy_file2.string ()); + + // Check all exist + ASSERT_TRUE (boost::filesystem::exists (dummy_directory)); + ASSERT_TRUE (boost::filesystem::exists (dummy_file1)); + ASSERT_TRUE (boost::filesystem::exists (dummy_file2)); + + // Should move only the files + nano::move_all_files_to_dir (dummy_directory, path); + + ASSERT_TRUE (boost::filesystem::exists (dummy_directory)); + ASSERT_TRUE (boost::filesystem::exists (path / "my_file1.txt")); + ASSERT_TRUE (boost::filesystem::exists (path / "my_file2.txt")); + ASSERT_FALSE (boost::filesystem::exists (dummy_file1)); + ASSERT_FALSE (boost::filesystem::exists (dummy_file2)); +} diff --git a/nano/core_test/wallets.cpp b/nano/core_test/wallets.cpp index 1826be76..ef727e7f 100644 --- a/nano/core_test/wallets.cpp +++ b/nano/core_test/wallets.cpp @@ -78,6 +78,7 @@ TEST (wallets, remove) } } +#if !NANO_ROCKSDB TEST (wallets, upgrade) { nano::system system (24000, 1); @@ -101,7 +102,7 @@ TEST (wallets, upgrade) nano::account_info info; ASSERT_FALSE (mdb_store.account_get (transaction_destination, nano::genesis_account, info)); nano::account_info_v13 account_info_v13 (info.head, info.rep_block, info.open_block, info.balance, info.modified, info.block_count, info.epoch); - auto status (mdb_put (mdb_store.env.tx (transaction_destination), mdb_store.get_account_db (info.epoch) == nano::block_store_partial::tables::accounts_v0 ? mdb_store.accounts_v0 : mdb_store.accounts_v1, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (account_info_v13), 0)); + auto status (mdb_put (mdb_store.env.tx (transaction_destination), mdb_store.get_account_db (info.epoch) == nano::tables::accounts_v0 ? mdb_store.accounts_v0 : mdb_store.accounts_v1, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (account_info_v13), 0)); (void)status; assert (status == 0); } @@ -118,6 +119,7 @@ TEST (wallets, upgrade) MDB_dbi new_handle; ASSERT_EQ (0, mdb_dbi_open (tx_new, id.pub.to_string ().c_str (), 0, &new_handle)); } +#endif // Keeps breaking whenever we add new DBs TEST (wallets, DISABLED_wallet_create_max) diff --git a/nano/lib/CMakeLists.txt b/nano/lib/CMakeLists.txt index 44ed9877..1f146a0b 100644 --- a/nano/lib/CMakeLists.txt +++ b/nano/lib/CMakeLists.txt @@ -21,6 +21,8 @@ add_library (nano_lib config.hpp config.cpp configbase.hpp + diagnosticsconfig.hpp + diagnosticsconfig.cpp errors.hpp errors.cpp ipc.hpp diff --git a/nano/node/diagnosticsconfig.cpp b/nano/lib/diagnosticsconfig.cpp similarity index 98% rename from nano/node/diagnosticsconfig.cpp rename to nano/lib/diagnosticsconfig.cpp index a4e23c68..8f2684a7 100644 --- a/nano/node/diagnosticsconfig.cpp +++ b/nano/lib/diagnosticsconfig.cpp @@ -1,6 +1,6 @@ +#include #include #include -#include nano::error nano::diagnostics_config::serialize_json (nano::jsonconfig & json) const { diff --git a/nano/node/diagnosticsconfig.hpp b/nano/lib/diagnosticsconfig.hpp similarity index 100% rename from nano/node/diagnosticsconfig.hpp rename to nano/lib/diagnosticsconfig.hpp diff --git a/nano/lib/utility.cpp b/nano/lib/utility.cpp index a983c970..439bd116 100644 --- a/nano/lib/utility.cpp +++ b/nano/lib/utility.cpp @@ -298,6 +298,30 @@ std::unique_ptr nano::collect_seq_con_info (nano:: return composite; } +void nano::remove_all_files_in_dir (boost::filesystem::path const & dir) +{ + for (auto & p : boost::filesystem::directory_iterator (dir)) + { + auto path = p.path (); + if (boost::filesystem::is_regular_file (path)) + { + boost::filesystem::remove (path); + } + } +} + +void nano::move_all_files_to_dir (boost::filesystem::path const & from, boost::filesystem::path const & to) +{ + for (auto & p : boost::filesystem::directory_iterator (from)) + { + auto path = p.path (); + if (boost::filesystem::is_regular_file (path)) + { + boost::filesystem::rename (path, to / path.filename ()); + } + } +} + /* * Backing code for "release_assert", which is itself a macro */ diff --git a/nano/lib/utility.hpp b/nano/lib/utility.hpp index 597ac0ff..f1cc299b 100644 --- a/nano/lib/utility.hpp +++ b/nano/lib/utility.hpp @@ -217,6 +217,9 @@ inline std::unique_ptr collect_seq_con_info (observer_se composite->add_component (std::make_unique (seq_con_info{ "observers", count, sizeof_element })); return composite; } + +void remove_all_files_in_dir (boost::filesystem::path const & dir); +void move_all_files_to_dir (boost::filesystem::path const & from, boost::filesystem::path const & to); } void release_assert_internal (bool check, const char * check_expr, const char * file, unsigned int line); diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index f36e16ac..3d57d86e 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -1,5 +1,6 @@ if (NANO_ROCKSDB) set (rocksdb_libs ${ROCKSDB_LIBRARIES} ${ZLIB_LIBRARIES}) + set (rocksdb_sources rocksdb/rocksdb.hpp rocksdb/rocksdb.cpp rocksdb/rocksdb_iterator.hpp rocksdb/rocksdb_txn.hpp rocksdb/rocksdb_txn.cpp) endif () if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") @@ -20,6 +21,7 @@ endif () add_library (node ${platform_sources} + ${rocksdb_sources} active_transactions.hpp active_transactions.cpp blockprocessor.cpp @@ -35,8 +37,6 @@ add_library (node confirmation_height_processor.cpp daemonconfig.hpp daemonconfig.cpp - diagnosticsconfig.hpp - diagnosticsconfig.cpp election.hpp election.cpp gap_cache.hpp diff --git a/nano/node/active_transactions.cpp b/nano/node/active_transactions.cpp index b209b901..49483b17 100644 --- a/nano/node/active_transactions.cpp +++ b/nano/node/active_transactions.cpp @@ -76,21 +76,23 @@ void nano::active_transactions::confirm_frontiers (nano::transaction const & tra lk.unlock (); nano::account_info info; auto error = node.store.account_get (transaction_a, cementable_account.account, info); - release_assert (!error); - uint64_t confirmation_height; - error = node.store.confirmation_height_get (transaction_a, cementable_account.account, confirmation_height); - release_assert (!error); - - if (info.block_count > confirmation_height && !this->node.pending_confirmation_height.is_processing_block (info.head)) + if (!error) { - auto block (this->node.store.block_get (transaction_a, info.head)); - if (!this->start (block)) + uint64_t confirmation_height; + error = node.store.confirmation_height_get (transaction_a, cementable_account.account, confirmation_height); + release_assert (!error); + + if (info.block_count > confirmation_height && !this->node.pending_confirmation_height.is_processing_block (info.head)) { - ++elections_count; - // Calculate votes for local representatives - if (representative) + auto block (this->node.store.block_get (transaction_a, info.head)); + if (!this->start (block)) { - this->node.block_processor.generator.add (block->hash ()); + ++elections_count; + // Calculate votes for local representatives + if (representative) + { + this->node.block_processor.generator.add (block->hash ()); + } } } } diff --git a/nano/node/blockprocessor.cpp b/nano/node/blockprocessor.cpp index a6389691..b361860f 100644 --- a/nano/node/blockprocessor.cpp +++ b/nano/node/blockprocessor.cpp @@ -262,7 +262,7 @@ void nano::block_processor::process_batch (std::unique_lock & lock_a } lock_a.unlock (); auto scoped_write_guard = write_database_queue.wait (nano::writer::process_batch); - auto transaction (node.store.tx_begin_write ()); + auto transaction (node.store.tx_begin_write ({ nano::tables::accounts_v0, nano::tables::accounts_v1, nano::tables::cached_counts, nano::tables::change_blocks, nano::tables::frontiers, nano::tables::open_blocks, nano::tables::pending_v0, nano::tables::pending_v1, nano::tables::receive_blocks, nano::tables::representation, nano::tables::send_blocks, nano::tables::state_blocks_v0, nano::tables::state_blocks_v1, nano::tables::unchecked }, { nano::tables::confirmation_height })); timer_l.restart (); lock_a.lock (); // Processing blocks @@ -348,8 +348,7 @@ void nano::block_processor::process_batch (std::unique_lock & lock_a } } number_of_blocks_processed++; - auto process_result (process_one (transaction, info)); - (void)process_result; + process_one (transaction, info); lock_a.lock (); /* Verify more state blocks if blocks deque is empty Because verification is long process, avoid large deque verification inside of write transaction */ @@ -407,7 +406,7 @@ void nano::block_processor::process_live (nano::block_hash const & hash_a, std:: }); } -nano::process_return nano::block_processor::process_one (nano::transaction const & transaction_a, nano::unchecked_info info_a, const bool watch_work_a) +nano::process_return nano::block_processor::process_one (nano::write_transaction const & transaction_a, nano::unchecked_info info_a, const bool watch_work_a) { nano::process_return result; auto hash (info_a.block->hash ()); @@ -540,14 +539,14 @@ nano::process_return nano::block_processor::process_one (nano::transaction const return result; } -nano::process_return nano::block_processor::process_one (nano::transaction const & transaction_a, std::shared_ptr block_a, const bool watch_work_a) +nano::process_return nano::block_processor::process_one (nano::write_transaction const & transaction_a, std::shared_ptr block_a, const bool watch_work_a) { nano::unchecked_info info (block_a, block_a->account (), 0, nano::signature_verification::unknown); auto result (process_one (transaction_a, info, watch_work_a)); return result; } -void nano::block_processor::queue_unchecked (nano::transaction const & transaction_a, nano::block_hash const & hash_a) +void nano::block_processor::queue_unchecked (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) { auto unchecked_blocks (node.store.unchecked_get (transaction_a, hash_a)); for (auto & info : unchecked_blocks) diff --git a/nano/node/blockprocessor.hpp b/nano/node/blockprocessor.hpp index 906b6c1f..61744c67 100644 --- a/nano/node/blockprocessor.hpp +++ b/nano/node/blockprocessor.hpp @@ -18,6 +18,7 @@ namespace nano { class node; class transaction; +class write_transaction; class write_database_queue; class rolled_hash @@ -47,14 +48,14 @@ public: bool should_log (bool); bool have_blocks (); void process_blocks (); - nano::process_return process_one (nano::transaction const &, nano::unchecked_info, const bool = false); - nano::process_return process_one (nano::transaction const &, std::shared_ptr, const bool = false); + nano::process_return process_one (nano::write_transaction const &, nano::unchecked_info, const bool = false); + nano::process_return process_one (nano::write_transaction const &, std::shared_ptr, const bool = false); nano::vote_generator generator; // Delay required for average network propagartion before requesting confirmation static std::chrono::milliseconds constexpr confirmation_request_delay{ 1500 }; private: - void queue_unchecked (nano::transaction const &, nano::block_hash const &); + void queue_unchecked (nano::write_transaction const &, nano::block_hash const &); void verify_state_blocks (nano::transaction const & transaction_a, std::unique_lock &, size_t = std::numeric_limits::max ()); void process_batch (std::unique_lock &); void process_live (nano::block_hash const &, std::shared_ptr, const bool = false); diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index 0e617a43..9a83602e 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -22,6 +22,8 @@ std::string nano::error_cli_messages::message (int ev) const return "Invalid arguments"; case nano::error_cli::unknown_command: return "Unknown command"; + case nano::error_cli::database_write_error: + return "Database write error"; } return "Invalid error code"; @@ -68,6 +70,57 @@ void nano::add_node_options (boost::program_options::options_description & descr // clang-format on } +namespace +{ +void database_write_lock_error (std::error_code & ec) +{ + std::cerr << "Write database error, this cannot be run while the node is already running\n"; + ec = nano::error_cli::database_write_error; +} + +bool copy_database (boost::filesystem::path const & data_path, boost::program_options::variables_map & vm, boost::filesystem::path const & output_path, std::error_code & ec) +{ + bool success = false; + bool needs_to_write = vm.count ("unchecked_clear") || vm.count ("clear_send_ids") || vm.count ("online_weight_clear") || vm.count ("peer_clear") || vm.count ("confirmation_height_clear"); + + nano::inactive_node node (data_path, 24000, !needs_to_write); + if (!node.init.error ()) + { + if (vm.count ("unchecked_clear")) + { + auto transaction (node.node->store.tx_begin_write ()); + node.node->store.unchecked_clear (transaction); + } + if (vm.count ("clear_send_ids")) + { + auto transaction (node.node->wallets.tx_begin_write ()); + node.node->wallets.clear_send_ids (transaction); + } + if (vm.count ("online_weight_clear")) + { + auto transaction (node.node->store.tx_begin_write ()); + node.node->store.online_weight_clear (transaction); + } + if (vm.count ("peer_clear")) + { + auto transaction (node.node->store.tx_begin_write ()); + node.node->store.peer_clear (transaction); + } + if (vm.count ("confirmation_height_clear")) + { + reset_confirmation_heights (node.node->store); + } + + success = node.node->copy_with_compaction (output_path); + } + else + { + database_write_lock_error (ec); + } + return success; +} +} + std::error_code nano::handle_node_options (boost::program_options::variables_map & vm) { std::error_code ec; @@ -151,53 +204,45 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map { try { - auto vacuum_path = data_path / "vacuumed.ldb"; - auto source_path = data_path / "data.ldb"; - auto backup_path = data_path / "backup.vacuum.ldb"; - - std::cout << "Vacuuming database copy in " << data_path << std::endl; - std::cout << "This may take a while..." << std::endl; - - // Scope the node so the mdb environment gets cleaned up properly before - // the original file is replaced with the vacuumed file. - bool success = false; + std::cout << "Vacuuming database copy in "; +#if NANO_ROCKSDB + auto source_path = data_path / "rocksdb"; + auto backup_path = source_path / "backup"; + auto vacuum_path = backup_path / "vacuumed"; + if (!boost::filesystem::exists (vacuum_path)) { - inactive_node node (data_path); - if (vm.count ("unchecked_clear")) - { - auto transaction (node.node->store.tx_begin_write ()); - node.node->store.unchecked_clear (transaction); - } - if (vm.count ("clear_send_ids")) - { - auto transaction (node.node->wallets.tx_begin_write ()); - node.node->wallets.clear_send_ids (transaction); - } - if (vm.count ("online_weight_clear")) - { - auto transaction (node.node->store.tx_begin_write ()); - node.node->store.online_weight_clear (transaction); - } - if (vm.count ("peer_clear")) - { - auto transaction (node.node->store.tx_begin_write ()); - node.node->store.peer_clear (transaction); - } - success = node.node->copy_with_compaction (vacuum_path); + boost::filesystem::create_directories (vacuum_path); } + std::cout << source_path << "\n"; +#else + auto source_path = data_path / "data.ldb"; + auto backup_path = data_path / "backup.vacuum.ldb"; + auto vacuum_path = data_path / "vacuumed.ldb"; + std::cout << data_path << "\n"; +#endif + std::cout << "This may take a while..." << std::endl; + + bool success = copy_database (data_path, vm, vacuum_path, ec); if (success) { // Note that these throw on failure std::cout << "Finalizing" << std::endl; +#ifdef NANO_ROCKSDB + nano::remove_all_files_in_dir (backup_path); + nano::move_all_files_to_dir (source_path, backup_path); + nano::move_all_files_to_dir (vacuum_path, source_path); + boost::filesystem::remove_all (vacuum_path); +#else boost::filesystem::remove (backup_path); boost::filesystem::rename (source_path, backup_path); boost::filesystem::rename (vacuum_path, source_path); +#endif std::cout << "Vacuum completed" << std::endl; } else { - std::cerr << "Vacuum failed (copy_with_compaction returned false)" << std::endl; + std::cerr << "Vacuum failed (copying returned false)" << std::endl; } } catch (const boost::filesystem::filesystem_error & ex) @@ -213,51 +258,24 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map { try { - boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as ()) : nano::working_path (); - +#if NANO_ROCKSDB + auto source_path = data_path / "rocksdb"; + auto snapshot_path = source_path / "backup"; +#else auto source_path = data_path / "data.ldb"; auto snapshot_path = data_path / "snapshot.ldb"; - +#endif std::cout << "Database snapshot of " << source_path << " to " << snapshot_path << " in progress" << std::endl; std::cout << "This may take a while..." << std::endl; - bool success = false; - { - inactive_node node (data_path); - if (vm.count ("unchecked_clear")) - { - auto transaction (node.node->store.tx_begin_write ()); - node.node->store.unchecked_clear (transaction); - } - if (vm.count ("clear_send_ids")) - { - auto transaction (node.node->wallets.tx_begin_write ()); - node.node->wallets.clear_send_ids (transaction); - } - if (vm.count ("online_weight_clear")) - { - auto transaction (node.node->store.tx_begin_write ()); - node.node->store.online_weight_clear (transaction); - } - if (vm.count ("peer_clear")) - { - auto transaction (node.node->store.tx_begin_write ()); - node.node->store.peer_clear (transaction); - } - if (vm.count ("confirmation_height_clear")) - { - reset_confirmation_heights (node.node->store); - } - - success = node.node->copy_with_compaction (snapshot_path); - } + bool success = copy_database (data_path, vm, snapshot_path, ec); if (success) { std::cout << "Snapshot completed, This can be found at " << snapshot_path << std::endl; } else { - std::cerr << "Snapshot Failed (copy_with_compaction returned false)" << std::endl; + std::cerr << "Snapshot Failed (copying returned false)" << std::endl; } } catch (const boost::filesystem::filesystem_error & ex) @@ -272,80 +290,115 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map else if (vm.count ("unchecked_clear")) { boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as ()) : nano::working_path (); - inactive_node node (data_path); - auto transaction (node.node->store.tx_begin_write ()); - node.node->store.unchecked_clear (transaction); - std::cout << "Unchecked blocks deleted" << std::endl; + inactive_node node (data_path, 24000, false); + if (!node.init.error ()) + { + auto transaction (node.node->store.tx_begin_write ()); + node.node->store.unchecked_clear (transaction); + std::cout << "Unchecked blocks deleted" << std::endl; + } + else + { + database_write_lock_error (ec); + } } else if (vm.count ("clear_send_ids")) { boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as ()) : nano::working_path (); - inactive_node node (data_path); - auto transaction (node.node->wallets.tx_begin_write ()); - node.node->wallets.clear_send_ids (transaction); - std::cout << "Send IDs deleted" << std::endl; + inactive_node node (data_path, 24000, false); + if (!node.init.error ()) + { + auto transaction (node.node->wallets.tx_begin_write ()); + node.node->wallets.clear_send_ids (transaction); + std::cout << "Send IDs deleted" << std::endl; + } + else + { + database_write_lock_error (ec); + } } else if (vm.count ("online_weight_clear")) { boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as ()) : nano::working_path (); - inactive_node node (data_path); - auto transaction (node.node->store.tx_begin_write ()); - node.node->store.online_weight_clear (transaction); - std::cout << "Onine weight records are removed" << std::endl; + inactive_node node (data_path, 24000, false); + if (!node.init.error ()) + { + auto transaction (node.node->store.tx_begin_write ()); + node.node->store.online_weight_clear (transaction); + std::cout << "Onine weight records are removed" << std::endl; + } + else + { + database_write_lock_error (ec); + } } else if (vm.count ("peer_clear")) { boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as ()) : nano::working_path (); - inactive_node node (data_path); - auto transaction (node.node->store.tx_begin_write ()); - node.node->store.peer_clear (transaction); - std::cout << "Database peers are removed" << std::endl; + inactive_node node (data_path, 24000, false); + if (!node.init.error ()) + { + auto transaction (node.node->store.tx_begin_write ()); + node.node->store.peer_clear (transaction); + std::cout << "Database peers are removed" << std::endl; + } + else + { + database_write_lock_error (ec); + } } else if (vm.count ("confirmation_height_clear")) { boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as ()) : nano::working_path (); - inactive_node node (data_path); - auto account_it = vm.find ("account"); - if (account_it != vm.cend ()) + inactive_node node (data_path, 24000, false); + if (!node.init.error ()) { - auto account_str = account_it->second.as (); - nano::account account; - if (!account.decode_account (account_str)) + auto account_it = vm.find ("account"); + if (account_it != vm.cend ()) { - uint64_t confirmation_height; - auto transaction (node.node->store.tx_begin_read ()); - if (!node.node->store.confirmation_height_get (transaction, account, confirmation_height)) + auto account_str = account_it->second.as (); + nano::account account; + if (!account.decode_account (account_str)) { - auto transaction (node.node->store.tx_begin_write ()); - auto conf_height_reset_num = 0; - if (account == node.node->network_params.ledger.genesis_account) + uint64_t confirmation_height; + auto transaction (node.node->store.tx_begin_read ()); + if (!node.node->store.confirmation_height_get (transaction, account, confirmation_height)) { - conf_height_reset_num = 1; - node.node->store.confirmation_height_put (transaction, account, confirmation_height); + auto transaction (node.node->store.tx_begin_write ()); + auto conf_height_reset_num = 0; + if (account == node.node->network_params.ledger.genesis_account) + { + conf_height_reset_num = 1; + node.node->store.confirmation_height_put (transaction, account, confirmation_height); + } + else + { + node.node->store.confirmation_height_clear (transaction, account, confirmation_height); + } + + std::cout << "Confirmation height of account " << account_str << " is set to " << conf_height_reset_num << std::endl; } else { - node.node->store.confirmation_height_clear (transaction, account, confirmation_height); + std::cerr << "Could not find account" << std::endl; + ec = nano::error_cli::generic; } - - std::cout << "Confirmation height of account " << account_str << " is set to " << conf_height_reset_num << std::endl; } else { - std::cerr << "Could not find account" << std::endl; - ec = nano::error_cli::generic; + std::cerr << "Invalid account id\n"; + ec = nano::error_cli::invalid_arguments; } } else { - std::cerr << "Invalid account id\n"; - ec = nano::error_cli::invalid_arguments; + reset_confirmation_heights (node.node->store); + std::cout << "Confirmation heights of all accounts (except genesis which is set to 1) are set to 0" << std::endl; } } else { - reset_confirmation_heights (node.node->store); - std::cout << "Confirmation heights of all accounts (except genesis) are set to 0" << std::endl; + database_write_lock_error (ec); } } else if (vm.count ("generate_config")) diff --git a/nano/node/cli.hpp b/nano/node/cli.hpp index 7e69b528..e9dde13f 100644 --- a/nano/node/cli.hpp +++ b/nano/node/cli.hpp @@ -12,7 +12,8 @@ enum class error_cli generic = 1, parse_error = 2, invalid_arguments = 3, - unknown_command = 4 + unknown_command = 4, + database_write_error = 5 }; void add_node_options (boost::program_options::options_description &); diff --git a/nano/node/confirmation_height_processor.cpp b/nano/node/confirmation_height_processor.cpp index cedfb7fa..1ecf8658 100644 --- a/nano/node/confirmation_height_processor.cpp +++ b/nano/node/confirmation_height_processor.cpp @@ -269,7 +269,8 @@ bool nano::confirmation_height_processor::write_pending (std::deque 0) { uint64_t num_accounts_processed = 0; - auto transaction (store.tx_begin_write ()); + + auto transaction (store.tx_begin_write ({}, { nano::tables::confirmation_height })); while (!all_pending_a.empty ()) { const auto & pending = all_pending_a.front (); diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index ac04136f..a1e9d9d0 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -4439,8 +4439,7 @@ void nano::json_handler::work_generate () { auto hash (hash_impl ()); auto difficulty (difficulty_optional_impl ()); - auto multiplier (multiplier_optional_impl (difficulty)); - (void)multiplier; + multiplier_optional_impl (difficulty); if (!ec && (difficulty > node.config.max_work_generate_difficulty || difficulty < node.network_params.network.publish_threshold)) { ec = nano::error_rpc::difficulty_limit; @@ -4540,8 +4539,7 @@ void nano::json_handler::work_validate () auto hash (hash_impl ()); auto work (work_optional_impl ()); auto difficulty (difficulty_optional_impl ()); - auto multiplier (multiplier_optional_impl (difficulty)); - (void)multiplier; + multiplier_optional_impl (difficulty); if (!ec) { uint64_t result_difficulty (0); diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 74ac53c6..ba40a66d 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -82,7 +82,7 @@ txn_tracking_enabled (txn_tracking_config_a.enable) if (!error_a && drop_unchecked) { - auto transaction (tx_begin_write ()); + auto transaction (tx_begin_write ({ nano::tables::cached_counts, tables::unchecked })); unchecked_clear (transaction); } } @@ -93,7 +93,7 @@ void nano::mdb_store::serialize_mdb_tracker (boost::property_tree::ptree & json, mdb_txn_tracker.serialize_json (json, min_read_time, min_write_time); } -nano::write_transaction nano::mdb_store::tx_begin_write () +nano::write_transaction nano::mdb_store::tx_begin_write (std::vector const &, std::vector const &) { return env.tx_begin_write (create_txn_callbacks ()); } @@ -189,7 +189,7 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, size return error; } -void nano::mdb_store::upgrade_v1_to_v2 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v1_to_v2 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 2); nano::account account (1); @@ -223,7 +223,7 @@ void nano::mdb_store::upgrade_v1_to_v2 (nano::transaction const & transaction_a) } } -void nano::mdb_store::upgrade_v2_to_v3 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v2_to_v3 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 3); mdb_drop (env.tx (transaction_a), representation, 0); @@ -241,7 +241,7 @@ void nano::mdb_store::upgrade_v2_to_v3 (nano::transaction const & transaction_a) } } -void nano::mdb_store::upgrade_v3_to_v4 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v3_to_v4 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 4); std::queue> items; @@ -259,7 +259,7 @@ void nano::mdb_store::upgrade_v3_to_v4 (nano::transaction const & transaction_a) } } -void nano::mdb_store::upgrade_v4_to_v5 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v4_to_v5 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 5); for (auto i (nano::store_iterator (std::make_unique> (transaction_a, accounts_v0))), n (nano::store_iterator (nullptr)); i != n; ++i) @@ -296,7 +296,7 @@ void nano::mdb_store::upgrade_v4_to_v5 (nano::transaction const & transaction_a) } } -void nano::mdb_store::upgrade_v5_to_v6 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v5_to_v6 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 6); std::deque> headers; @@ -322,20 +322,20 @@ void nano::mdb_store::upgrade_v5_to_v6 (nano::transaction const & transaction_a) } } -void nano::mdb_store::upgrade_v6_to_v7 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v6_to_v7 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 7); mdb_drop (env.tx (transaction_a), unchecked, 0); } -void nano::mdb_store::upgrade_v7_to_v8 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v7_to_v8 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 8); mdb_drop (env.tx (transaction_a), unchecked, 1); mdb_dbi_open (env.tx (transaction_a), "unchecked", MDB_CREATE | MDB_DUPSORT, &unchecked); } -void nano::mdb_store::upgrade_v8_to_v9 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v8_to_v9 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 9); MDB_dbi sequence; @@ -363,7 +363,7 @@ void nano::mdb_store::upgrade_v8_to_v9 (nano::transaction const & transaction_a) mdb_drop (env.tx (transaction_a), sequence, 1); } -void nano::mdb_store::upgrade_v10_to_v11 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v10_to_v11 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 11); MDB_dbi unsynced; @@ -371,7 +371,7 @@ void nano::mdb_store::upgrade_v10_to_v11 (nano::transaction const & transaction_ mdb_drop (env.tx (transaction_a), unsynced, 1); } -void nano::mdb_store::upgrade_v11_to_v12 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v11_to_v12 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 12); mdb_drop (env.tx (transaction_a), unchecked, 1); @@ -443,7 +443,7 @@ void nano::mdb_store::upgrade_v12_to_v13 (nano::write_transaction & transaction_ } } -void nano::mdb_store::upgrade_v13_to_v14 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v13_to_v14 (nano::write_transaction const & transaction_a) { // Upgrade all accounts to have a confirmation of 0 (except genesis which should have 1) version_put (transaction_a, 14); @@ -476,7 +476,7 @@ void nano::mdb_store::upgrade_v13_to_v14 (nano::transaction const & transaction_ release_assert (!error || error == MDB_NOTFOUND); } -void nano::mdb_store::upgrade_v14_to_v15 (nano::transaction const & transaction_a) +void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction const & transaction_a) { version_put (transaction_a, 15); @@ -530,7 +530,7 @@ void nano::mdb_store::create_backup_file (nano::mdb_env & env_a, boost::filesyst } } -void nano::mdb_store::version_put (nano::transaction const & transaction_a, int version_a) +void nano::mdb_store::version_put (nano::write_transaction const & transaction_a, int version_a) { nano::uint256_union version_key (1); nano::uint256_union version_value (version_a); @@ -584,22 +584,22 @@ int nano::mdb_store::get (nano::transaction const & transaction_a, tables table_ return mdb_get (env.tx (transaction_a), table_to_dbi (table_a), key_a, value_a); } -int nano::mdb_store::put (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, const nano::mdb_val & value_a) const +int nano::mdb_store::put (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, const nano::mdb_val & value_a) const { return (mdb_put (env.tx (transaction_a), table_to_dbi (table_a), key_a, value_a, 0)); } -int nano::mdb_store::del (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const +int nano::mdb_store::del (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const { return (mdb_del (env.tx (transaction_a), table_to_dbi (table_a), key_a, nullptr)); } -int nano::mdb_store::drop (nano::transaction const & transaction_a, tables table_a) +int nano::mdb_store::drop (nano::write_transaction const & transaction_a, tables table_a) { return clear (transaction_a, table_to_dbi (table_a)); } -int nano::mdb_store::clear (nano::transaction const & transaction_a, MDB_dbi handle_a) +int nano::mdb_store::clear (nano::write_transaction const & transaction_a, MDB_dbi handle_a) { return mdb_drop (env.tx (transaction_a), handle_a, 0); } @@ -667,7 +667,7 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const bool nano::mdb_store::not_found (int status) const { - return (MDB_NOTFOUND == status); + return (status_code_not_found () == status); } bool nano::mdb_store::success (int status) const @@ -679,3 +679,8 @@ int nano::mdb_store::status_code_not_found () const { return MDB_NOTFOUND; } + +bool nano::mdb_store::copy_db (boost::filesystem::path const & destination_file) +{ + return !mdb_env_copy2 (env.environment, destination_file.string ().c_str (), MDB_CP_COMPACT); +} \ No newline at end of file diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index 4b0456e5..165e6e84 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -1,9 +1,9 @@ #pragma once #include +#include #include #include -#include #include #include #include @@ -33,12 +33,12 @@ public: using block_store_partial::unchecked_put; mdb_store (bool &, nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512, bool backup_before_upgrade = false); - nano::write_transaction tx_begin_write () override; + nano::write_transaction tx_begin_write (std::vector const & tables_requiring_lock = {}, std::vector const & tables_no_lock = {}) override; nano::read_transaction tx_begin_read () override; bool block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) const override; - void version_put (nano::transaction const &, int) override; + void version_put (nano::write_transaction const &, int) override; void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) override; @@ -165,52 +165,54 @@ public: bool exists (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const; int get (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, nano::mdb_val & value_a) const; - int put (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, const nano::mdb_val & value_a) const; - int del (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const; + int put (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, const nano::mdb_val & value_a) const; + int del (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const; + + bool copy_db (boost::filesystem::path const & destination_file) override; template - nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a) + nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a) const { return nano::store_iterator (std::make_unique> (transaction_a, table_to_dbi (table_a))); } template - nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key) + nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key) const { return nano::store_iterator (std::make_unique> (transaction_a, table_to_dbi (table_a), key)); } template - nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, nano::mdb_val const & key) + nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, nano::mdb_val const & key) const { return nano::store_iterator (std::make_unique> (transaction_a, table_to_dbi (table1_a), table_to_dbi (table2_a), key)); } template - nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a) + nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a) const { return nano::store_iterator (std::make_unique> (transaction_a, table_to_dbi (table1_a), table_to_dbi (table2_a))); } private: bool do_upgrades (nano::write_transaction &, size_t); - void upgrade_v1_to_v2 (nano::transaction const &); - void upgrade_v2_to_v3 (nano::transaction const &); - void upgrade_v3_to_v4 (nano::transaction const &); - void upgrade_v4_to_v5 (nano::transaction const &); - void upgrade_v5_to_v6 (nano::transaction const &); - void upgrade_v6_to_v7 (nano::transaction const &); - void upgrade_v7_to_v8 (nano::transaction const &); - void upgrade_v8_to_v9 (nano::transaction const &); - void upgrade_v10_to_v11 (nano::transaction const &); - void upgrade_v11_to_v12 (nano::transaction const &); + void upgrade_v1_to_v2 (nano::write_transaction const &); + void upgrade_v2_to_v3 (nano::write_transaction const &); + void upgrade_v3_to_v4 (nano::write_transaction const &); + void upgrade_v4_to_v5 (nano::write_transaction const &); + void upgrade_v5_to_v6 (nano::write_transaction const &); + void upgrade_v6_to_v7 (nano::write_transaction const &); + void upgrade_v7_to_v8 (nano::write_transaction const &); + void upgrade_v8_to_v9 (nano::write_transaction const &); + void upgrade_v10_to_v11 (nano::write_transaction const &); + void upgrade_v11_to_v12 (nano::write_transaction const &); void upgrade_v12_to_v13 (nano::write_transaction &, size_t); - void upgrade_v13_to_v14 (nano::transaction const &); - void upgrade_v14_to_v15 (nano::transaction const &); + void upgrade_v13_to_v14 (nano::write_transaction const &); + void upgrade_v14_to_v15 (nano::write_transaction const &); void open_databases (bool &, nano::transaction const &, unsigned); - int drop (nano::transaction const & transaction_a, tables table_a) override; - int clear (nano::transaction const & transaction_a, MDB_dbi handle_a); + int drop (nano::write_transaction const & transaction_a, tables table_a) override; + int clear (nano::write_transaction const & transaction_a, MDB_dbi handle_a); bool not_found (int status) const override; bool success (int status) const override; diff --git a/nano/node/lmdb/lmdb_txn.cpp b/nano/node/lmdb/lmdb_txn.cpp index 1c77b436..85ef541f 100644 --- a/nano/node/lmdb/lmdb_txn.cpp +++ b/nano/node/lmdb/lmdb_txn.cpp @@ -110,6 +110,12 @@ void * nano::write_mdb_txn::get_handle () const return handle; } +bool nano::write_mdb_txn::contains (nano::tables table_a) const +{ + // LMDB locks on every write + return true; +} + nano::mdb_txn_tracker::mdb_txn_tracker (nano::logger_mt & logger_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a) : logger (logger_a), txn_tracking_config (txn_tracking_config_a), diff --git a/nano/node/lmdb/lmdb_txn.hpp b/nano/node/lmdb/lmdb_txn.hpp index 3830f60e..a6ca82c1 100644 --- a/nano/node/lmdb/lmdb_txn.hpp +++ b/nano/node/lmdb/lmdb_txn.hpp @@ -1,7 +1,7 @@ #pragma once +#include #include -#include #include #include @@ -46,6 +46,7 @@ public: void commit () const override; void renew () override; void * get_handle () const override; + bool contains (nano::tables table_a) const override; MDB_txn * handle; nano::mdb_env const & env; mdb_txn_callbacks txn_callbacks; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 1e508184..18efaa80 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -5,6 +5,10 @@ #include #include +#if NANO_ROCKSDB +#include +#endif + #include #include @@ -123,7 +127,7 @@ flags (flags_a), alarm (alarm_a), work (work_a), logger (config_a.logging.min_time_between_log_output), -store_impl (std::make_unique (init_a.block_store_init, logger, application_path_a / "data.ldb", config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_max_dbs, !flags.disable_unchecked_drop, flags.sideband_batch_size, config_a.backup_before_upgrade)), +store_impl (nano::make_store (init_a.block_store_init, logger, application_path_a, flags.read_only, true, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_max_dbs, !flags.disable_unchecked_drop, flags.sideband_batch_size, config_a.backup_before_upgrade)), store (*store_impl), wallets_store_impl (std::make_unique (init_a.wallets_store_init, application_path_a / "wallets.ldb", config_a.lmdb_max_dbs)), wallets_store (*wallets_store_impl), @@ -143,7 +147,7 @@ block_processor_thread ([this]() { nano::thread_role::set (nano::thread_role::name::block_processing); this->block_processor.process_blocks (); }), -online_reps (*this, config.online_weight_minimum.number ()), +online_reps (*this, config.online_weight_minimum.number (), init_a.block_store_init), vote_uniquer (block_uniquer), active (*this), confirmation_height_processor (pending_confirmation_height, store, ledger.stats, active, ledger.epoch_link, write_database_queue, config.conf_height_processor_batch_min_time, logger), @@ -389,6 +393,7 @@ startup_time (std::chrono::steady_clock::now ()) nano::genesis genesis; if (!is_initialized) { + release_assert (!flags.read_only); auto transaction (store.tx_begin_write ()); // Store was empty meaning we just created it, add the genesis block store.initialize (transaction, genesis); @@ -397,7 +402,16 @@ startup_time (std::chrono::steady_clock::now ()) auto transaction (store.tx_begin_read ()); if (!store.block_exists (transaction, genesis.hash ())) { - logger.always_log ("Genesis block not found. Make sure the node network ID is correct."); + std::stringstream ss; + ss << "Genesis block not found. Make sure the node network ID is correct."; + if (network_params.network.is_beta_network ()) + { + ss << " Beta network may have reset, try clearing database files"; + } + auto str = ss.str (); + + logger.always_log (str); + std::cerr << str << std::endl; std::exit (1); } @@ -520,9 +534,9 @@ void nano::node::do_rpc_callback (boost::asio::ip::tcp::resolver::iterator i_a, } } -bool nano::node::copy_with_compaction (boost::filesystem::path const & destination_file) +bool nano::node::copy_with_compaction (boost::filesystem::path const & destination) { - return !mdb_env_copy2 (boost::polymorphic_downcast (store_impl.get ())->env.environment, destination_file.string ().c_str (), MDB_CP_COMPACT); + return store.copy_db (destination); } void nano::node::process_fork (nano::transaction const & transaction_a, std::shared_ptr block_a) @@ -602,7 +616,7 @@ void nano::node::process_active (std::shared_ptr incoming) nano::process_return nano::node::process (nano::block const & block_a) { - auto transaction (store.tx_begin_write ()); + auto transaction (store.tx_begin_write ({ tables::accounts_v0, tables::accounts_v1, tables::cached_counts, tables::change_blocks, tables::frontiers, tables::open_blocks, tables::pending_v0, tables::pending_v1, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks_v0, tables::state_blocks_v1 }, { tables::confirmation_height })); auto result (ledger.process (transaction, block_a)); return result; } @@ -791,7 +805,7 @@ void nano::node::ongoing_bootstrap () void nano::node::ongoing_store_flush () { { - auto transaction (store.tx_begin_write ()); + auto transaction (store.tx_begin_write ({ tables::vote })); store.flush (transaction); } std::weak_ptr node_w (shared_from_this ()); @@ -1625,7 +1639,7 @@ int nano::node::store_version () return store.version_get (transaction); } -nano::inactive_node::inactive_node (boost::filesystem::path const & path_a, uint16_t peering_port_a) : +nano::inactive_node::inactive_node (boost::filesystem::path const & path_a, uint16_t peering_port_a, bool read_only_a) : path (path_a), io_context (std::make_shared ()), alarm (*io_context), @@ -1641,7 +1655,9 @@ peering_port (peering_port_a) nano::set_secure_perm_directory (path, error_chmod); logging.max_size = std::numeric_limits::max (); logging.init (path); - node = std::make_shared (init, *io_context, peering_port, path, alarm, logging, work); + auto node_flags = nano::node_flags (); + node_flags.read_only = read_only_a; + node = std::make_shared (init, *io_context, peering_port, path, alarm, logging, work, node_flags); node->active.stop (); } @@ -1650,7 +1666,11 @@ nano::inactive_node::~inactive_node () node->stop (); } -std::unique_ptr nano::make_store (bool & init, nano::logger_mt & logger, boost::filesystem::path const & path) +std::unique_ptr nano::make_store (bool & init, nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, bool drop_unchecked, size_t batch_size, bool backup_before_upgrade) { - return std::make_unique (init, logger, path); +#if NANO_ROCKSDB + return std::make_unique (init, logger, add_db_postfix ? path / "rocksdb" : path, drop_unchecked, read_only); +#else + return std::make_unique (init, logger, add_db_postfix ? path / "data.ldb" : path, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_max_dbs, drop_unchecked, batch_size, backup_before_upgrade); +#endif } diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 2546e091..d7e9e4c9 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -195,7 +195,7 @@ std::unique_ptr collect_seq_con_info (node & node, const class inactive_node final { public: - inactive_node (boost::filesystem::path const & path = nano::working_path (), uint16_t = 24000); + inactive_node (boost::filesystem::path const & path = nano::working_path (), uint16_t = 24000, bool read_only_a = true); ~inactive_node (); boost::filesystem::path path; std::shared_ptr io_context; diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index 2e53becc..020924ce 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -1,11 +1,11 @@ #pragma once #include +#include #include #include #include #include -#include #include #include #include diff --git a/nano/node/online_reps.cpp b/nano/node/online_reps.cpp index dfcad8ef..aef598fd 100644 --- a/nano/node/online_reps.cpp +++ b/nano/node/online_reps.cpp @@ -3,12 +3,15 @@ #include -nano::online_reps::online_reps (nano::node & node_a, nano::uint128_t minimum_a) : +nano::online_reps::online_reps (nano::node & node_a, nano::uint128_t minimum_a, bool block_store_error_a) : node (node_a), minimum (minimum_a) { - auto transaction (node.ledger.store.tx_begin_read ()); - online = trend (transaction); + if (!block_store_error_a) + { + auto transaction (node.ledger.store.tx_begin_read ()); + online = trend (transaction); + } } void nano::online_reps::observe (nano::account const & rep_a) diff --git a/nano/node/online_reps.hpp b/nano/node/online_reps.hpp index 27488bbe..9dc3b5f5 100644 --- a/nano/node/online_reps.hpp +++ b/nano/node/online_reps.hpp @@ -16,7 +16,7 @@ class transaction; class online_reps final { public: - online_reps (nano::node &, nano::uint128_t); + online_reps (nano::node &, nano::uint128_t, bool); /** Add voting account \p rep_account to the set of online representatives */ void observe (nano::account const & rep_account); diff --git a/nano/node/rocksdb/rocksdb.cpp b/nano/node/rocksdb/rocksdb.cpp new file mode 100644 index 00000000..f50b006e --- /dev/null +++ b/nano/node/rocksdb/rocksdb.cpp @@ -0,0 +1,616 @@ +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace nano +{ +template <> +void * rocksdb_val::data () const +{ + return (void *)value.data (); +} + +template <> +size_t rocksdb_val::size () const +{ + return value.size (); +} + +template <> +rocksdb_val::db_val (size_t size_a, void * data_a, nano::epoch epoch_a) : +value (static_cast (data_a), size_a), +epoch (epoch_a) +{ +} + +template <> +void rocksdb_val::convert_buffer_to_value () +{ + value = rocksdb::Slice (reinterpret_cast (buffer->data ()), buffer->size ()); +} +} + +nano::rocksdb_store::rocksdb_store (bool & error_a, nano::logger_mt & logger_a, boost::filesystem::path const & path_a, bool drop_unchecked_a, bool open_read_only_a) : +logger (logger_a) +{ + boost::system::error_code error_mkdir, error_chmod; + boost::filesystem::create_directories (path_a, error_mkdir); + nano::set_secure_perm_directory (path_a, error_chmod); + error_a |= static_cast (error_mkdir); + + if (!error_a) + { + auto table_options = get_table_options (); + table_factory.reset (rocksdb::NewBlockBasedTableFactory (table_options)); + if (!open_read_only_a) + { + construct_column_family_mutexes (); + } + open (error_a, path_a, open_read_only_a); + } + + if (!error_a && !open_read_only_a && drop_unchecked_a) + { + auto transaction (tx_begin_write ({ nano::tables::cached_counts, tables::unchecked })); + unchecked_clear (transaction); + } +} + +nano::rocksdb_store::~rocksdb_store () +{ + for (auto handle : handles) + { + delete handle; + } + + delete db; +} + +void nano::rocksdb_store::open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a) +{ + std::initializer_list names{ rocksdb::kDefaultColumnFamilyName.c_str (), "frontiers", "accounts", "accounts_v1", "send", "receive", "open", "change", "state", "state_v1", "pending", "pending_v1", "representation", "unchecked", "vote", "online_weight", "meta", "peers", "cached_counts", "confirmation_height" }; + std::vector column_families; + for (const auto & cf_name : names) + { + column_families.emplace_back (cf_name, get_cf_options ()); + } + + auto options = get_db_options (); + rocksdb::Status s; + + if (open_read_only_a) + { + s = rocksdb::DB::OpenForReadOnly (options, path_a.string (), column_families, &handles, &db); + } + else + { + s = rocksdb::OptimisticTransactionDB::Open (options, path_a.string (), column_families, &handles, &optimistic_db); + if (optimistic_db) + { + db = optimistic_db; + } + } + + // Assign handles to supplied + error_a |= !s.ok (); + + if (!error_a) + { + auto transaction = tx_begin_read (); + auto version_l = version_get (transaction); + if (version_l > version) + { + error_a = true; + logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); + } + } +} + +nano::write_transaction nano::rocksdb_store::tx_begin_write (std::vector const & tables_requiring_locks_a, std::vector const & tables_no_locks_a) +{ + std::unique_ptr txn; + release_assert (optimistic_db != nullptr); + if (tables_requiring_locks_a.empty () && tables_no_locks_a.empty ()) + { + // Use all tables if none are specified + txn = std::make_unique (optimistic_db, all_tables (), tables_no_locks_a, write_lock_mutexes); + } + else + { + txn = std::make_unique (optimistic_db, tables_requiring_locks_a, tables_no_locks_a, write_lock_mutexes); + } + + // Tables must be kept in alphabetical order. These can be used for mutex locking, so order is important to prevent deadlocking + assert (std::is_sorted (tables_requiring_locks_a.begin (), tables_requiring_locks_a.end ())); + + return nano::write_transaction{ std::move (txn) }; +} + +nano::read_transaction nano::rocksdb_store::tx_begin_read () +{ + return nano::read_transaction{ std::make_unique (db) }; +} + +rocksdb::ColumnFamilyHandle * nano::rocksdb_store::table_to_column_family (tables table_a) const +{ + auto & handles_l = handles; + auto get_handle = [&handles_l](const char * name) { + auto iter = std::find_if (handles_l.begin (), handles_l.end (), [name](auto handle) { + return (handle->GetName () == name); + }); + assert (iter != handles_l.end ()); + return *iter; + }; + + switch (table_a) + { + case tables::frontiers: + return get_handle ("frontiers"); + case tables::accounts_v0: + return get_handle ("accounts"); + case tables::accounts_v1: + return get_handle ("accounts_v1"); + case tables::send_blocks: + return get_handle ("send"); + case tables::receive_blocks: + return get_handle ("receive"); + case tables::open_blocks: + return get_handle ("open"); + case tables::change_blocks: + return get_handle ("change"); + case tables::state_blocks_v0: + return get_handle ("state"); + case tables::state_blocks_v1: + return get_handle ("state_v1"); + case tables::pending_v0: + return get_handle ("pending"); + case tables::pending_v1: + return get_handle ("pending_v1"); + case tables::blocks_info: + assert (false); + case tables::representation: + return get_handle ("representation"); + case tables::unchecked: + return get_handle ("unchecked"); + case tables::vote: + return get_handle ("vote"); + case tables::online_weight: + return get_handle ("online_weight"); + case tables::meta: + return get_handle ("meta"); + case tables::peers: + return get_handle ("peers"); + case tables::cached_counts: + return get_handle ("cached_counts"); + case tables::confirmation_height: + return get_handle ("confirmation_height"); + default: + release_assert (false); + return get_handle ("peers"); + } +} + +bool nano::rocksdb_store::exists (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a) const +{ + rocksdb::PinnableSlice slice; + rocksdb::Status status; + if (is_read (transaction_a)) + { + status = db->Get (snapshot_options (transaction_a), table_to_column_family (table_a), key_a, &slice); + } + else + { + rocksdb::ReadOptions options; + status = tx (transaction_a)->Get (options, table_to_column_family (table_a), key_a, &slice); + } + + return (status.ok ()); +} + +int nano::rocksdb_store::del (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a) +{ + assert (transaction_a.contains (table_a)); + if (!exists (transaction_a, table_a, key_a)) + { + return status_code_not_found (); + } + else + { + // Adding a new entry so counts need adjusting (use RMW otherwise known as merge) + if (is_caching_counts (table_a)) + { + decrement (transaction_a, tables::cached_counts, rocksdb_val (rocksdb::Slice (table_to_column_family (table_a)->GetName ())), 1); + } + } + + return tx (transaction_a)->Delete (table_to_column_family (table_a), key_a).code (); +} + +bool nano::rocksdb_store::block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) const +{ + // Should not be called + assert (false); + return true; +} + +void nano::rocksdb_store::version_put (nano::write_transaction const & transaction_a, int version_a) +{ + assert (transaction_a.contains (tables::meta)); + nano::uint256_union version_key (1); + nano::uint256_union version_value (version_a); + auto status (put (transaction_a, tables::meta, version_key, nano::rocksdb_val (version_value))); + release_assert (success (status)); +} + +rocksdb::Transaction * nano::rocksdb_store::tx (nano::transaction const & transaction_a) const +{ + assert (!is_read (transaction_a)); + return static_cast (transaction_a.get_handle ()); +} + +int nano::rocksdb_store::get (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val & value_a) const +{ + rocksdb::ReadOptions options; + rocksdb::PinnableSlice slice; + auto handle = table_to_column_family (table_a); + rocksdb::Status status; + if (is_read (transaction_a)) + { + status = db->Get (snapshot_options (transaction_a), handle, key_a, &slice); + } + else + { + status = tx (transaction_a)->Get (options, handle, key_a, &slice); + } + + if (status.ok ()) + { + value_a.buffer = std::make_shared> (slice.size ()); + std::memcpy (value_a.buffer->data (), slice.data (), slice.size ()); + value_a.convert_buffer_to_value (); + } + return status.code (); +} + +/** The column families which need to have their counts cached for later querying */ +bool nano::rocksdb_store::is_caching_counts (nano::tables table_a) const +{ + switch (table_a) + { + case tables::accounts_v0: + case tables::accounts_v1: + case tables::unchecked: + case tables::send_blocks: + case tables::receive_blocks: + case tables::open_blocks: + case tables::change_blocks: + case tables::state_blocks_v0: + case tables::state_blocks_v1: + return true; + default: + return false; + } +} + +int nano::rocksdb_store::increment (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, uint64_t amount_a) +{ + release_assert (transaction_a.contains (table_a)); + uint64_t base; + nano::rocksdb_val value; + if (!success (get (transaction_a, table_a, key_a, value))) + { + base = 0; + } + else + { + base = static_cast (value); + } + + return put (transaction_a, table_a, key_a, nano::rocksdb_val (base + amount_a)); +} + +int nano::rocksdb_store::decrement (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, uint64_t amount_a) +{ + release_assert (transaction_a.contains (table_a)); + nano::rocksdb_val value; + auto status = get (transaction_a, table_a, key_a, value); + release_assert (success (status)); + auto base = static_cast (value); + return put (transaction_a, table_a, key_a, nano::rocksdb_val (base - amount_a)); +} + +int nano::rocksdb_store::put (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val const & value_a) +{ + assert (transaction_a.contains (table_a)); + + auto txn = tx (transaction_a); + if (is_caching_counts (table_a)) + { + if (!exists (transaction_a, table_a, key_a)) + { + // Adding a new entry so counts need adjusting + increment (transaction_a, tables::cached_counts, rocksdb_val (rocksdb::Slice (table_to_column_family (table_a)->GetName ())), 1); + } + } + + return txn->Put (table_to_column_family (table_a), key_a, value_a).code (); +} + +bool nano::rocksdb_store::not_found (int status) const +{ + return (status_code_not_found () == status); +} + +bool nano::rocksdb_store::success (int status) const +{ + return (static_cast (rocksdb::Status::Code::kOk) == status); +} + +int nano::rocksdb_store::status_code_not_found () const +{ + return static_cast (rocksdb::Status::Code::kNotFound); +} + +uint64_t nano::rocksdb_store::count (nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle) const +{ + uint64_t count = 0; + nano::rocksdb_val val; + auto const & key = handle->GetName (); + auto status = get (transaction_a, tables::cached_counts, nano::rocksdb_val (key.size (), (void *)key.data ()), val); + if (success (status)) + { + count = static_cast (val); + } + + release_assert (success (status) || not_found (status)); + return count; +} + +size_t nano::rocksdb_store::count (nano::transaction const & transaction_a, tables table_a) const +{ + size_t sum = 0; + // Some column families are small enough that they can just be iterated, rather than doing extra io caching counts + if (table_a == tables::peers) + { + for (auto i (peers_begin (transaction_a)), n (peers_end ()); i != n; ++i) + { + ++sum; + } + } + else if (table_a == tables::online_weight) + { + for (auto i (online_weight_begin (transaction_a)), n (online_weight_end ()); i != n; ++i) + { + ++sum; + } + } + else + { + return count (transaction_a, table_to_column_family (table_a)); + } + + return sum; +} + +int nano::rocksdb_store::drop (nano::write_transaction const & transaction_a, tables table_a) +{ + assert (transaction_a.contains (table_a)); + auto col = table_to_column_family (table_a); + + int status = static_cast (rocksdb::Status::Code::kOk); + if (is_caching_counts (table_a)) + { + // Reset counter to 0 + status = put (transaction_a, tables::cached_counts, nano::rocksdb_val (rocksdb::Slice (col->GetName ())), nano::rocksdb_val (uint64_t{ 0 })); + } + + if (success (status)) + { + // Dropping/Creating families like in node::ongoing_peer_clear can cause write stalls, just delete them manually. + if (table_a == tables::peers) + { + int status = 0; + for (auto i = peers_begin (transaction_a), n = peers_end (); i != n; ++i) + { + status = del (transaction_a, tables::peers, nano::rocksdb_val (i->first)); + release_assert (success (status)); + } + return status; + } + else + { + return clear (col); + } + } + return status; +} + +int nano::rocksdb_store::clear (rocksdb::ColumnFamilyHandle * column_family) +{ + // Dropping completely removes the column + auto name = column_family->GetName (); + auto status = db->DropColumnFamily (column_family); + release_assert (status.ok ()); + delete column_family; + + // Need to add it back as we just want to clear the contents + auto handle_it = std::find (handles.begin (), handles.end (), column_family); + assert (handle_it != handles.cend ()); + status = db->CreateColumnFamily (get_cf_options (), name, &column_family); + release_assert (status.ok ()); + *handle_it = column_family; + return status.code (); +} + +void nano::rocksdb_store::construct_column_family_mutexes () +{ + for (auto table : all_tables ()) + { + write_lock_mutexes.emplace (std::piecewise_construct, std::forward_as_tuple (table), std::forward_as_tuple ()); + } +} + +rocksdb::Options nano::rocksdb_store::get_db_options () const +{ + rocksdb::Options db_options; + db_options.create_if_missing = true; + db_options.create_missing_column_families = true; + + // Sets the compaction priority + db_options.compaction_pri = rocksdb::CompactionPri::kMinOverlappingRatio; + + // Start agressively flushing WAL files when they reach over 1GB + db_options.max_total_wal_size = 1 * 1024 * 1024 * 1024LL; + + if (!low_end_system ()) + { + // Adds a separate write queue for memtable/WAL + db_options.enable_pipelined_write = true; + + // Optimize RocksDB. This is the easiest way to get RocksDB to perform well + db_options.IncreaseParallelism (std::thread::hardware_concurrency ()); + db_options.OptimizeLevelStyleCompaction (); + } + return db_options; +} + +/** As options are currently hardcoded, a heuristic is taken based off the number of cores to modify config options */ +bool nano::rocksdb_store::low_end_system () const +{ + return (std::thread::hardware_concurrency () < 2); +} + +rocksdb::BlockBasedTableOptions nano::rocksdb_store::get_table_options () const +{ + rocksdb::BlockBasedTableOptions table_options; + if (!low_end_system ()) + { + // 512MB block cache + table_options.block_cache = rocksdb::NewLRUCache (512 * 1024 * 1024LL); + + // Bloom filter to help with point reads + table_options.filter_policy.reset (rocksdb::NewBloomFilterPolicy (10, false)); + table_options.block_size = 16 * 1024; + table_options.cache_index_and_filter_blocks = true; + table_options.pin_l0_filter_and_index_blocks_in_cache = true; + } + return table_options; +} + +rocksdb::ColumnFamilyOptions nano::rocksdb_store::get_cf_options () const +{ + rocksdb::ColumnFamilyOptions cf_options; + cf_options.table_factory = table_factory; + + if (!low_end_system ()) + { + cf_options.level_compaction_dynamic_level_bytes = true; + + // Number of files in level which triggers compaction. Size of L0 and L1 should be kept similar as this is the only compaction which is single threaded + cf_options.level0_file_num_compaction_trigger = 4; + + // L1 size, compaction is triggered for L0 at this size (512MB) + cf_options.max_bytes_for_level_base = 512 * 1024 * 1024LL; + + // Each level is a multiple of the above. L1 will be 512MB. Le will be 512 * 8 = 2GB. L3 will be 2GB * 8 = 16GB, and so on... + cf_options.max_bytes_for_level_multiplier = 8; + + // Size of level 1 sst files (128MB) + cf_options.target_file_size_base = 128 * 1024 * 1024LL; + + // Size of each memtable (128MB) + cf_options.write_buffer_size = 128 * 1024 * 1024LL; + + // Number of memtables to keep in memory (1 active, rest inactive/immutable) + cf_options.max_write_buffer_number = 3; + + // Files older than this (1 day) will be scheduled for compaction when there is no other background work. This can lead to more writes however. + cf_options.ttl = 1 * 24 * 60 * 60; + } + + return cf_options; +} + +std::vector nano::rocksdb_store::all_tables () const +{ + return std::vector{ tables::accounts_v0, tables::accounts_v1, tables::cached_counts, tables::change_blocks, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::open_blocks, tables::peers, tables::pending_v0, tables::pending_v1, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks_v0, tables::state_blocks_v1, tables::unchecked, tables::vote }; +} + +bool nano::rocksdb_store::copy_db (boost::filesystem::path const & destination_path) +{ + std::unique_ptr backup_engine; + { + rocksdb::BackupEngine * backup_engine_raw; + rocksdb::BackupableDBOptions backup_options (destination_path.string ()); + // Use incremental backups (default) + backup_options.share_table_files = true; + + // Increase number of threads used for copying + backup_options.max_background_operations = std::thread::hardware_concurrency (); + auto status = rocksdb::BackupEngine::Open (rocksdb::Env::Default (), backup_options, &backup_engine_raw); + backup_engine.reset (backup_engine_raw); + if (!status.ok ()) + { + return false; + } + } + + auto status = backup_engine->CreateNewBackup (db); + if (!status.ok ()) + { + return false; + } + + std::vector backup_infos; + backup_engine->GetBackupInfo (&backup_infos); + + for (auto const & backup_info : backup_infos) + { + status = backup_engine->VerifyBackup (backup_info.backup_id); + if (!status.ok ()) + { + return false; + } + } + + rocksdb::BackupEngineReadOnly * backup_engine_read; + status = rocksdb::BackupEngineReadOnly::Open (rocksdb::Env::Default (), rocksdb::BackupableDBOptions (destination_path.string ()), &backup_engine_read); + if (!status.ok ()) + { + delete backup_engine_read; + return false; + } + + // First remove all files (not directories) in the destination + for (boost::filesystem::directory_iterator end_dir_it, it (destination_path); it != end_dir_it; ++it) + { + auto path = it->path (); + if (boost::filesystem::is_regular_file (path)) + { + boost::filesystem::remove (it->path ()); + } + } + + // Now generate the relevant files from the backup + status = backup_engine->RestoreDBFromLatestBackup (destination_path.string (), destination_path.string ()); + delete backup_engine_read; + + // Open it so that it flushes all WAL files + if (status.ok ()) + { + auto error{ false }; + nano::rocksdb_store rocksdb_store (error, logger, destination_path.string (), false, false); + return !error; + } + return false; +} diff --git a/nano/node/rocksdb/rocksdb.hpp b/nano/node/rocksdb/rocksdb.hpp new file mode 100644 index 00000000..765d5554 --- /dev/null +++ b/nano/node/rocksdb/rocksdb.hpp @@ -0,0 +1,104 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace nano +{ +class logging_mt; +/** + * rocksdb implementation of the block store + */ +class rocksdb_store : public block_store_partial +{ +public: + rocksdb_store (bool &, nano::logger_mt &, boost::filesystem::path const &, bool drop_unchecked = false, bool open_read_only = false); + ~rocksdb_store (); + nano::write_transaction tx_begin_write (std::vector const & tables_requiring_lock = {}, std::vector const & tables_no_lock = {}) override; + nano::read_transaction tx_begin_read () override; + + bool block_info_get (nano::transaction const &, nano::block_hash const &, nano::block_info &) const override; + size_t count (nano::transaction const & transaction_a, tables table_a) const override; + void version_put (nano::write_transaction const &, int) override; + + bool exists (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a) const; + int get (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val & value_a) const; + int put (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val const & value_a); + int del (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a); + + void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) override + { + // Do nothing + } + + bool copy_db (boost::filesystem::path const & destination) override; + + template + nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a) const + { + return nano::store_iterator (std::make_unique> (db, transaction_a, table_to_column_family (table_a))); + } + + template + nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key) const + { + return nano::store_iterator (std::make_unique> (db, transaction_a, table_to_column_family (table_a), key)); + } + + template + nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, rocksdb_val const & key) const + { + return nano::store_iterator (std::make_unique> (db, transaction_a, table_to_column_family (table1_a), table_to_column_family (table2_a), key)); + } + + template + nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a) const + { + return nano::store_iterator (std::make_unique> (db, transaction_a, table_to_column_family (table1_a), table_to_column_family (table2_a))); + } + +private: + nano::logger_mt & logger; + std::vector handles; + // Optimistic transactions are used in write mode + rocksdb::OptimisticTransactionDB * optimistic_db = nullptr; + rocksdb::DB * db = nullptr; + std::shared_ptr table_factory; + std::unordered_map write_lock_mutexes; + + rocksdb::Transaction * tx (nano::transaction const & transaction_a) const; + std::vector all_tables () const; + + bool not_found (int status) const override; + bool success (int status) const override; + int status_code_not_found () const override; + int drop (nano::write_transaction const &, tables) override; + + rocksdb::ColumnFamilyHandle * table_to_column_family (tables table_a) const; + int clear (rocksdb::ColumnFamilyHandle * column_family); + + void open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a); + uint64_t count (nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle) const; + bool is_caching_counts (nano::tables table_a) const; + + int increment (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, uint64_t amount_a); + int decrement (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, uint64_t amount_a); + bool low_end_system () const; + rocksdb::ColumnFamilyOptions get_cf_options () const; + void construct_column_family_mutexes (); + rocksdb::Options get_db_options () const; + rocksdb::BlockBasedTableOptions get_table_options () const; +}; +} diff --git a/nano/node/rocksdb/rocksdb_iterator.hpp b/nano/node/rocksdb/rocksdb_iterator.hpp new file mode 100644 index 00000000..78e9f743 --- /dev/null +++ b/nano/node/rocksdb/rocksdb_iterator.hpp @@ -0,0 +1,313 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace +{ +inline bool is_read (nano::transaction const & transaction_a) +{ + return (dynamic_cast (&transaction_a) != nullptr); +} + +inline rocksdb::ReadOptions const & snapshot_options (nano::transaction const & transaction_a) +{ + assert (is_read (transaction_a)); + return *static_cast (transaction_a.get_handle ()); +} +} + +namespace nano +{ +using rocksdb_val = db_val; + +template +class rocksdb_iterator : public store_iterator_impl +{ +public: + rocksdb_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle_a, nano::epoch epoch_a = nano::epoch::unspecified) : + cursor (nullptr) + { + current.first.epoch = epoch_a; + current.second.epoch = epoch_a; + + rocksdb::Iterator * iter; + if (is_read (transaction_a)) + { + iter = db->NewIterator (snapshot_options (transaction_a), handle_a); + } + else + { + rocksdb::ReadOptions ropts; + ropts.fill_cache = false; + iter = tx (transaction_a)->GetIterator (ropts, handle_a); + } + + cursor.reset (iter); + cursor->SeekToFirst (); + + if (cursor->Valid ()) + { + current.first.value = cursor->key (); + current.second.value = cursor->value (); + } + else + { + clear (); + } + } + + rocksdb_iterator (std::nullptr_t, nano::epoch epoch_a = nano::epoch::unspecified) : + cursor (nullptr) + { + current.first.epoch = epoch_a; + current.second.epoch = epoch_a; + } + + rocksdb_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * handle_a, rocksdb_val const & val_a, nano::epoch epoch_a = nano::epoch::unspecified) : + cursor (nullptr) + { + current.first.epoch = epoch_a; + current.second.epoch = epoch_a; + + rocksdb::Iterator * iter; + if (is_read (transaction_a)) + { + iter = db->NewIterator (snapshot_options (transaction_a), handle_a); + } + else + { + iter = tx (transaction_a)->GetIterator (rocksdb::ReadOptions (), handle_a); + } + + cursor.reset (iter); + cursor->Seek (val_a); + + if (cursor->Valid ()) + { + current.first = cursor->key (); + current.second = cursor->value (); + } + else + { + clear (); + } + } + + rocksdb_iterator (nano::rocksdb_iterator && other_a) + { + cursor = other_a.cursor; + other_a.cursor = nullptr; + current = other_a.current; + } + + rocksdb_iterator (nano::rocksdb_iterator const &) = delete; + + nano::store_iterator_impl & operator++ () override + { + cursor->Next (); + if (cursor->Valid ()) + { + current.first = cursor->key (); + current.second = cursor->value (); + + if (current.first.size () != sizeof (T)) + { + clear (); + } + } + else + { + clear (); + } + + return *this; + } + + std::pair * operator-> () + { + return ¤t; + } + + bool operator== (nano::store_iterator_impl const & base_a) const override + { + auto const other_a (boost::polymorphic_downcast const *> (&base_a)); + + if (!current.first.data () && !other_a->current.first.data ()) + { + return true; + } + else if (!current.first.data () || !other_a->current.first.data ()) + { + return false; + } + + auto result (std::memcmp (current.first.data (), other_a->current.first.data (), current.first.size ()) == 0); + assert (!result || (current.first.size () == other_a->current.first.size ())); + assert (!result || (current.second.data () == other_a->current.second.data ())); + assert (!result || (current.second.size () == other_a->current.second.size ())); + return result; + } + + bool is_end_sentinal () const override + { + return current.first.size () == 0; + } + + void fill (std::pair & value_a) const override + { + { + if (current.first.size () != 0) + { + value_a.first = static_cast (current.first); + } + else + { + value_a.first = T (); + } + if (current.second.size () != 0) + { + value_a.second = static_cast (current.second); + } + else + { + value_a.second = U (); + } + } + } + void clear () + { + current.first = nano::rocksdb_val (current.first.epoch); + current.second = nano::rocksdb_val (current.second.epoch); + assert (is_end_sentinal ()); + } + nano::rocksdb_iterator & operator= (nano::rocksdb_iterator && other_a) + { + cursor = std::move (other_a.cursor); + current = other_a.current; + return *this; + } + nano::store_iterator_impl & operator= (nano::store_iterator_impl const &) = delete; + + std::unique_ptr cursor; + std::pair current; + +private: + rocksdb::Transaction * tx (nano::transaction const & transaction_a) const + { + return static_cast (transaction_a.get_handle ()); + } +}; + +/** + * Iterates the key/value pairs of two stores merged together + */ + +template +class rocksdb_merge_iterator : public store_iterator_impl +{ +public: + rocksdb_merge_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * db1_a, rocksdb::ColumnFamilyHandle * db2_a) : + impl1 (std::make_unique> (db, transaction_a, db1_a, nano::epoch::epoch_0)), + impl2 (std::make_unique> (db, transaction_a, db2_a, nano::epoch::epoch_1)) + { + } + rocksdb_merge_iterator (std::nullptr_t) : + impl1 (std::make_unique> (nullptr, nano::epoch::epoch_0)), + impl2 (std::make_unique> (nullptr, nano::epoch::epoch_1)) + { + } + + rocksdb_merge_iterator (rocksdb::DB * db, nano::transaction const & transaction_a, rocksdb::ColumnFamilyHandle * db1_a, rocksdb::ColumnFamilyHandle * db2_a, rocksdb::Slice const & val_a) : + impl1 (std::make_unique> (db, transaction_a, db1_a, val_a, nano::epoch::epoch_0)), + impl2 (std::make_unique> (db, transaction_a, db2_a, val_a, nano::epoch::epoch_1)) + { + } + rocksdb_merge_iterator (nano::rocksdb_merge_iterator && other_a) + { + impl1 = std::move (other_a.impl1); + impl2 = std::move (other_a.impl2); + } + + rocksdb_merge_iterator (nano::rocksdb_merge_iterator const &) = delete; + nano::store_iterator_impl & operator++ () override + { + ++least_iterator (); + return *this; + } + std::pair * operator-> (); + bool operator== (nano::store_iterator_impl const & base_a) const override + { + assert ((dynamic_cast const *> (&base_a) != nullptr) && "Incompatible iterator comparison"); + auto & other (static_cast const &> (base_a)); + return *impl1 == *other.impl1 && *impl2 == *other.impl2; + } + + bool is_end_sentinal () const override + { + return least_iterator ().is_end_sentinal (); + } + + void fill (std::pair & value_a) const override + { + auto & current (least_iterator ()); + if (current->first.size () != 0) + { + value_a.first = static_cast (current->first); + } + else + { + value_a.first = T (); + } + if (current->second.size () != 0) + { + value_a.second = static_cast (current->second); + } + else + { + value_a.second = U (); + } + } + nano::rocksdb_merge_iterator & operator= (nano::rocksdb_merge_iterator &&) = default; + nano::rocksdb_merge_iterator & operator= (nano::rocksdb_merge_iterator const &) = delete; + +private: + nano::rocksdb_iterator & least_iterator () const + { + nano::rocksdb_iterator * result; + if (impl1->is_end_sentinal ()) + { + result = impl2.get (); + } + else if (impl2->is_end_sentinal ()) + { + result = impl1.get (); + } + else + { + if (impl1->current.first.value.compare (impl2->current.first.value) < 0) + { + result = impl1.get (); + } + else if (impl1->current.first.value.compare (impl2->current.first.value) > 0) + { + result = impl2.get (); + } + else + { + result = impl1->current.second.value.compare (impl2->current.second.value) < 0 ? impl1.get () : impl2.get (); + } + } + return *result; + } + + std::unique_ptr> impl1; + std::unique_ptr> impl2; +}; +} diff --git a/nano/node/rocksdb/rocksdb_txn.cpp b/nano/node/rocksdb/rocksdb_txn.cpp new file mode 100644 index 00000000..6e7abfd7 --- /dev/null +++ b/nano/node/rocksdb/rocksdb_txn.cpp @@ -0,0 +1,95 @@ +#include + +nano::read_rocksdb_txn::read_rocksdb_txn (rocksdb::DB * db_a) : +db (db_a) +{ + options.snapshot = db_a->GetSnapshot (); +} + +nano::read_rocksdb_txn::~read_rocksdb_txn () +{ + reset (); +} + +void nano::read_rocksdb_txn::reset () +{ + db->ReleaseSnapshot (options.snapshot); +} + +void nano::read_rocksdb_txn::renew () +{ + options.snapshot = db->GetSnapshot (); +} + +void * nano::read_rocksdb_txn::get_handle () const +{ + return (void *)&options; +} + +nano::write_rocksdb_txn::write_rocksdb_txn (rocksdb::OptimisticTransactionDB * db_a, std::vector const & tables_requiring_locks_a, std::vector const & tables_no_locks_a, std::unordered_map & mutexes_a) : +db (db_a), +tables_requiring_locks (tables_requiring_locks_a), +tables_no_locks (tables_no_locks_a), +mutexes (mutexes_a) +{ + lock (); + rocksdb::OptimisticTransactionOptions txn_options; + txn_options.set_snapshot = true; + txn = db->BeginTransaction (rocksdb::WriteOptions (), txn_options); +} + +nano::write_rocksdb_txn::~write_rocksdb_txn () +{ + commit (); + delete txn; + unlock (); +} + +void nano::write_rocksdb_txn::commit () const +{ + auto status = txn->Commit (); + + // If there are no available memtables try again a few more times + constexpr auto num_attempts = 10; + auto attempt_num = 0; + while (status.IsTryAgain () && attempt_num < num_attempts) + { + status = txn->Commit (); + ++attempt_num; + } + + release_assert (status.ok ()); +} + +void nano::write_rocksdb_txn::renew () +{ + rocksdb::OptimisticTransactionOptions txn_options; + txn_options.set_snapshot = true; + db->BeginTransaction (rocksdb::WriteOptions (), txn_options, txn); +} + +void * nano::write_rocksdb_txn::get_handle () const +{ + return txn; +} + +void nano::write_rocksdb_txn::lock () +{ + for (auto table : tables_requiring_locks) + { + mutexes.at (table).lock (); + } +} + +void nano::write_rocksdb_txn::unlock () +{ + for (auto table : tables_requiring_locks) + { + mutexes.at (table).unlock (); + } +} + +bool nano::write_rocksdb_txn::contains (nano::tables table_a) const +{ + return (std::find (tables_requiring_locks.begin (), tables_requiring_locks.end (), table_a) != tables_requiring_locks.end ()) || (std::find (tables_no_locks.begin (), tables_no_locks.end (), table_a) != tables_no_locks.end ()); +} diff --git a/nano/node/rocksdb/rocksdb_txn.hpp b/nano/node/rocksdb/rocksdb_txn.hpp new file mode 100644 index 00000000..f642ee56 --- /dev/null +++ b/nano/node/rocksdb/rocksdb_txn.hpp @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace nano +{ +class read_rocksdb_txn final : public read_transaction_impl +{ +public: + read_rocksdb_txn (rocksdb::DB * db); + ~read_rocksdb_txn (); + void reset () override; + void renew () override; + void * get_handle () const override; + +private: + rocksdb::DB * db; + rocksdb::ReadOptions options; +}; + +class write_rocksdb_txn final : public write_transaction_impl +{ +public: + write_rocksdb_txn (rocksdb::OptimisticTransactionDB * db_a, std::vector const & tables_requiring_locks_a, std::vector const & tables_no_locks_a, std::unordered_map & mutexes_a); + ~write_rocksdb_txn (); + void commit () const override; + void renew () override; + void * get_handle () const override; + bool contains (nano::tables table_a) const override; + +private: + rocksdb::Transaction * txn; + rocksdb::OptimisticTransactionDB * db; + std::vector tables_requiring_locks; + std::vector tables_no_locks; + std::unordered_map & mutexes; + + void lock (); + void unlock (); +}; +} diff --git a/nano/node/transport/tcp.cpp b/nano/node/transport/tcp.cpp index c2819e74..58f6be37 100644 --- a/nano/node/transport/tcp.cpp +++ b/nano/node/transport/tcp.cpp @@ -191,7 +191,7 @@ bool nano::transport::tcp_channels::store_all (bool clear_peers) if (!endpoints.empty ()) { // Clear all peers then refresh with the current list of peers - auto transaction (node.store.tx_begin_write ()); + auto transaction (node.store.tx_begin_write ({ tables::peers })); if (clear_peers) { node.store.peer_clear (transaction); diff --git a/nano/node/transport/udp.cpp b/nano/node/transport/udp.cpp index 1b589540..949ee2ea 100644 --- a/nano/node/transport/udp.cpp +++ b/nano/node/transport/udp.cpp @@ -188,7 +188,7 @@ bool nano::transport::udp_channels::store_all (bool clear_peers) if (!endpoints.empty ()) { // Clear all peers then refresh with the current list of peers - auto transaction (node.store.tx_begin_write ()); + auto transaction (node.store.tx_begin_write ({ tables::peers })); if (clear_peers) { node.store.peer_clear (transaction); diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index 0a1f0ccc..349425c8 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -6646,11 +6646,6 @@ TEST (rpc, block_confirmed) auto send = std::make_shared (latest, key.pub, 10, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (latest)); node->process_active (send); node->block_processor.flush (); - system.deadline_set (10s); - while (!node->pending_confirmation_height.is_processing_block (send->hash ())) - { - ASSERT_NO_ERROR (system.poll ()); - } // Wait until the confirmation height has been set system.deadline_set (10s); @@ -6681,6 +6676,7 @@ TEST (rpc, block_confirmed) ASSERT_TRUE (response3.json.get ("confirmed")); } +#if !NANO_ROCKSDB TEST (rpc, database_txn_tracker) { // First try when database tracking is disabled @@ -6808,6 +6804,7 @@ TEST (rpc, database_txn_tracker) ASSERT_TRUE (!std::get<3> (json_l.front ()).empty ()); thread.join (); } +#endif TEST (rpc, active_difficulty) { diff --git a/nano/secure/blockstore.cpp b/nano/secure/blockstore.cpp index 15cb88ae..92dcb7fa 100644 --- a/nano/secure/blockstore.cpp +++ b/nano/secure/blockstore.cpp @@ -414,3 +414,8 @@ void nano::write_transaction::renew () { impl->renew (); } + +bool nano::write_transaction::contains (nano::tables table_a) const +{ + return impl->contains (table_a); +} diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index f12d6bc6..7c5a464e 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -476,6 +477,31 @@ private: std::unique_ptr> impl; }; +// Keep this in alphabetical order +enum class tables +{ + accounts_v0, + accounts_v1, + blocks_info, // LMDB only + cached_counts, // RocksDB only + change_blocks, + confirmation_height, + frontiers, + meta, + online_weight, + open_blocks, + peers, + pending_v0, + pending_v1, + receive_blocks, + representation, + send_blocks, + state_blocks_v0, + state_blocks_v1, + unchecked, + vote +}; + class transaction_impl { public: @@ -495,6 +521,7 @@ class write_transaction_impl : public transaction_impl public: virtual void commit () const = 0; virtual void renew () = 0; + virtual bool contains (nano::tables table_a) const = 0; }; class transaction @@ -532,6 +559,7 @@ public: void * get_handle () const override; void commit () const; void renew (); + bool contains (nano::tables table_a) const; private: std::unique_ptr impl; @@ -543,37 +571,14 @@ private: class block_store { public: - enum class tables - { - frontiers, - accounts_v0, - accounts_v1, - send_blocks, - receive_blocks, - open_blocks, - change_blocks, - state_blocks_v0, - state_blocks_v1, - pending_v0, - pending_v1, - blocks_info, - representation, - unchecked, - vote, - online_weight, - meta, - peers, - confirmation_height - }; - virtual ~block_store () = default; - virtual void initialize (nano::transaction const &, nano::genesis const &) = 0; - virtual void block_put (nano::transaction const &, nano::block_hash const &, nano::block const &, nano::block_sideband const &, nano::epoch version = nano::epoch::epoch_0) = 0; + virtual void initialize (nano::write_transaction const &, nano::genesis const &) = 0; + virtual void block_put (nano::write_transaction const &, nano::block_hash const &, nano::block const &, nano::block_sideband const &, nano::epoch version = nano::epoch::epoch_0) = 0; virtual nano::block_hash block_successor (nano::transaction const &, nano::block_hash const &) const = 0; - virtual void block_successor_clear (nano::transaction const &, nano::block_hash const &) = 0; + virtual void block_successor_clear (nano::write_transaction const &, nano::block_hash const &) = 0; virtual std::shared_ptr block_get (nano::transaction const &, nano::block_hash const &, nano::block_sideband * = nullptr) const = 0; virtual std::shared_ptr block_random (nano::transaction const &) = 0; - virtual void block_del (nano::transaction const &, nano::block_hash const &) = 0; + virtual void block_del (nano::write_transaction const &, nano::block_hash const &) = 0; virtual bool block_exists (nano::transaction const &, nano::block_hash const &) = 0; virtual bool block_exists (nano::transaction const &, nano::block_type, nano::block_hash const &) = 0; virtual nano::block_counts block_count (nano::transaction const &) = 0; @@ -581,17 +586,17 @@ public: virtual bool source_exists (nano::transaction const &, nano::block_hash const &) = 0; virtual nano::account block_account (nano::transaction const &, nano::block_hash const &) const = 0; - virtual void frontier_put (nano::transaction const &, nano::block_hash const &, nano::account const &) = 0; + virtual void frontier_put (nano::write_transaction const &, nano::block_hash const &, nano::account const &) = 0; virtual nano::account frontier_get (nano::transaction const &, nano::block_hash const &) const = 0; - virtual void frontier_del (nano::transaction const &, nano::block_hash const &) = 0; + virtual void frontier_del (nano::write_transaction const &, nano::block_hash const &) = 0; - virtual void account_put (nano::transaction const &, nano::account const &, nano::account_info const &) = 0; + virtual void account_put (nano::write_transaction const &, nano::account const &, nano::account_info const &) = 0; virtual bool account_get (nano::transaction const &, nano::account const &, nano::account_info &) = 0; - virtual void account_del (nano::transaction const &, nano::account const &) = 0; + virtual void account_del (nano::write_transaction const &, nano::account const &) = 0; virtual bool account_exists (nano::transaction const &, nano::account const &) = 0; virtual size_t account_count (nano::transaction const &) = 0; - virtual void confirmation_height_clear (nano::transaction const &, nano::account const & account, uint64_t existing_confirmation_height) = 0; - virtual void confirmation_height_clear (nano::transaction const &) = 0; + virtual void confirmation_height_clear (nano::write_transaction const &, nano::account const & account, uint64_t existing_confirmation_height) = 0; + virtual void confirmation_height_clear (nano::write_transaction const &) = 0; virtual uint64_t cemented_count (nano::transaction const &) = 0; virtual nano::store_iterator latest_v0_begin (nano::transaction const &, nano::account const &) = 0; virtual nano::store_iterator latest_v0_begin (nano::transaction const &) = 0; @@ -603,8 +608,8 @@ public: virtual nano::store_iterator latest_begin (nano::transaction const &) = 0; virtual nano::store_iterator latest_end () = 0; - virtual void pending_put (nano::transaction const &, nano::pending_key const &, nano::pending_info const &) = 0; - virtual void pending_del (nano::transaction const &, nano::pending_key const &) = 0; + virtual void pending_put (nano::write_transaction const &, nano::pending_key const &, nano::pending_info const &) = 0; + virtual void pending_del (nano::write_transaction const &, nano::pending_key const &) = 0; virtual bool pending_get (nano::transaction const &, nano::pending_key const &, nano::pending_info &) = 0; virtual bool pending_exists (nano::transaction const &, nano::pending_key const &) = 0; virtual nano::store_iterator pending_v0_begin (nano::transaction const &, nano::pending_key const &) = 0; @@ -623,16 +628,16 @@ public: virtual nano::epoch block_version (nano::transaction const &, nano::block_hash const &) = 0; virtual nano::uint128_t representation_get (nano::transaction const &, nano::account const &) = 0; - virtual void representation_put (nano::transaction const &, nano::account const &, nano::uint128_union const &) = 0; - virtual void representation_add (nano::transaction const &, nano::account const &, nano::uint128_t const &) = 0; + virtual void representation_put (nano::write_transaction const &, nano::account const &, nano::uint128_union const &) = 0; + virtual void representation_add (nano::write_transaction const &, nano::account const &, nano::uint128_t const &) = 0; virtual nano::store_iterator representation_begin (nano::transaction const &) = 0; virtual nano::store_iterator representation_end () = 0; - virtual void unchecked_clear (nano::transaction const &) = 0; - virtual void unchecked_put (nano::transaction const &, nano::unchecked_key const &, nano::unchecked_info const &) = 0; - virtual void unchecked_put (nano::transaction const &, nano::block_hash const &, std::shared_ptr const &) = 0; + virtual void unchecked_clear (nano::write_transaction const &) = 0; + virtual void unchecked_put (nano::write_transaction const &, nano::unchecked_key const &, nano::unchecked_info const &) = 0; + virtual void unchecked_put (nano::write_transaction const &, nano::block_hash const &, std::shared_ptr const &) = 0; virtual std::vector unchecked_get (nano::transaction const &, nano::block_hash const &) = 0; - virtual void unchecked_del (nano::transaction const &, nano::unchecked_key const &) = 0; + virtual void unchecked_del (nano::write_transaction const &, nano::unchecked_key const &) = 0; virtual nano::store_iterator unchecked_begin (nano::transaction const &) = 0; virtual nano::store_iterator unchecked_begin (nano::transaction const &, nano::unchecked_key const &) = 0; virtual nano::store_iterator unchecked_end () = 0; @@ -647,32 +652,32 @@ public: virtual std::shared_ptr vote_max (nano::transaction const &, std::shared_ptr) = 0; // Return latest vote for an account considering the vote cache virtual std::shared_ptr vote_current (nano::transaction const &, nano::account const &) = 0; - virtual void flush (nano::transaction const &) = 0; + virtual void flush (nano::write_transaction const &) = 0; virtual nano::store_iterator> vote_begin (nano::transaction const &) = 0; virtual nano::store_iterator> vote_end () = 0; - virtual void online_weight_put (nano::transaction const &, uint64_t, nano::amount const &) = 0; - virtual void online_weight_del (nano::transaction const &, uint64_t) = 0; - virtual nano::store_iterator online_weight_begin (nano::transaction const &) = 0; - virtual nano::store_iterator online_weight_end () = 0; + virtual void online_weight_put (nano::write_transaction const &, uint64_t, nano::amount const &) = 0; + virtual void online_weight_del (nano::write_transaction const &, uint64_t) = 0; + virtual nano::store_iterator online_weight_begin (nano::transaction const &) const = 0; + virtual nano::store_iterator online_weight_end () const = 0; virtual size_t online_weight_count (nano::transaction const &) const = 0; - virtual void online_weight_clear (nano::transaction const &) = 0; + virtual void online_weight_clear (nano::write_transaction const &) = 0; - virtual void version_put (nano::transaction const &, int) = 0; + virtual void version_put (nano::write_transaction const &, int) = 0; virtual int version_get (nano::transaction const &) const = 0; - virtual void peer_put (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0; - virtual void peer_del (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0; + virtual void peer_put (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0; + virtual void peer_del (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0; virtual bool peer_exists (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const = 0; virtual size_t peer_count (nano::transaction const & transaction_a) const = 0; - virtual void peer_clear (nano::transaction const & transaction_a) = 0; - virtual nano::store_iterator peers_begin (nano::transaction const & transaction_a) = 0; - virtual nano::store_iterator peers_end () = 0; + virtual void peer_clear (nano::write_transaction const & transaction_a) = 0; + virtual nano::store_iterator peers_begin (nano::transaction const & transaction_a) const = 0; + virtual nano::store_iterator peers_end () const = 0; - virtual void confirmation_height_put (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) = 0; + virtual void confirmation_height_put (nano::write_transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) = 0; virtual bool confirmation_height_get (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t & confirmation_height_a) = 0; virtual bool confirmation_height_exists (nano::transaction const & transaction_a, nano::account const & account_a) const = 0; - virtual void confirmation_height_del (nano::transaction const & transaction_a, nano::account const & account_a) = 0; + virtual void confirmation_height_del (nano::write_transaction const & transaction_a, nano::account const & account_a) = 0; virtual uint64_t confirmation_height_count (nano::transaction const & transaction_a) = 0; virtual nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a, nano::account const & account_a) = 0; virtual nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a) = 0; @@ -681,14 +686,29 @@ public: virtual uint64_t block_account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0; virtual std::mutex & get_cache_mutex () = 0; + virtual bool copy_db (boost::filesystem::path const & destination) = 0; + + /** Not applicable to all sub-classes */ virtual void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) = 0; /** Start read-write transaction */ - virtual nano::write_transaction tx_begin_write () = 0; + virtual nano::write_transaction tx_begin_write (std::vector const & tables_to_lock = {}, std::vector const & tables_no_lock = {}) = 0; /** Start read-only transaction */ virtual nano::read_transaction tx_begin_read () = 0; }; -std::unique_ptr make_store (bool & init, nano::logger_mt & logger, boost::filesystem::path const & path); +std::unique_ptr make_store (bool & init, nano::logger_mt & logger, boost::filesystem::path const & path, bool open_read_only = false, bool add_db_postfix = false, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512, bool backup_before_upgrade = false); +} + +namespace std +{ +template <> +struct hash<::nano::tables> +{ + size_t operator() (::nano::tables const & table_a) const + { + return static_cast (table_a); + } +}; } diff --git a/nano/secure/blockstore_partial.hpp b/nano/secure/blockstore_partial.hpp index 63c07030..be900ad2 100644 --- a/nano/secure/blockstore_partial.hpp +++ b/nano/secure/blockstore_partial.hpp @@ -23,7 +23,7 @@ public: * If using a different store version than the latest then you may need * to modify some of the objects in the store to be appropriate for the version before an upgrade. */ - void initialize (nano::transaction const & transaction_a, nano::genesis const & genesis_a) override + void initialize (nano::write_transaction const & transaction_a, nano::genesis const & genesis_a) override { auto hash_l (genesis_a.hash ()); assert (latest_v0_begin (transaction_a) == latest_v0_end ()); @@ -44,7 +44,7 @@ public: return result; } - void representation_add (nano::transaction const & transaction_a, nano::block_hash const & source_a, nano::uint128_t const & amount_a) override + void representation_add (nano::write_transaction const & transaction_a, nano::block_hash const & source_a, nano::uint128_t const & amount_a) override { auto source_block (block_get (transaction_a, source_a)); assert (source_block != nullptr); @@ -59,7 +59,7 @@ public: return iterator != latest_end () && nano::account (iterator->first) == account_a; } - void confirmation_height_clear (nano::transaction const & transaction_a, nano::account const & account, uint64_t existing_confirmation_height) override + void confirmation_height_clear (nano::write_transaction const & transaction_a, nano::account const & account, uint64_t existing_confirmation_height) override { if (existing_confirmation_height > 0) { @@ -67,7 +67,7 @@ public: } } - void confirmation_height_clear (nano::transaction const & transaction_a) override + void confirmation_height_clear (nano::write_transaction const & transaction_a) override { for (auto i (confirmation_height_begin (transaction_a)), n (confirmation_height_end ()); i != n; ++i) { @@ -92,7 +92,7 @@ public: return result; } - void block_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a, nano::block_sideband const & sideband_a, nano::epoch epoch_a = nano::epoch::epoch_0) override + void block_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a, nano::block_sideband const & sideband_a, nano::epoch epoch_a = nano::epoch::epoch_0) override { assert (block_a.type () == sideband_a.type); assert (sideband_a.successor.is_zero () || block_exists (transaction_a, sideband_a.successor)); @@ -241,7 +241,7 @@ public: return version_get (transaction_a) > 12; } - void block_successor_clear (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override + void block_successor_clear (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override { nano::block_type type; auto value (block_raw_get (transaction_a, hash_a, type)); @@ -262,7 +262,7 @@ public: return sum; } - void unchecked_put (nano::transaction const & transaction_a, nano::block_hash const & hash_a, std::shared_ptr const & block_a) override + void unchecked_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, std::shared_ptr const & block_a) override { nano::unchecked_key key (hash_a, block_a->hash ()); nano::unchecked_info info (block_a, block_a->account (), nano::seconds_since_epoch (), nano::signature_verification::unknown); @@ -342,7 +342,7 @@ public: return nano::store_iterator> (nullptr); } - nano::store_iterator peers_end () override + nano::store_iterator peers_end () const override { return nano::store_iterator (nullptr); } @@ -362,7 +362,7 @@ public: return nano::store_iterator (nullptr); } - nano::store_iterator online_weight_end () override + nano::store_iterator online_weight_end () const override { return nano::store_iterator (nullptr); } @@ -392,7 +392,7 @@ public: return cache_mutex; } - void block_del (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override + void block_del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override { auto status = del (transaction_a, tables::state_blocks_v1, hash_a); release_assert (success (status) || not_found (status)); @@ -446,7 +446,7 @@ public: return status == 0 ? nano::epoch::epoch_1 : nano::epoch::epoch_0; } - void block_raw_put (nano::transaction const & transaction_a, std::vector const & data, nano::block_type block_type_a, nano::epoch epoch_a, nano::block_hash const & hash_a) + void block_raw_put (nano::write_transaction const & transaction_a, std::vector const & data, nano::block_type block_type_a, nano::epoch epoch_a, nano::block_hash const & hash_a) { auto database_a = block_database (block_type_a, epoch_a); nano::db_val value{ data.size (), (void *)data.data () }; @@ -454,14 +454,14 @@ public: release_assert (success (status)); } - void pending_put (nano::transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_a) override + void pending_put (nano::write_transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_a) override { nano::db_val pending (pending_a); auto status = put (transaction_a, get_pending_db (pending_a.epoch), key_a, pending); release_assert (success (status)); } - void pending_del (nano::transaction const & transaction_a, nano::pending_key const & key_a) override + void pending_del (nano::write_transaction const & transaction_a, nano::pending_key const & key_a) override { auto status1 = del (transaction_a, tables::pending_v1, key_a); if (!success (status1)) @@ -506,7 +506,7 @@ public: return result; } - void frontier_put (nano::transaction const & transaction_a, nano::block_hash const & block_a, nano::account const & account_a) override + void frontier_put (nano::write_transaction const & transaction_a, nano::block_hash const & block_a, nano::account const & account_a) override { nano::db_val account (account_a); auto status (put (transaction_a, tables::frontiers, block_a, account)); @@ -526,7 +526,7 @@ public: return result; } - void frontier_del (nano::transaction const & transaction_a, nano::block_hash const & block_a) override + void frontier_del (nano::write_transaction const & transaction_a, nano::block_hash const & block_a) override { auto status (del (transaction_a, tables::frontiers, block_a)); release_assert (success (status)); @@ -550,21 +550,21 @@ public: return result; } - void representation_put (nano::transaction const & transaction_a, nano::account const & account_a, nano::uint128_union const & representation_a) override + void representation_put (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::uint128_union const & representation_a) override { nano::db_val rep (representation_a); auto status (put (transaction_a, tables::representation, account_a, rep)); release_assert (success (status)); } - void unchecked_put (nano::transaction const & transaction_a, nano::unchecked_key const & key_a, nano::unchecked_info const & info_a) override + void unchecked_put (nano::write_transaction const & transaction_a, nano::unchecked_key const & key_a, nano::unchecked_info const & info_a) override { nano::db_val info (info_a); auto status (put (transaction_a, tables::unchecked, key_a, info)); release_assert (success (status)); } - void unchecked_del (nano::transaction const & transaction_a, nano::unchecked_key const & key_a) override + void unchecked_del (nano::write_transaction const & transaction_a, nano::unchecked_key const & key_a) override { auto status (del (transaction_a, tables::unchecked, key_a)); release_assert (success (status) || not_found (status)); @@ -584,7 +584,7 @@ public: return nullptr; } - void flush (nano::transaction const & transaction_a) override + void flush (nano::write_transaction const & transaction_a) override { { std::lock_guard lock (cache_mutex); @@ -604,14 +604,14 @@ public: } } - void online_weight_put (nano::transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override + void online_weight_put (nano::write_transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override { nano::db_val value (amount_a); auto status (put (transaction_a, tables::online_weight, time_a, value)); release_assert (success (status)); } - void online_weight_del (nano::transaction const & transaction_a, uint64_t time_a) override + void online_weight_del (nano::write_transaction const & transaction_a, uint64_t time_a) override { auto status (del (transaction_a, tables::online_weight, time_a)); release_assert (success (status)); @@ -635,7 +635,7 @@ public: return db; } - void account_put (nano::transaction const & transaction_a, nano::account const & account_a, nano::account_info const & info_a) override + void account_put (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::account_info const & info_a) override { // Check we are still in sync with other tables assert (confirmation_height_exists (transaction_a, account_a)); @@ -644,7 +644,7 @@ public: release_assert (success (status)); } - void account_del (nano::transaction const & transaction_a, nano::account const & account_a) override + void account_del (nano::write_transaction const & transaction_a, nano::account const & account_a) override { auto status1 = del (transaction_a, tables::accounts_v1, account_a); if (!success (status1)) @@ -689,7 +689,7 @@ public: return result; } - void unchecked_clear (nano::transaction const & transaction_a) override + void unchecked_clear (nano::write_transaction const & transaction_a) override { auto status = drop (transaction_a, tables::unchecked); release_assert (success (status)); @@ -700,20 +700,20 @@ public: return count (transaction_a, tables::online_weight); } - void online_weight_clear (nano::transaction const & transaction_a) override + void online_weight_clear (nano::write_transaction const & transaction_a) override { auto status (drop (transaction_a, tables::online_weight)); release_assert (success (status)); } - void peer_put (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override + void peer_put (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override { nano::db_val zero (static_cast (0)); auto status = put (transaction_a, tables::peers, endpoint_a, zero); release_assert (success (status)); } - void peer_del (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override + void peer_del (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override { auto status (del (transaction_a, tables::peers, endpoint_a)); release_assert (success (status)); @@ -729,7 +729,7 @@ public: return count (transaction_a, tables::peers); } - void peer_clear (nano::transaction const & transaction_a) override + void peer_clear (nano::write_transaction const & transaction_a) override { auto status = drop (transaction_a, tables::peers); release_assert (success (status)); @@ -813,7 +813,7 @@ public: return count (transaction_a, tables::confirmation_height); } - void confirmation_height_put (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) override + void confirmation_height_put (nano::write_transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) override { nano::db_val confirmation_height (confirmation_height_a); auto status = put (transaction_a, tables::confirmation_height, account_a, confirmation_height); @@ -833,7 +833,7 @@ public: return (!success (status)); } - void confirmation_height_del (nano::transaction const & transaction_a, nano::account const & account_a) override + void confirmation_height_del (nano::write_transaction const & transaction_a, nano::account const & account_a) override { auto status (del (transaction_a, tables::confirmation_height, nano::db_val (account_a))); release_assert (success (status)); @@ -924,12 +924,12 @@ public: return make_iterator> (transaction_a, tables::vote); } - nano::store_iterator online_weight_begin (nano::transaction const & transaction_a) override + nano::store_iterator online_weight_begin (nano::transaction const & transaction_a) const override { return make_iterator (transaction_a, tables::online_weight); } - nano::store_iterator peers_begin (nano::transaction const & transaction_a) override + nano::store_iterator peers_begin (nano::transaction const & transaction_a) const override { return make_iterator (transaction_a, tables::peers); } @@ -971,27 +971,27 @@ protected: } template - nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a) + nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a) const { - return static_cast (*this).template make_iterator (transaction_a, table_a); + return static_cast (*this).template make_iterator (transaction_a, table_a); } template - nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a, nano::db_val const & key) + nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a, nano::db_val const & key) const { - return static_cast (*this).template make_iterator (transaction_a, table_a, key); + return static_cast (*this).template make_iterator (transaction_a, table_a, key); } template - nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a) + nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a) const { - return static_cast (*this).template make_merge_iterator (transaction_a, table1_a, table2_a); + return static_cast (*this).template make_merge_iterator (transaction_a, table1_a, table2_a); } template - nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, nano::db_val const & key) + nano::store_iterator make_merge_iterator (nano::transaction const & transaction_a, tables table1_a, tables table2_a, nano::db_val const & key) const { - return static_cast (*this).template make_merge_iterator (transaction_a, table1_a, table2_a, key); + return static_cast (*this).template make_merge_iterator (transaction_a, table1_a, table2_a, key); } bool entry_has_sideband (size_t entry_size_a, nano::block_type type_a) const @@ -1214,18 +1214,18 @@ protected: return static_cast (*this).get (transaction_a, table_a, key_a, value_a); } - int put (nano::transaction const & transaction_a, tables table_a, nano::db_val const & key_a, nano::db_val const & value_a) + int put (nano::write_transaction const & transaction_a, tables table_a, nano::db_val const & key_a, nano::db_val const & value_a) { return static_cast (*this).put (transaction_a, table_a, key_a, value_a); } - int del (nano::transaction const & transaction_a, tables table_a, nano::db_val const & key_a) + int del (nano::write_transaction const & transaction_a, tables table_a, nano::db_val const & key_a) { return static_cast (*this).del (transaction_a, table_a, key_a); } virtual size_t count (nano::transaction const & transaction_a, tables table_a) const = 0; - virtual int drop (nano::transaction const & transaction_a, tables table_a) = 0; + virtual int drop (nano::write_transaction const & transaction_a, tables table_a) = 0; virtual bool not_found (int status) const = 0; virtual bool success (int status) const = 0; virtual int status_code_not_found () const = 0; @@ -1238,7 +1238,7 @@ template class block_predecessor_set : public nano::block_visitor { public: - block_predecessor_set (nano::transaction const & transaction_a, nano::block_store_partial & store_a) : + block_predecessor_set (nano::write_transaction const & transaction_a, nano::block_store_partial & store_a) : transaction (transaction_a), store (store_a) { @@ -1278,7 +1278,7 @@ public: fill_value (block_a); } } - nano::transaction const & transaction; + nano::write_transaction const & transaction; nano::block_store_partial & store; }; } \ No newline at end of file diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index 79274d43..07df8256 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -12,7 +12,7 @@ namespace class rollback_visitor : public nano::block_visitor { public: - rollback_visitor (nano::transaction const & transaction_a, nano::ledger & ledger_a, std::vector> & list_a) : + rollback_visitor (nano::write_transaction const & transaction_a, nano::ledger & ledger_a, std::vector> & list_a) : transaction (transaction_a), ledger (ledger_a), list (list_a) @@ -154,7 +154,7 @@ public: } ledger.store.block_del (transaction, hash); } - nano::transaction const & transaction; + nano::write_transaction const & transaction; nano::ledger & ledger; std::vector> & list; bool error{ false }; @@ -163,7 +163,7 @@ public: class ledger_processor : public nano::block_visitor { public: - ledger_processor (nano::ledger &, nano::transaction const &, nano::signature_verification = nano::signature_verification::unknown); + ledger_processor (nano::ledger &, nano::write_transaction const &, nano::signature_verification = nano::signature_verification::unknown); virtual ~ledger_processor () = default; void send_block (nano::send_block const &) override; void receive_block (nano::receive_block const &) override; @@ -173,7 +173,7 @@ public: void state_block_impl (nano::state_block const &); void epoch_block_impl (nano::state_block const &); nano::ledger & ledger; - nano::transaction const & transaction; + nano::write_transaction const & transaction; nano::signature_verification verification; nano::process_return result; }; @@ -647,7 +647,7 @@ void ledger_processor::open_block (nano::open_block const & block_a) } } -ledger_processor::ledger_processor (nano::ledger & ledger_a, nano::transaction const & transaction_a, nano::signature_verification verification_a) : +ledger_processor::ledger_processor (nano::ledger & ledger_a, nano::write_transaction const & transaction_a, nano::signature_verification verification_a) : ledger (ledger_a), transaction (transaction_a), verification (verification_a) @@ -713,7 +713,7 @@ nano::uint128_t nano::ledger::account_pending (nano::transaction const & transac return result; } -nano::process_return nano::ledger::process (nano::transaction const & transaction_a, nano::block const & block_a, nano::signature_verification verification) +nano::process_return nano::ledger::process (nano::write_transaction const & transaction_a, nano::block const & block_a, nano::signature_verification verification) { assert (!nano::work_validate (block_a)); ledger_processor processor (*this, transaction_a, verification); @@ -839,7 +839,7 @@ nano::uint128_t nano::ledger::weight (nano::transaction const & transaction_a, n } // Rollback blocks until `block_a' doesn't exist or it tries to penetrate the confirmation height -bool nano::ledger::rollback (nano::transaction const & transaction_a, nano::block_hash const & block_a, std::vector> & list_a) +bool nano::ledger::rollback (nano::write_transaction const & transaction_a, nano::block_hash const & block_a, std::vector> & list_a) { assert (store.block_exists (transaction_a, block_a)); auto account_l (account (transaction_a, block_a)); @@ -870,7 +870,7 @@ bool nano::ledger::rollback (nano::transaction const & transaction_a, nano::bloc return error; } -bool nano::ledger::rollback (nano::transaction const & transaction_a, nano::block_hash const & block_a) +bool nano::ledger::rollback (nano::write_transaction const & transaction_a, nano::block_hash const & block_a) { std::vector> rollback_list; return rollback (transaction_a, block_a, rollback_list); @@ -989,7 +989,7 @@ bool nano::ledger::is_epoch_link (nano::uint256_union const & link_a) return link_a == epoch_link; } -void nano::ledger::change_latest (nano::transaction const & transaction_a, nano::account const & account_a, nano::block_hash const & hash_a, nano::block_hash const & rep_block_a, nano::amount const & balance_a, uint64_t block_count_a, bool is_state, nano::epoch epoch_a) +void nano::ledger::change_latest (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::block_hash const & hash_a, nano::block_hash const & rep_block_a, nano::amount const & balance_a, uint64_t block_count_a, bool is_state, nano::epoch epoch_a) { nano::account_info info; auto exists (!store.account_get (transaction_a, account_a, info)); diff --git a/nano/secure/ledger.hpp b/nano/secure/ledger.hpp index 778f3f7a..e21be446 100644 --- a/nano/secure/ledger.hpp +++ b/nano/secure/ledger.hpp @@ -40,10 +40,10 @@ public: bool is_send (nano::transaction const &, nano::state_block const &) const; nano::block_hash block_destination (nano::transaction const &, nano::block const &); nano::block_hash block_source (nano::transaction const &, nano::block const &); - nano::process_return process (nano::transaction const &, nano::block const &, nano::signature_verification = nano::signature_verification::unknown); - bool rollback (nano::transaction const &, nano::block_hash const &, std::vector> &); - bool rollback (nano::transaction const &, nano::block_hash const &); - void change_latest (nano::transaction const &, nano::account const &, nano::block_hash const &, nano::account const &, nano::uint128_union const &, uint64_t, bool = false, nano::epoch = nano::epoch::epoch_0); + nano::process_return process (nano::write_transaction const &, nano::block const &, nano::signature_verification = nano::signature_verification::unknown); + bool rollback (nano::write_transaction const &, nano::block_hash const &, std::vector> &); + bool rollback (nano::write_transaction const &, nano::block_hash const &); + void change_latest (nano::write_transaction const &, nano::account const &, nano::block_hash const &, nano::account const &, nano::uint128_union const &, uint64_t, bool = false, nano::epoch = nano::epoch::epoch_0); void dump_account_chain (nano::account const &); bool could_fit (nano::transaction const &, nano::block const &); bool is_epoch_link (nano::uint256_union const &);