diff --git a/.gitmodules b/.gitmodules index 20b18827..7f846995 100644 --- a/.gitmodules +++ b/.gitmodules @@ -24,3 +24,7 @@ [submodule "flatbuffers"] path = flatbuffers url = https://github.com/google/flatbuffers.git +[submodule "rocksdb"] + path = rocksdb + url = https://github.com/nanocurrency/rocksdb.git + branch = 6.11.4 diff --git a/CMakeLists.txt b/CMakeLists.txt index 2592adcd..d8152ba7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ endif () # compatibility for osx sierra and on # needs to be set before project -set (CMAKE_OSX_DEPLOYMENT_TARGET 10.12 CACHE STRING "") +set (CMAKE_OSX_DEPLOYMENT_TARGET 10.14 CACHE STRING "") project (nano-node) @@ -59,13 +59,13 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) set (NANO_GUI OFF CACHE BOOL "") set (NANO_TEST OFF CACHE BOOL "") set (NANO_SECURE_RPC OFF CACHE BOOL "") -set (NANO_ROCKSDB OFF CACHE BOOL "") set (NANO_POW_SERVER OFF CACHE BOOL "") set (NANO_WARN_TO_ERR OFF CACHE BOOL "") set (NANO_TIMED_LOCKS 0 CACHE STRING "") set (NANO_TIMED_LOCKS_IGNORE_BLOCKED OFF CACHE BOOL "") set (NANO_FUZZER_TEST OFF CACHE BOOL "") set (NANO_ASIO_HANDLER_TRACKING 0 CACHE STRING "") +set (NANO_ROCKSDB_TOOLS OFF CACHE BOOL "") option (NANO_STACKTRACE_BACKTRACE "Use BOOST_STACKTRACE_USE_BACKTRACE in stacktraces, for POSIX" OFF) if (NANO_STACKTRACE_BACKTRACE) @@ -86,8 +86,6 @@ if (${NANO_ASIO_HANDLER_TRACKING} GREATER 0) add_definitions (-DNANO_ASIO_HANDLER_TRACKING=${NANO_ASIO_HANDLER_TRACKING} -DBOOST_ASIO_ENABLE_HANDLER_TRACKING) endif () -add_definitions (-DNANO_ROCKSDB=$) - option(NANO_ASAN_INT "Enable ASan+UBSan+Integer overflow" OFF) option(NANO_ASAN "Enable ASan+UBSan" OFF) option(NANO_TSAN "Enable TSan" OFF) @@ -252,7 +250,7 @@ if (NANO_SHARED_BOOST) SET(Boost_USE_STATIC_LIBS OFF) SET(Boost_USE_STATIC_RUNTIME OFF) SET(Boost_NO_BOOST_CMAKE ON) - add_definitions( -DBOOST_ALL_DYN_LINK -DBoost_ALL_NO_LIB) + add_definitions(-DBOOST_ALL_DYN_LINK -DBoost_ALL_NO_LIB) else() set(Boost_USE_STATIC_LIBS ON) endif() @@ -262,11 +260,18 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules") find_package (Boost 1.69.0 REQUIRED COMPONENTS coroutine context filesystem log log_setup thread program_options system) -if (NANO_ROCKSDB) - find_package (RocksDB REQUIRED) - find_package (ZLIB REQUIRED) - include_directories (${ROCKSDB_INCLUDE_DIRS}) -endif () +# RocksDB +include_directories(rocksdb/include) +set(USE_RTTI ON CACHE BOOL "") +set(WITH_GFLAGS OFF CACHE BOOL "") +set(WITH_TESTS OFF CACHE BOOL "") +set(WITH_BENCHMARK_TOOLS OFF CACHE BOOL "") +set(ROCKSDB_BUILD_SHARED OFF CACHE BOOL "") +set(WITH_CORE_TOOLS OFF CACHE BOOL "") +set(WITH_TOOLS OFF CACHE BOOL "") +set(WITH_CORE_TOOLS ${NANO_ROCKSDB_TOOLS}) +set(WITH_TOOLS ${NANO_ROCKSDB_TOOLS}) +add_subdirectory(rocksdb) # There is a compile bug with boost 1.69 interprocess headers on Mac if (APPLE AND Boost_VERSION EQUAL 106900) diff --git a/ci/actions/osx/install_deps.sh b/ci/actions/osx/install_deps.sh index 8f740877..cde72c6d 100755 --- a/ci/actions/osx/install_deps.sh +++ b/ci/actions/osx/install_deps.sh @@ -3,10 +3,5 @@ brew update brew install coreutils brew cask install xquartz -if [[ ${TEST-0} -eq 1 ]]; then - brew install rocksdb; -else - util/build_prep/fetch_rocksdb.sh -fi sudo util/build_prep/fetch_boost.sh util/build_prep/macosx/build_qt.sh diff --git a/ci/actions/windows/configure.bat b/ci/actions/windows/configure.bat index b37de8b2..7a3f08cb 100644 --- a/ci/actions/windows/configure.bat +++ b/ci/actions/windows/configure.bat @@ -8,12 +8,8 @@ cmake .. ^ -Ax64 ^ %NANO_TEST% ^ %CI% ^ - -DNANO_ROCKSDB=ON ^ %ROCKS_LIB% ^ - -DROCKSDB_INCLUDE_DIRS="c:\vcpkg\installed\x64-windows-static\include" ^ - -DZLIB_LIBRARY_RELEASE="c:\vcpkg\installed\x64-windows-static\lib\zlib.lib" ^ - -DZLIB_LIBRARY_DEBUG="c:\vcpkg\installed\x64-windows-static\debug\lib\zlibd.lib" ^ - -DZLIB_INCLUDE_DIR="c:\vcpkg\installed\x64-windows-static\include" ^ + -DPORTABLE=1 ^ -DQt5_DIR="c:\qt\5.13.1\msvc2017_64\lib\cmake\Qt5" ^ -DNANO_GUI=ON ^ -DCMAKE_BUILD_TYPE=%BUILD_TYPE% ^ diff --git a/ci/build-deploy.sh b/ci/build-deploy.sh index 8a3d5d36..634c6681 100755 --- a/ci/build-deploy.sh +++ b/ci/build-deploy.sh @@ -24,9 +24,7 @@ cmake \ -DACTIVE_NETWORK=nano_${NETWORK_CFG}_network \ -DNANO_POW_SERVER=ON \ -DNANO_GUI=ON \ - -DNANO_ROCKSDB=ON \ - -DROCKSDB_LIBRARIES=/tmp/rocksdb/lib/librocksdb.a \ - -DROCKSDB_INCLUDE_DIRS=/tmp/rocksdb/include \ + -DPORTABLE=1 \ -DCMAKE_BUILD_TYPE=${CONFIGURATION} \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DBOOST_ROOT=/tmp/boost/ \ diff --git a/ci/build-travis.sh b/ci/build-travis.sh index 24ae3a6b..d3c871bd 100755 --- a/ci/build-travis.sh +++ b/ci/build-travis.sh @@ -48,8 +48,6 @@ fi ulimit -S -n 8192 if [[ "$OS" == 'Linux' ]]; then - ROCKSDB="-DROCKSDB_LIBRARIES=/tmp/rocksdb/lib/librocksdb.a \ - -DROCKSDB_INCLUDE_DIRS=/tmp/rocksdb/include" if clang --version; then BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON \ -DBACKTRACE_INCLUDE=" @@ -57,7 +55,6 @@ if [[ "$OS" == 'Linux' ]]; then BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON" fi else - ROCKSDB="" BACKTRACE="" fi @@ -66,8 +63,7 @@ cmake \ -DACTIVE_NETWORK=nano_dev_network \ -DNANO_TEST=ON \ -DNANO_GUI=ON \ - -DNANO_ROCKSDB=ON \ - ${ROCKSDB} \ + -DPORTABLE=1 \ -DNANO_WARN_TO_ERR=ON \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_VERBOSE_MAKEFILE=ON \ diff --git a/cmake/Modules/FindRocksDB.cmake b/cmake/Modules/FindRocksDB.cmake deleted file mode 100644 index 1bbfd599..00000000 --- a/cmake/Modules/FindRocksDB.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# Try to find RocksDB headers and library. -# -# Usage of this module as follows: -# -# find_package(RocksDB) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# ROCKSDB_ROOT_DIR Set this variable to the root installation of -# RocksDB if the module has problems finding the -# proper installation path. -# -# Variables defined by this module: -# -# ROCKSDB_FOUND System has RocksDB library/headers. -# ROCKSDB_LIBRARIES The RocksDB library. -# ROCKSDB_INCLUDE_DIRS The location of RocksDB headers. - -find_path(ROCKSDB_ROOT_DIR - NAMES include/rocksdb/db.h -) - -find_library(ROCKSDB_LIBRARIES - NAMES rocksdb - HINTS ${ROCKSDB_ROOT_DIR}/lib -) - -find_path(ROCKSDB_INCLUDE_DIRS - NAMES rocksdb/db.h - HINTS ${ROCKSDB_ROOT_DIR}/include -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(RocksDB DEFAULT_MSG - ROCKSDB_LIBRARIES - ROCKSDB_INCLUDE_DIRS -) - -mark_as_advanced( - ROCKSDB_ROOT_DIR - ROCKSDB_LIBRARIES - ROCKSDB_INCLUDE_DIRS -) diff --git a/docker/ci/Dockerfile-clang b/docker/ci/Dockerfile-clang index adf04df7..2f2aa6d6 100644 --- a/docker/ci/Dockerfile-clang +++ b/docker/ci/Dockerfile-clang @@ -3,9 +3,6 @@ FROM nanocurrency/nano-env:base RUN apt-get update -qq && apt-get install -yqq \ clang-3.9 lldb-3.9 git -ADD util/build_prep/fetch_rocksdb.sh fetch_rocksdb.sh -RUN ./fetch_rocksdb.sh - ENV CXX=/usr/bin/clang++ ENV CC=/usr/bin/clang RUN ln -s /usr/bin/clang-3.9 /usr/bin/clang diff --git a/docker/ci/Dockerfile-clang-6 b/docker/ci/Dockerfile-clang-6 index e244b21e..47c47052 100644 --- a/docker/ci/Dockerfile-clang-6 +++ b/docker/ci/Dockerfile-clang-6 @@ -8,9 +8,6 @@ RUN apt-get update && apt-get install -yqq software-properties-common && \ apt-get update -qq && apt-get install -yqq \ clang-6.0 lldb-6.0 libfuzzer-6.0-dev git -ADD util/build_prep/fetch_rocksdb.sh fetch_rocksdb.sh -RUN ./fetch_rocksdb.sh - ENV CXX=/usr/bin/clang++ ENV CC=/usr/bin/clang RUN ln -s /usr/bin/clang-6.0 /usr/bin/clang diff --git a/docker/ci/Dockerfile-gcc b/docker/ci/Dockerfile-gcc index 8b588f22..5b012fb7 100644 --- a/docker/ci/Dockerfile-gcc +++ b/docker/ci/Dockerfile-gcc @@ -2,9 +2,6 @@ FROM nanocurrency/nano-env:base RUN apt-get install -yqq git -ADD util/build_prep/fetch_rocksdb.sh fetch_rocksdb.sh -RUN ./fetch_rocksdb.sh - ENV BOOST_ROOT=/tmp/boost ADD util/build_prep/fetch_boost.sh fetch_boost.sh diff --git a/docker/node/Dockerfile b/docker/node/Dockerfile index 1d070be5..2018c59b 100644 --- a/docker/node/Dockerfile +++ b/docker/node/Dockerfile @@ -7,9 +7,10 @@ ADD ./ /tmp/src RUN mkdir /tmp/build && \ cd /tmp/build && \ - cmake /tmp/src -DCI_BUILD=${CI_BUILD} -DBOOST_ROOT=${BOOST_ROOT} -DACTIVE_NETWORK=nano_${NETWORK}_network \ - -DNANO_ROCKSDB=ON -DNANO_POW_SERVER=ON -DROCKSDB_LIBRARIES=/tmp/rocksdb/lib/librocksdb.a \ - -DROCKSDB_INCLUDE_DIRS=/tmp/rocksdb/include -DNANO_SHARED_BOOST=ON && \ + cmake /tmp/src -DCI_BUILD=${CI_BUILD} \ + -DBOOST_ROOT=${BOOST_ROOT} -DPORTABLE=1 \ + -DACTIVE_NETWORK=nano_${NETWORK}_network \ + -DNANO_POW_SERVER=ON -DNANO_SHARED_BOOST=ON && \ make nano_node -j $(nproc) && \ make nano_rpc -j $(nproc) && \ make nano_pow_server -j $(nproc) && \ diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 07faab0f..6a488f08 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -1,26 +1,22 @@ #include #include +#include #include #include #include #include +#include +#include #include #include #include #include #include -#include - -#if NANO_ROCKSDB -#include -#endif - -#include -#include - #include +#include + #include #include @@ -1914,8 +1910,6 @@ TEST (block_store, rocksdb_force_test_env_variable) auto store = nano::make_store (logger, nano::unique_path ()); auto mdb_cast = dynamic_cast (store.get ()); - -#if NANO_ROCKSDB if (value && boost::lexical_cast (value) == 1) { ASSERT_NE (boost::polymorphic_downcast (store.get ()), nullptr); @@ -1924,16 +1918,12 @@ TEST (block_store, rocksdb_force_test_env_variable) { ASSERT_NE (mdb_cast, nullptr); } -#else - ASSERT_NE (mdb_cast, nullptr); -#endif } namespace nano { TEST (rocksdb_block_store, tombstone_count) { -#if NANO_ROCKSDB if (nano::using_rocksdb_in_tests ()) { nano::logger_mt logger; @@ -1946,7 +1936,6 @@ TEST (rocksdb_block_store, tombstone_count) store->unchecked_del (transaction, nano::unchecked_key (block1->previous (), block1->hash ())); ASSERT_EQ (store->tombstone_map.at (nano::tables::unchecked).num_since_last_flush.load (), 1); } -#endif } } diff --git a/nano/lib/errors.cpp b/nano/lib/errors.cpp index b4bedbf4..0860fc94 100644 --- a/nano/lib/errors.cpp +++ b/nano/lib/errors.cpp @@ -273,8 +273,6 @@ std::string nano::error_config_messages::message (int ev) const return "Invalid configuration value"; case nano::error_config::missing_value: return "Missing value in configuration"; - case nano::error_config::rocksdb_enabled_but_not_supported: - return "RocksDB has been enabled, but the node has not been built with RocksDB support. Set the CMake flag -DNANO_ROCKSDB=ON"; } return "Invalid error code"; diff --git a/nano/lib/errors.hpp b/nano/lib/errors.hpp index 4a495430..129b2491 100644 --- a/nano/lib/errors.hpp +++ b/nano/lib/errors.hpp @@ -149,8 +149,7 @@ enum class error_config { generic = 1, invalid_value, - missing_value, - rocksdb_enabled_but_not_supported + missing_value }; } // nano namespace diff --git a/nano/nano_wallet/entry.cpp b/nano/nano_wallet/entry.cpp index 290808e3..067b3a1b 100644 --- a/nano/nano_wallet/entry.cpp +++ b/nano/nano_wallet/entry.cpp @@ -87,13 +87,6 @@ int run_wallet (QApplication & application, int argc, char * const * argv, boost error = read_wallet_config (wallet_config, data_path); } -#if !NANO_ROCKSDB - if (!error && config.node.rocksdb_config.enable) - { - error = nano::error_config::rocksdb_enabled_but_not_supported; - } -#endif - if (!error) { nano::set_use_memory_pools (config.node.use_memory_pools); diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index 4c18dc66..f31d8ef5 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -1,16 +1,9 @@ -if (NANO_ROCKSDB) - set (rocksdb_libs ${ROCKSDB_LIBRARIES} ${ZLIB_LIBRARIES}) - set (rocksdb_sources rocksdb/rocksdb.hpp rocksdb/rocksdb.cpp rocksdb/rocksdb_iterator.hpp rocksdb/rocksdb_txn.hpp rocksdb/rocksdb_txn.cpp) -endif () if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") # No opencl elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows") set (platform_sources plat/windows/openclapi.cpp) set (psapi_lib Psapi) - if (NANO_ROCKSDB) - set (rocksdb_libs ${rocksdb_libs} Shlwapi Rpcrt4) - endif () elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux") set (platform_sources plat/posix/openclapi.cpp) elseif (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") @@ -21,7 +14,6 @@ endif () add_library (node ${platform_sources} - ${rocksdb_sources} active_transactions.hpp active_transactions.cpp blockprocessor.hpp @@ -121,6 +113,11 @@ add_library (node repcrawler.cpp request_aggregator.hpp request_aggregator.cpp + rocksdb/rocksdb.hpp + rocksdb/rocksdb.cpp + rocksdb/rocksdb_iterator.hpp + rocksdb/rocksdb_txn.hpp + rocksdb/rocksdb_txn.cpp signatures.hpp signatures.cpp socket.hpp @@ -167,7 +164,7 @@ target_link_libraries (node Boost::system Boost::thread Boost::boost - ${rocksdb_libs} + rocksdb ${CMAKE_DL_LIBS} ${psapi_lib} ) diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index c52dce7b..f0e22ed4 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -1239,14 +1239,7 @@ bool is_using_rocksdb (boost::filesystem::path const & data_path, std::error_cod auto error = nano::read_node_config_toml (data_path, config); if (!error) { - bool use_rocksdb = config.node.rocksdb_config.enable; - if (use_rocksdb) - { -#if !NANO_ROCKSDB - ec = nano::error_cli::database_write_error; -#endif - return (NANO_ROCKSDB == 1); - } + return config.node.rocksdb_config.enable; } else { diff --git a/nano/node/node.cpp b/nano/node/node.cpp index eac23c07..7e9a4e74 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -4,16 +4,13 @@ #include #include #include +#include #include #include #include #include #include -#if NANO_ROCKSDB -#include -#endif - #include #include @@ -1706,30 +1703,9 @@ nano::node_flags const & nano::inactive_node_flag_defaults () std::unique_ptr nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade, bool use_rocksdb_backend) { -#if NANO_ROCKSDB - auto make_rocksdb = [&logger, add_db_postfix, &path, &rocksdb_config, read_only]() { + if (use_rocksdb_backend || using_rocksdb_in_tests ()) + { return std::make_unique (logger, add_db_postfix ? path / "rocksdb" : path, rocksdb_config, read_only); - }; -#endif - - if (use_rocksdb_backend) - { -#if NANO_ROCKSDB - return make_rocksdb (); -#else - logger.always_log (std::error_code (nano::error_config::rocksdb_enabled_but_not_supported).message ()); - release_assert (false); - return nullptr; -#endif - } - else - { -#if NANO_ROCKSDB - if (using_rocksdb_in_tests ()) - { - return make_rocksdb (); - } -#endif } return std::make_unique (logger, add_db_postfix ? path / "data.ldb" : path, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_config_a, backup_before_upgrade); diff --git a/nano/node/testing.hpp b/nano/node/testing.hpp index b3f72eba..58c863e6 100644 --- a/nano/node/testing.hpp +++ b/nano/node/testing.hpp @@ -59,7 +59,7 @@ std::unique_ptr upgrade_epoch (nano::work_pool &, nano::ledge void blocks_confirm (nano::node &, std::vector> const &); uint16_t get_available_port (); void cleanup_dev_directories_on_exit (); -/** To use RocksDB in tests make sure the node is built with the cmake variable -DNANO_ROCKSDB=ON and the environment variable TEST_USE_ROCKSDB=1 is set */ +/** To use RocksDB in tests make sure the environment variable TEST_USE_ROCKSDB=1 is set */ bool using_rocksdb_in_tests (); } REGISTER_ERROR_CODES (nano, error_system); diff --git a/rocksdb b/rocksdb new file mode 160000 index 00000000..5daf2d72 --- /dev/null +++ b/rocksdb @@ -0,0 +1 @@ +Subproject commit 5daf2d724d1f67efa0764e5789c4e20d62f3c4b8 diff --git a/util/build_prep/fetch_rocksdb.sh b/util/build_prep/fetch_rocksdb.sh deleted file mode 100755 index 5095085f..00000000 --- a/util/build_prep/fetch_rocksdb.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -OS=`uname` - -pushd /tmp -wget -O rocksdb.tgz https://s3.us-east-2.amazonaws.com/repo.nano.org/artifacts/rocksdb-$OS-6.5.2.tgz - -tar -zxf rocksdb.tgz -mv tmp/* . -rm -fr tmp -popd