Add RocksDB as a submodule (#2943)

* Add RocksDB as a submodule

* Update CMakeLists.txt setting cache default first before overriding

* finish removing cached rocksdb from CI pipeline using submodule instead
always build with PORTABLE=1 for those without AVX2 support

Co-authored-by: Russel <russel@nano.org>

.gitmodules

@@ -24,3 +24,7 @@
 [submodule "flatbuffers"]
 	path = flatbuffers
 	url = https://github.com/google/flatbuffers.git
+[submodule "rocksdb"]
+	path = rocksdb
+	url = https://github.com/nanocurrency/rocksdb.git
+	branch = 6.11.4

CMakeLists.txt

@@ -16,7 +16,7 @@ endif ()
 
 # compatibility for osx sierra and on
 # needs to be set before project
-set (CMAKE_OSX_DEPLOYMENT_TARGET 10.12 CACHE STRING "")
+set (CMAKE_OSX_DEPLOYMENT_TARGET 10.14 CACHE STRING "")
 
 project (nano-node)
 
@@ -59,13 +59,13 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 set (NANO_GUI OFF CACHE BOOL "")
 set (NANO_TEST OFF CACHE BOOL "")
 set (NANO_SECURE_RPC OFF CACHE BOOL "")
-set (NANO_ROCKSDB OFF CACHE BOOL "")
 set (NANO_POW_SERVER OFF CACHE BOOL "")
 set (NANO_WARN_TO_ERR OFF CACHE BOOL "")
 set (NANO_TIMED_LOCKS 0 CACHE STRING "")
 set (NANO_TIMED_LOCKS_IGNORE_BLOCKED OFF CACHE BOOL "")
 set (NANO_FUZZER_TEST OFF CACHE BOOL "")
 set (NANO_ASIO_HANDLER_TRACKING 0 CACHE STRING "")
+set (NANO_ROCKSDB_TOOLS OFF CACHE BOOL "")
 
 option (NANO_STACKTRACE_BACKTRACE "Use BOOST_STACKTRACE_USE_BACKTRACE in stacktraces, for POSIX" OFF)
 if (NANO_STACKTRACE_BACKTRACE)
@@ -86,8 +86,6 @@ if (${NANO_ASIO_HANDLER_TRACKING} GREATER 0)
 	add_definitions (-DNANO_ASIO_HANDLER_TRACKING=${NANO_ASIO_HANDLER_TRACKING} -DBOOST_ASIO_ENABLE_HANDLER_TRACKING)
 endif ()
 
-add_definitions (-DNANO_ROCKSDB=$<STREQUAL:${NANO_ROCKSDB},ON>)
-
 option(NANO_ASAN_INT "Enable ASan+UBSan+Integer overflow" OFF)
 option(NANO_ASAN "Enable ASan+UBSan" OFF)
 option(NANO_TSAN "Enable TSan" OFF)
@@ -252,7 +250,7 @@ if (NANO_SHARED_BOOST)
 	SET(Boost_USE_STATIC_LIBS OFF)
 	SET(Boost_USE_STATIC_RUNTIME OFF)
 	SET(Boost_NO_BOOST_CMAKE ON)
-	add_definitions( -DBOOST_ALL_DYN_LINK -DBoost_ALL_NO_LIB)
+	add_definitions(-DBOOST_ALL_DYN_LINK -DBoost_ALL_NO_LIB)
 else()
 	set(Boost_USE_STATIC_LIBS ON)
 endif()
@@ -262,11 +260,18 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules")
 
 find_package (Boost 1.69.0 REQUIRED COMPONENTS coroutine context filesystem log log_setup thread program_options system)
 
-if (NANO_ROCKSDB)
-	find_package (RocksDB REQUIRED)
-	find_package (ZLIB REQUIRED)
-	include_directories (${ROCKSDB_INCLUDE_DIRS})
-endif ()
+# RocksDB
+include_directories(rocksdb/include)
+set(USE_RTTI ON CACHE BOOL "")
+set(WITH_GFLAGS OFF CACHE BOOL "")
+set(WITH_TESTS OFF CACHE BOOL "")
+set(WITH_BENCHMARK_TOOLS OFF CACHE BOOL "")
+set(ROCKSDB_BUILD_SHARED OFF CACHE BOOL "")
+set(WITH_CORE_TOOLS OFF CACHE BOOL "")
+set(WITH_TOOLS OFF CACHE BOOL "")
+set(WITH_CORE_TOOLS ${NANO_ROCKSDB_TOOLS})
+set(WITH_TOOLS ${NANO_ROCKSDB_TOOLS})
+add_subdirectory(rocksdb)
 
 # There is a compile bug with boost 1.69 interprocess headers on Mac
 if (APPLE AND Boost_VERSION EQUAL 106900)

ci/actions/osx/install_deps.sh

@@ -3,10 +3,5 @@
 brew update
 brew install coreutils
 brew cask install xquartz
-if [[ ${TEST-0} -eq 1 ]]; then
-    brew install rocksdb;
-else
-    util/build_prep/fetch_rocksdb.sh
-fi
 sudo util/build_prep/fetch_boost.sh 
 util/build_prep/macosx/build_qt.sh

ci/actions/windows/configure.bat

@@ -8,12 +8,8 @@ cmake .. ^
   -Ax64 ^
   %NANO_TEST% ^
   %CI% ^
-  -DNANO_ROCKSDB=ON ^
   %ROCKS_LIB% ^
-  -DROCKSDB_INCLUDE_DIRS="c:\vcpkg\installed\x64-windows-static\include" ^
-  -DZLIB_LIBRARY_RELEASE="c:\vcpkg\installed\x64-windows-static\lib\zlib.lib" ^
-  -DZLIB_LIBRARY_DEBUG="c:\vcpkg\installed\x64-windows-static\debug\lib\zlibd.lib" ^
-  -DZLIB_INCLUDE_DIR="c:\vcpkg\installed\x64-windows-static\include" ^
+  -DPORTABLE=1 ^
   -DQt5_DIR="c:\qt\5.13.1\msvc2017_64\lib\cmake\Qt5" ^
   -DNANO_GUI=ON ^
   -DCMAKE_BUILD_TYPE=%BUILD_TYPE% ^

ci/build-deploy.sh

@@ -24,9 +24,7 @@ cmake \
     -DACTIVE_NETWORK=nano_${NETWORK_CFG}_network \
     -DNANO_POW_SERVER=ON \
     -DNANO_GUI=ON \
-    -DNANO_ROCKSDB=ON \
-    -DROCKSDB_LIBRARIES=/tmp/rocksdb/lib/librocksdb.a \
-    -DROCKSDB_INCLUDE_DIRS=/tmp/rocksdb/include \
+    -DPORTABLE=1 \
     -DCMAKE_BUILD_TYPE=${CONFIGURATION} \
     -DCMAKE_VERBOSE_MAKEFILE=ON \
     -DBOOST_ROOT=/tmp/boost/ \

ci/build-travis.sh

@@ -48,8 +48,6 @@ fi
 ulimit -S -n 8192
 
 if [[ "$OS" == 'Linux' ]]; then
-    ROCKSDB="-DROCKSDB_LIBRARIES=/tmp/rocksdb/lib/librocksdb.a \
-    -DROCKSDB_INCLUDE_DIRS=/tmp/rocksdb/include"
     if clang --version; then
         BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON \
         -DBACKTRACE_INCLUDE=</tmp/backtrace.h>"
@@ -57,7 +55,6 @@ if [[ "$OS" == 'Linux' ]]; then
         BACKTRACE="-DNANO_STACKTRACE_BACKTRACE=ON"
     fi
 else
-    ROCKSDB=""
     BACKTRACE=""
 fi
 
@@ -66,8 +63,7 @@ cmake \
     -DACTIVE_NETWORK=nano_dev_network \
     -DNANO_TEST=ON \
     -DNANO_GUI=ON \
-    -DNANO_ROCKSDB=ON \
-    ${ROCKSDB} \
+    -DPORTABLE=1 \
     -DNANO_WARN_TO_ERR=ON \
     -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
     -DCMAKE_VERBOSE_MAKEFILE=ON \

cmake/Modules/FindRocksDB.cmake

@@ -1,44 +0,0 @@
-# Try to find RocksDB headers and library.
-#
-# Usage of this module as follows:
-#
-#     find_package(RocksDB)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-#  ROCKSDB_ROOT_DIR          Set this variable to the root installation of
-#                            RocksDB if the module has problems finding the
-#                            proper installation path.
-#
-# Variables defined by this module:
-#
-#  ROCKSDB_FOUND               System has RocksDB library/headers.
-#  ROCKSDB_LIBRARIES           The RocksDB library.
-#  ROCKSDB_INCLUDE_DIRS        The location of RocksDB headers.
-
-find_path(ROCKSDB_ROOT_DIR
-    NAMES include/rocksdb/db.h
-)
-
-find_library(ROCKSDB_LIBRARIES
-    NAMES rocksdb
-    HINTS ${ROCKSDB_ROOT_DIR}/lib
-)
-
-find_path(ROCKSDB_INCLUDE_DIRS
-    NAMES rocksdb/db.h
-    HINTS ${ROCKSDB_ROOT_DIR}/include
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(RocksDB DEFAULT_MSG
-    ROCKSDB_LIBRARIES
-    ROCKSDB_INCLUDE_DIRS
-)
-
-mark_as_advanced(
-    ROCKSDB_ROOT_DIR
-    ROCKSDB_LIBRARIES
-    ROCKSDB_INCLUDE_DIRS
-)

docker/ci/Dockerfile-clang

@@ -3,9 +3,6 @@ FROM nanocurrency/nano-env:base
 RUN apt-get update -qq && apt-get install -yqq \
     clang-3.9 lldb-3.9 git
 
-ADD util/build_prep/fetch_rocksdb.sh fetch_rocksdb.sh
-RUN ./fetch_rocksdb.sh
-
 ENV CXX=/usr/bin/clang++
 ENV CC=/usr/bin/clang
 RUN ln -s /usr/bin/clang-3.9 /usr/bin/clang

docker/ci/Dockerfile-clang-6

@@ -8,9 +8,6 @@ RUN apt-get update && apt-get install -yqq software-properties-common && \
     apt-get update -qq && apt-get install -yqq \
     clang-6.0 lldb-6.0 libfuzzer-6.0-dev git
 
-ADD util/build_prep/fetch_rocksdb.sh fetch_rocksdb.sh
-RUN ./fetch_rocksdb.sh
-
 ENV CXX=/usr/bin/clang++
 ENV CC=/usr/bin/clang
 RUN ln -s /usr/bin/clang-6.0 /usr/bin/clang

docker/ci/Dockerfile-gcc

@@ -2,9 +2,6 @@ FROM nanocurrency/nano-env:base
 
 RUN apt-get install -yqq git
 
-ADD util/build_prep/fetch_rocksdb.sh fetch_rocksdb.sh
-RUN ./fetch_rocksdb.sh
-
 ENV BOOST_ROOT=/tmp/boost
 
 ADD util/build_prep/fetch_boost.sh fetch_boost.sh

docker/node/Dockerfile

@@ -7,9 +7,10 @@ ADD ./ /tmp/src
 
 RUN mkdir /tmp/build && \
     cd /tmp/build && \
-    cmake /tmp/src -DCI_BUILD=${CI_BUILD} -DBOOST_ROOT=${BOOST_ROOT} -DACTIVE_NETWORK=nano_${NETWORK}_network \
-    -DNANO_ROCKSDB=ON -DNANO_POW_SERVER=ON -DROCKSDB_LIBRARIES=/tmp/rocksdb/lib/librocksdb.a \
-    -DROCKSDB_INCLUDE_DIRS=/tmp/rocksdb/include -DNANO_SHARED_BOOST=ON && \
+    cmake /tmp/src -DCI_BUILD=${CI_BUILD} \
+    -DBOOST_ROOT=${BOOST_ROOT} -DPORTABLE=1 \
+    -DACTIVE_NETWORK=nano_${NETWORK}_network \
+    -DNANO_POW_SERVER=ON -DNANO_SHARED_BOOST=ON && \
     make nano_node -j $(nproc) && \
     make nano_rpc -j $(nproc) && \
     make nano_pow_server -j $(nproc) && \

nano/core_test/block_store.cpp

@@ -1,26 +1,22 @@
 #include <nano/crypto_lib/random_pool.hpp>
 #include <nano/lib/lmdbconfig.hpp>
+#include <nano/lib/logger_mt.hpp>
 #include <nano/lib/stats.hpp>
 #include <nano/lib/utility.hpp>
 #include <nano/lib/work.hpp>
 #include <nano/node/common.hpp>
+#include <nano/node/lmdb/lmdb.hpp>
+#include <nano/node/rocksdb/rocksdb.hpp>
 #include <nano/node/testing.hpp>
 #include <nano/secure/ledger.hpp>
 #include <nano/secure/utility.hpp>
 #include <nano/secure/versioning.hpp>
 #include <nano/test_common/testutil.hpp>
 
-#include <boost/filesystem.hpp>
-
-#if NANO_ROCKSDB
-#include <nano/node/rocksdb/rocksdb.hpp>
-#endif
-
-#include <nano/lib/logger_mt.hpp>
-#include <nano/node/lmdb/lmdb.hpp>
-
 #include <gtest/gtest.h>
 
+#include <boost/filesystem.hpp>
+
 #include <fstream>
 #include <unordered_set>
 
@@ -1914,8 +1910,6 @@ TEST (block_store, rocksdb_force_test_env_variable)
 	auto store = nano::make_store (logger, nano::unique_path ());
 
 	auto mdb_cast = dynamic_cast<nano::mdb_store *> (store.get ());
-
-#if NANO_ROCKSDB
 	if (value && boost::lexical_cast<int> (value) == 1)
 	{
 		ASSERT_NE (boost::polymorphic_downcast<nano::rocksdb_store *> (store.get ()), nullptr);
@@ -1924,16 +1918,12 @@ TEST (block_store, rocksdb_force_test_env_variable)
 	{
 		ASSERT_NE (mdb_cast, nullptr);
 	}
-#else
-	ASSERT_NE (mdb_cast, nullptr);
-#endif
 }
 
 namespace nano
 {
 TEST (rocksdb_block_store, tombstone_count)
 {
-#if NANO_ROCKSDB
 	if (nano::using_rocksdb_in_tests ())
 	{
 		nano::logger_mt logger;
@@ -1946,7 +1936,6 @@ TEST (rocksdb_block_store, tombstone_count)
 		store->unchecked_del (transaction, nano::unchecked_key (block1->previous (), block1->hash ()));
 		ASSERT_EQ (store->tombstone_map.at (nano::tables::unchecked).num_since_last_flush.load (), 1);
 	}
-#endif
 }
 }
 

nano/lib/errors.cpp

@@ -273,8 +273,6 @@ std::string nano::error_config_messages::message (int ev) const
 			return "Invalid configuration value";
 		case nano::error_config::missing_value:
 			return "Missing value in configuration";
-		case nano::error_config::rocksdb_enabled_but_not_supported:
-			return "RocksDB has been enabled, but the node has not been built with RocksDB support. Set the CMake flag -DNANO_ROCKSDB=ON";
 	}
 
 	return "Invalid error code";

nano/lib/errors.hpp

@@ -149,8 +149,7 @@ enum class error_config
 {
 	generic = 1,
 	invalid_value,
-	missing_value,
-	rocksdb_enabled_but_not_supported
+	missing_value
 };
 } // nano namespace
 

nano/nano_wallet/entry.cpp

@@ -87,13 +87,6 @@ int run_wallet (QApplication & application, int argc, char * const * argv, boost
 		error = read_wallet_config (wallet_config, data_path);
 	}
 
-#if !NANO_ROCKSDB
-	if (!error && config.node.rocksdb_config.enable)
-	{
-		error = nano::error_config::rocksdb_enabled_but_not_supported;
-	}
-#endif
-
 	if (!error)
 	{
 		nano::set_use_memory_pools (config.node.use_memory_pools);

nano/node/CMakeLists.txt

@@ -1,16 +1,9 @@
-if (NANO_ROCKSDB)
-	set (rocksdb_libs ${ROCKSDB_LIBRARIES} ${ZLIB_LIBRARIES})
-	set (rocksdb_sources rocksdb/rocksdb.hpp rocksdb/rocksdb.cpp rocksdb/rocksdb_iterator.hpp rocksdb/rocksdb_txn.hpp rocksdb/rocksdb_txn.cpp)
-endif ()
 
 if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
 	# No opencl
 elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
 	set (platform_sources plat/windows/openclapi.cpp)
 	set (psapi_lib Psapi)
-	if (NANO_ROCKSDB)
-		set (rocksdb_libs ${rocksdb_libs} Shlwapi Rpcrt4)
-	endif ()
 elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
 	set (platform_sources plat/posix/openclapi.cpp)
 elseif (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
@@ -21,7 +14,6 @@ endif ()
 
 add_library (node
 	${platform_sources}
-	${rocksdb_sources}
 	active_transactions.hpp
 	active_transactions.cpp
 	blockprocessor.hpp
@@ -121,6 +113,11 @@ add_library (node
 	repcrawler.cpp
 	request_aggregator.hpp
 	request_aggregator.cpp
+	rocksdb/rocksdb.hpp
+	rocksdb/rocksdb.cpp
+	rocksdb/rocksdb_iterator.hpp
+	rocksdb/rocksdb_txn.hpp
+	rocksdb/rocksdb_txn.cpp
 	signatures.hpp
 	signatures.cpp
 	socket.hpp
@@ -167,7 +164,7 @@ target_link_libraries (node
 	Boost::system
 	Boost::thread
 	Boost::boost
-	${rocksdb_libs}
+	rocksdb
 	${CMAKE_DL_LIBS}
 	${psapi_lib}
 	)

nano/node/cli.cpp

@@ -1239,14 +1239,7 @@ bool is_using_rocksdb (boost::filesystem::path const & data_path, std::error_cod
 	auto error = nano::read_node_config_toml (data_path, config);
 	if (!error)
 	{
-		bool use_rocksdb = config.node.rocksdb_config.enable;
-		if (use_rocksdb)
-		{
-#if !NANO_ROCKSDB
-			ec = nano::error_cli::database_write_error;
-#endif
-			return (NANO_ROCKSDB == 1);
-		}
+		return config.node.rocksdb_config.enable;
 	}
 	else
 	{

nano/node/node.cpp

@@ -4,16 +4,13 @@
 #include <nano/node/common.hpp>
 #include <nano/node/daemonconfig.hpp>
 #include <nano/node/node.hpp>
+#include <nano/node/rocksdb/rocksdb.hpp>
 #include <nano/node/telemetry.hpp>
 #include <nano/node/testing.hpp>
 #include <nano/node/websocket.hpp>
 #include <nano/rpc/rpc.hpp>
 #include <nano/secure/buffer.hpp>
 
-#if NANO_ROCKSDB
-#include <nano/node/rocksdb/rocksdb.hpp>
-#endif
-
 #include <boost/filesystem.hpp>
 #include <boost/property_tree/json_parser.hpp>
 
@@ -1706,30 +1703,9 @@ nano::node_flags const & nano::inactive_node_flag_defaults ()
 
 std::unique_ptr<nano::block_store> nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade, bool use_rocksdb_backend)
 {
-#if NANO_ROCKSDB
-	auto make_rocksdb = [&logger, add_db_postfix, &path, &rocksdb_config, read_only]() {
+	if (use_rocksdb_backend || using_rocksdb_in_tests ())
+	{
 		return std::make_unique<nano::rocksdb_store> (logger, add_db_postfix ? path / "rocksdb" : path, rocksdb_config, read_only);
-	};
-#endif
-
-	if (use_rocksdb_backend)
-	{
-#if NANO_ROCKSDB
-		return make_rocksdb ();
-#else
-		logger.always_log (std::error_code (nano::error_config::rocksdb_enabled_but_not_supported).message ());
-		release_assert (false);
-		return nullptr;
-#endif
-	}
-	else
-	{
-#if NANO_ROCKSDB
-		if (using_rocksdb_in_tests ())
-		{
-			return make_rocksdb ();
-		}
-#endif
 	}
 
 	return std::make_unique<nano::mdb_store> (logger, add_db_postfix ? path / "data.ldb" : path, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_config_a, backup_before_upgrade);

nano/node/testing.hpp

@@ -59,7 +59,7 @@ std::unique_ptr<nano::state_block> upgrade_epoch (nano::work_pool &, nano::ledge
 void blocks_confirm (nano::node &, std::vector<std::shared_ptr<nano::block>> const &);
 uint16_t get_available_port ();
 void cleanup_dev_directories_on_exit ();
-/** To use RocksDB in tests make sure the node is built with the cmake variable -DNANO_ROCKSDB=ON and the environment variable TEST_USE_ROCKSDB=1 is set */
+/** To use RocksDB in tests make sure the environment variable TEST_USE_ROCKSDB=1 is set */
 bool using_rocksdb_in_tests ();
 }
 REGISTER_ERROR_CODES (nano, error_system);

rocksdb

@@ -0,0 +1 @@
+Subproject commit 5daf2d724d1f67efa0764e5789c4e20d62f3c4b8

util/build_prep/fetch_rocksdb.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-OS=`uname`
-
-pushd /tmp
-wget -O rocksdb.tgz https://s3.us-east-2.amazonaws.com/repo.nano.org/artifacts/rocksdb-$OS-6.5.2.tgz
-
-tar -zxf rocksdb.tgz
-mv tmp/* .
-rm -fr tmp
-popd