Add rocksdb to build system (#2157)

2019-07-16 15:02:40 +01:00 · 2019-07-16 15:02:40 +01:00 · 52033b0906
commit 52033b0906
parent 0bbc02b9fa
11 changed files with 129 additions and 28 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -46,7 +46,7 @@ jobs:
      os: osx
      compiler: clang
      before_install:
-        - brew update && brew install qt5 && brew cask install xquartz && brew upgrade cmake;
+        - brew update && brew install qt5 && brew cask install xquartz && brew upgrade cmake && brew install rocksdb;
      install:
        - brew install ccache;
        - export PATH="/usr/local/opt/ccache/libexec:$PATH";
@ -70,4 +70,4 @@ script:
  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then docker run -v $TRAVIS_BUILD_DIR:/workspace -v $HOME/.ccache:/ccache nanocurrency/nano-env:$TRAVIS_COMPILER /bin/bash -c "apt install ccache; cd /workspace && ASAN=${ASAN} TSAN=${TSAN} CCACHE_DIR=/ccache ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"; fi

 env: 
-  - secure: "k8kmpD+xRS57ukdvlvaXT0WN4H0rr/aHSjV+l5IUUFpKx5N+DEsb+7ElIepKzqQrGG6qE71cFwDyn6rDwW/Objb9aiEITnvJBzk1XrOVgbc5AnlqDm8LKvqToD/VnQiojyXhBQe2wa//nEZ3PC9dv7hE5zb/K/U5z+LaE9T1cPPk1jHQMCUAFT7QGCK0YeX/gAZqPbLZdHHQChEi+Gu/XY0gc5Bl8Idbp8W7Aky9Ps06lKXPORkE1G2xQfJFrNPB3CKjxev/eoXGBJmNYzxkJlUHmyenjwgdDh9TWiOK2uKH1K6olLIx/qFuIgFRVJFv0QFzCjqqjOJJF1EN9i1M21Lm4wi1iJxYShAP86ZKkC/WmtRn1xNTEMHZJeZ3TXX+B3ybLEWTamHS1Ia8HOif18nrQE3O0aRC/NNfH1kewX+94UNxmSfHtL5Waa41shxeG5waemyQg+HR5zCEtrb5l1btgbfGrR8BMbHYLLe4ywJqMx3n8Iy6ZzC6Xx8+X1zTZZ3zDYPBHUalA+ZoYu2rrFG2+SARP0W/VKqCIKaB+lQKYpbv7ojXGrrDJe7MA/raTLL2pTfSkcx0qxJvcsbPLGI1MdG3mD7M8HncrZbw+sKI1LZ04gyWt3til6d3vSlbIkd6kCxxZh69nd1/KJy8rYrMYjqxxNSTctkGyVb2DtY=" 
+  - secure: "k8kmpD+xRS57ukdvlvaXT0WN4H0rr/aHSjV+l5IUUFpKx5N+DEsb+7ElIepKzqQrGG6qE71cFwDyn6rDwW/Objb9aiEITnvJBzk1XrOVgbc5AnlqDm8LKvqToD/VnQiojyXhBQe2wa//nEZ3PC9dv7hE5zb/K/U5z+LaE9T1cPPk1jHQMCUAFT7QGCK0YeX/gAZqPbLZdHHQChEi+Gu/XY0gc5Bl8Idbp8W7Aky9Ps06lKXPORkE1G2xQfJFrNPB3CKjxev/eoXGBJmNYzxkJlUHmyenjwgdDh9TWiOK2uKH1K6olLIx/qFuIgFRVJFv0QFzCjqqjOJJF1EN9i1M21Lm4wi1iJxYShAP86ZKkC/WmtRn1xNTEMHZJeZ3TXX+B3ybLEWTamHS1Ia8HOif18nrQE3O0aRC/NNfH1kewX+94UNxmSfHtL5Waa41shxeG5waemyQg+HR5zCEtrb5l1btgbfGrR8BMbHYLLe4ywJqMx3n8Iy6ZzC6Xx8+X1zTZZ3zDYPBHUalA+ZoYu2rrFG2+SARP0W/VKqCIKaB+lQKYpbv7ojXGrrDJe7MA/raTLL2pTfSkcx0qxJvcsbPLGI1MdG3mD7M8HncrZbw+sKI1LZ04gyWt3til6d3vSlbIkd6kCxxZh69nd1/KJy8rYrMYjqxxNSTctkGyVb2DtY=" 
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -13,6 +13,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 set (NANO_GUI OFF CACHE BOOL "")
 set (NANO_TEST OFF CACHE BOOL "")
 set (NANO_SECURE_RPC OFF CACHE BOOL "")
+set (NANO_ROCKSDB OFF CACHE BOOL "")

 option(NANO_ASAN_INT "Enable ASan+UBSan+Integer overflow" OFF)
 option(NANO_ASAN "Enable ASan+UBSan" OFF)
@ -152,11 +153,14 @@ include_directories (${CMAKE_SOURCE_DIR})
 set(Boost_USE_STATIC_LIBS        ON)
 set(Boost_USE_MULTITHREADED      ON)

-if (BOOST_ROOT)
-	list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules")
-endif ()
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules")

 find_package (Boost 1.67.0 REQUIRED COMPONENTS filesystem log thread program_options system)
+if (NANO_ROCKSDB)
+	find_package (RocksDB REQUIRED)
+	find_package (ZLIB REQUIRED)
+	include_directories (${ROCKSDB_INCLUDE_DIRS})
+endif ()

 # There is a compile bug with boost 1.69 interprocess headers on Mac
 if (APPLE AND Boost_VERSION EQUAL 106900)
@ -301,21 +305,21 @@ add_subdirectory(nano/rpc)
 add_subdirectory(nano/nano_rpc)

 if (NANO_TEST OR RAIBLOCKS_TEST)
-    if(WIN32)
-        if(MSVC_VERSION)
-            if(MSVC_VERSION GREATER_EQUAL 1910)
-                add_definitions(-DGTEST_LANG_CXX11=1)
-                add_definitions(-DGTEST_HAS_TR1_TUPLE=0)
-            endif()
-        endif()
-        set (gtest_force_shared_crt ON)
-    else ()
-        set (gtest_force_shared_crt OFF)
-    endif()
+	if(WIN32)
+		if(MSVC_VERSION)
+			if(MSVC_VERSION GREATER_EQUAL 1910)
+				add_definitions(-DGTEST_LANG_CXX11=1)
+				add_definitions(-DGTEST_HAS_TR1_TUPLE=0)
+			endif()
+		endif()
+		set (gtest_force_shared_crt ON)
+	else ()
+		set (gtest_force_shared_crt OFF)
+endif()

-    add_subdirectory(nano/load_test)
+	add_subdirectory(nano/load_test)

-    add_subdirectory (gtest/googletest)
+	add_subdirectory (gtest/googletest)
 	# FIXME: This fixes gtest include directories without modifying gtest's
 	# CMakeLists.txt. Ideally we should use GTest::GTest and GTest::Main as
 	# dependencies but it requires building gtest differently
--- a/appveyor.yml
+++ b/appveyor.yml
@ -7,19 +7,25 @@ image: Visual Studio 2017
 configuration: Release
 platform: x64
 environment:
+  VCPKG_DIR: C:\Tools\vcpkg
  matrix:
    - network: nano_live_network
      configuration: Release
    - network: nano_beta_network
      configuration: RelWithDebInfo
 clone_folder: C:\projects\myproject
+cache: $VCPKG_DIR%\installed\
 install:
- cmd: >-
-    git submodule update --init --recursive
+  - git submodule update --init --recursive
+  - echo set (VCPKG_LIBRARY_LINKAGE static)  > temp.txt
+  - type %VCPKG_DIR%\ports\rocksdb\portfile.cmake >> temp.txt
+  - type temp.txt > %VCPKG_DIR%\ports\rocksdb\portfile.cmake
+  - del temp.txt
+  - echo set(VCPKG_BUILD_TYPE release) >> C:\Tools\vcpkg\triplets\x64-windows.cmake
+  - vcpkg install rocksdb:%PLATFORM%-windows
+  - cmake -DNANO_GUI=ON -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DACTIVE_NETWORK=%NETWORK% -DQt5_DIR="C:\Qt\5.9\msvc2017_64\lib\cmake\Qt5" -DNANO_SIMD_OPTIMIZATIONS=TRUE -DBoost_COMPILER="-vc141" -DBOOST_ROOT="C:/Libraries/boost_1_67_0" -DBOOST_LIBRARYDIR="C:/Libraries/boost_1_67_0/lib64-msvc-14.1" -G "Visual Studio 15 2017 Win64" -DNANO_ROCKSDB=ON -DROCKSDB_LIBRARIES="%VCPKG_DIR%\installed\x64-windows\lib\rocksdb.lib" -DROCKSDB_INCLUDE_DIRS="%VCPKG_DIR%\installed\x64_windows\include" -DZLIB_LIBRARY=%VCPKG_DIR%\installed\x64-windows\lib\zlib.lib -DZLIB_INCLUDE_DIR=%VCPKG_DIR%\installed\x64-windows\include -DIPHLPAPI_LIBRARY="C:/Program Files (x86)/Windows Kits/10/Lib/10.0.14393.0/um/x64/iphlpapi.lib" -DWINSOCK2_LIBRARY="C:/Program Files (x86)/Windows Kits/10/Lib/10.0.14393.0/um/x64/WS2_32.lib" .

-    cmake -DNANO_GUI=ON -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DACTIVE_NETWORK=%NETWORK% -DQt5_DIR="C:\Qt\5.9\msvc2017_64\lib\cmake\Qt5" -DNANO_SIMD_OPTIMIZATIONS=TRUE -DBoost_COMPILER="-vc141" -DBOOST_ROOT="C:/Libraries/boost_1_67_0" -DBOOST_LIBRARYDIR="C:/Libraries/boost_1_67_0/lib64-msvc-14.1" -G "Visual Studio 15 2017 Win64" -DIPHLPAPI_LIBRARY="C:/Program Files (x86)/Windows Kits/10/Lib/10.0.14393.0/um/x64/iphlpapi.lib" -DWINSOCK2_LIBRARY="C:/Program Files (x86)/Windows Kits/10/Lib/10.0.14393.0/um/x64/WS2_32.lib" .
-
- ps: Invoke-WebRequest -Uri https://aka.ms/vs/15/release/vc_redist.x64.exe -OutFile .\vc_redist.x64.exe
+  - ps: Invoke-WebRequest -Uri https://aka.ms/vs/15/release/vc_redist.x64.exe -OutFile .\vc_redist.x64.exe
 build:
  project: INSTALL.vcxproj
  parallel: true
--- a/ci/build-travis.sh
+++ b/ci/build-travis.sh
@ -26,6 +26,7 @@ cmake \
    -DACTIVE_NETWORK=nano_test_network \
    -DNANO_TEST=ON \
    -DNANO_GUI=ON \
+    -DNANO_ROCKSDB=ON \
    -DCMAKE_BUILD_TYPE=Debug \
    -DCMAKE_VERBOSE_MAKEFILE=ON \
    -DBOOST_ROOT=/usr/local \
--- a/cmake/Modules/FindRocksDB.cmake
+++ b/cmake/Modules/FindRocksDB.cmake
@ -0,0 +1,44 @@
+# Try to find RocksDB headers and library.
+#
+# Usage of this module as follows:
+#
+#     find_package(RocksDB)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  ROCKSDB_ROOT_DIR          Set this variable to the root installation of
+#                            RocksDB if the module has problems finding the
+#                            proper installation path.
+#
+# Variables defined by this module:
+#
+#  ROCKSDB_FOUND               System has RocksDB library/headers.
+#  ROCKSDB_LIBRARIES           The RocksDB library.
+#  ROCKSDB_INCLUDE_DIRS        The location of RocksDB headers.
+
+find_path(ROCKSDB_ROOT_DIR
+    NAMES include/rocksdb/db.h
+)
+
+find_library(ROCKSDB_LIBRARIES
+    NAMES rocksdb
+    HINTS ${ROCKSDB_ROOT_DIR}/lib
+)
+
+find_path(ROCKSDB_INCLUDE_DIRS
+    NAMES rocksdb/db.h
+    HINTS ${ROCKSDB_ROOT_DIR}/include
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(RocksDB DEFAULT_MSG
+    ROCKSDB_LIBRARIES
+    ROCKSDB_INCLUDE_DIRS
+)
+
+mark_as_advanced(
+    ROCKSDB_ROOT_DIR
+    ROCKSDB_LIBRARIES
+    ROCKSDB_INCLUDE_DIRS
+)
--- a/docker/ci/Dockerfile-base
+++ b/docker/ci/Dockerfile-base
@ -12,4 +12,4 @@ RUN apt-get update -qq && apt-get install -yqq \
 RUN apt-get update -qq && apt-get install -yqq \
    qt5-default \
    valgrind \
-    xorg xvfb xauth xfonts-100dpi xfonts-75dpi xfonts-scalable xfonts-cyrillic
+    xorg xvfb xauth xfonts-100dpi xfonts-75dpi xfonts-scalable xfonts-cyrillic
--- a/docker/ci/Dockerfile-clang
+++ b/docker/ci/Dockerfile-clang
@ -1,7 +1,7 @@
 FROM nanocurrency/nano-env:base

 RUN apt-get update -qq && apt-get install -yqq \
-    clang-3.9 lldb-3.9
+    clang-3.9 lldb-3.9 librocksdb-dev

 ENV CXX=/usr/bin/clang++
 ENV CC=/usr/bin/clang
@ -14,4 +14,4 @@ ADD util/build_prep/bootstrap_boost.sh bootstrap_boost.sh

 RUN ./bootstrap_boost.sh -m -c

-RUN rm bootstrap_boost.sh
+RUN rm bootstrap_boost.sh
--- a/docker/ci/Dockerfile-gcc
+++ b/docker/ci/Dockerfile-gcc
@ -1,7 +1,8 @@
 FROM nanocurrency/nano-env:base

+RUN apt-get install -yqq librocksdb-dev
+
 ENV BOOST_ROOT=/usr/local
 ADD util/build_prep/bootstrap_boost.sh bootstrap_boost.sh
 RUN ./bootstrap_boost.sh -m
 RUN rm bootstrap_boost.sh
-
--- a/nano/core_test/CMakeLists.txt
+++ b/nano/core_test/CMakeLists.txt
@ -1,3 +1,8 @@
+if (NANO_ROCKSDB)
+  set (rocksdb_test rocksdb_test.cpp)
+  set (rocksdb_libs ${ROCKSDB_LIBRARIES} ${ZLIB_LIBRARIES})
+endif ()
+
 add_executable (core_test
 	core_test_main.cc
 	testutil.hpp
@ -18,6 +23,7 @@ add_executable (core_test
 	memory_pool.cpp
 	processor_service.cpp
 	peer_container.cpp
+	${rocksdb_test}
 	signing.cpp
 	socket.cpp
 	timer.cpp
@ -34,4 +40,4 @@ target_compile_definitions(core_test
 			-DNANO_VERSION_MINOR=${CPACK_PACKAGE_VERSION_MINOR}
 			-DNANO_VERSION_PATCH=${CPACK_PACKAGE_VERSION_PATCH}
 			-DBOOST_PROCESS_SUPPORTED=${BOOST_PROCESS_SUPPORTED})
-target_link_libraries (core_test node secure gtest libminiupnpc-static Boost::boost)
+target_link_libraries (core_test node secure gtest libminiupnpc-static Boost::boost ${rocksdb_libs})
--- a/nano/core_test/rocksdb_test.cpp
+++ b/nano/core_test/rocksdb_test.cpp
@ -0,0 +1,31 @@
+#include <nano/secure/utility.hpp>
+
+#include <gtest/gtest.h>
+
+#include <thread>
+
+#include <rocksdb/db.h>
+#include <rocksdb/options.h>
+#include <rocksdb/utilities/optimistic_transaction_db.h>
+#include <rocksdb/utilities/transaction.h>
+#include <rocksdb/utilities/transaction_db.h>
+
+TEST (rocksdb, build_test)
+{
+	auto path = nano::unique_path ();
+
+	std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
+	column_families.push_back (rocksdb::ColumnFamilyDescriptor (
+	rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions ()));
+
+	rocksdb::Options options;
+	options.create_if_missing = true;
+	options.IncreaseParallelism (std::thread::hardware_concurrency ());
+	options.OptimizeLevelStyleCompaction ();
+	options.OptimizeUniversalStyleCompaction ();
+
+	std::vector<rocksdb::ColumnFamilyHandle *> handles;
+	rocksdb::OptimisticTransactionDB * db;
+	auto s = rocksdb::OptimisticTransactionDB::Open (options, path.string (), column_families, &handles, &db);
+	ASSERT_TRUE (s.ok ());
+}
--- a/nano/node/CMakeLists.txt
+++ b/nano/node/CMakeLists.txt
@ -1,8 +1,15 @@
+if (NANO_ROCKSDB)
+	set (rocksdb_libs ${ROCKSDB_LIBRARIES} ${ZLIB_LIBRARIES})
+endif ()
+
 if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
 	# No opencl
 elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
 	set (platform_sources plat/windows/openclapi.cpp)
 	set (psapi_lib Psapi)
+	if (NANO_ROCKSDB)
+		set (rocksdb_libs ${rocksdb_libs} Shlwapi Rpcrt4)
+	endif ()
 elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
 	set (platform_sources plat/posix/openclapi.cpp)
 elseif (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
@ -110,6 +117,7 @@ target_link_libraries (node
 	Boost::system
 	Boost::thread
 	Boost::boost
+	${rocksdb_libs}
 	${CMAKE_DL_LIBS}
 	${psapi_lib}
 	)