From 52033b0906d25e58d7fa45af19392aa1721cbf11 Mon Sep 17 00:00:00 2001 From: Wesley Shillingford Date: Tue, 16 Jul 2019 15:02:40 +0100 Subject: [PATCH] Add rocksdb to build system (#2157) --- .travis.yml | 4 +-- CMakeLists.txt | 36 +++++++++++++++------------ appveyor.yml | 16 ++++++++---- ci/build-travis.sh | 1 + cmake/Modules/FindRocksDB.cmake | 44 +++++++++++++++++++++++++++++++++ docker/ci/Dockerfile-base | 2 +- docker/ci/Dockerfile-clang | 4 +-- docker/ci/Dockerfile-gcc | 3 ++- nano/core_test/CMakeLists.txt | 8 +++++- nano/core_test/rocksdb_test.cpp | 31 +++++++++++++++++++++++ nano/node/CMakeLists.txt | 8 ++++++ 11 files changed, 129 insertions(+), 28 deletions(-) create mode 100644 cmake/Modules/FindRocksDB.cmake create mode 100644 nano/core_test/rocksdb_test.cpp diff --git a/.travis.yml b/.travis.yml index cb97fdee..ece5f160 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ jobs: os: osx compiler: clang before_install: - - brew update && brew install qt5 && brew cask install xquartz && brew upgrade cmake; + - brew update && brew install qt5 && brew cask install xquartz && brew upgrade cmake && brew install rocksdb; install: - brew install ccache; - export PATH="/usr/local/opt/ccache/libexec:$PATH"; @@ -70,4 +70,4 @@ script: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then docker run -v $TRAVIS_BUILD_DIR:/workspace -v $HOME/.ccache:/ccache nanocurrency/nano-env:$TRAVIS_COMPILER /bin/bash -c "apt install ccache; cd /workspace && ASAN=${ASAN} TSAN=${TSAN} CCACHE_DIR=/ccache ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"; fi env: - - secure: "k8kmpD+xRS57ukdvlvaXT0WN4H0rr/aHSjV+l5IUUFpKx5N+DEsb+7ElIepKzqQrGG6qE71cFwDyn6rDwW/Objb9aiEITnvJBzk1XrOVgbc5AnlqDm8LKvqToD/VnQiojyXhBQe2wa//nEZ3PC9dv7hE5zb/K/U5z+LaE9T1cPPk1jHQMCUAFT7QGCK0YeX/gAZqPbLZdHHQChEi+Gu/XY0gc5Bl8Idbp8W7Aky9Ps06lKXPORkE1G2xQfJFrNPB3CKjxev/eoXGBJmNYzxkJlUHmyenjwgdDh9TWiOK2uKH1K6olLIx/qFuIgFRVJFv0QFzCjqqjOJJF1EN9i1M21Lm4wi1iJxYShAP86ZKkC/WmtRn1xNTEMHZJeZ3TXX+B3ybLEWTamHS1Ia8HOif18nrQE3O0aRC/NNfH1kewX+94UNxmSfHtL5Waa41shxeG5waemyQg+HR5zCEtrb5l1btgbfGrR8BMbHYLLe4ywJqMx3n8Iy6ZzC6Xx8+X1zTZZ3zDYPBHUalA+ZoYu2rrFG2+SARP0W/VKqCIKaB+lQKYpbv7ojXGrrDJe7MA/raTLL2pTfSkcx0qxJvcsbPLGI1MdG3mD7M8HncrZbw+sKI1LZ04gyWt3til6d3vSlbIkd6kCxxZh69nd1/KJy8rYrMYjqxxNSTctkGyVb2DtY=" \ No newline at end of file + - secure: "k8kmpD+xRS57ukdvlvaXT0WN4H0rr/aHSjV+l5IUUFpKx5N+DEsb+7ElIepKzqQrGG6qE71cFwDyn6rDwW/Objb9aiEITnvJBzk1XrOVgbc5AnlqDm8LKvqToD/VnQiojyXhBQe2wa//nEZ3PC9dv7hE5zb/K/U5z+LaE9T1cPPk1jHQMCUAFT7QGCK0YeX/gAZqPbLZdHHQChEi+Gu/XY0gc5Bl8Idbp8W7Aky9Ps06lKXPORkE1G2xQfJFrNPB3CKjxev/eoXGBJmNYzxkJlUHmyenjwgdDh9TWiOK2uKH1K6olLIx/qFuIgFRVJFv0QFzCjqqjOJJF1EN9i1M21Lm4wi1iJxYShAP86ZKkC/WmtRn1xNTEMHZJeZ3TXX+B3ybLEWTamHS1Ia8HOif18nrQE3O0aRC/NNfH1kewX+94UNxmSfHtL5Waa41shxeG5waemyQg+HR5zCEtrb5l1btgbfGrR8BMbHYLLe4ywJqMx3n8Iy6ZzC6Xx8+X1zTZZ3zDYPBHUalA+ZoYu2rrFG2+SARP0W/VKqCIKaB+lQKYpbv7ojXGrrDJe7MA/raTLL2pTfSkcx0qxJvcsbPLGI1MdG3mD7M8HncrZbw+sKI1LZ04gyWt3til6d3vSlbIkd6kCxxZh69nd1/KJy8rYrMYjqxxNSTctkGyVb2DtY=" diff --git a/CMakeLists.txt b/CMakeLists.txt index 9955ca5c..4abda7ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) set (NANO_GUI OFF CACHE BOOL "") set (NANO_TEST OFF CACHE BOOL "") set (NANO_SECURE_RPC OFF CACHE BOOL "") +set (NANO_ROCKSDB OFF CACHE BOOL "") option(NANO_ASAN_INT "Enable ASan+UBSan+Integer overflow" OFF) option(NANO_ASAN "Enable ASan+UBSan" OFF) @@ -152,11 +153,14 @@ include_directories (${CMAKE_SOURCE_DIR}) set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_MULTITHREADED ON) -if (BOOST_ROOT) - list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules") -endif () +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules") find_package (Boost 1.67.0 REQUIRED COMPONENTS filesystem log thread program_options system) +if (NANO_ROCKSDB) + find_package (RocksDB REQUIRED) + find_package (ZLIB REQUIRED) + include_directories (${ROCKSDB_INCLUDE_DIRS}) +endif () # There is a compile bug with boost 1.69 interprocess headers on Mac if (APPLE AND Boost_VERSION EQUAL 106900) @@ -301,21 +305,21 @@ add_subdirectory(nano/rpc) add_subdirectory(nano/nano_rpc) if (NANO_TEST OR RAIBLOCKS_TEST) - if(WIN32) - if(MSVC_VERSION) - if(MSVC_VERSION GREATER_EQUAL 1910) - add_definitions(-DGTEST_LANG_CXX11=1) - add_definitions(-DGTEST_HAS_TR1_TUPLE=0) - endif() - endif() - set (gtest_force_shared_crt ON) - else () - set (gtest_force_shared_crt OFF) - endif() + if(WIN32) + if(MSVC_VERSION) + if(MSVC_VERSION GREATER_EQUAL 1910) + add_definitions(-DGTEST_LANG_CXX11=1) + add_definitions(-DGTEST_HAS_TR1_TUPLE=0) + endif() + endif() + set (gtest_force_shared_crt ON) + else () + set (gtest_force_shared_crt OFF) +endif() - add_subdirectory(nano/load_test) + add_subdirectory(nano/load_test) - add_subdirectory (gtest/googletest) + add_subdirectory (gtest/googletest) # FIXME: This fixes gtest include directories without modifying gtest's # CMakeLists.txt. Ideally we should use GTest::GTest and GTest::Main as # dependencies but it requires building gtest differently diff --git a/appveyor.yml b/appveyor.yml index 57533a93..74723c59 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,19 +7,25 @@ image: Visual Studio 2017 configuration: Release platform: x64 environment: + VCPKG_DIR: C:\Tools\vcpkg matrix: - network: nano_live_network configuration: Release - network: nano_beta_network configuration: RelWithDebInfo clone_folder: C:\projects\myproject +cache: $VCPKG_DIR%\installed\ install: -- cmd: >- - git submodule update --init --recursive + - git submodule update --init --recursive + - echo set (VCPKG_LIBRARY_LINKAGE static) > temp.txt + - type %VCPKG_DIR%\ports\rocksdb\portfile.cmake >> temp.txt + - type temp.txt > %VCPKG_DIR%\ports\rocksdb\portfile.cmake + - del temp.txt + - echo set(VCPKG_BUILD_TYPE release) >> C:\Tools\vcpkg\triplets\x64-windows.cmake + - vcpkg install rocksdb:%PLATFORM%-windows + - cmake -DNANO_GUI=ON -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DACTIVE_NETWORK=%NETWORK% -DQt5_DIR="C:\Qt\5.9\msvc2017_64\lib\cmake\Qt5" -DNANO_SIMD_OPTIMIZATIONS=TRUE -DBoost_COMPILER="-vc141" -DBOOST_ROOT="C:/Libraries/boost_1_67_0" -DBOOST_LIBRARYDIR="C:/Libraries/boost_1_67_0/lib64-msvc-14.1" -G "Visual Studio 15 2017 Win64" -DNANO_ROCKSDB=ON -DROCKSDB_LIBRARIES="%VCPKG_DIR%\installed\x64-windows\lib\rocksdb.lib" -DROCKSDB_INCLUDE_DIRS="%VCPKG_DIR%\installed\x64_windows\include" -DZLIB_LIBRARY=%VCPKG_DIR%\installed\x64-windows\lib\zlib.lib -DZLIB_INCLUDE_DIR=%VCPKG_DIR%\installed\x64-windows\include -DIPHLPAPI_LIBRARY="C:/Program Files (x86)/Windows Kits/10/Lib/10.0.14393.0/um/x64/iphlpapi.lib" -DWINSOCK2_LIBRARY="C:/Program Files (x86)/Windows Kits/10/Lib/10.0.14393.0/um/x64/WS2_32.lib" . - cmake -DNANO_GUI=ON -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DACTIVE_NETWORK=%NETWORK% -DQt5_DIR="C:\Qt\5.9\msvc2017_64\lib\cmake\Qt5" -DNANO_SIMD_OPTIMIZATIONS=TRUE -DBoost_COMPILER="-vc141" -DBOOST_ROOT="C:/Libraries/boost_1_67_0" -DBOOST_LIBRARYDIR="C:/Libraries/boost_1_67_0/lib64-msvc-14.1" -G "Visual Studio 15 2017 Win64" -DIPHLPAPI_LIBRARY="C:/Program Files (x86)/Windows Kits/10/Lib/10.0.14393.0/um/x64/iphlpapi.lib" -DWINSOCK2_LIBRARY="C:/Program Files (x86)/Windows Kits/10/Lib/10.0.14393.0/um/x64/WS2_32.lib" . - -- ps: Invoke-WebRequest -Uri https://aka.ms/vs/15/release/vc_redist.x64.exe -OutFile .\vc_redist.x64.exe + - ps: Invoke-WebRequest -Uri https://aka.ms/vs/15/release/vc_redist.x64.exe -OutFile .\vc_redist.x64.exe build: project: INSTALL.vcxproj parallel: true diff --git a/ci/build-travis.sh b/ci/build-travis.sh index 76b0b44a..ccc3b069 100755 --- a/ci/build-travis.sh +++ b/ci/build-travis.sh @@ -26,6 +26,7 @@ cmake \ -DACTIVE_NETWORK=nano_test_network \ -DNANO_TEST=ON \ -DNANO_GUI=ON \ + -DNANO_ROCKSDB=ON \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DBOOST_ROOT=/usr/local \ diff --git a/cmake/Modules/FindRocksDB.cmake b/cmake/Modules/FindRocksDB.cmake new file mode 100644 index 00000000..1bbfd599 --- /dev/null +++ b/cmake/Modules/FindRocksDB.cmake @@ -0,0 +1,44 @@ +# Try to find RocksDB headers and library. +# +# Usage of this module as follows: +# +# find_package(RocksDB) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# ROCKSDB_ROOT_DIR Set this variable to the root installation of +# RocksDB if the module has problems finding the +# proper installation path. +# +# Variables defined by this module: +# +# ROCKSDB_FOUND System has RocksDB library/headers. +# ROCKSDB_LIBRARIES The RocksDB library. +# ROCKSDB_INCLUDE_DIRS The location of RocksDB headers. + +find_path(ROCKSDB_ROOT_DIR + NAMES include/rocksdb/db.h +) + +find_library(ROCKSDB_LIBRARIES + NAMES rocksdb + HINTS ${ROCKSDB_ROOT_DIR}/lib +) + +find_path(ROCKSDB_INCLUDE_DIRS + NAMES rocksdb/db.h + HINTS ${ROCKSDB_ROOT_DIR}/include +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(RocksDB DEFAULT_MSG + ROCKSDB_LIBRARIES + ROCKSDB_INCLUDE_DIRS +) + +mark_as_advanced( + ROCKSDB_ROOT_DIR + ROCKSDB_LIBRARIES + ROCKSDB_INCLUDE_DIRS +) diff --git a/docker/ci/Dockerfile-base b/docker/ci/Dockerfile-base index 55fb6166..0afda04a 100644 --- a/docker/ci/Dockerfile-base +++ b/docker/ci/Dockerfile-base @@ -12,4 +12,4 @@ RUN apt-get update -qq && apt-get install -yqq \ RUN apt-get update -qq && apt-get install -yqq \ qt5-default \ valgrind \ - xorg xvfb xauth xfonts-100dpi xfonts-75dpi xfonts-scalable xfonts-cyrillic \ No newline at end of file + xorg xvfb xauth xfonts-100dpi xfonts-75dpi xfonts-scalable xfonts-cyrillic diff --git a/docker/ci/Dockerfile-clang b/docker/ci/Dockerfile-clang index cc0641fb..2ff79eda 100644 --- a/docker/ci/Dockerfile-clang +++ b/docker/ci/Dockerfile-clang @@ -1,7 +1,7 @@ FROM nanocurrency/nano-env:base RUN apt-get update -qq && apt-get install -yqq \ - clang-3.9 lldb-3.9 + clang-3.9 lldb-3.9 librocksdb-dev ENV CXX=/usr/bin/clang++ ENV CC=/usr/bin/clang @@ -14,4 +14,4 @@ ADD util/build_prep/bootstrap_boost.sh bootstrap_boost.sh RUN ./bootstrap_boost.sh -m -c -RUN rm bootstrap_boost.sh \ No newline at end of file +RUN rm bootstrap_boost.sh diff --git a/docker/ci/Dockerfile-gcc b/docker/ci/Dockerfile-gcc index 62e3cc66..adf956f3 100644 --- a/docker/ci/Dockerfile-gcc +++ b/docker/ci/Dockerfile-gcc @@ -1,7 +1,8 @@ FROM nanocurrency/nano-env:base +RUN apt-get install -yqq librocksdb-dev + ENV BOOST_ROOT=/usr/local ADD util/build_prep/bootstrap_boost.sh bootstrap_boost.sh RUN ./bootstrap_boost.sh -m RUN rm bootstrap_boost.sh - diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index 89bd18c0..f0eaf92c 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -1,3 +1,8 @@ +if (NANO_ROCKSDB) + set (rocksdb_test rocksdb_test.cpp) + set (rocksdb_libs ${ROCKSDB_LIBRARIES} ${ZLIB_LIBRARIES}) +endif () + add_executable (core_test core_test_main.cc testutil.hpp @@ -18,6 +23,7 @@ add_executable (core_test memory_pool.cpp processor_service.cpp peer_container.cpp + ${rocksdb_test} signing.cpp socket.cpp timer.cpp @@ -34,4 +40,4 @@ target_compile_definitions(core_test -DNANO_VERSION_MINOR=${CPACK_PACKAGE_VERSION_MINOR} -DNANO_VERSION_PATCH=${CPACK_PACKAGE_VERSION_PATCH} -DBOOST_PROCESS_SUPPORTED=${BOOST_PROCESS_SUPPORTED}) -target_link_libraries (core_test node secure gtest libminiupnpc-static Boost::boost) +target_link_libraries (core_test node secure gtest libminiupnpc-static Boost::boost ${rocksdb_libs}) diff --git a/nano/core_test/rocksdb_test.cpp b/nano/core_test/rocksdb_test.cpp new file mode 100644 index 00000000..0c578398 --- /dev/null +++ b/nano/core_test/rocksdb_test.cpp @@ -0,0 +1,31 @@ +#include + +#include + +#include + +#include +#include +#include +#include +#include + +TEST (rocksdb, build_test) +{ + auto path = nano::unique_path (); + + std::vector column_families; + column_families.push_back (rocksdb::ColumnFamilyDescriptor ( + rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions ())); + + rocksdb::Options options; + options.create_if_missing = true; + options.IncreaseParallelism (std::thread::hardware_concurrency ()); + options.OptimizeLevelStyleCompaction (); + options.OptimizeUniversalStyleCompaction (); + + std::vector handles; + rocksdb::OptimisticTransactionDB * db; + auto s = rocksdb::OptimisticTransactionDB::Open (options, path.string (), column_families, &handles, &db); + ASSERT_TRUE (s.ok ()); +} diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index 204995f9..f4b30140 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -1,8 +1,15 @@ +if (NANO_ROCKSDB) + set (rocksdb_libs ${ROCKSDB_LIBRARIES} ${ZLIB_LIBRARIES}) +endif () + if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") # No opencl elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows") set (platform_sources plat/windows/openclapi.cpp) set (psapi_lib Psapi) + if (NANO_ROCKSDB) + set (rocksdb_libs ${rocksdb_libs} Shlwapi Rpcrt4) + endif () elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux") set (platform_sources plat/posix/openclapi.cpp) elseif (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") @@ -110,6 +117,7 @@ target_link_libraries (node Boost::system Boost::thread Boost::boost + ${rocksdb_libs} ${CMAKE_DL_LIBS} ${psapi_lib} )