From 98d6f3d8e1832597903172441f16c06dacbf3f60 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Tue, 6 Feb 2018 19:23:13 -0700 Subject: [PATCH] Docker deploy (for real this time) * Run ci/clang-format-all.sh * Build and deploy Docker image in Travis * Quiet doxygen --- .travis.yml | 7 +++++-- ci/build-docker-image.sh | 8 ++++++++ ci/build-travis.sh | 6 ------ ci/custom-timeout.sh | 9 +++++++++ ci/deploy-docker.sh | 13 +++++++++++++ docker/ci/Dockerfile | 10 +++++----- docker/node/Dockerfile | 8 ++++---- doxygen.config | 4 ++-- rai/core_test/block_store.cpp | 6 +++--- rai/core_test/node.cpp | 6 +++--- rai/secure.cpp | 31 +++++++++++++++---------------- rai/secure.hpp | 2 +- 12 files changed, 68 insertions(+), 42 deletions(-) create mode 100755 ci/build-docker-image.sh create mode 100755 ci/custom-timeout.sh create mode 100755 ci/deploy-docker.sh diff --git a/.travis.yml b/.travis.yml index 681ab2b8..a2edeb98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,10 +34,13 @@ compiler: before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo mkdir -p /etc/docker && echo '{"ipv6":true,"fixed-cidr-v6":"2001:db8:1::/64"}' | sudo tee /etc/docker/daemon.json && sudo service docker restart; fi - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew update && brew install qt5 && brew cask install xquartz && brew upgrade boost && brew install rust; fi - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then docker pull lukealonso/raiblocks-ci:latest; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then ci/build-docker-image.sh docker/ci/Dockerfile nanocurrency/nano-ci; fi script: - if [ -n "$ONE_TIME_TESTS" ]; then ci/check-commit-format.sh; fi - if [ -n "$ONE_TIME_TESTS" ]; then doxygen doxygen.config; fi # TODO also deploy the built HTML - if [ "$TRAVIS_OS_NAME" = "osx" ]; then ci/build-travis.sh "/usr/local/opt/qt5/lib/cmake/Qt5"; fi - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then docker run -v $TRAVIS_BUILD_DIR:/workspace lukealonso/raiblocks-ci:latest /bin/bash -c "cd /workspace && ASAN=${ASAN} TSAN=${TSAN} ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then docker run -v $TRAVIS_BUILD_DIR:/workspace nanocurrency/nano-ci /bin/bash -c "cd /workspace && ASAN=${ASAN} TSAN=${TSAN} ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"; fi + +after_script: + - if [ -n "$ONE_TIME_TESTS" ] && [ -n "$DOCKER_PASSWORD" ] && [ "$TRAVIS_BRANCH" = "master" ]; then ci/deploy-docker.sh; fi diff --git a/ci/build-docker-image.sh b/ci/build-docker-image.sh new file mode 100755 index 00000000..e5e34488 --- /dev/null +++ b/ci/build-docker-image.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -eu + +scripts="$(dirname "$0")" + +"$scripts"/custom-timeout.sh 20 docker pull "$2" || true +echo "Building $2" +"$scripts"/custom-timeout.sh 30 docker build -f "$1" -t "$2" --cache-from "$2" . diff --git a/ci/build-travis.sh b/ci/build-travis.sh index 032c5422..6189c723 100755 --- a/ci/build-travis.sh +++ b/ci/build-travis.sh @@ -48,12 +48,6 @@ else TRUE_CMD=true fi -if ! cargo --version &>/dev/null; then - # We'll update the docker image once this PR gets merged. - # If you're reading this comment on master, contact @PlasmaPower - apt-get update && apt-get install -yq cargo -fi - pushd load-tester cargo build --release popd diff --git a/ci/custom-timeout.sh b/ci/custom-timeout.sh new file mode 100755 index 00000000..69006474 --- /dev/null +++ b/ci/custom-timeout.sh @@ -0,0 +1,9 @@ +#!/bin/bash +delay="$1" +shift +while true; do + printf ".\b" + sleep 120 +done & +timeout -k 5 "$(( $delay * 60 ))" "$@" +exit "$?" diff --git a/ci/deploy-docker.sh b/ci/deploy-docker.sh new file mode 100755 index 00000000..21930b54 --- /dev/null +++ b/ci/deploy-docker.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -eu + +scripts="$(dirname "$0")" + +docker login -u nanocurrency -p "$DOCKER_PASSWORD" + +# We push this just so it can be a cache next time +"$scripts"/custom-timeout.sh 30 docker push nanocurrency/nano-ci + +# We don't need to build this unless we're deploying it +ci/build-docker-image.sh docker/node/Dockerfile nanocurrency/nano +"$scripts"/custom-timeout.sh 30 docker push nanocurrency/nano diff --git a/docker/ci/Dockerfile b/docker/ci/Dockerfile index 9eb59e85..4bd6065a 100644 --- a/docker/ci/Dockerfile +++ b/docker/ci/Dockerfile @@ -1,9 +1,9 @@ FROM ubuntu:16.04 ENV BOOST_BASENAME=boost_1_66_0 \ - BOOST_URL=http://sourceforge.net/projects/boost/files/boost/1.66.0/boost_1_66_0.tar.gz/download + BOOST_URL=https://sourceforge.net/projects/boost/files/boost/1.66.0/boost_1_66_0.tar.gz/download -RUN apt-get update && apt-get install -yq \ +RUN apt-get update -qq && apt-get install -yqq \ build-essential \ cmake \ g++ \ @@ -11,11 +11,11 @@ RUN apt-get update && apt-get install -yq \ WORKDIR "/tmp" -RUN wget -O ${BOOST_BASENAME}.tar.gz ${BOOST_URL} && \ - tar xzvf ${BOOST_BASENAME}.tar.gz && \ +RUN wget -qO ${BOOST_BASENAME}.tar.gz ${BOOST_URL} && \ + tar xzf ${BOOST_BASENAME}.tar.gz && \ cd ${BOOST_BASENAME} && \ ./bootstrap.sh && \ - ./b2 link=static install && \ + ./b2 -d0 link=static install && \ rm -rf ${BOOST_BASENAME} && \ rm -f ${BOOST_BASENAME}.tar.gz && \ cd .. diff --git a/docker/node/Dockerfile b/docker/node/Dockerfile index 0f589da1..ff7467f4 100644 --- a/docker/node/Dockerfile +++ b/docker/node/Dockerfile @@ -4,16 +4,16 @@ ENV BOOST_BASENAME=boost_1_66_0 \ BOOST_ROOT=/tmp/boost_install \ BOOST_URL=https://sourceforge.net/projects/boost/files/boost/1.66.0/boost_1_66_0.tar.gz/download -RUN apt-get update && apt-get install -yq \ +RUN apt-get update -qq && apt-get install -yqq \ build-essential \ cmake \ g++ \ wget && \ - wget -O ${BOOST_BASENAME}.tar.gz ${BOOST_URL} && \ - tar xzvf ${BOOST_BASENAME}.tar.gz && \ + wget -qO ${BOOST_BASENAME}.tar.gz ${BOOST_URL} && \ + tar xzf ${BOOST_BASENAME}.tar.gz && \ cd ${BOOST_BASENAME} && \ ./bootstrap.sh && \ - ./b2 --prefix=${BOOST_ROOT} link=static install && \ + ./b2 -d0 --prefix=${BOOST_ROOT} link=static install && \ rm -rf ${BOOST_BASENAME} && \ rm -f ${BOOST_BASENAME}.tar.gz && \ cd .. && \ diff --git a/doxygen.config b/doxygen.config index a85b121e..92e72fb0 100644 --- a/doxygen.config +++ b/doxygen.config @@ -725,7 +725,7 @@ CITE_BIB_FILES = # messages are off. # The default value is: NO. -QUIET = NO +QUIET = YES # The WARNINGS tag can be used to turn on/off the warning messages that are # generated to standard error (stderr) by doxygen. If WARNINGS is set to YES @@ -741,7 +741,7 @@ WARNINGS = YES # will automatically be disabled. # The default value is: YES. -WARN_IF_UNDOCUMENTED = YES +WARN_IF_UNDOCUMENTED = NO # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for # potential errors in the documentation, such as not documenting some parameters diff --git a/rai/core_test/block_store.cpp b/rai/core_test/block_store.cpp index 09ff1b75..bfc78f58 100644 --- a/rai/core_test/block_store.cpp +++ b/rai/core_test/block_store.cpp @@ -981,7 +981,7 @@ TEST (block_store, hash2) rai::transaction transaction (store.environment, nullptr, true); ASSERT_TRUE (store.hash2_get (transaction, rai::block_hash (1)).is_zero ()); store.hash2_put (transaction, rai::block_hash (1), rai::block_hash (2)); - auto value (store.hash2_get(transaction, rai::block_hash (1))); + auto value (store.hash2_get (transaction, rai::block_hash (1))); ASSERT_EQ (rai::block_hash (2), value); } @@ -1061,13 +1061,13 @@ TEST (block_store, hash2_upgrade_multi) genesis.initialize (transaction, store); ASSERT_EQ (rai::process_result::progress, ledger.process (transaction, send).code); ASSERT_EQ (send.hash (), store.block_successor (transaction, genesis.hash ())); - + auto hash3 (store.hash2_get (transaction, send.hash ())); auto successor2 (store.block_successor (transaction, hash3)); store.block_del (transaction, hash3); store.hash2_del (transaction, send.hash ()); store.block_put (transaction, send.hash (), send, successor2); - + auto hash2 (store.hash2_get (transaction, genesis.hash ())); auto successor (store.block_successor (transaction, hash2)); store.block_del (transaction, hash2); diff --git a/rai/core_test/node.cpp b/rai/core_test/node.cpp index a96a1a30..848ce830 100644 --- a/rai/core_test/node.cpp +++ b/rai/core_test/node.cpp @@ -1485,10 +1485,10 @@ TEST (node, bootstrap_connection_scaling) TEST (node, hash2_upgrade) { rai::system system (24000, 1); - auto & node1 (*system.nodes [0]); + auto & node1 (*system.nodes[0]); rai::genesis genesis; { - rai::transaction transaction (system.nodes [0]->store.environment, nullptr, true); + rai::transaction transaction (system.nodes[0]->store.environment, nullptr, true); node1.store.version_put (transaction, 10); auto hash2 (node1.store.hash2_get (transaction, genesis.hash ())); ASSERT_FALSE (hash2.is_zero ()); @@ -1501,7 +1501,7 @@ TEST (node, hash2_upgrade) ASSERT_EQ (10, node1.store_version ()); node1.store_update (); ASSERT_EQ (11, node1.store_version ()); - rai::transaction transaction (system.nodes [0]->store.environment, nullptr, false); + rai::transaction transaction (system.nodes[0]->store.environment, nullptr, false); auto count (node1.store.block_count (transaction)); ASSERT_EQ (2, count.hash2); } diff --git a/rai/secure.cpp b/rai/secure.cpp index 312dd3f7..af390937 100644 --- a/rai/secure.cpp +++ b/rai/secure.cpp @@ -788,12 +788,12 @@ void rai::block_store::upgrade_v9_to_v10 (MDB_txn * transaction_a) void rai::block_store::upgrade_v10_to_v11 () { - std::unordered_multimap dependencies; + std::unordered_multimap dependencies; { rai::transaction transaction (environment, nullptr, false); - dependencies = (block_dependencies (transaction)); + dependencies = (block_dependencies (transaction)); } - std::deque remaining; + std::deque remaining; rai::genesis genesis; remaining.push_back (genesis.hash ()); auto total (0); @@ -819,8 +819,7 @@ void rai::block_store::upgrade_v10_to_v11 () auto successor (block_successor (transaction, hash1)); block_del (transaction, hash1); block_put (transaction, hash2_new, *block, successor); - std::for_each (range.first, range.second, [&] (std::pair item_a) - { + std::for_each (range.first, range.second, [&](std::pair item_a) { remaining.push_front (item_a.second); }); ++count; @@ -1150,9 +1149,9 @@ rai::block_counts rai::block_store::block_count (MDB_txn * transaction_a) return result; } -std::unordered_multimap rai::block_store::block_dependencies (MDB_txn * transaction_a) +std::unordered_multimap rai::block_store::block_dependencies (MDB_txn * transaction_a) { - std::unordered_multimap result; + std::unordered_multimap result; // For every block type for (auto type : { rai::block_type::send, rai::block_type::receive, rai::block_type::open, rai::block_type::change }) { @@ -1708,7 +1707,7 @@ public: assert (status == 0); rai::uint256_union preamble (1); blake2b_update (&hash_l, preamble.bytes.data (), preamble.bytes.size ()); - + rai::block_hash previous (store.hash2_get (transaction, block_a.hashables.previous)); if (!previous.is_zero ()) { @@ -1718,7 +1717,7 @@ public: assert (status == 0); status = blake2b_update (&hash_l, block_a.hashables.balance.bytes.data (), block_a.hashables.balance.bytes.size ()); assert (status == 0); - + status = blake2b_final (&hash_l, result.bytes.data (), result.bytes.size ()); assert (status == 0); } @@ -1730,7 +1729,7 @@ public: assert (status == 0); rai::uint256_union preamble (2); blake2b_update (&hash_l, preamble.bytes.data (), preamble.bytes.size ()); - + rai::block_hash previous (store.hash2_get (transaction, block_a.hashables.previous)); if (!previous.is_zero ()) { @@ -1741,7 +1740,7 @@ public: { status = blake2b_update (&hash_l, source.bytes.data (), source.bytes.size ()); assert (status == 0); - + status = blake2b_final (&hash_l, result.bytes.data (), result.bytes.size ()); assert (status == 0); } @@ -1754,7 +1753,7 @@ public: assert (status == 0); rai::uint256_union preamble (3); blake2b_update (&hash_l, preamble.bytes.data (), preamble.bytes.size ()); - + rai::block_hash source (store.hash2_get (transaction, block_a.hashables.source)); if (!source.is_zero ()) { @@ -1764,7 +1763,7 @@ public: assert (status == 0); status = blake2b_update (&hash_l, block_a.hashables.account.bytes.data (), block_a.hashables.account.bytes.size ()); assert (status == 0); - + status = blake2b_final (&hash_l, result.bytes.data (), result.bytes.size ()); assert (status == 0); } @@ -1776,7 +1775,7 @@ public: assert (status == 0); rai::uint256_union preamble (4); blake2b_update (&hash_l, preamble.bytes.data (), preamble.bytes.size ()); - + rai::block_hash previous (store.hash2_get (transaction, block_a.hashables.previous)); if (!previous.is_zero ()) { @@ -1784,7 +1783,7 @@ public: assert (status == 0); status = blake2b_update (&hash_l, block_a.hashables.representative.bytes.data (), block_a.hashables.representative.bytes.size ()); assert (status == 0); - + status = blake2b_final (&hash_l, result.bytes.data (), result.bytes.size ()); assert (status == 0); } @@ -1805,7 +1804,7 @@ rai::block_hash rai::block_store::hash2_calc (MDB_txn * transaction_a, rai::bloc rai::block_hash rai::block_store::hash2_get (MDB_txn * transaction_a, rai::block_hash const & hash_a) { rai::block_hash result; - for (auto & i: result.qwords) + for (auto & i : result.qwords) { i = 0; } diff --git a/rai/secure.hpp b/rai/secure.hpp index ff217e53..a37264ae 100644 --- a/rai/secure.hpp +++ b/rai/secure.hpp @@ -192,7 +192,7 @@ public: void block_del (MDB_txn *, rai::block_hash const &); bool block_exists (MDB_txn *, rai::block_hash const &); rai::block_counts block_count (MDB_txn *); - std::unordered_multimap block_dependencies (MDB_txn *); + std::unordered_multimap block_dependencies (MDB_txn *); void frontier_put (MDB_txn *, rai::block_hash const &, rai::account const &); rai::account frontier_get (MDB_txn *, rai::block_hash const &);