From 4599b43a5d7364fe9019594a0339ad5a035f6cd7 Mon Sep 17 00:00:00 2001 From: Russel Waters Date: Wed, 3 Jul 2019 12:17:56 -0400 Subject: [PATCH] Docker cache (#2121) * Build new unified docker base build clang and gcc docker test env's off base image Build releases based off gcc base deploy across multiple jobs * use correct tag for base image * add 2 more stages, one to build base if necessary, split environment building before testing, then split deploy live/beta releases * move clang deps back to clang specific dockerfile * re factor secure variable --- .travis.yml | 32 +++++++++++++++++++++++++------- ci/deploy-docker.sh | 38 +++++++++++++++++--------------------- docker/ci/Dockerfile-base | 15 +++++++++++++++ docker/ci/Dockerfile-clang | 12 +----------- docker/ci/Dockerfile-gcc | 15 +-------------- docker/node/Dockerfile | 10 +--------- 6 files changed, 60 insertions(+), 62 deletions(-) create mode 100644 docker/ci/Dockerfile-base diff --git a/.travis.yml b/.travis.yml index 012a25541..61ba46075 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,24 @@ language: cpp stages: + - build_base + - build_env - test - deploy jobs: include: + - stage: build_base + name: "base" + if: branch = docker_cache + script: + - if [ -n "$DOCKER_PASSWORD" ]; then ci/deploy-docker.sh; fi + - stage: build_env + name: "clang" + script: + - if [ -n "$DOCKER_PASSWORD" ]; then ci/deploy-docker.sh; fi + - name: "gcc" + script: + - if [ -n "$DOCKER_PASSWORD" ]; then ci/deploy-docker.sh; fi - stage: test name: "GCC + ONE_TIME_TESTS" os: linux @@ -19,7 +33,7 @@ jobs: - doxygen before_install: - sudo mkdir -p /etc/docker && echo '{"ipv6":true,"fixed-cidr-v6":"2001:db8:1::/64"}' | sudo tee /etc/docker/daemon.json && sudo service docker restart; - - ci/build-docker-image.sh docker/ci/Dockerfile-gcc nanocurrency/nano-ci-gcc; + - ci/build-docker-image.sh docker/ci/Dockerfile-gcc nanocurrency/nano-env:gcc; - name: "clang" os: linux compiler: clang @@ -27,7 +41,7 @@ jobs: sudo: required before_install: - sudo mkdir -p /etc/docker && echo '{"ipv6":true,"fixed-cidr-v6":"2001:db8:1::/64"}' | sudo tee /etc/docker/daemon.json && sudo service docker restart; - - ci/build-docker-image.sh docker/ci/Dockerfile-clang nanocurrency/nano-ci-clang; + - ci/build-docker-image.sh docker/ci/Dockerfile-clang nanocurrency/nano-env:clang; - name: "osx" os: osx compiler: clang @@ -36,12 +50,13 @@ jobs: install: - brew install ccache; - export PATH="/usr/local/opt/ccache/libexec:$PATH"; - - stage: deploy + - stage: deploy + name: "live" + script: + - if [ -n "$DOCKER_PASSWORD" ]; then ci/deploy-docker.sh; fi + - name: "beta" script: - if [ -n "$DOCKER_PASSWORD" ]; then ci/deploy-docker.sh; fi - env: - - secure: "jYlg5yC8plC4zbV4sCt8BNXEHOgvF4tydJbbUsyPvmWqIKU4rScQOKvN4euDUa1UFc9dz7UxNahGuyEGQAWjG2jmvdgDG97+ImXOZa/kRnCsyJX/3R+k6dQZ/8MN9Ax6FOpXfOoHxHOZLBzE5PBIbaeF+Okcw9ks6eawXZDu2JLckGDxLDWXTz6x+NbZLE2wlm7UI9aladZBsAR/KHlYH3ms+xgZAxcrzZ2p2BFfhHpnplOYcNhivpLn0BP66DZ6/7HtpPYkfXmqTCZZ0HkxzJ2ayZ9PuvbVXL/cHXCxRJCNweRQsPYsdowzv+k/J74X2yN0jNCsSbWU619MXxzBPJGmTq20iSvEYvX0I+uEP1PCocP7PWfvfkjlj5Ts5vPJcpBZeAUgl1yjPxA4NrwRQ7iJni14VrK8dSxjQvubV/zyK2ey2vUU7e8nOpzqw/66UPDQmvDe7byadgHIacSG55ge7zZIeI6AeK1QGh5nRnLWaMMqb0drT7ZaM6BS7iwRTBVBzw2ZjKV67MRRU0O7PY+jsNx8JG58ycXqKvP4hB6tKEV+8YMSdSDl/f06FFZJT96p4sCQmUaVecXokzTyfq7ugFgWuiRRwRqcH/wi3LMLHY7JseHBw9S8bJ98dOFEcoZCUoviwnxNSXOqA3tXGudduQh4KsCDYDUKJNzDtAo=" - cache: - ccache: true - directories: @@ -52,4 +67,7 @@ script: - if [ -n "$ONE_TIME_TESTS" ]; then ci/check-commit-format.sh; fi - if [ -n "$ONE_TIME_TESTS" ]; then doxygen doxygen.config; fi # TODO also deploy the built HTML - if [ "$TRAVIS_OS_NAME" = "osx" ]; then ci/build-travis.sh "/usr/local/opt/qt5/lib/cmake/Qt5"; fi - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then docker run -v $TRAVIS_BUILD_DIR:/workspace -v $HOME/.ccache:/ccache nanocurrency/nano-ci-$TRAVIS_COMPILER /bin/bash -c "apt install ccache; cd /workspace && ASAN=${ASAN} TSAN=${TSAN} CCACHE_DIR=/ccache ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then docker run -v $TRAVIS_BUILD_DIR:/workspace -v $HOME/.ccache:/ccache nanocurrency/nano-env:$TRAVIS_COMPILER /bin/bash -c "apt install ccache; cd /workspace && ASAN=${ASAN} TSAN=${TSAN} CCACHE_DIR=/ccache ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"; fi + +env: + - secure: "jYlg5yC8plC4zbV4sCt8BNXEHOgvF4tydJbbUsyPvmWqIKU4rScQOKvN4euDUa1UFc9dz7UxNahGuyEGQAWjG2jmvdgDG97+ImXOZa/kRnCsyJX/3R+k6dQZ/8MN9Ax6FOpXfOoHxHOZLBzE5PBIbaeF+Okcw9ks6eawXZDu2JLckGDxLDWXTz6x+NbZLE2wlm7UI9aladZBsAR/KHlYH3ms+xgZAxcrzZ2p2BFfhHpnplOYcNhivpLn0BP66DZ6/7HtpPYkfXmqTCZZ0HkxzJ2ayZ9PuvbVXL/cHXCxRJCNweRQsPYsdowzv+k/J74X2yN0jNCsSbWU619MXxzBPJGmTq20iSvEYvX0I+uEP1PCocP7PWfvfkjlj5Ts5vPJcpBZeAUgl1yjPxA4NrwRQ7iJni14VrK8dSxjQvubV/zyK2ey2vUU7e8nOpzqw/66UPDQmvDe7byadgHIacSG55ge7zZIeI6AeK1QGh5nRnLWaMMqb0drT7ZaM6BS7iwRTBVBzw2ZjKV67MRRU0O7PY+jsNx8JG58ycXqKvP4hB6tKEV+8YMSdSDl/f06FFZJT96p4sCQmUaVecXokzTyfq7ugFgWuiRRwRqcH/wi3LMLHY7JseHBw9S8bJ98dOFEcoZCUoviwnxNSXOqA3tXGudduQh4KsCDYDUKJNzDtAo=" \ No newline at end of file diff --git a/ci/deploy-docker.sh b/ci/deploy-docker.sh index 68f4e4f48..4efc51c70 100755 --- a/ci/deploy-docker.sh +++ b/ci/deploy-docker.sh @@ -7,33 +7,30 @@ if [ -n "$DOCKER_PASSWORD" ]; then echo "$DOCKER_PASSWORD" | docker login -u nanoreleaseteam --password-stdin # We push this just so it can be a cache next time - if [ "$TRAVIS_BRANCH" = "master" ]; then - for compiler in gcc clang; do - ci_image_name="nanocurrency/nano-ci-${compiler}" - ci/build-docker-image.sh docker/ci/Dockerfile-${compiler} nanocurrency/nano-ci-${compiler}; - "$scripts"/custom-timeout.sh 30 docker push "$ci_image_name" - done + if [[ "$TRAVIS_BRANCH" == "master" || "$TRAVIS_BRANCH" == "docker_cache" ]] && [[ "${TRAVIS_BUILD_STAGE_NAME}" =~ 'Build' ]]; then + ci_image_name="nanocurrency/nano-env:$TRAVIS_JOB_NAME" + ci/build-docker-image.sh docker/ci/Dockerfile-$TRAVIS_JOB_NAME "$ci_image_name"; + "$scripts"/custom-timeout.sh 30 docker push "$ci_image_name" fi - tags=() - if [[ "${TRAVIS_TAG}" =~ 'RC' ]]; then - tags+=("$TRAVIS_TAG" latest-including-rc) - elif [ -n "$TRAVIS_TAG" ]; then - tags+=("$TRAVIS_TAG" latest latest-including-rc) - elif [ -n "$TRAVIS_BRANCH" ]; then - tags+=("$TRAVIS_BRANCH") - fi + if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Deploy" ]]; then + tags=() + if [[ "${TRAVIS_TAG}" =~ 'RC' ]]; then + tags+=("$TRAVIS_TAG" latest-including-rc) + elif [ -n "$TRAVIS_TAG" ]; then + tags+=("$TRAVIS_TAG" latest latest-including-rc) + elif [ -n "$TRAVIS_BRANCH" ]; then + tags+=("$TRAVIS_BRANCH") + fi - for network in live beta; do - if [ "${network}" = 'live' ]; then + if [[ "$TRAVIS_JOB_NAME" == "live" ]]; then network_tag_suffix='' else - network_tag_suffix="-${network}" + network_tag_suffix="-$TRAVIS_JOB_NAME" fi docker_image_name="nanocurrency/nano${network_tag_suffix}" - - "$scripts"/custom-timeout.sh 30 docker build --build-arg NETWORK="$network" -f docker/node/Dockerfile -t "$docker_image_name" . + "$scripts"/custom-timeout.sh 30 docker build --build-arg NETWORK="$TRAVIS_JOB_NAME" -f docker/node/Dockerfile -t "$docker_image_name" . for tag in "${tags[@]}"; do # Sanitize docker tag # https://docs.docker.com/engine/reference/commandline/tag/ @@ -43,6 +40,5 @@ if [ -n "$DOCKER_PASSWORD" ]; then fi "$scripts"/custom-timeout.sh 30 docker push "${docker_image_name}:$tag" done - done - + fi fi \ No newline at end of file diff --git a/docker/ci/Dockerfile-base b/docker/ci/Dockerfile-base new file mode 100644 index 000000000..55fb61660 --- /dev/null +++ b/docker/ci/Dockerfile-base @@ -0,0 +1,15 @@ +FROM ubuntu:16.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -qq && apt-get install -yqq \ + build-essential \ + cmake \ + g++ \ + wget \ + python + +RUN apt-get update -qq && apt-get install -yqq \ + qt5-default \ + valgrind \ + xorg xvfb xauth xfonts-100dpi xfonts-75dpi xfonts-scalable xfonts-cyrillic \ No newline at end of file diff --git a/docker/ci/Dockerfile-clang b/docker/ci/Dockerfile-clang index d165a686f..cc0641fb9 100644 --- a/docker/ci/Dockerfile-clang +++ b/docker/ci/Dockerfile-clang @@ -1,14 +1,4 @@ -FROM ubuntu:16.04 - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update -qq && apt-get install -yqq \ - cmake wget bzip2 - -RUN apt-get update -qq && apt-get install -yqq \ - qt5-default \ - valgrind \ - xorg xvfb xauth xfonts-100dpi xfonts-75dpi xfonts-scalable xfonts-cyrillic +FROM nanocurrency/nano-env:base RUN apt-get update -qq && apt-get install -yqq \ clang-3.9 lldb-3.9 diff --git a/docker/ci/Dockerfile-gcc b/docker/ci/Dockerfile-gcc index a1dc493f8..62e3cc66c 100644 --- a/docker/ci/Dockerfile-gcc +++ b/docker/ci/Dockerfile-gcc @@ -1,20 +1,7 @@ -FROM ubuntu:16.04 - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update -qq && apt-get install -yqq \ - build-essential \ - cmake \ - g++ \ - wget \ - python +FROM nanocurrency/nano-env:base ENV BOOST_ROOT=/usr/local ADD util/build_prep/bootstrap_boost.sh bootstrap_boost.sh RUN ./bootstrap_boost.sh -m RUN rm bootstrap_boost.sh -RUN apt-get update -qq && apt-get install -yqq \ - qt5-default \ - valgrind \ - xorg xvfb xauth xfonts-100dpi xfonts-75dpi xfonts-scalable xfonts-cyrillic diff --git a/docker/node/Dockerfile b/docker/node/Dockerfile index 90b45c065..8ac8e76f1 100644 --- a/docker/node/Dockerfile +++ b/docker/node/Dockerfile @@ -1,14 +1,6 @@ -FROM ubuntu:16.04 +FROM nanocurrency/nano-env:gcc ARG NETWORK=live - -ENV BOOST_ROOT=/tmp/boost_install - -ADD ci /tmp/ci -ADD util /tmp/util - -RUN /tmp/util/build_prep/update-common && /tmp/util/build_prep/ubuntu/prep.sh - ADD ./ /tmp/src RUN mkdir /tmp/build && \