Docker cache (#2121)

* Build new unified docker base
build clang and gcc docker test env's off base image
Build releases based off gcc base
deploy across multiple jobs

* use correct tag for base image

* add 2 more stages, one to build base if necessary, split environment building before testing, then split deploy live/beta releases

* move clang deps back to clang specific dockerfile

* re factor secure variable

.travis.yml

@@ -1,10 +1,24 @@
 language: cpp
 
 stages:
+  - build_base
+  - build_env
   - test
   - deploy
 jobs:
   include:
+    - stage: build_base
+      name: "base"
+      if: branch = docker_cache
+      script:
+        - if [ -n "$DOCKER_PASSWORD" ]; then ci/deploy-docker.sh; fi
+    - stage: build_env
+      name: "clang"
+      script: 
+        - if [ -n "$DOCKER_PASSWORD" ]; then ci/deploy-docker.sh; fi
+    - name: "gcc"
+      script: 
+        - if [ -n "$DOCKER_PASSWORD" ]; then ci/deploy-docker.sh; fi
     - stage: test
       name: "GCC + ONE_TIME_TESTS"      
       os: linux
@@ -19,7 +33,7 @@ jobs:
             - doxygen
       before_install:
         - sudo mkdir -p /etc/docker && echo '{"ipv6":true,"fixed-cidr-v6":"2001:db8:1::/64"}' | sudo tee /etc/docker/daemon.json && sudo service docker restart;
-        - ci/build-docker-image.sh docker/ci/Dockerfile-gcc nanocurrency/nano-ci-gcc;
+        - ci/build-docker-image.sh docker/ci/Dockerfile-gcc nanocurrency/nano-env:gcc;
     - name: "clang"
       os: linux
       compiler: clang
@@ -27,7 +41,7 @@ jobs:
       sudo: required
       before_install:
         - sudo mkdir -p /etc/docker && echo '{"ipv6":true,"fixed-cidr-v6":"2001:db8:1::/64"}' | sudo tee /etc/docker/daemon.json && sudo service docker restart;
-        - ci/build-docker-image.sh docker/ci/Dockerfile-clang nanocurrency/nano-ci-clang;
+        - ci/build-docker-image.sh docker/ci/Dockerfile-clang nanocurrency/nano-env:clang;
     - name: "osx"
       os: osx
       compiler: clang
@@ -36,12 +50,13 @@ jobs:
       install:
         - brew install ccache;
         - export PATH="/usr/local/opt/ccache/libexec:$PATH";
-    - stage: deploy
+    - stage: deploy    
+      name: "live"
+      script: 
+        - if [ -n "$DOCKER_PASSWORD" ]; then ci/deploy-docker.sh; fi
+    - name: "beta"
       script: 
         - if [ -n "$DOCKER_PASSWORD" ]; then ci/deploy-docker.sh; fi
-      env: 
-        - secure: "jYlg5yC8plC4zbV4sCt8BNXEHOgvF4tydJbbUsyPvmWqIKU4rScQOKvN4euDUa1UFc9dz7UxNahGuyEGQAWjG2jmvdgDG97+ImXOZa/kRnCsyJX/3R+k6dQZ/8MN9Ax6FOpXfOoHxHOZLBzE5PBIbaeF+Okcw9ks6eawXZDu2JLckGDxLDWXTz6x+NbZLE2wlm7UI9aladZBsAR/KHlYH3ms+xgZAxcrzZ2p2BFfhHpnplOYcNhivpLn0BP66DZ6/7HtpPYkfXmqTCZZ0HkxzJ2ayZ9PuvbVXL/cHXCxRJCNweRQsPYsdowzv+k/J74X2yN0jNCsSbWU619MXxzBPJGmTq20iSvEYvX0I+uEP1PCocP7PWfvfkjlj5Ts5vPJcpBZeAUgl1yjPxA4NrwRQ7iJni14VrK8dSxjQvubV/zyK2ey2vUU7e8nOpzqw/66UPDQmvDe7byadgHIacSG55ge7zZIeI6AeK1QGh5nRnLWaMMqb0drT7ZaM6BS7iwRTBVBzw2ZjKV67MRRU0O7PY+jsNx8JG58ycXqKvP4hB6tKEV+8YMSdSDl/f06FFZJT96p4sCQmUaVecXokzTyfq7ugFgWuiRRwRqcH/wi3LMLHY7JseHBw9S8bJ98dOFEcoZCUoviwnxNSXOqA3tXGudduQh4KsCDYDUKJNzDtAo="
-
 cache:
   - ccache: true
   - directories:
@@ -52,4 +67,7 @@ script:
   - if [ -n "$ONE_TIME_TESTS" ]; then ci/check-commit-format.sh; fi
   - if [ -n "$ONE_TIME_TESTS" ]; then doxygen doxygen.config; fi # TODO also deploy the built HTML
   - if [ "$TRAVIS_OS_NAME" = "osx" ]; then ci/build-travis.sh "/usr/local/opt/qt5/lib/cmake/Qt5"; fi
-  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then docker run -v $TRAVIS_BUILD_DIR:/workspace -v $HOME/.ccache:/ccache nanocurrency/nano-ci-$TRAVIS_COMPILER /bin/bash -c "apt install ccache; cd /workspace && ASAN=${ASAN} TSAN=${TSAN} CCACHE_DIR=/ccache ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"; fi
+  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then docker run -v $TRAVIS_BUILD_DIR:/workspace -v $HOME/.ccache:/ccache nanocurrency/nano-env:$TRAVIS_COMPILER /bin/bash -c "apt install ccache; cd /workspace && ASAN=${ASAN} TSAN=${TSAN} CCACHE_DIR=/ccache ./ci/build-travis.sh /usr/lib/x86_64-linux-gnu/cmake/Qt5 ${PWD}"; fi
+
+env: 
+  - secure: "jYlg5yC8plC4zbV4sCt8BNXEHOgvF4tydJbbUsyPvmWqIKU4rScQOKvN4euDUa1UFc9dz7UxNahGuyEGQAWjG2jmvdgDG97+ImXOZa/kRnCsyJX/3R+k6dQZ/8MN9Ax6FOpXfOoHxHOZLBzE5PBIbaeF+Okcw9ks6eawXZDu2JLckGDxLDWXTz6x+NbZLE2wlm7UI9aladZBsAR/KHlYH3ms+xgZAxcrzZ2p2BFfhHpnplOYcNhivpLn0BP66DZ6/7HtpPYkfXmqTCZZ0HkxzJ2ayZ9PuvbVXL/cHXCxRJCNweRQsPYsdowzv+k/J74X2yN0jNCsSbWU619MXxzBPJGmTq20iSvEYvX0I+uEP1PCocP7PWfvfkjlj5Ts5vPJcpBZeAUgl1yjPxA4NrwRQ7iJni14VrK8dSxjQvubV/zyK2ey2vUU7e8nOpzqw/66UPDQmvDe7byadgHIacSG55ge7zZIeI6AeK1QGh5nRnLWaMMqb0drT7ZaM6BS7iwRTBVBzw2ZjKV67MRRU0O7PY+jsNx8JG58ycXqKvP4hB6tKEV+8YMSdSDl/f06FFZJT96p4sCQmUaVecXokzTyfq7ugFgWuiRRwRqcH/wi3LMLHY7JseHBw9S8bJ98dOFEcoZCUoviwnxNSXOqA3tXGudduQh4KsCDYDUKJNzDtAo=" 

ci/deploy-docker.sh

@@ -7,33 +7,30 @@ if [ -n "$DOCKER_PASSWORD" ]; then
     echo "$DOCKER_PASSWORD" | docker login -u nanoreleaseteam --password-stdin
 
     # We push this just so it can be a cache next time
-    if [ "$TRAVIS_BRANCH" = "master" ]; then
-        for compiler in gcc clang; do
-            ci_image_name="nanocurrency/nano-ci-${compiler}"
-            ci/build-docker-image.sh docker/ci/Dockerfile-${compiler} nanocurrency/nano-ci-${compiler};
-            "$scripts"/custom-timeout.sh 30 docker push "$ci_image_name"
-        done
+    if [[ "$TRAVIS_BRANCH" == "master" || "$TRAVIS_BRANCH" == "docker_cache" ]] && [[ "${TRAVIS_BUILD_STAGE_NAME}" =~ 'Build' ]]; then
+        ci_image_name="nanocurrency/nano-env:$TRAVIS_JOB_NAME"
+        ci/build-docker-image.sh docker/ci/Dockerfile-$TRAVIS_JOB_NAME "$ci_image_name";
+        "$scripts"/custom-timeout.sh 30 docker push "$ci_image_name"
     fi
 
-    tags=()
-    if [[ "${TRAVIS_TAG}" =~ 'RC' ]]; then
-        tags+=("$TRAVIS_TAG" latest-including-rc)
-    elif [ -n "$TRAVIS_TAG" ]; then
-        tags+=("$TRAVIS_TAG" latest latest-including-rc)
-    elif [ -n "$TRAVIS_BRANCH" ]; then
-        tags+=("$TRAVIS_BRANCH")
-    fi
+    if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Deploy" ]]; then
+        tags=()
+        if [[ "${TRAVIS_TAG}" =~ 'RC' ]]; then
+            tags+=("$TRAVIS_TAG" latest-including-rc)
+        elif [ -n "$TRAVIS_TAG" ]; then
+            tags+=("$TRAVIS_TAG" latest latest-including-rc)
+        elif [ -n "$TRAVIS_BRANCH" ]; then
+            tags+=("$TRAVIS_BRANCH")
+        fi
 
-    for network in live beta; do
-        if [ "${network}" = 'live' ]; then
+        if [[ "$TRAVIS_JOB_NAME" == "live" ]]; then
             network_tag_suffix=''
         else
-            network_tag_suffix="-${network}"
+            network_tag_suffix="-$TRAVIS_JOB_NAME"
         fi
 
         docker_image_name="nanocurrency/nano${network_tag_suffix}"
-
-        "$scripts"/custom-timeout.sh 30 docker build --build-arg NETWORK="$network" -f docker/node/Dockerfile -t "$docker_image_name" .
+        "$scripts"/custom-timeout.sh 30 docker build --build-arg NETWORK="$TRAVIS_JOB_NAME" -f docker/node/Dockerfile -t "$docker_image_name" .
         for tag in "${tags[@]}"; do
             # Sanitize docker tag
             # https://docs.docker.com/engine/reference/commandline/tag/
@@ -43,6 +40,5 @@ if [ -n "$DOCKER_PASSWORD" ]; then
             fi
             "$scripts"/custom-timeout.sh 30 docker push "${docker_image_name}:$tag"
         done
-    done
-
+    fi
 fi

docker/ci/Dockerfile-base

@@ -0,0 +1,15 @@
+FROM ubuntu:16.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -qq && apt-get install -yqq \
+    build-essential \
+    cmake \
+    g++ \
+    wget \
+    python
+
+RUN apt-get update -qq && apt-get install -yqq \
+    qt5-default \
+    valgrind \
+    xorg xvfb xauth xfonts-100dpi xfonts-75dpi xfonts-scalable xfonts-cyrillic

docker/ci/Dockerfile-clang

@@ -1,14 +1,4 @@
-FROM ubuntu:16.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update -qq && apt-get install -yqq \
-    cmake wget bzip2
-
-RUN apt-get update -qq && apt-get install -yqq \
-    qt5-default \
-    valgrind \
-    xorg xvfb xauth xfonts-100dpi xfonts-75dpi xfonts-scalable xfonts-cyrillic
+FROM nanocurrency/nano-env:base
 
 RUN apt-get update -qq && apt-get install -yqq \
     clang-3.9 lldb-3.9

docker/ci/Dockerfile-gcc

@@ -1,20 +1,7 @@
-FROM ubuntu:16.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update -qq && apt-get install -yqq \
-    build-essential \
-    cmake \
-    g++ \
-    wget \
-    python
+FROM nanocurrency/nano-env:base
 
 ENV BOOST_ROOT=/usr/local
 ADD util/build_prep/bootstrap_boost.sh bootstrap_boost.sh
 RUN ./bootstrap_boost.sh -m
 RUN rm bootstrap_boost.sh
 
-RUN apt-get update -qq && apt-get install -yqq \
-    qt5-default \
-    valgrind \
-    xorg xvfb xauth xfonts-100dpi xfonts-75dpi xfonts-scalable xfonts-cyrillic

docker/node/Dockerfile

@@ -1,14 +1,6 @@
-FROM ubuntu:16.04
+FROM nanocurrency/nano-env:gcc
 
 ARG NETWORK=live
-
-ENV BOOST_ROOT=/tmp/boost_install
-
-ADD ci /tmp/ci
-ADD util /tmp/util
-
-RUN /tmp/util/build_prep/update-common && /tmp/util/build_prep/ubuntu/prep.sh
-
 ADD ./ /tmp/src
 
 RUN mkdir /tmp/build && \