From 8a2f0d39db2eebb02b7b6ba04ed43eb5bc22bb40 Mon Sep 17 00:00:00 2001 From: Russel Waters Date: Tue, 1 Mar 2022 23:37:25 -0800 Subject: [PATCH 1/9] initial centos cleanup rpm build image, and signing container rpm build updates --- .github/workflows/live_artifacts.yml | 28 ++++ ci/actions/linux/install_deps.sh | 1 + ci/build-centos.sh | 13 +- docker/ci/Dockerfile-centos | 8 + docker/sign/Dockerfile-signer | 24 +++ docker/sign/README.MD | 21 +++ docker/sign/entry.sh | 57 +++++++ util/build_prep/centos/prep.sh.in | 4 +- util/changelog.py | 225 ++++++++++++++++----------- util/makesrc | 36 ++--- 10 files changed, 301 insertions(+), 116 deletions(-) create mode 100644 docker/ci/Dockerfile-centos create mode 100644 docker/sign/Dockerfile-signer create mode 100644 docker/sign/README.MD create mode 100755 docker/sign/entry.sh diff --git a/.github/workflows/live_artifacts.yml b/.github/workflows/live_artifacts.yml index fc5c9599..cd5d9abd 100644 --- a/.github/workflows/live_artifacts.yml +++ b/.github/workflows/live_artifacts.yml @@ -66,6 +66,34 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-2 + linux_rpm_job: + runs-on: ubuntu-20.04 + timeout-minutes: 90 + steps: + - name: tag + run: | + echo "TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV + - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + with: + submodules: "recursive" + ref: ${{ github.event.inputs.ref }} + repository: ${{ github.event.inputs.repo }} + - name: centos base + run: ci/build-docker-image.sh docker/ci/Dockerfile-centos nanocurrency/nano-env:centos + - name: Build Artifact + run: | + docker run -e LIVE=1 -v ${GITHUB_WORKSPACE}:/workspace \ + -v artifacts:/root/rpmbuild/RPMS -v source:/root/rpmbuild/SOURCES \ + nanocurrency/nano-env:centos /bin/bash -c "ci/build-centos.sh" + env: + PAT: ${{ secrets.GITHUB_TOKEN }} + - name: Deploy Artifact + run: ci/actions/deploy.sh + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-2 + linux_docker_job: runs-on: ubuntu-20.04 timeout-minutes: 90 diff --git a/ci/actions/linux/install_deps.sh b/ci/actions/linux/install_deps.sh index fb421c44..52a82a51 100755 --- a/ci/actions/linux/install_deps.sh +++ b/ci/actions/linux/install_deps.sh @@ -8,4 +8,5 @@ if [[ "${COMPILER:-}" != "" ]]; then else ci/build-docker-image.sh docker/ci/Dockerfile-gcc nanocurrency/nano-env:gcc ci/build-docker-image.sh docker/ci/Dockerfile-clang-6 nanocurrency/nano-env:clang-6 + ci/build-docker-image.sh docker/ci/Dockerfile-centos nanocurrency/nano-env:centos fi diff --git a/ci/build-centos.sh b/ci/build-centos.sh index 19b6bee1..6f9c7cf4 100755 --- a/ci/build-centos.sh +++ b/ci/build-centos.sh @@ -1,17 +1,20 @@ #!/usr/bin/env bash -TAGS=$(git describe --abbrev=0 --tags) -VERSIONS=${TAGS//V/} +TAG=$(echo $TAG) +VERSIONS=${TAG//V/} RELEASE=$(echo $CI_JOB_ID) run_source() { - ./util/makesrc $TAGS + ./util/makesrc $TAG $(echo $PAT) } run_build() { mkdir -p ~/rpmbuild/SOURCES/ mv -f ~/nano-${VERSIONS}.tar.gz ~/rpmbuild/SOURCES/. - scl enable devtoolset-7 'rpmbuild -ba nanocurrency.spec' - scl enable devtoolset-7 'rpmbuild -ba nanocurrency-beta.spec' + if [ "${LIVE:-}" == "1" ]; then + scl enable devtoolset-7 'rpmbuild -ba nanocurrency.spec' + else + scl enable devtoolset-7 'rpmbuild -ba nanocurrency-beta.spec' + fi } run_update() { diff --git a/docker/ci/Dockerfile-centos b/docker/ci/Dockerfile-centos new file mode 100644 index 00000000..f295845e --- /dev/null +++ b/docker/ci/Dockerfile-centos @@ -0,0 +1,8 @@ +FROM centos:7 + +WORKDIR /workspace + +ADD util /build +RUN bash /build/build_prep/update-common +RUN bash /build/build_prep/centos/prep.sh +RUN rm -rf /build diff --git a/docker/sign/Dockerfile-signer b/docker/sign/Dockerfile-signer new file mode 100644 index 00000000..41a18201 --- /dev/null +++ b/docker/sign/Dockerfile-signer @@ -0,0 +1,24 @@ +FROM python:3.9-buster + + +ENV DEBIAN_FRONTEND="noninteractive" +ENV TZ="America/Los_Angeles" +ENV GPG_TTY=/dev/console + +ARG GPG_PRIVATE_KEY +ARG GPG_SIGNER + +## Install required for rpm signing +RUN apt-get update -yqq && \ + apt-get install -y librpmsign8 gnupg2 wget rpm + +## Additional for mkrepo support +RUN python3 -m pip install mkrepo boto3 + +#WORKDIR /signing +#RUN python3 -m venv && .venv/bin/pip install mkrepo + +## Control Entrypoint +ADD docker/sign/entry.sh /usr/local/bin/entry.sh + +ENTRYPOINT [ "entry.sh" ] diff --git a/docker/sign/README.MD b/docker/sign/README.MD new file mode 100644 index 00000000..769cc60f --- /dev/null +++ b/docker/sign/README.MD @@ -0,0 +1,21 @@ +## Usage + +### Build + +`docker build . -f docker/sign/Dockerfile-signer -t ` + +### Required Variables +* GPG_PRIVATE_KEY -> base64 encoded GPG private signing key +* GPG_SIGNER -> key id to sign with, must match GPG_PRIVATE_KEY +* S3_ACCESS_KEY_ID -> S3 Key with access to rpm/repo +* S3_SECRET_ACCES_KEY -> S3 Secret for S3_ACCESS_KEY_ID + +### Sign RPM artifacts +`docker run --rm -it -e GPG_PRIVATE_KEY -e GPG_SIGNER -v :/tmp rpm-sign /tmp` + +### Upload RPM artifacts +Upload artifacts to appropriate location + +### Update Repo Metadata Files + +`docker run --rm -it -e GPG_PRIVATE_KEY -e GPG_SIGNER -e S3_ACCESS_KEY_ID -e S3_SECRET_ACCESS_KEY repo-update ` \ No newline at end of file diff --git a/docker/sign/entry.sh b/docker/sign/entry.sh new file mode 100755 index 00000000..ed5c5a03 --- /dev/null +++ b/docker/sign/entry.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +if [ -z "${GPG_PRIVATE_KEY}" ]; then + echo "GPG_PRIVATE_KEY not set" + exit 1 +fi + +if [ -z "${GPG_SIGNER}" ]; then + echo "GPG_SIGNER not set" + exit 1 +fi + +echo "${GPG_PRIVATE_KEY}" | base64 -d | gpg2 --import --no-tty --yes + +cat < /root/.rpmmacros +%_signature gpg +%_gpg_name ${GPG_SIGNER} +%_gpg_path /root/.gnupg +%_gpgbin /usr/bin/gpg2 +EOF + + +if [ "${1}" == "rpm-sign" ]; then + if [ -z "${2}" ]; then + echo "Usage: ${0} rpm-sign " + exit 1 + fi + for a in `ls ${2}/*.rpm`; do + rpm --addsign "${a}" + done + exit 0 +elif [ "${1}" == "repo-update" ]; then + if [ -z "${S3_ACCESS_KEY_ID}" ]; then + echo "S3_ACCESS_KEY_ID not set" + exit 1 + fi + if [ -z "${S3_SECRET_ACCESS_KEY}" ]; then + echo "S3_SECRET_ACCESS_KEY not set" + exit 1 + fi + if [ -z "${S3_REGION}" ]; then + echo "Defaulting S3_REGION to us-east-2" + export S3_REGION="us-east-2" + fi + if [ -z "${2}" ]; then + echo "Usage: ${0} repo-update " + exit 1 + fi + mkrepo --s3-access-key-id ${S3_ACCESS_KEY_ID} \ + --s3-secret-access-key ${S3_SECRET_ACCESS_KEY} \ + --s3-public-read --s3-region ${S3_REGION} \ + ${2} + +else + echo "Usage: ${0} " + exit 1 +fi \ No newline at end of file diff --git a/util/build_prep/centos/prep.sh.in b/util/build_prep/centos/prep.sh.in index a321cfd3..75034f70 100644 --- a/util/build_prep/centos/prep.sh.in +++ b/util/build_prep/centos/prep.sh.in @@ -14,6 +14,8 @@ yes | yum install -y rpm-build || exit 1 yes | yum install -y glibc-devel glibc-headers make which libstdc++-static || exit 1 yes | yum install -y centos-release-scl || exit 1 yes | yum install -y devtoolset-7-gcc-c++ || exit 1 +yes | yum install -y rh-python38 || exit 1 + wget -O cmake_install.sh https://github.com/Kitware/CMake/releases/download/v3.15.4/cmake-3.15.4-Linux-x86_64.sh chmod +x cmake_install.sh @@ -25,7 +27,7 @@ rm -f ./cmake_install.sh ( eval "$(scl enable devtoolset-7 "bash -c 'set | grep ^PATH='")" if ! have boost; then - bootstrap_boost -m -s + bootstrap_boost -m -s -j`nproc` fi if ! have boost; then diff --git a/util/changelog.py b/util/changelog.py index dd61cc0d..a4081d89 100755 --- a/util/changelog.py +++ b/util/changelog.py @@ -1,18 +1,35 @@ import argparse import copy import sys +import re +from typing import Tuple """ Changelog generation script, requires PAT with public_repo access, see https://github.com/settings/tokens -Caveats V20 and prior release tags are tips on their respective release branches -If you try to use a start tag with one of these a full changelog will be generated -since the commit wont appear in your iterations +usage: changelog [-h] [-e END] [-m {full,incremental}] -p PAT [-r REPO] [-s START] [-t TAG] + +Generate Changelogs between tags or commits + +optional arguments: + -h, --help show this help message and exit + -e END, --end END Ending reference for Changelog(newest) + -m {full,incremental}, --mode {full,incremental} + Mode to run changelog for [full, incremental] + -p PAT, --pat PAT Personal Access Token + -r REPO, --repo REPO to generate logs for + -s START, --start START + Starting reference for Changelog(oldest) + -t TAG, --tag TAG Tag to use for changelog generation """ + +full = re.compile(r"^(V(\d)+.(\d)+.?(\d)?)$") +incremental = re.compile(r"^(V(\d)+.(\d)+.?(\d)?(RC(\d)+)?(DB(\d)+)?)$") + try: - from github import Github,UnknownObjectException + from github import Github, UnknownObjectException from mdutils import MdUtils except BaseException: sys.exit("Error: run 'pip install PyGithub mdutils'") @@ -80,11 +97,33 @@ SECTIONS = { class cliArgs(): - def __init__(self): + def __init__(self) -> dict: + + changelog_choices = ["full", "incremental"] + parse = argparse.ArgumentParser( prog="changelog", description="Generate Changelogs between tags or commits" ) + + parse.add_argument( + '-e', '--end', + help="Ending reference for Changelog(newest)", + type=str, action="store", + ) + parse.add_argument( + "-m", "--mode", + help="Mode to run changelog for [full, incremental]", + type=str, action="store", + default="incremental", + choices=changelog_choices + ) + parse.add_argument( + '-p', '--pat', + help="Personal Access Token", + type=str, action="store", + required=True, + ) parse.add_argument( '-r', '--repo', help=" to generate logs for", @@ -93,101 +132,97 @@ class cliArgs(): ) parse.add_argument( '-s', '--start', - help="Starting reference for Changelog", + help="Starting reference for Changelog(oldest)", type=str, action="store", - required=True, ) parse.add_argument( - '-e', '--end', - help="Ending reference for Changelog", - type=str, action="store", - required=True, - ) - parse.add_argument( - '--pat', - help="Personal Access Token", - type=str, action="store", - required=True, + '-t', '--tag', + help="Tag to use for changelog generation", + type=str, action="store" ) options = parse.parse_args() + self.end = options.end + self.mode = options.mode + self.pat = options.pat self.repo = options.repo.rstrip("/") self.start = options.start - self.end = options.end - self.pat = options.pat - def __repr__(self): - return "" \ - .format(self.repo, self.start, self.end, self.pat) - def __str__(self): - return "Generating a changelog for {0} starting with {1} " \ - "and ending with {2}".format(self.repo, self.start, self.end) + self.tag = options.tag + + class generateTree: def __init__(self, args): github = Github(args.pat) self.name = args.repo - self.repo = github.get_repo(args.repo) - self.start = args.start - self.end = args.end - try: - self.startCommit = self.repo.get_commit(args.start) - except BaseException: - exit("Error finding commit for " + args.start) - try: - self.endCommit = self.repo.get_commit(args.end) - except BaseException: - exit("Error finding commit for " + args.end) - commits = self.repo.get_commits(sha=self.endCommit.sha) + self.repo = github.get_repo(self.name) + if args.tag: + self.tag = args.tag + self.end = self.repo.get_commit(args.tag).sha + elif args.end: + self.end = self.repo.get_commit(args.end).sha + else: + print("need end or tag") + exit(1) + if args.start: + self.start = self.repo.get_commit(args.start).sha + else: + self.start = self.get_common(args.mode) self.commits = {} - self.other_commits = [] # for commits that do not have an associated pull + self.other_commits = [] + commits = self.repo.get_commits(sha=self.end) for commit in commits: - if commit.sha == self.startCommit.sha: + if commit.sha == self.start: break - else: - message = commit.commit.message.partition('\n')[0] - try: - pr_number = int( - message[message.rfind('#')+1:message.rfind(')')]) + m = commit.commit.message.partition('\n')[0] + try: + pr_number = int(m[m.rfind('#')+1:m.rfind(')')]) + pull = self.repo.get_pull(pr_number) + except (ValueError, UnknownObjectException): + p = commit.get_pulls() + if p.totalCount > 0: + pr_number = p[0].number pull = self.repo.get_pull(pr_number) - except (ValueError, UnknownObjectException): - pulls = commit.get_pulls() - if pulls.totalCount > 0: - # no commits with more than 1 PR associated to it were found in V23 release - # but targeting first entry only if that ends up being the case - pr_number = pulls[0].number - pull = self.repo.get_pull(pr_number) - else: - print("Commit has no associated PR {}: \"{}\"".format( - commit.sha, message)) - self.other_commits.append((commit.sha, message)) - continue + else: + print( + f"Commit has no associated PR {commit.sha}: \"{m}\"") + self.other_commits.append((commit.sha, m)) + continue + + labels = [] + for label in pull.labels: + labels.append(label.name) + self.commits[pull.number] = { + "Title": pull.title, + "Url": pull.html_url, + "labels": labels + } + + def get_common(self, mode) -> str: + tags = [] + for tag in self.repo.get_tags(): + if mode == "full": + found = full.match(tag.name) + else: + found = incremental.match(tag.name) + if found: + tags.append(tag) + tree = self.repo.compare(self.end, tags[1].commit.sha) + return tree.merge_base_commit.sha + - labels = [] - for label in pull.labels: - labels.append(label.name) - self.commits[pull.number] = { - "Title": pull.title, - "Url": pull.html_url, - "labels": labels - } - def __repr__(self): - return " Tuple[str, bool]: for section, values in SECTIONS.items(): for label in labels: if label in values: @@ -233,7 +272,8 @@ class generateMarkdown(): else: return section, False return 'Other', False - def pull_to_section(self, commits): + + def pull_to_section(self, commits) -> dict: sect = copy.deepcopy(SECTIONS) result = {} for a in sect: @@ -248,7 +288,8 @@ class generateMarkdown(): if len(sect[a]) > 0: result[a] = sect[a] return result -if __name__ == "__main__": - args = cliArgs() - repo = generateTree(args) - generateMarkdown(repo) + + +arg = cliArgs() +trees = generateTree(arg) +generateMarkdown(trees) diff --git a/util/makesrc b/util/makesrc index 50bb6fa9..f42a06ba 100755 --- a/util/makesrc +++ b/util/makesrc @@ -1,23 +1,24 @@ #!/bin/bash -# makesrc -# requires ruby gem github_changelog_generator -# API_KEY generated here https://github.com/settings/tokens/new?description=GitHub%20Changelog%20Generator%20token +# makesrc +# personal access token allows around rate limit -if [ -e $1 ]; then - echo "makesrc " >&2 +if [ -e $2 ]; then + echo "makesrc " >&2 echo " tag valid for nanocurrency/nano-node" >&2 + echo " pat personal access token" >&2 exit 1 fi TAG="$1" +PAT="$2" VERSION=$(echo $TAG | sed 's/V//') TAG_DATE="" VERSION_MAJOR=$(echo $VERSION | cut -d "." -f 1) scriptDir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" function make_source() { - git clone --recursive --single-branch --branch releases/v${VERSION_MAJOR} https://github.com/nanocurrency/nano-node nano-$VERSION + git clone --recursive --single-branch --branch ${TAG} https://github.com/nanocurrency/nano-node nano-$VERSION cd nano-$VERSION git pull --tags COUNT=$(git tag -l "${TAG}" | wc -l) @@ -39,19 +40,18 @@ function source_information() { COMMIT=$(git log | head -1 | cut -d " " -f 2) TAG_DATE=$(TZ=UTC date -d"$DATE" +%s) - case "${VERSION}" in - *RC*) - "${scriptDir}/changelog_generator" nanocurrency/nano-node "V${VERSION}" only - ;; - *) - "${scriptDir}/changelog_generator" nanocurrency/nano-node "V${VERSION}" - ;; - esac + ( + eval "$(scl enable rh-python38 'echo -e "source /opt/rh/rh-python38/enable\n"')" + python -m pip install PyGithub mdutils - if [ ! -s CHANGELOG.md ]; then - echo "CHANGELOG not generated" - exit 1 - fi + python ${scriptDir}/changelog.py -p ${PAT} -t ${TAG} -m full + + if [ ! -s CHANGELOG.md ]; then + echo "CHANGELOG not generated" + exit 1 + fi + exit 0 + ) || exit 1 export TAG_DATE } From 8bcc3818b63d7678810e5052c7b5be228ca9a401 Mon Sep 17 00:00:00 2001 From: Thiago Silva Date: Wed, 4 May 2022 18:17:30 -0300 Subject: [PATCH 2/9] Fixes to let it run in the CI --- .github/workflows/live_artifacts.yml | 15 ++++++++------- ci/actions/deploy.sh | 18 ++++++++++++++++-- util/build_prep/centos/prep.sh.in | 2 +- util/makesrc | 26 +++++++++++++------------- 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/.github/workflows/live_artifacts.yml b/.github/workflows/live_artifacts.yml index cd5d9abd..633b216e 100644 --- a/.github/workflows/live_artifacts.yml +++ b/.github/workflows/live_artifacts.yml @@ -70,7 +70,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 steps: - - name: tag + - name: Checkout the Tag run: | echo "TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f @@ -78,18 +78,19 @@ jobs: submodules: "recursive" ref: ${{ github.event.inputs.ref }} repository: ${{ github.event.inputs.repo }} - - name: centos base + - name: CentOS Base run: ci/build-docker-image.sh docker/ci/Dockerfile-centos nanocurrency/nano-env:centos - name: Build Artifact run: | - docker run -e LIVE=1 -v ${GITHUB_WORKSPACE}:/workspace \ - -v artifacts:/root/rpmbuild/RPMS -v source:/root/rpmbuild/SOURCES \ + mkdir -p ${GITHUB_WORKSPACE}/artifacts + docker run -e LIVE=1 -e PAT=${{ secrets.GITHUB_TOKEN }} \ + -e TAG=$TAG -v ${GITHUB_WORKSPACE}:/workspace \ + -v ${GITHUB_WORKSPACE}/artifacts:/root/rpmbuild \ nanocurrency/nano-env:centos /bin/bash -c "ci/build-centos.sh" - env: - PAT: ${{ secrets.GITHUB_TOKEN }} - - name: Deploy Artifact + - name: Deploy Artifacts run: ci/actions/deploy.sh env: + LINUX_RPM: 1 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-2 diff --git a/ci/actions/deploy.sh b/ci/actions/deploy.sh index 88bee52e..6895108c 100755 --- a/ci/actions/deploy.sh +++ b/ci/actions/deploy.sh @@ -4,6 +4,7 @@ set -o errexit set -o nounset set -o xtrace OS=$(uname) +IS_RPM_DEPLOY="${LINUX_RPM:-0}" if [[ "${BETA:-0}" -eq 1 ]]; then BUILD="beta" @@ -19,7 +20,20 @@ else DIRECTORY="${S3_BUILD_DIRECTORY}/${BUILD}" fi -if [[ "$OS" == 'Linux' ]]; then +if [[ "$OS" == 'Linux' && "$IS_RPM_DEPLOY" -eq "1" ]]; then + RPMS=$(find ${GITHUB_WORKSPACE}/artifacts/RPMS -type f -name '*.rpm') + SRPMS=$(find ${GITHUB_WORKSPACE}/artifacts/SRPMS -type f -name '*.src.rpm') + for rpm in $RPMS; do + sha256sum $rpm > "${GITHUB_WORKSPACE}/${rpm}.sha256" + aws s3 cp "${rpm}" s3://repo.nano.org/$DIRECTORY/binaries/$(basename "${rpm}") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers + aws s3 cp "${GITHUB_WORKSPACE}/${rpm}.sha256" s3://repo.nano.org/$DIRECTORY/binaries/$(basename "${rpm}.sha256") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers + done + for srpm in $SRPMS; do + sha256sum $srpm > "${GITHUB_WORKSPACE}/${srpm}.sha256" + aws s3 cp "${srpm}" s3://repo.nano.org/$DIRECTORY/source/$(basename "${srpm}") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers + aws s3 cp "${GITHUB_WORKSPACE}/${srpm}.sha256" s3://repo.nano.org/$DIRECTORY/source/$(basename "${srpm}.sha256") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers + done +elif [[ "$OS" == 'Linux' ]]; then sha256sum $GITHUB_WORKSPACE/build/nano-node-*-Linux.tar.bz2 >$GITHUB_WORKSPACE/nano-node-$TAG-Linux.tar.bz2.sha256 sha256sum $GITHUB_WORKSPACE/build/nano-node-*-Linux.deb >$GITHUB_WORKSPACE/nano-node-$TAG-Linux.deb.sha256 aws s3 cp $GITHUB_WORKSPACE/build/nano-node-*-Linux.tar.bz2 s3://repo.nano.org/$DIRECTORY/binaries/nano-node-$TAG-Linux.tar.bz2 --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers @@ -30,4 +44,4 @@ else sha256sum $GITHUB_WORKSPACE/build/nano-node-*-Darwin.dmg >$GITHUB_WORKSPACE/build/nano-node-$TAG-Darwin.dmg.sha256 aws s3 cp $GITHUB_WORKSPACE/build/nano-node-*-Darwin.dmg s3://repo.nano.org/$DIRECTORY/binaries/nano-node-$TAG-Darwin.dmg --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers aws s3 cp $GITHUB_WORKSPACE/build/nano-node-$TAG-Darwin.dmg.sha256 s3://repo.nano.org/$DIRECTORY/binaries/nano-node-$TAG-Darwin.dmg.sha256 --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers -fi \ No newline at end of file +fi diff --git a/util/build_prep/centos/prep.sh.in b/util/build_prep/centos/prep.sh.in index 75034f70..8835fa2c 100644 --- a/util/build_prep/centos/prep.sh.in +++ b/util/build_prep/centos/prep.sh.in @@ -7,7 +7,7 @@ yes | yum update -y yes | yum install -y git wget openssl bzip2 # -yes | yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm # epel 7 +yes | yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm # epel 7 yes | yum install -y jq || exit 1 yes | yum install -y rpm-build || exit 1 diff --git a/util/makesrc b/util/makesrc index f42a06ba..ca0a32aa 100755 --- a/util/makesrc +++ b/util/makesrc @@ -20,7 +20,7 @@ scriptDir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" function make_source() { git clone --recursive --single-branch --branch ${TAG} https://github.com/nanocurrency/nano-node nano-$VERSION cd nano-$VERSION - git pull --tags + git fetch --tags COUNT=$(git tag -l "${TAG}" | wc -l) if [ "$COUNT" -eq 0 ]; then echo "tag $TAG not found" @@ -40,18 +40,18 @@ function source_information() { COMMIT=$(git log | head -1 | cut -d " " -f 2) TAG_DATE=$(TZ=UTC date -d"$DATE" +%s) - ( - eval "$(scl enable rh-python38 'echo -e "source /opt/rh/rh-python38/enable\n"')" - python -m pip install PyGithub mdutils - - python ${scriptDir}/changelog.py -p ${PAT} -t ${TAG} -m full - - if [ ! -s CHANGELOG.md ]; then - echo "CHANGELOG not generated" - exit 1 - fi - exit 0 - ) || exit 1 +# ( +# eval "$(scl enable rh-python38 'echo -e "source /opt/rh/rh-python38/enable\n"')" +# python -m pip install PyGithub mdutils +# +# python ${scriptDir}/changelog.py -p ${PAT} -t ${TAG} -m full +# +# if [ ! -s CHANGELOG.md ]; then +# echo "CHANGELOG not generated" +# exit 1 +# fi +# exit 0 +# ) || exit 1 export TAG_DATE } From a437e12afd62f4893538586882537fa2167897b0 Mon Sep 17 00:00:00 2001 From: Thiago Silva Date: Mon, 16 May 2022 13:31:00 -0300 Subject: [PATCH 3/9] Changes the repo to be passed as argument --- .github/workflows/live_artifacts.yml | 6 ++++-- ci/build-centos.sh | 3 ++- util/makesrc | 8 +++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/live_artifacts.yml b/.github/workflows/live_artifacts.yml index 633b216e..de1fc72f 100644 --- a/.github/workflows/live_artifacts.yml +++ b/.github/workflows/live_artifacts.yml @@ -70,9 +70,10 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 90 steps: - - name: Checkout the Tag + - name: Sets the tag and repo variables (to build) run: | echo "TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV + echo "REPO_TO_BUILD=${{ github.event.inputs.repo }}" >> $GITHUB_ENV - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f with: submodules: "recursive" @@ -84,7 +85,8 @@ jobs: run: | mkdir -p ${GITHUB_WORKSPACE}/artifacts docker run -e LIVE=1 -e PAT=${{ secrets.GITHUB_TOKEN }} \ - -e TAG=$TAG -v ${GITHUB_WORKSPACE}:/workspace \ + -e TAG=$TAG -e REPO_TO_BUILD=$REPO_TO_BUILD + -v ${GITHUB_WORKSPACE}:/workspace \ -v ${GITHUB_WORKSPACE}/artifacts:/root/rpmbuild \ nanocurrency/nano-env:centos /bin/bash -c "ci/build-centos.sh" - name: Deploy Artifacts diff --git a/ci/build-centos.sh b/ci/build-centos.sh index 6f9c7cf4..39157c31 100755 --- a/ci/build-centos.sh +++ b/ci/build-centos.sh @@ -2,9 +2,10 @@ TAG=$(echo $TAG) VERSIONS=${TAG//V/} RELEASE=$(echo $CI_JOB_ID) +REPO_TO_BUILD=$(echo $REPO_TO_BUILD) run_source() { - ./util/makesrc $TAG $(echo $PAT) + ./util/makesrc $TAG $(echo $PAT) $REPO_TO_BUILD } run_build() { diff --git a/util/makesrc b/util/makesrc index ca0a32aa..a4e22ce8 100755 --- a/util/makesrc +++ b/util/makesrc @@ -4,21 +4,23 @@ # personal access token allows around rate limit if [ -e $2 ]; then - echo "makesrc " >&2 - echo " tag valid for nanocurrency/nano-node" >&2 + echo "makesrc " >&2 + echo " tag valid for " >&2 echo " pat personal access token" >&2 + echo " repo repository to build" >&2 exit 1 fi TAG="$1" PAT="$2" +repository="${3:-nanocurrency/nano-node}" VERSION=$(echo $TAG | sed 's/V//') TAG_DATE="" VERSION_MAJOR=$(echo $VERSION | cut -d "." -f 1) scriptDir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" function make_source() { - git clone --recursive --single-branch --branch ${TAG} https://github.com/nanocurrency/nano-node nano-$VERSION + git clone --recursive --single-branch --branch "${TAG}" "https://github.com/${repository}" "nano-${VERSION}" cd nano-$VERSION git fetch --tags COUNT=$(git tag -l "${TAG}" | wc -l) From 7808d518c61cf3a0c4597558a3d75881c85609f6 Mon Sep 17 00:00:00 2001 From: Thiago Silva Date: Fri, 20 May 2022 13:56:14 -0300 Subject: [PATCH 4/9] Removes the changelog generation from the RPM workflow --- .github/workflows/live_artifacts.yml | 3 +-- ci/build-centos.sh | 2 +- util/makesrc | 25 +++---------------------- 3 files changed, 5 insertions(+), 25 deletions(-) diff --git a/.github/workflows/live_artifacts.yml b/.github/workflows/live_artifacts.yml index de1fc72f..600700a0 100644 --- a/.github/workflows/live_artifacts.yml +++ b/.github/workflows/live_artifacts.yml @@ -84,8 +84,7 @@ jobs: - name: Build Artifact run: | mkdir -p ${GITHUB_WORKSPACE}/artifacts - docker run -e LIVE=1 -e PAT=${{ secrets.GITHUB_TOKEN }} \ - -e TAG=$TAG -e REPO_TO_BUILD=$REPO_TO_BUILD + docker run -e LIVE=1 -e TAG=$TAG -e REPO_TO_BUILD=$REPO_TO_BUILD \ -v ${GITHUB_WORKSPACE}:/workspace \ -v ${GITHUB_WORKSPACE}/artifacts:/root/rpmbuild \ nanocurrency/nano-env:centos /bin/bash -c "ci/build-centos.sh" diff --git a/ci/build-centos.sh b/ci/build-centos.sh index 39157c31..37e5ebb7 100755 --- a/ci/build-centos.sh +++ b/ci/build-centos.sh @@ -5,7 +5,7 @@ RELEASE=$(echo $CI_JOB_ID) REPO_TO_BUILD=$(echo $REPO_TO_BUILD) run_source() { - ./util/makesrc $TAG $(echo $PAT) $REPO_TO_BUILD + ./util/makesrc $TAG $REPO_TO_BUILD } run_build() { diff --git a/util/makesrc b/util/makesrc index a4e22ce8..9ba5ba9e 100755 --- a/util/makesrc +++ b/util/makesrc @@ -1,19 +1,14 @@ #!/bin/bash -# makesrc -# personal access token allows around rate limit - if [ -e $2 ]; then - echo "makesrc " >&2 + echo "makesrc " >&2 echo " tag valid for " >&2 - echo " pat personal access token" >&2 echo " repo repository to build" >&2 exit 1 fi TAG="$1" -PAT="$2" -repository="${3:-nanocurrency/nano-node}" +repository="${2:-nanocurrency/nano-node}" VERSION=$(echo $TAG | sed 's/V//') TAG_DATE="" VERSION_MAJOR=$(echo $VERSION | cut -d "." -f 1) @@ -41,28 +36,14 @@ function source_information() { DATE=$(git log --tags --simplify-by-decoration --pretty="format:%ai %d" | head -1 | cut -d " " -f1-3) COMMIT=$(git log | head -1 | cut -d " " -f 2) TAG_DATE=$(TZ=UTC date -d"$DATE" +%s) - -# ( -# eval "$(scl enable rh-python38 'echo -e "source /opt/rh/rh-python38/enable\n"')" -# python -m pip install PyGithub mdutils -# -# python ${scriptDir}/changelog.py -p ${PAT} -t ${TAG} -m full -# -# if [ ! -s CHANGELOG.md ]; then -# echo "CHANGELOG not generated" -# exit 1 -# fi -# exit 0 -# ) || exit 1 - export TAG_DATE } function cleanup_source() { mv nano-$VERSION.tar.gz ~/. echo "ARCHIVE MOVDED TO HOME..." rm -fr nano-$TAG/ - } + function tarball_creation() { cd .. ARCHIVE_FILE_NAME="nano-${VERSION}.tar.gz" From c15a9c152c5a868afd1aea8e7283ef8aa3b78366 Mon Sep 17 00:00:00 2001 From: Thiago Silva Date: Mon, 23 May 2022 15:32:58 -0300 Subject: [PATCH 5/9] Sets the workflow scripts to use its ref/repo --- .github/workflows/live_artifacts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/live_artifacts.yml b/.github/workflows/live_artifacts.yml index 600700a0..94af9500 100644 --- a/.github/workflows/live_artifacts.yml +++ b/.github/workflows/live_artifacts.yml @@ -77,8 +77,8 @@ jobs: - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f with: submodules: "recursive" - ref: ${{ github.event.inputs.ref }} - repository: ${{ github.event.inputs.repo }} + ref: ${{ github.ref }} + repository: ${{ github.repository }} - name: CentOS Base run: ci/build-docker-image.sh docker/ci/Dockerfile-centos nanocurrency/nano-env:centos - name: Build Artifact From 50ab261a4e7d07cfa036c9be31d76c704747f5e2 Mon Sep 17 00:00:00 2001 From: Thiago Silva Date: Mon, 23 May 2022 17:43:22 -0300 Subject: [PATCH 6/9] Better name for the RPM release variable --- .github/workflows/live_artifacts.yml | 2 +- ci/build-centos.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/live_artifacts.yml b/.github/workflows/live_artifacts.yml index 94af9500..bd429e6e 100644 --- a/.github/workflows/live_artifacts.yml +++ b/.github/workflows/live_artifacts.yml @@ -84,7 +84,7 @@ jobs: - name: Build Artifact run: | mkdir -p ${GITHUB_WORKSPACE}/artifacts - docker run -e LIVE=1 -e TAG=$TAG -e REPO_TO_BUILD=$REPO_TO_BUILD \ + docker run -e LIVE=1 -e TAG=$TAG -e REPO_TO_BUILD=$REPO_TO_BUILD -e RPM_RELEASE=1 \ -v ${GITHUB_WORKSPACE}:/workspace \ -v ${GITHUB_WORKSPACE}/artifacts:/root/rpmbuild \ nanocurrency/nano-env:centos /bin/bash -c "ci/build-centos.sh" diff --git a/ci/build-centos.sh b/ci/build-centos.sh index 37e5ebb7..117a7555 100755 --- a/ci/build-centos.sh +++ b/ci/build-centos.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash TAG=$(echo $TAG) VERSIONS=${TAG//V/} -RELEASE=$(echo $CI_JOB_ID) +RPM_RELEASE=$(echo $RPM_RELEASE) REPO_TO_BUILD=$(echo $REPO_TO_BUILD) run_source() { @@ -25,7 +25,7 @@ run_update() { echo "Updating \"${outfile}\"..." rm -f "${file}.new" - awk -v srch="@VERSION@" -v repl="$VERSIONS" -v srch2="@RELEASE@" -v repl2="$RELEASE" '{ sub(srch,repl,$0); sub(srch2,repl2, $0); print $0}' <${file} >${file}.new + awk -v srch="@VERSION@" -v repl="$VERSIONS" -v srch2="@RELEASE@" -v repl2="$RPM_RELEASE" '{ sub(srch,repl,$0); sub(srch2,repl2, $0); print $0}' <${file} >${file}.new rm -fr "${outfile}" cat "${file}.new" >"${outfile}" rm -f "${file}.new" From c7103e6544863de1787c9b37b351c21595bd1219 Mon Sep 17 00:00:00 2001 From: Thiago Silva Date: Mon, 6 Jun 2022 17:10:48 -0300 Subject: [PATCH 7/9] Improve maksrc build-centos.sh scripts --- ci/build-centos.sh | 8 +++++--- util/makesrc | 22 ++++++++++++---------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/ci/build-centos.sh b/ci/build-centos.sh index 117a7555..bfd3f707 100755 --- a/ci/build-centos.sh +++ b/ci/build-centos.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash TAG=$(echo $TAG) -VERSIONS=${TAG//V/} +VERSION=$(echo "${TAG}" | sed 's/V//' | sed 's/-/_/g') RPM_RELEASE=$(echo $RPM_RELEASE) REPO_TO_BUILD=$(echo $REPO_TO_BUILD) @@ -10,7 +10,7 @@ run_source() { run_build() { mkdir -p ~/rpmbuild/SOURCES/ - mv -f ~/nano-${VERSIONS}.tar.gz ~/rpmbuild/SOURCES/. + mv -f ~/nano-${VERSION}.tar.gz ~/rpmbuild/SOURCES/. if [ "${LIVE:-}" == "1" ]; then scl enable devtoolset-7 'rpmbuild -ba nanocurrency.spec' else @@ -25,7 +25,7 @@ run_update() { echo "Updating \"${outfile}\"..." rm -f "${file}.new" - awk -v srch="@VERSION@" -v repl="$VERSIONS" -v srch2="@RELEASE@" -v repl2="$RPM_RELEASE" '{ sub(srch,repl,$0); sub(srch2,repl2, $0); print $0}' <${file} >${file}.new + awk -v srch="@VERSION@" -v repl="$VERSION" -v srch2="@RELEASE@" -v repl2="$RPM_RELEASE" '{ sub(srch,repl,$0); sub(srch2,repl2, $0); print $0}' <${file} >${file}.new rm -fr "${outfile}" cat "${file}.new" >"${outfile}" rm -f "${file}.new" @@ -33,6 +33,8 @@ run_update() { done } +set -x + run_update run_source run_build diff --git a/util/makesrc b/util/makesrc index 9ba5ba9e..8c33e714 100755 --- a/util/makesrc +++ b/util/makesrc @@ -1,26 +1,25 @@ #!/bin/bash -if [ -e $2 ]; then +if [ -e "${2}" ]; then echo "makesrc " >&2 echo " tag valid for " >&2 echo " repo repository to build" >&2 exit 1 fi -TAG="$1" +TAG="${1}" repository="${2:-nanocurrency/nano-node}" -VERSION=$(echo $TAG | sed 's/V//') +VERSION=$(echo "${TAG}" | sed 's/V//' | sed 's/-/_/g') TAG_DATE="" -VERSION_MAJOR=$(echo $VERSION | cut -d "." -f 1) scriptDir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" function make_source() { git clone --recursive --single-branch --branch "${TAG}" "https://github.com/${repository}" "nano-${VERSION}" - cd nano-$VERSION + cd "nano-${VERSION}" git fetch --tags COUNT=$(git tag -l "${TAG}" | wc -l) if [ "$COUNT" -eq 0 ]; then - echo "tag $TAG not found" + echo "tag ${TAG} not found" exit 1 else git checkout "${TAG}" @@ -35,13 +34,13 @@ function make_source() { function source_information() { DATE=$(git log --tags --simplify-by-decoration --pretty="format:%ai %d" | head -1 | cut -d " " -f1-3) COMMIT=$(git log | head -1 | cut -d " " -f 2) - TAG_DATE=$(TZ=UTC date -d"$DATE" +%s) + TAG_DATE=$(TZ=UTC date -d"${DATE}" +%s) export TAG_DATE } function cleanup_source() { - mv nano-$VERSION.tar.gz ~/. + mv "nano-${VERSION}.tar.gz" ~/. echo "ARCHIVE MOVDED TO HOME..." - rm -fr nano-$TAG/ + rm -fr "nano-${VERSION}"/ } function tarball_creation() { @@ -61,7 +60,10 @@ function tarball_creation() { tarArgs=("${tarArgs[@]}" --mtime="@${TAG_DATE}") fi fi - TZ=UTC LANG=C LC_ALL=C tar "${tarArgs[@]}" -cvf - nano-${VERSION} | TZ=UTC gzip --no-name -9c >"${ARCHIVE_FILE_NAME}" || exit 1 + TZ=UTC LANG=C LC_ALL=C tar "${tarArgs[@]}" -cvf - "nano-${VERSION}" | TZ=UTC gzip --no-name -9c >"${ARCHIVE_FILE_NAME}" || exit 1 } + +set -x + make_source cleanup_source From 7fda7e3f015f918024318717218dc092374ec6e3 Mon Sep 17 00:00:00 2001 From: Thiago Silva Date: Thu, 9 Jun 2022 08:17:30 -0300 Subject: [PATCH 8/9] Missing S3_BUILD_DIRECTORY var definition --- .github/workflows/live_artifacts.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/live_artifacts.yml b/.github/workflows/live_artifacts.yml index bd429e6e..f797b893 100644 --- a/.github/workflows/live_artifacts.yml +++ b/.github/workflows/live_artifacts.yml @@ -92,6 +92,7 @@ jobs: run: ci/actions/deploy.sh env: LINUX_RPM: 1 + S3_BUILD_DIRECTORY: ${{ secrets.S3_BUILD_DIRECTORY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-2 From 0110911d1c1b5065f8fd8993ba88d6eac991695d Mon Sep 17 00:00:00 2001 From: Thiago Silva Date: Mon, 13 Jun 2022 17:10:37 -0300 Subject: [PATCH 9/9] Fix the checksum basenames --- ci/actions/deploy.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/actions/deploy.sh b/ci/actions/deploy.sh index 6895108c..19e860b4 100755 --- a/ci/actions/deploy.sh +++ b/ci/actions/deploy.sh @@ -24,14 +24,14 @@ if [[ "$OS" == 'Linux' && "$IS_RPM_DEPLOY" -eq "1" ]]; then RPMS=$(find ${GITHUB_WORKSPACE}/artifacts/RPMS -type f -name '*.rpm') SRPMS=$(find ${GITHUB_WORKSPACE}/artifacts/SRPMS -type f -name '*.src.rpm') for rpm in $RPMS; do - sha256sum $rpm > "${GITHUB_WORKSPACE}/${rpm}.sha256" - aws s3 cp "${rpm}" s3://repo.nano.org/$DIRECTORY/binaries/$(basename "${rpm}") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - aws s3 cp "${GITHUB_WORKSPACE}/${rpm}.sha256" s3://repo.nano.org/$DIRECTORY/binaries/$(basename "${rpm}.sha256") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers + sha256sum ${rpm} > ${GITHUB_WORKSPACE}/$(basename "${rpm}.sha256") + aws s3 cp ${rpm} s3://repo.nano.org/$DIRECTORY/binaries/$(basename "${rpm}") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers + aws s3 cp ${GITHUB_WORKSPACE}/$(basename "${rpm}.sha256") s3://repo.nano.org/$DIRECTORY/binaries/$(basename "${rpm}.sha256") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers done for srpm in $SRPMS; do - sha256sum $srpm > "${GITHUB_WORKSPACE}/${srpm}.sha256" - aws s3 cp "${srpm}" s3://repo.nano.org/$DIRECTORY/source/$(basename "${srpm}") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - aws s3 cp "${GITHUB_WORKSPACE}/${srpm}.sha256" s3://repo.nano.org/$DIRECTORY/source/$(basename "${srpm}.sha256") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers + sha256sum ${srpm} > ${GITHUB_WORKSPACE}/$(basename "${srpm}).sha256") + aws s3 cp ${srpm} s3://repo.nano.org/$DIRECTORY/source/$(basename "${srpm}") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers + aws s3 cp ${GITHUB_WORKSPACE}/$(basename "${srpm}).sha256") s3://repo.nano.org/$DIRECTORY/source/$(basename "${srpm}.sha256") --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers done elif [[ "$OS" == 'Linux' ]]; then sha256sum $GITHUB_WORKSPACE/build/nano-node-*-Linux.tar.bz2 >$GITHUB_WORKSPACE/nano-node-$TAG-Linux.tar.bz2.sha256