diff --git a/.github/workflows/live_artifacts.yml b/.github/workflows/live_artifacts.yml index fc5c9599..cd5d9abd 100644 --- a/.github/workflows/live_artifacts.yml +++ b/.github/workflows/live_artifacts.yml @@ -66,6 +66,34 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-2 + linux_rpm_job: + runs-on: ubuntu-20.04 + timeout-minutes: 90 + steps: + - name: tag + run: | + echo "TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV + - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + with: + submodules: "recursive" + ref: ${{ github.event.inputs.ref }} + repository: ${{ github.event.inputs.repo }} + - name: centos base + run: ci/build-docker-image.sh docker/ci/Dockerfile-centos nanocurrency/nano-env:centos + - name: Build Artifact + run: | + docker run -e LIVE=1 -v ${GITHUB_WORKSPACE}:/workspace \ + -v artifacts:/root/rpmbuild/RPMS -v source:/root/rpmbuild/SOURCES \ + nanocurrency/nano-env:centos /bin/bash -c "ci/build-centos.sh" + env: + PAT: ${{ secrets.GITHUB_TOKEN }} + - name: Deploy Artifact + run: ci/actions/deploy.sh + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-2 + linux_docker_job: runs-on: ubuntu-20.04 timeout-minutes: 90 diff --git a/ci/actions/linux/install_deps.sh b/ci/actions/linux/install_deps.sh index fb421c44..52a82a51 100755 --- a/ci/actions/linux/install_deps.sh +++ b/ci/actions/linux/install_deps.sh @@ -8,4 +8,5 @@ if [[ "${COMPILER:-}" != "" ]]; then else ci/build-docker-image.sh docker/ci/Dockerfile-gcc nanocurrency/nano-env:gcc ci/build-docker-image.sh docker/ci/Dockerfile-clang-6 nanocurrency/nano-env:clang-6 + ci/build-docker-image.sh docker/ci/Dockerfile-centos nanocurrency/nano-env:centos fi diff --git a/ci/build-centos.sh b/ci/build-centos.sh index 19b6bee1..6f9c7cf4 100755 --- a/ci/build-centos.sh +++ b/ci/build-centos.sh @@ -1,17 +1,20 @@ #!/usr/bin/env bash -TAGS=$(git describe --abbrev=0 --tags) -VERSIONS=${TAGS//V/} +TAG=$(echo $TAG) +VERSIONS=${TAG//V/} RELEASE=$(echo $CI_JOB_ID) run_source() { - ./util/makesrc $TAGS + ./util/makesrc $TAG $(echo $PAT) } run_build() { mkdir -p ~/rpmbuild/SOURCES/ mv -f ~/nano-${VERSIONS}.tar.gz ~/rpmbuild/SOURCES/. - scl enable devtoolset-7 'rpmbuild -ba nanocurrency.spec' - scl enable devtoolset-7 'rpmbuild -ba nanocurrency-beta.spec' + if [ "${LIVE:-}" == "1" ]; then + scl enable devtoolset-7 'rpmbuild -ba nanocurrency.spec' + else + scl enable devtoolset-7 'rpmbuild -ba nanocurrency-beta.spec' + fi } run_update() { diff --git a/docker/ci/Dockerfile-centos b/docker/ci/Dockerfile-centos new file mode 100644 index 00000000..f295845e --- /dev/null +++ b/docker/ci/Dockerfile-centos @@ -0,0 +1,8 @@ +FROM centos:7 + +WORKDIR /workspace + +ADD util /build +RUN bash /build/build_prep/update-common +RUN bash /build/build_prep/centos/prep.sh +RUN rm -rf /build diff --git a/docker/sign/Dockerfile-signer b/docker/sign/Dockerfile-signer new file mode 100644 index 00000000..41a18201 --- /dev/null +++ b/docker/sign/Dockerfile-signer @@ -0,0 +1,24 @@ +FROM python:3.9-buster + + +ENV DEBIAN_FRONTEND="noninteractive" +ENV TZ="America/Los_Angeles" +ENV GPG_TTY=/dev/console + +ARG GPG_PRIVATE_KEY +ARG GPG_SIGNER + +## Install required for rpm signing +RUN apt-get update -yqq && \ + apt-get install -y librpmsign8 gnupg2 wget rpm + +## Additional for mkrepo support +RUN python3 -m pip install mkrepo boto3 + +#WORKDIR /signing +#RUN python3 -m venv && .venv/bin/pip install mkrepo + +## Control Entrypoint +ADD docker/sign/entry.sh /usr/local/bin/entry.sh + +ENTRYPOINT [ "entry.sh" ] diff --git a/docker/sign/README.MD b/docker/sign/README.MD new file mode 100644 index 00000000..769cc60f --- /dev/null +++ b/docker/sign/README.MD @@ -0,0 +1,21 @@ +## Usage + +### Build + +`docker build . -f docker/sign/Dockerfile-signer -t ` + +### Required Variables +* GPG_PRIVATE_KEY -> base64 encoded GPG private signing key +* GPG_SIGNER -> key id to sign with, must match GPG_PRIVATE_KEY +* S3_ACCESS_KEY_ID -> S3 Key with access to rpm/repo +* S3_SECRET_ACCES_KEY -> S3 Secret for S3_ACCESS_KEY_ID + +### Sign RPM artifacts +`docker run --rm -it -e GPG_PRIVATE_KEY -e GPG_SIGNER -v :/tmp rpm-sign /tmp` + +### Upload RPM artifacts +Upload artifacts to appropriate location + +### Update Repo Metadata Files + +`docker run --rm -it -e GPG_PRIVATE_KEY -e GPG_SIGNER -e S3_ACCESS_KEY_ID -e S3_SECRET_ACCESS_KEY repo-update ` \ No newline at end of file diff --git a/docker/sign/entry.sh b/docker/sign/entry.sh new file mode 100755 index 00000000..ed5c5a03 --- /dev/null +++ b/docker/sign/entry.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +if [ -z "${GPG_PRIVATE_KEY}" ]; then + echo "GPG_PRIVATE_KEY not set" + exit 1 +fi + +if [ -z "${GPG_SIGNER}" ]; then + echo "GPG_SIGNER not set" + exit 1 +fi + +echo "${GPG_PRIVATE_KEY}" | base64 -d | gpg2 --import --no-tty --yes + +cat < /root/.rpmmacros +%_signature gpg +%_gpg_name ${GPG_SIGNER} +%_gpg_path /root/.gnupg +%_gpgbin /usr/bin/gpg2 +EOF + + +if [ "${1}" == "rpm-sign" ]; then + if [ -z "${2}" ]; then + echo "Usage: ${0} rpm-sign " + exit 1 + fi + for a in `ls ${2}/*.rpm`; do + rpm --addsign "${a}" + done + exit 0 +elif [ "${1}" == "repo-update" ]; then + if [ -z "${S3_ACCESS_KEY_ID}" ]; then + echo "S3_ACCESS_KEY_ID not set" + exit 1 + fi + if [ -z "${S3_SECRET_ACCESS_KEY}" ]; then + echo "S3_SECRET_ACCESS_KEY not set" + exit 1 + fi + if [ -z "${S3_REGION}" ]; then + echo "Defaulting S3_REGION to us-east-2" + export S3_REGION="us-east-2" + fi + if [ -z "${2}" ]; then + echo "Usage: ${0} repo-update " + exit 1 + fi + mkrepo --s3-access-key-id ${S3_ACCESS_KEY_ID} \ + --s3-secret-access-key ${S3_SECRET_ACCESS_KEY} \ + --s3-public-read --s3-region ${S3_REGION} \ + ${2} + +else + echo "Usage: ${0} " + exit 1 +fi \ No newline at end of file diff --git a/util/build_prep/centos/prep.sh.in b/util/build_prep/centos/prep.sh.in index a321cfd3..75034f70 100644 --- a/util/build_prep/centos/prep.sh.in +++ b/util/build_prep/centos/prep.sh.in @@ -14,6 +14,8 @@ yes | yum install -y rpm-build || exit 1 yes | yum install -y glibc-devel glibc-headers make which libstdc++-static || exit 1 yes | yum install -y centos-release-scl || exit 1 yes | yum install -y devtoolset-7-gcc-c++ || exit 1 +yes | yum install -y rh-python38 || exit 1 + wget -O cmake_install.sh https://github.com/Kitware/CMake/releases/download/v3.15.4/cmake-3.15.4-Linux-x86_64.sh chmod +x cmake_install.sh @@ -25,7 +27,7 @@ rm -f ./cmake_install.sh ( eval "$(scl enable devtoolset-7 "bash -c 'set | grep ^PATH='")" if ! have boost; then - bootstrap_boost -m -s + bootstrap_boost -m -s -j`nproc` fi if ! have boost; then diff --git a/util/changelog.py b/util/changelog.py index dd61cc0d..a4081d89 100755 --- a/util/changelog.py +++ b/util/changelog.py @@ -1,18 +1,35 @@ import argparse import copy import sys +import re +from typing import Tuple """ Changelog generation script, requires PAT with public_repo access, see https://github.com/settings/tokens -Caveats V20 and prior release tags are tips on their respective release branches -If you try to use a start tag with one of these a full changelog will be generated -since the commit wont appear in your iterations +usage: changelog [-h] [-e END] [-m {full,incremental}] -p PAT [-r REPO] [-s START] [-t TAG] + +Generate Changelogs between tags or commits + +optional arguments: + -h, --help show this help message and exit + -e END, --end END Ending reference for Changelog(newest) + -m {full,incremental}, --mode {full,incremental} + Mode to run changelog for [full, incremental] + -p PAT, --pat PAT Personal Access Token + -r REPO, --repo REPO to generate logs for + -s START, --start START + Starting reference for Changelog(oldest) + -t TAG, --tag TAG Tag to use for changelog generation """ + +full = re.compile(r"^(V(\d)+.(\d)+.?(\d)?)$") +incremental = re.compile(r"^(V(\d)+.(\d)+.?(\d)?(RC(\d)+)?(DB(\d)+)?)$") + try: - from github import Github,UnknownObjectException + from github import Github, UnknownObjectException from mdutils import MdUtils except BaseException: sys.exit("Error: run 'pip install PyGithub mdutils'") @@ -80,11 +97,33 @@ SECTIONS = { class cliArgs(): - def __init__(self): + def __init__(self) -> dict: + + changelog_choices = ["full", "incremental"] + parse = argparse.ArgumentParser( prog="changelog", description="Generate Changelogs between tags or commits" ) + + parse.add_argument( + '-e', '--end', + help="Ending reference for Changelog(newest)", + type=str, action="store", + ) + parse.add_argument( + "-m", "--mode", + help="Mode to run changelog for [full, incremental]", + type=str, action="store", + default="incremental", + choices=changelog_choices + ) + parse.add_argument( + '-p', '--pat', + help="Personal Access Token", + type=str, action="store", + required=True, + ) parse.add_argument( '-r', '--repo', help=" to generate logs for", @@ -93,101 +132,97 @@ class cliArgs(): ) parse.add_argument( '-s', '--start', - help="Starting reference for Changelog", + help="Starting reference for Changelog(oldest)", type=str, action="store", - required=True, ) parse.add_argument( - '-e', '--end', - help="Ending reference for Changelog", - type=str, action="store", - required=True, - ) - parse.add_argument( - '--pat', - help="Personal Access Token", - type=str, action="store", - required=True, + '-t', '--tag', + help="Tag to use for changelog generation", + type=str, action="store" ) options = parse.parse_args() + self.end = options.end + self.mode = options.mode + self.pat = options.pat self.repo = options.repo.rstrip("/") self.start = options.start - self.end = options.end - self.pat = options.pat - def __repr__(self): - return "" \ - .format(self.repo, self.start, self.end, self.pat) - def __str__(self): - return "Generating a changelog for {0} starting with {1} " \ - "and ending with {2}".format(self.repo, self.start, self.end) + self.tag = options.tag + + class generateTree: def __init__(self, args): github = Github(args.pat) self.name = args.repo - self.repo = github.get_repo(args.repo) - self.start = args.start - self.end = args.end - try: - self.startCommit = self.repo.get_commit(args.start) - except BaseException: - exit("Error finding commit for " + args.start) - try: - self.endCommit = self.repo.get_commit(args.end) - except BaseException: - exit("Error finding commit for " + args.end) - commits = self.repo.get_commits(sha=self.endCommit.sha) + self.repo = github.get_repo(self.name) + if args.tag: + self.tag = args.tag + self.end = self.repo.get_commit(args.tag).sha + elif args.end: + self.end = self.repo.get_commit(args.end).sha + else: + print("need end or tag") + exit(1) + if args.start: + self.start = self.repo.get_commit(args.start).sha + else: + self.start = self.get_common(args.mode) self.commits = {} - self.other_commits = [] # for commits that do not have an associated pull + self.other_commits = [] + commits = self.repo.get_commits(sha=self.end) for commit in commits: - if commit.sha == self.startCommit.sha: + if commit.sha == self.start: break - else: - message = commit.commit.message.partition('\n')[0] - try: - pr_number = int( - message[message.rfind('#')+1:message.rfind(')')]) + m = commit.commit.message.partition('\n')[0] + try: + pr_number = int(m[m.rfind('#')+1:m.rfind(')')]) + pull = self.repo.get_pull(pr_number) + except (ValueError, UnknownObjectException): + p = commit.get_pulls() + if p.totalCount > 0: + pr_number = p[0].number pull = self.repo.get_pull(pr_number) - except (ValueError, UnknownObjectException): - pulls = commit.get_pulls() - if pulls.totalCount > 0: - # no commits with more than 1 PR associated to it were found in V23 release - # but targeting first entry only if that ends up being the case - pr_number = pulls[0].number - pull = self.repo.get_pull(pr_number) - else: - print("Commit has no associated PR {}: \"{}\"".format( - commit.sha, message)) - self.other_commits.append((commit.sha, message)) - continue + else: + print( + f"Commit has no associated PR {commit.sha}: \"{m}\"") + self.other_commits.append((commit.sha, m)) + continue + + labels = [] + for label in pull.labels: + labels.append(label.name) + self.commits[pull.number] = { + "Title": pull.title, + "Url": pull.html_url, + "labels": labels + } + + def get_common(self, mode) -> str: + tags = [] + for tag in self.repo.get_tags(): + if mode == "full": + found = full.match(tag.name) + else: + found = incremental.match(tag.name) + if found: + tags.append(tag) + tree = self.repo.compare(self.end, tags[1].commit.sha) + return tree.merge_base_commit.sha + - labels = [] - for label in pull.labels: - labels.append(label.name) - self.commits[pull.number] = { - "Title": pull.title, - "Url": pull.html_url, - "labels": labels - } - def __repr__(self): - return " Tuple[str, bool]: for section, values in SECTIONS.items(): for label in labels: if label in values: @@ -233,7 +272,8 @@ class generateMarkdown(): else: return section, False return 'Other', False - def pull_to_section(self, commits): + + def pull_to_section(self, commits) -> dict: sect = copy.deepcopy(SECTIONS) result = {} for a in sect: @@ -248,7 +288,8 @@ class generateMarkdown(): if len(sect[a]) > 0: result[a] = sect[a] return result -if __name__ == "__main__": - args = cliArgs() - repo = generateTree(args) - generateMarkdown(repo) + + +arg = cliArgs() +trees = generateTree(arg) +generateMarkdown(trees) diff --git a/util/makesrc b/util/makesrc index 50bb6fa9..f42a06ba 100755 --- a/util/makesrc +++ b/util/makesrc @@ -1,23 +1,24 @@ #!/bin/bash -# makesrc -# requires ruby gem github_changelog_generator -# API_KEY generated here https://github.com/settings/tokens/new?description=GitHub%20Changelog%20Generator%20token +# makesrc +# personal access token allows around rate limit -if [ -e $1 ]; then - echo "makesrc " >&2 +if [ -e $2 ]; then + echo "makesrc " >&2 echo " tag valid for nanocurrency/nano-node" >&2 + echo " pat personal access token" >&2 exit 1 fi TAG="$1" +PAT="$2" VERSION=$(echo $TAG | sed 's/V//') TAG_DATE="" VERSION_MAJOR=$(echo $VERSION | cut -d "." -f 1) scriptDir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" function make_source() { - git clone --recursive --single-branch --branch releases/v${VERSION_MAJOR} https://github.com/nanocurrency/nano-node nano-$VERSION + git clone --recursive --single-branch --branch ${TAG} https://github.com/nanocurrency/nano-node nano-$VERSION cd nano-$VERSION git pull --tags COUNT=$(git tag -l "${TAG}" | wc -l) @@ -39,19 +40,18 @@ function source_information() { COMMIT=$(git log | head -1 | cut -d " " -f 2) TAG_DATE=$(TZ=UTC date -d"$DATE" +%s) - case "${VERSION}" in - *RC*) - "${scriptDir}/changelog_generator" nanocurrency/nano-node "V${VERSION}" only - ;; - *) - "${scriptDir}/changelog_generator" nanocurrency/nano-node "V${VERSION}" - ;; - esac + ( + eval "$(scl enable rh-python38 'echo -e "source /opt/rh/rh-python38/enable\n"')" + python -m pip install PyGithub mdutils - if [ ! -s CHANGELOG.md ]; then - echo "CHANGELOG not generated" - exit 1 - fi + python ${scriptDir}/changelog.py -p ${PAT} -t ${TAG} -m full + + if [ ! -s CHANGELOG.md ]; then + echo "CHANGELOG not generated" + exit 1 + fi + exit 0 + ) || exit 1 export TAG_DATE }