initial centos cleanup

rpm build image, and signing container

rpm build updates

.github/workflows/live_artifacts.yml

@@ -66,6 +66,34 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: us-east-2
 
+  linux_rpm_job:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 90
+    steps:
+      - name: tag
+        run: |
+          echo "TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV
+      - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+        with:
+          submodules: "recursive"
+          ref: ${{ github.event.inputs.ref }}
+          repository: ${{ github.event.inputs.repo }}
+      - name: centos base
+        run: ci/build-docker-image.sh docker/ci/Dockerfile-centos nanocurrency/nano-env:centos
+      - name: Build Artifact
+        run: |
+          docker run -e LIVE=1 -v ${GITHUB_WORKSPACE}:/workspace \
+          -v artifacts:/root/rpmbuild/RPMS -v source:/root/rpmbuild/SOURCES \
+          nanocurrency/nano-env:centos /bin/bash -c "ci/build-centos.sh"
+        env:
+          PAT: ${{ secrets.GITHUB_TOKEN }}
+      - name: Deploy Artifact
+        run: ci/actions/deploy.sh
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-2
+
   linux_docker_job:
     runs-on: ubuntu-20.04
     timeout-minutes: 90

ci/actions/linux/install_deps.sh

@@ -8,4 +8,5 @@ if [[ "${COMPILER:-}" != "" ]]; then
 else
     ci/build-docker-image.sh docker/ci/Dockerfile-gcc nanocurrency/nano-env:gcc
     ci/build-docker-image.sh docker/ci/Dockerfile-clang-6 nanocurrency/nano-env:clang-6
+    ci/build-docker-image.sh docker/ci/Dockerfile-centos nanocurrency/nano-env:centos
 fi

ci/build-centos.sh

@@ -1,17 +1,20 @@
 #!/usr/bin/env bash
-TAGS=$(git describe --abbrev=0 --tags)
+TAG=$(echo $TAG)
-VERSIONS=${TAGS//V/}
+VERSIONS=${TAG//V/}
 RELEASE=$(echo $CI_JOB_ID)
 
 run_source() {
-	./util/makesrc $TAGS
+	./util/makesrc $TAG $(echo $PAT)
 }
 
 run_build() {
 	mkdir -p ~/rpmbuild/SOURCES/
 	mv -f ~/nano-${VERSIONS}.tar.gz ~/rpmbuild/SOURCES/.
-	scl enable devtoolset-7 'rpmbuild -ba nanocurrency.spec'
+	if [ "${LIVE:-}" == "1" ]; then
-	scl enable devtoolset-7 'rpmbuild -ba nanocurrency-beta.spec'
+		scl enable devtoolset-7 'rpmbuild -ba nanocurrency.spec'
+	else
+		scl enable devtoolset-7 'rpmbuild -ba nanocurrency-beta.spec'
+	fi
 }
 
 run_update() {

docker/ci/Dockerfile-centos

@@ -0,0 +1,8 @@
+FROM centos:7
+
+WORKDIR /workspace
+
+ADD util /build
+RUN bash /build/build_prep/update-common
+RUN bash /build/build_prep/centos/prep.sh
+RUN rm -rf /build

docker/sign/Dockerfile-signer

@@ -0,0 +1,24 @@
+FROM python:3.9-buster
+
+
+ENV DEBIAN_FRONTEND="noninteractive"
+ENV TZ="America/Los_Angeles"
+ENV GPG_TTY=/dev/console
+
+ARG GPG_PRIVATE_KEY
+ARG GPG_SIGNER
+
+## Install required for rpm signing
+RUN apt-get update -yqq && \
+    apt-get install -y librpmsign8 gnupg2 wget rpm
+
+## Additional for mkrepo support
+RUN python3 -m pip install mkrepo boto3
+
+#WORKDIR /signing
+#RUN python3 -m venv && .venv/bin/pip install mkrepo
+
+## Control Entrypoint
+ADD docker/sign/entry.sh /usr/local/bin/entry.sh
+
+ENTRYPOINT [ "entry.sh" ]

docker/sign/README.MD

@@ -0,0 +1,21 @@
+## Usage
+
+### Build
+
+`docker build . -f docker/sign/Dockerfile-signer -t <docker/image/tag>`
+
+### Required Variables
+* GPG_PRIVATE_KEY -> base64 encoded GPG private signing key
+* GPG_SIGNER -> key id to sign with, must match GPG_PRIVATE_KEY
+* S3_ACCESS_KEY_ID -> S3 Key with access to rpm/repo
+* S3_SECRET_ACCES_KEY -> S3 Secret for S3_ACCESS_KEY_ID
+
+### Sign RPM artifacts
+`docker run --rm -it -e GPG_PRIVATE_KEY -e GPG_SIGNER -v </directory/containing/rpm/to/sign/>:/tmp <docker/image/tag> rpm-sign /tmp`
+
+### Upload RPM artifacts
+Upload artifacts to appropriate location
+
+### Update Repo Metadata Files
+
+`docker run --rm -it -e GPG_PRIVATE_KEY -e GPG_SIGNER -e S3_ACCESS_KEY_ID -e S3_SECRET_ACCESS_KEY <docker/image/tag> repo-update <s3://root/of/rpm/repo>`

docker/sign/entry.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+if [ -z "${GPG_PRIVATE_KEY}" ]; then
+    echo "GPG_PRIVATE_KEY not set"
+    exit 1
+fi
+
+if [ -z "${GPG_SIGNER}" ]; then
+    echo "GPG_SIGNER not set"
+    exit 1
+fi
+
+echo "${GPG_PRIVATE_KEY}" | base64 -d | gpg2 --import --no-tty --yes
+
+cat <<EOF > /root/.rpmmacros
+%_signature gpg
+%_gpg_name ${GPG_SIGNER}
+%_gpg_path /root/.gnupg
+%_gpgbin /usr/bin/gpg2
+EOF
+
+
+if [ "${1}" == "rpm-sign" ]; then
+   if [ -z "${2}" ]; then
+        echo "Usage: ${0} rpm-sign <base-dir-to-sign>"
+        exit 1
+    fi
+    for a in `ls ${2}/*.rpm`; do
+        rpm --addsign "${a}"
+    done
+    exit 0
+elif [ "${1}" == "repo-update" ]; then
+    if [ -z "${S3_ACCESS_KEY_ID}" ]; then
+        echo "S3_ACCESS_KEY_ID not set"
+        exit 1
+    fi
+    if [ -z "${S3_SECRET_ACCESS_KEY}" ]; then
+        echo "S3_SECRET_ACCESS_KEY not set"
+        exit 1
+    fi
+    if [ -z "${S3_REGION}" ]; then
+        echo "Defaulting S3_REGION to us-east-2"
+        export S3_REGION="us-east-2"
+    fi
+    if [ -z "${2}" ]; then
+        echo "Usage: ${0} repo-update <s3 path>"
+        exit 1
+    fi
+    mkrepo --s3-access-key-id ${S3_ACCESS_KEY_ID} \
+        --s3-secret-access-key ${S3_SECRET_ACCESS_KEY} \
+        --s3-public-read --s3-region ${S3_REGION} \
+        ${2}
+
+else
+    echo "Usage: ${0} <rpm-sign|repo-update>"
+    exit 1
+fi

util/build_prep/centos/prep.sh.in

@@ -14,6 +14,8 @@ yes | yum install -y rpm-build || exit 1
 yes | yum install -y glibc-devel glibc-headers make which libstdc++-static || exit 1
 yes | yum install -y centos-release-scl || exit 1
 yes | yum install -y devtoolset-7-gcc-c++ || exit 1
+yes | yum install -y rh-python38 || exit 1
+
 
 wget -O cmake_install.sh https://github.com/Kitware/CMake/releases/download/v3.15.4/cmake-3.15.4-Linux-x86_64.sh
 chmod +x cmake_install.sh
@@ -25,7 +27,7 @@ rm -f ./cmake_install.sh
 (
 	eval "$(scl enable devtoolset-7 "bash -c 'set | grep ^PATH='")"
 	if ! have boost; then
-		bootstrap_boost -m -s
+		bootstrap_boost -m -s -j`nproc`
 	fi
 
 	if ! have boost; then

util/changelog.py

@@ -1,18 +1,35 @@
 import argparse
 import copy
 import sys
+import re
+from typing import Tuple
 
 """
 Changelog generation script, requires PAT with public_repo access, 
 see https://github.com/settings/tokens
 
-Caveats V20 and prior release tags are tips on their respective release branches
+usage: changelog [-h] [-e END] [-m {full,incremental}] -p PAT [-r REPO] [-s START] [-t TAG]
-If you try to use a start tag with one of these a full changelog will be generated
+
-since the commit wont appear in your iterations
+Generate Changelogs between tags or commits
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -e END, --end END     Ending reference for Changelog(newest)
+  -m {full,incremental}, --mode {full,incremental}
+                        Mode to run changelog for [full, incremental]
+  -p PAT, --pat PAT     Personal Access Token
+  -r REPO, --repo REPO  <org/repo> to generate logs for
+  -s START, --start START
+                        Starting reference for Changelog(oldest)
+  -t TAG, --tag TAG     Tag to use for changelog generation
 """
 
+
+full = re.compile(r"^(V(\d)+.(\d)+.?(\d)?)$")
+incremental = re.compile(r"^(V(\d)+.(\d)+.?(\d)?(RC(\d)+)?(DB(\d)+)?)$")
+
 try:
-    from github import Github,UnknownObjectException
+    from github import Github, UnknownObjectException
     from mdutils import MdUtils
 except BaseException:
     sys.exit("Error: run 'pip install PyGithub mdutils'")
@@ -80,11 +97,33 @@ SECTIONS = {
 
 
 class cliArgs():
-    def __init__(self):
+    def __init__(self) -> dict:
+
+        changelog_choices = ["full", "incremental"]
+
         parse = argparse.ArgumentParser(
             prog="changelog",
             description="Generate Changelogs between tags or commits"
         )
+
+        parse.add_argument(
+            '-e', '--end',
+            help="Ending reference for Changelog(newest)",
+            type=str, action="store",
+        )
+        parse.add_argument(
+            "-m", "--mode",
+            help="Mode to run changelog for [full, incremental]",
+            type=str, action="store",
+            default="incremental",
+            choices=changelog_choices
+        )
+        parse.add_argument(
+            '-p', '--pat',
+            help="Personal Access Token",
+            type=str, action="store",
+            required=True,
+        )
         parse.add_argument(
             '-r', '--repo',
             help="<org/repo> to generate logs for",
@@ -93,101 +132,97 @@ class cliArgs():
         )
         parse.add_argument(
             '-s', '--start',
-            help="Starting reference for Changelog",
+            help="Starting reference for Changelog(oldest)",
             type=str, action="store",
-            required=True,
         )
         parse.add_argument(
-            '-e', '--end',
+            '-t', '--tag',
-            help="Ending reference for Changelog",
+            help="Tag to use for changelog generation",
-            type=str, action="store",
+            type=str, action="store"
-            required=True,
-        )
-        parse.add_argument(
-            '--pat',
-            help="Personal Access Token",
-            type=str, action="store",
-            required=True,
         )
         options = parse.parse_args()
+        self.end = options.end
+        self.mode = options.mode
+        self.pat = options.pat
         self.repo = options.repo.rstrip("/")
         self.start = options.start
-        self.end = options.end
+        self.tag = options.tag
-        self.pat = options.pat
+
-    def __repr__(self):
+
-        return "<cliArgs(repo='{0}', start='{1}', end='{2}', pat='{3}')>" \
-            .format(self.repo, self.start, self.end, self.pat)
-    def __str__(self):
-        return "Generating a changelog for {0} starting with {1} " \
-            "and ending with {2}".format(self.repo, self.start, self.end)
 class generateTree:
     def __init__(self, args):
         github = Github(args.pat)
         self.name = args.repo
-        self.repo = github.get_repo(args.repo)
+        self.repo = github.get_repo(self.name)
-        self.start = args.start
+        if args.tag:
-        self.end = args.end
+            self.tag = args.tag
-        try:
+            self.end = self.repo.get_commit(args.tag).sha
-            self.startCommit = self.repo.get_commit(args.start)
+        elif args.end:
-        except BaseException:
+            self.end = self.repo.get_commit(args.end).sha
-            exit("Error finding commit for " + args.start)
+        else:
-        try:
+            print("need end or tag")
-            self.endCommit = self.repo.get_commit(args.end)
+            exit(1)
-        except BaseException:
+        if args.start:
-            exit("Error finding commit for " + args.end)
+            self.start = self.repo.get_commit(args.start).sha
-        commits = self.repo.get_commits(sha=self.endCommit.sha)
+        else:
+            self.start = self.get_common(args.mode)
         self.commits = {}
-        self.other_commits = []  # for commits that do not have an associated pull
+        self.other_commits = []
+        commits = self.repo.get_commits(sha=self.end)
         for commit in commits:
-            if commit.sha == self.startCommit.sha:
+            if commit.sha == self.start:
                 break
-            else:
+            m = commit.commit.message.partition('\n')[0]
-                message = commit.commit.message.partition('\n')[0]
+            try:
-                try:
+                pr_number = int(m[m.rfind('#')+1:m.rfind(')')])
-                    pr_number = int(
+                pull = self.repo.get_pull(pr_number)
-                        message[message.rfind('#')+1:message.rfind(')')])
+            except (ValueError, UnknownObjectException):
+                p = commit.get_pulls()
+                if p.totalCount > 0:
+                    pr_number = p[0].number
                     pull = self.repo.get_pull(pr_number)
-                except (ValueError, UnknownObjectException):
+                else:
-                    pulls = commit.get_pulls()
+                    print(
-                    if pulls.totalCount > 0:
+                        f"Commit has no associated PR {commit.sha}: \"{m}\"")
-                        # no commits with more than 1 PR associated to it were found in V23 release
+                    self.other_commits.append((commit.sha, m))
-                        # but targeting first entry only if that ends up being the case
+                    continue
-                        pr_number = pulls[0].number
+
-                        pull = self.repo.get_pull(pr_number)
+            labels = []
-                    else:
+            for label in pull.labels:
-                        print("Commit has no associated PR {}: \"{}\"".format(
+                labels.append(label.name)
-                        commit.sha, message))
+            self.commits[pull.number] = {
-                        self.other_commits.append((commit.sha, message))
+                "Title": pull.title,
-                        continue
+                "Url": pull.html_url,
+                "labels": labels
+            }
+
+    def get_common(self, mode) -> str:
+        tags = []
+        for tag in self.repo.get_tags():
+            if mode == "full":
+                found = full.match(tag.name)
+            else:
+                found = incremental.match(tag.name)
+            if found:
+                tags.append(tag)
+        tree = self.repo.compare(self.end, tags[1].commit.sha)
+        return tree.merge_base_commit.sha
+
 
-                labels = []
-                for label in pull.labels:
-                    labels.append(label.name)
-                self.commits[pull.number] = {
-                    "Title": pull.title,
-                    "Url": pull.html_url,
-                    "labels": labels
-                }
-    def __repr__(self):
-        return "<generateTree(repo='{0}', start='{1}', startCommit='{2}', " \
-            "end='{3}', endCommit='{4}', tree='{5}', commits='{6}".format(
-                self.repo, self.start, self.startCommit, self.end,
-                self.endCommit, self.tree, self.commits,
-            )
 class generateMarkdown():
-    def __repr__(self):
+    def __init__(self, repo: generateTree):
-        return "<generateMarkdown(mdFile={0})>".format(
-            self.mdFile
-        )
-    def __init__(self, repo):
         self.mdFile = MdUtils(
             file_name='CHANGELOG', title='CHANGELOG'
         )
-        self.mdFile.new_line(
+        if repo.tag:
-            "## Release " +
+            self.mdFile.new_line(
-            "[{0}](https://github.com/nanocurrency/nano-node/tree/{0})".format(repo.end), wrap_width=0)
+                "## Release " +
-        self.mdFile.new_line("[Full Changelog](https://github.com/nanocurrency"
+                f"[{repo.tag}](https://github.com/{repo.name}/tree/{repo.tag})", wrap_width=0)
-                             "/nano-node/compare/{0}...{1})".format(repo.start, repo.end), wrap_width=0)
+        else:
+            self.mdFile.new_line(
+                f"[{repo.end}](https://github.com/{repo.name}/tree/{repo.end})", wrap_width=0)
+        self.mdFile.new_line(f"[Full Changelog](https://github.com/{repo.name}"
+                             f"/compare/{repo.start}...{repo.end})", wrap_width=0)
         sort = self.pull_to_section(repo.commits)
         for section, prs in sort.items():
             self.write_header_PR(section)
@@ -198,30 +233,34 @@ class generateMarkdown():
             for sha, message in repo.other_commits:
                 self.write_no_PR(repo, sha, message)
         self.mdFile.create_md_file()
+
     def write_header_PR(self, section):
         self.mdFile.new_line("---")
-        self.mdFile.new_header(level=3, title=section, add_table_of_contents='n')
+        self.mdFile.new_header(level=3, title=section,
+                               add_table_of_contents='n')
         self.mdFile.new_line(
             "|Pull Request|Title")
         self.mdFile.new_line("|:-:|:--")
+
     def write_header_no_PR(self):
         self.mdFile.new_line()
         self.mdFile.new_line(
             "|Commit|Title")
         self.mdFile.new_line("|:-:|:--")
+
     def write_PR(self, pr, info):
         imp = ""
         if pr[1]:
             imp = "**BREAKING** "
         self.mdFile.new_line(
-            "|[#{0}]({1})|{2}{3}".format(
+            f"|[#{pr[0]}]({info['Url']})|{imp}{info['Title']}", wrap_width=0)
-                pr[0], info['Url'], imp, info['Title']), wrap_width=0)
+
     def write_no_PR(self, repo, sha, message):
-        url = "https://github.com/{0}/commit/{1}".format(repo.name, sha)
+        url = f"https://github.com/{repo.name}/commit/{sha}"
         self.mdFile.new_line(
-            "|[{0}]({1})|{2}".format(
+            f"|[{sha[:8]}]({url})|{message}", wrap_width=0)
-                sha[:8], url, message), wrap_width=0)
+
-    def handle_labels(self, labels):
+    def handle_labels(self, labels) -> Tuple[str, bool]:
         for section, values in SECTIONS.items():
             for label in labels:
                 if label in values:
@@ -233,7 +272,8 @@ class generateMarkdown():
                     else:
                         return section, False
         return 'Other', False
-    def pull_to_section(self, commits):
+
+    def pull_to_section(self, commits) -> dict:
         sect = copy.deepcopy(SECTIONS)
         result = {}
         for a in sect:
@@ -248,7 +288,8 @@ class generateMarkdown():
             if len(sect[a]) > 0:
                 result[a] = sect[a]
         return result
-if __name__ == "__main__":
+
-    args = cliArgs()
+
-    repo = generateTree(args)
+arg = cliArgs()
-    generateMarkdown(repo)
+trees = generateTree(arg)
+generateMarkdown(trees)

util/makesrc

@@ -1,23 +1,24 @@
 #!/bin/bash
 
-#    makesrc <tag>
+#    makesrc <tag> <pat> 
-#    requires ruby gem github_changelog_generator
+#    personal access token allows around rate limit
-#    API_KEY generated here https://github.com/settings/tokens/new?description=GitHub%20Changelog%20Generator%20token
 
-if [ -e $1 ]; then
+if [ -e $2 ]; then
-	echo "makesrc <tag>" >&2
+	echo "makesrc <tag> <pat>" >&2
 	echo "    tag      valid <tag> for nanocurrency/nano-node" >&2
+	echo "    pat      personal access token" >&2
 	exit 1
 fi
 
 TAG="$1"
+PAT="$2"
 VERSION=$(echo $TAG | sed 's/V//')
 TAG_DATE=""
 VERSION_MAJOR=$(echo $VERSION | cut -d "." -f 1)
 scriptDir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
 
 function make_source() {
-	git clone --recursive --single-branch --branch releases/v${VERSION_MAJOR} https://github.com/nanocurrency/nano-node nano-$VERSION
+	git clone --recursive --single-branch --branch ${TAG} https://github.com/nanocurrency/nano-node nano-$VERSION
 	cd nano-$VERSION
 	git pull --tags
 	COUNT=$(git tag -l "${TAG}" | wc -l)
@@ -39,19 +40,18 @@ function source_information() {
 	COMMIT=$(git log | head -1 | cut -d " " -f 2)
 	TAG_DATE=$(TZ=UTC date -d"$DATE" +%s)
 
-	case "${VERSION}" in
+	(
-	*RC*)
+		eval "$(scl enable rh-python38 'echo -e "source /opt/rh/rh-python38/enable\n"')"
-		"${scriptDir}/changelog_generator" nanocurrency/nano-node "V${VERSION}" only
+		python -m pip install PyGithub mdutils
-		;;
-	*)
-		"${scriptDir}/changelog_generator" nanocurrency/nano-node "V${VERSION}"
-		;;
-	esac
 
-	if [ ! -s CHANGELOG.md ]; then
+		python ${scriptDir}/changelog.py -p ${PAT} -t ${TAG} -m full
-		echo "CHANGELOG not generated"
+
-		exit 1
+		if [ ! -s CHANGELOG.md ]; then
-	fi
+			echo "CHANGELOG not generated"
+			exit 1
+		fi
+		exit 0
+	) || exit 1
 
 	export TAG_DATE
 }