Minecon724/dncurrency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

initial centos cleanup

Browse source 
rpm build image, and signing container

rpm build updates
This commit is contained in:
Russel Waters 2022-03-01 23:37:25 -08:00 • committed by Thiago Silva
commit 8a2f0d39db
No known key found for this signature in database
GPG key ID: 034303EB8F453169
10 changed files with 301 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

28
.github/workflows/live_artifacts.yml vendored
View file

@ -66,6 +66,34 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-2
linux_rpm_job:
runs-on: ubuntu-20.04
timeout-minutes: 90
steps:
- name: tag
run: |
echo "TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV
- uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
with:
submodules: "recursive"
ref: ${{ github.event.inputs.ref }}
repository: ${{ github.event.inputs.repo }}
- name: centos base
run: ci/build-docker-image.sh docker/ci/Dockerfile-centos nanocurrency/nano-env:centos
- name: Build Artifact
run: |
docker run -e LIVE=1 -v ${GITHUB_WORKSPACE}:/workspace \
-v artifacts:/root/rpmbuild/RPMS -v source:/root/rpmbuild/SOURCES \
nanocurrency/nano-env:centos /bin/bash -c "ci/build-centos.sh"
env:
PAT: ${{ secrets.GITHUB_TOKEN }}
- name: Deploy Artifact
run: ci/actions/deploy.sh
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-2
linux_docker_job:
runs-on: ubuntu-20.04
timeout-minutes: 90

1
ci/actions/linux/install_deps.sh
View file

@ -8,4 +8,5 @@ if [[ "${COMPILER:-}" != "" ]]; then
else
ci/build-docker-image.sh docker/ci/Dockerfile-gcc nanocurrency/nano-env:gcc
ci/build-docker-image.sh docker/ci/Dockerfile-clang-6 nanocurrency/nano-env:clang-6
ci/build-docker-image.sh docker/ci/Dockerfile-centos nanocurrency/nano-env:centos
fi

13
ci/build-centos.sh
View file

@ -1,17 +1,20 @@
#!/usr/bin/env bash
TAGS=$(git describe --abbrev=0 --tags)
VERSIONS=${TAGS//V/}
TAG=$(echo $TAG)
VERSIONS=${TAG//V/}
RELEASE=$(echo $CI_JOB_ID)
run_source() {
./util/makesrc $TAGS
./util/makesrc $TAG $(echo $PAT)
}
run_build() {
mkdir -p ~/rpmbuild/SOURCES/
mv -f ~/nano-${VERSIONS}.tar.gz ~/rpmbuild/SOURCES/.
scl enable devtoolset-7 'rpmbuild -ba nanocurrency.spec'
scl enable devtoolset-7 'rpmbuild -ba nanocurrency-beta.spec'
if [ "${LIVE:-}" == "1" ]; then
scl enable devtoolset-7 'rpmbuild -ba nanocurrency.spec'
else
scl enable devtoolset-7 'rpmbuild -ba nanocurrency-beta.spec'
fi
}
run_update() {

8
docker/ci/Dockerfile-centos Normal file
View file

@ -0,0 +1,8 @@
FROM centos:7
WORKDIR /workspace
ADD util /build
RUN bash /build/build_prep/update-common
RUN bash /build/build_prep/centos/prep.sh
RUN rm -rf /build

24
docker/sign/Dockerfile-signer Normal file
View file

@ -0,0 +1,24 @@
FROM python:3.9-buster
ENV DEBIAN_FRONTEND="noninteractive"
ENV TZ="America/Los_Angeles"
ENV GPG_TTY=/dev/console
ARG GPG_PRIVATE_KEY
ARG GPG_SIGNER
## Install required for rpm signing
RUN apt-get update -yqq && \
apt-get install -y librpmsign8 gnupg2 wget rpm
## Additional for mkrepo support
RUN python3 -m pip install mkrepo boto3
#WORKDIR /signing
#RUN python3 -m venv && .venv/bin/pip install mkrepo
## Control Entrypoint
ADD docker/sign/entry.sh /usr/local/bin/entry.sh
ENTRYPOINT [ "entry.sh" ]

21
docker/sign/README.MD Normal file
View file

@ -0,0 +1,21 @@
## Usage
### Build
`docker build . -f docker/sign/Dockerfile-signer -t <docker/image/tag>`
### Required Variables
* GPG_PRIVATE_KEY -> base64 encoded GPG private signing key
* GPG_SIGNER -> key id to sign with, must match GPG_PRIVATE_KEY
* S3_ACCESS_KEY_ID -> S3 Key with access to rpm/repo
* S3_SECRET_ACCES_KEY -> S3 Secret for S3_ACCESS_KEY_ID
### Sign RPM artifacts
`docker run --rm -it -e GPG_PRIVATE_KEY -e GPG_SIGNER -v </directory/containing/rpm/to/sign/>:/tmp <docker/image/tag> rpm-sign /tmp`
### Upload RPM artifacts
Upload artifacts to appropriate location
### Update Repo Metadata Files
`docker run --rm -it -e GPG_PRIVATE_KEY -e GPG_SIGNER -e S3_ACCESS_KEY_ID -e S3_SECRET_ACCESS_KEY <docker/image/tag> repo-update <s3://root/of/rpm/repo>`

57
docker/sign/entry.sh Executable file
View file

@ -0,0 +1,57 @@
#!/bin/bash
if [ -z "${GPG_PRIVATE_KEY}" ]; then
echo "GPG_PRIVATE_KEY not set"
exit 1
fi
if [ -z "${GPG_SIGNER}" ]; then
echo "GPG_SIGNER not set"
exit 1
fi
echo "${GPG_PRIVATE_KEY}" | base64 -d | gpg2 --import --no-tty --yes
cat <<EOF > /root/.rpmmacros
%_signature gpg
%_gpg_name ${GPG_SIGNER}
%_gpg_path /root/.gnupg
%_gpgbin /usr/bin/gpg2
EOF
if [ "${1}" == "rpm-sign" ]; then
if [ -z "${2}" ]; then
echo "Usage: ${0} rpm-sign <base-dir-to-sign>"
exit 1
fi
for a in `ls ${2}/*.rpm`; do
rpm --addsign "${a}"
done
exit 0
elif [ "${1}" == "repo-update" ]; then
if [ -z "${S3_ACCESS_KEY_ID}" ]; then
echo "S3_ACCESS_KEY_ID not set"
exit 1
fi
if [ -z "${S3_SECRET_ACCESS_KEY}" ]; then
echo "S3_SECRET_ACCESS_KEY not set"
exit 1
fi
if [ -z "${S3_REGION}" ]; then
echo "Defaulting S3_REGION to us-east-2"
export S3_REGION="us-east-2"
fi
if [ -z "${2}" ]; then
echo "Usage: ${0} repo-update <s3 path>"
exit 1
fi
mkrepo --s3-access-key-id ${S3_ACCESS_KEY_ID} \
--s3-secret-access-key ${S3_SECRET_ACCESS_KEY} \
--s3-public-read --s3-region ${S3_REGION} \
${2}
else
echo "Usage: ${0} <rpm-sign|repo-update>"
exit 1
fi

4
util/build_prep/centos/prep.sh.in
View file

@ -14,6 +14,8 @@ yes | yum install -y rpm-build || exit 1
yes | yum install -y glibc-devel glibc-headers make which libstdc++-static || exit 1
yes | yum install -y centos-release-scl || exit 1
yes | yum install -y devtoolset-7-gcc-c++ || exit 1
yes | yum install -y rh-python38 || exit 1
wget -O cmake_install.sh https://github.com/Kitware/CMake/releases/download/v3.15.4/cmake-3.15.4-Linux-x86_64.sh
chmod +x cmake_install.sh
@ -25,7 +27,7 @@ rm -f ./cmake_install.sh
(
eval "$(scl enable devtoolset-7 "bash -c 'set | grep ^PATH='")"
if ! have boost; then
bootstrap_boost -m -s
bootstrap_boost -m -s -j`nproc`
fi
if ! have boost; then

225
util/changelog.py
View file

@ -1,18 +1,35 @@
import argparse
import copy
import sys
import re
from typing import Tuple
"""
Changelog generation script, requires PAT with public_repo access,
see https://github.com/settings/tokens
Caveats V20 and prior release tags are tips on their respective release branches
If you try to use a start tag with one of these a full changelog will be generated
since the commit wont appear in your iterations
usage: changelog [-h] [-e END] [-m {full,incremental}] -p PAT [-r REPO] [-s START] [-t TAG]
Generate Changelogs between tags or commits
optional arguments:
-h, --help show this help message and exit
-e END, --end END Ending reference for Changelog(newest)
-m {full,incremental}, --mode {full,incremental}
Mode to run changelog for [full, incremental]
-p PAT, --pat PAT Personal Access Token
-r REPO, --repo REPO <org/repo> to generate logs for
-s START, --start START
Starting reference for Changelog(oldest)
-t TAG, --tag TAG Tag to use for changelog generation
"""
full = re.compile(r"^(V(\d)+.(\d)+.?(\d)?)$")
incremental = re.compile(r"^(V(\d)+.(\d)+.?(\d)?(RC(\d)+)?(DB(\d)+)?)$")
try:
from github import Github,UnknownObjectException
from github import Github, UnknownObjectException
from mdutils import MdUtils
except BaseException:
sys.exit("Error: run 'pip install PyGithub mdutils'")
@ -80,11 +97,33 @@ SECTIONS = {
class cliArgs():
def __init__(self):
def __init__(self) -> dict:
changelog_choices = ["full", "incremental"]
parse = argparse.ArgumentParser(
prog="changelog",
description="Generate Changelogs between tags or commits"
)
parse.add_argument(
'-e', '--end',
help="Ending reference for Changelog(newest)",
type=str, action="store",
)
parse.add_argument(
"-m", "--mode",
help="Mode to run changelog for [full, incremental]",
type=str, action="store",
default="incremental",
choices=changelog_choices
)
parse.add_argument(
'-p', '--pat',
help="Personal Access Token",
type=str, action="store",
required=True,
)
parse.add_argument(
'-r', '--repo',
help="<org/repo> to generate logs for",
@ -93,101 +132,97 @@ class cliArgs():
)
parse.add_argument(
'-s', '--start',
help="Starting reference for Changelog",
help="Starting reference for Changelog(oldest)",
type=str, action="store",
required=True,
)
parse.add_argument(
'-e', '--end',
help="Ending reference for Changelog",
type=str, action="store",
required=True,
)
parse.add_argument(
'--pat',
help="Personal Access Token",
type=str, action="store",
required=True,
'-t', '--tag',
help="Tag to use for changelog generation",
type=str, action="store"
)
options = parse.parse_args()
self.end = options.end
self.mode = options.mode
self.pat = options.pat
self.repo = options.repo.rstrip("/")
self.start = options.start
self.end = options.end
self.pat = options.pat
def __repr__(self):
return "<cliArgs(repo='{0}', start='{1}', end='{2}', pat='{3}')>" \
.format(self.repo, self.start, self.end, self.pat)
def __str__(self):
return "Generating a changelog for {0} starting with {1} " \
"and ending with {2}".format(self.repo, self.start, self.end)
self.tag = options.tag
class generateTree:
def __init__(self, args):
github = Github(args.pat)
self.name = args.repo
self.repo = github.get_repo(args.repo)
self.start = args.start
self.end = args.end
try:
self.startCommit = self.repo.get_commit(args.start)
except BaseException:
exit("Error finding commit for " + args.start)
try:
self.endCommit = self.repo.get_commit(args.end)
except BaseException:
exit("Error finding commit for " + args.end)
commits = self.repo.get_commits(sha=self.endCommit.sha)
self.repo = github.get_repo(self.name)
if args.tag:
self.tag = args.tag
self.end = self.repo.get_commit(args.tag).sha
elif args.end:
self.end = self.repo.get_commit(args.end).sha
else:
print("need end or tag")
exit(1)
if args.start:
self.start = self.repo.get_commit(args.start).sha
else:
self.start = self.get_common(args.mode)
self.commits = {}
self.other_commits = [] # for commits that do not have an associated pull
self.other_commits = []
commits = self.repo.get_commits(sha=self.end)
for commit in commits:
if commit.sha == self.startCommit.sha:
if commit.sha == self.start:
break
else:
message = commit.commit.message.partition('\n')[0]
try:
pr_number = int(
message[message.rfind('#')+1:message.rfind(')')])
m = commit.commit.message.partition('\n')[0]
try:
pr_number = int(m[m.rfind('#')+1:m.rfind(')')])
pull = self.repo.get_pull(pr_number)
except (ValueError, UnknownObjectException):
p = commit.get_pulls()
if p.totalCount > 0:
pr_number = p[0].number
pull = self.repo.get_pull(pr_number)
except (ValueError, UnknownObjectException):
pulls = commit.get_pulls()
if pulls.totalCount > 0:
# no commits with more than 1 PR associated to it were found in V23 release
# but targeting first entry only if that ends up being the case
pr_number = pulls[0].number
pull = self.repo.get_pull(pr_number)
else:
print("Commit has no associated PR {}: \"{}\"".format(
commit.sha, message))
self.other_commits.append((commit.sha, message))
continue
else:
print(
f"Commit has no associated PR {commit.sha}: \"{m}\"")
self.other_commits.append((commit.sha, m))
continue
labels = []
for label in pull.labels:
labels.append(label.name)
self.commits[pull.number] = {
"Title": pull.title,
"Url": pull.html_url,
"labels": labels
}
def get_common(self, mode) -> str:
tags = []
for tag in self.repo.get_tags():
if mode == "full":
found = full.match(tag.name)
else:
found = incremental.match(tag.name)
if found:
tags.append(tag)
tree = self.repo.compare(self.end, tags[1].commit.sha)
return tree.merge_base_commit.sha
labels = []
for label in pull.labels:
labels.append(label.name)
self.commits[pull.number] = {
"Title": pull.title,
"Url": pull.html_url,
"labels": labels
}
def __repr__(self):
return "<generateTree(repo='{0}', start='{1}', startCommit='{2}', " \
"end='{3}', endCommit='{4}', tree='{5}', commits='{6}".format(
self.repo, self.start, self.startCommit, self.end,
self.endCommit, self.tree, self.commits,
)
class generateMarkdown():
def __repr__(self):
return "<generateMarkdown(mdFile={0})>".format(
self.mdFile
)
def __init__(self, repo):
def __init__(self, repo: generateTree):
self.mdFile = MdUtils(
file_name='CHANGELOG', title='CHANGELOG'
)
self.mdFile.new_line(
"## Release " +
"[{0}](https://github.com/nanocurrency/nano-node/tree/{0})".format(repo.end), wrap_width=0)
self.mdFile.new_line("[Full Changelog](https://github.com/nanocurrency"
"/nano-node/compare/{0}...{1})".format(repo.start, repo.end), wrap_width=0)
if repo.tag:
self.mdFile.new_line(
"## Release " +
f"[{repo.tag}](https://github.com/{repo.name}/tree/{repo.tag})", wrap_width=0)
else:
self.mdFile.new_line(
f"[{repo.end}](https://github.com/{repo.name}/tree/{repo.end})", wrap_width=0)
self.mdFile.new_line(f"[Full Changelog](https://github.com/{repo.name}"
f"/compare/{repo.start}...{repo.end})", wrap_width=0)
sort = self.pull_to_section(repo.commits)
for section, prs in sort.items():
self.write_header_PR(section)
@ -198,30 +233,34 @@ class generateMarkdown():
for sha, message in repo.other_commits:
self.write_no_PR(repo, sha, message)
self.mdFile.create_md_file()
def write_header_PR(self, section):
self.mdFile.new_line("---")
self.mdFile.new_header(level=3, title=section, add_table_of_contents='n')
self.mdFile.new_header(level=3, title=section,
add_table_of_contents='n')
self.mdFile.new_line(
"|Pull Request|Title")
self.mdFile.new_line("|:-:|:--")
def write_header_no_PR(self):
self.mdFile.new_line()
self.mdFile.new_line(
"|Commit|Title")
self.mdFile.new_line("|:-:|:--")
def write_PR(self, pr, info):
imp = ""
if pr[1]:
imp = "**BREAKING** "
self.mdFile.new_line(
"|[#{0}]({1})|{2}{3}".format(
pr[0], info['Url'], imp, info['Title']), wrap_width=0)
f"|[#{pr[0]}]({info['Url']})|{imp}{info['Title']}", wrap_width=0)
def write_no_PR(self, repo, sha, message):
url = "https://github.com/{0}/commit/{1}".format(repo.name, sha)
url = f"https://github.com/{repo.name}/commit/{sha}"
self.mdFile.new_line(
"|[{0}]({1})|{2}".format(
sha[:8], url, message), wrap_width=0)
def handle_labels(self, labels):
f"|[{sha[:8]}]({url})|{message}", wrap_width=0)
def handle_labels(self, labels) -> Tuple[str, bool]:
for section, values in SECTIONS.items():
for label in labels:
if label in values:
@ -233,7 +272,8 @@ class generateMarkdown():
else:
return section, False
return 'Other', False
def pull_to_section(self, commits):
def pull_to_section(self, commits) -> dict:
sect = copy.deepcopy(SECTIONS)
result = {}
for a in sect:
@ -248,7 +288,8 @@ class generateMarkdown():
if len(sect[a]) > 0:
result[a] = sect[a]
return result
if __name__ == "__main__":
args = cliArgs()
repo = generateTree(args)
generateMarkdown(repo)
arg = cliArgs()
trees = generateTree(arg)
generateMarkdown(trees)

36
util/makesrc
View file

@ -1,23 +1,24 @@
#!/bin/bash
# makesrc <tag>
# requires ruby gem github_changelog_generator
# API_KEY generated here https://github.com/settings/tokens/new?description=GitHub%20Changelog%20Generator%20token
# makesrc <tag> <pat>
# personal access token allows around rate limit
if [ -e $1 ]; then
echo "makesrc <tag>" >&2
if [ -e $2 ]; then
echo "makesrc <tag> <pat>" >&2
echo " tag valid <tag> for nanocurrency/nano-node" >&2
echo " pat personal access token" >&2
exit 1
fi
TAG="$1"
PAT="$2"
VERSION=$(echo $TAG | sed 's/V//')
TAG_DATE=""
VERSION_MAJOR=$(echo $VERSION | cut -d "." -f 1)
scriptDir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
function make_source() {
git clone --recursive --single-branch --branch releases/v${VERSION_MAJOR} https://github.com/nanocurrency/nano-node nano-$VERSION
git clone --recursive --single-branch --branch ${TAG} https://github.com/nanocurrency/nano-node nano-$VERSION
cd nano-$VERSION
git pull --tags
COUNT=$(git tag -l "${TAG}" | wc -l)
@ -39,19 +40,18 @@ function source_information() {
COMMIT=$(git log | head -1 | cut -d " " -f 2)
TAG_DATE=$(TZ=UTC date -d"$DATE" +%s)
case "${VERSION}" in
*RC*)
"${scriptDir}/changelog_generator" nanocurrency/nano-node "V${VERSION}" only
;;
*)
"${scriptDir}/changelog_generator" nanocurrency/nano-node "V${VERSION}"
;;
esac
(
eval "$(scl enable rh-python38 'echo -e "source /opt/rh/rh-python38/enable\n"')"
python -m pip install PyGithub mdutils
if [ ! -s CHANGELOG.md ]; then
echo "CHANGELOG not generated"
exit 1
fi
python ${scriptDir}/changelog.py -p ${PAT} -t ${TAG} -m full
if [ ! -s CHANGELOG.md ]; then
echo "CHANGELOG not generated"
exit 1
fi
exit 0
) || exit 1
export TAG_DATE
}