Adds a changelog workflow (#3819)

* Adds the changelog generation workflow

* Allow changelog output directory in secrets

* Missing S3_BUILD_DIRECTORY var definition

* Improve scrpt and fix automatic version backtrack issues

* Improvements in the changelog script

* Set changelog.py to use git merge-base --octopus

* Update workflow and remove uneeded scripts

.github/workflows/changelog.yml

@@ -0,0 +1,65 @@
+name: Changelog
+on:
+  workflow_dispatch:
+    inputs:
+      repo:
+        description: "Repository"
+        default: "nanocurrency/nano-node"
+        required: true
+      ref:
+        description: "Version (VX.Y tag)"
+        default: "VX.Y"
+        required: true
+      ref_start:
+        description: "Start (VX.Y tag)"
+        default: ""
+        required: false
+      mode:
+        description: "Find tag as start"
+        type: choice
+        options:
+          - final
+          - beta
+        default: final
+        required: true
+env:
+  artifact: 1
+
+jobs:
+  changelog_job:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 90
+    steps:
+      - name: Sets the tag and repo variables (to build)
+        run: |
+          echo "TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV
+          echo "REF_START=${{ github.event.inputs.ref_start }}" >> $GITHUB_ENV
+          echo "REPO_TO_RUN=${{ github.event.inputs.repo }}" >> $GITHUB_ENV
+          echo "MODE=${{ github.event.inputs.mode }}" >> $GITHUB_ENV
+      - name: Checks out the required workflow files (workflow repo)
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
+        with:
+          ref: ${{ github.ref }}
+          repository: ${{ github.repository }}
+      - name: Install Dependencies
+        run: sudo apt-get install -yqq uuid
+      - name: Setup Python 3.x
+        uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 #v4.3.0
+        with:
+          python-version: '3.x'
+      - name: Installs PIP Packages
+        run: |
+          pip install PyGithub
+          pip install mdutils
+      - name: Generating the CHANGELOG file (automatic interval)
+        if: ${{ github.event.inputs.ref_start == '' }}
+        run: python util/changelog.py -v --repo $REPO_TO_RUN -m $MODE -t $TAG -p "${{ secrets.GITHUB_TOKEN }}"
+      - name: Generating the CHANGELOG file (selected interval)
+        if: ${{ github.event.inputs.ref_start != '' }}
+        run: python util/changelog.py -v --repo $REPO_TO_RUN --start-tag $REF_START -t $TAG -p "${{ secrets.GITHUB_TOKEN }}"
+      - name: Upload the Changelog Artifact
+        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb #v3.1.1
+        with:
+          name: CHANGELOG_${{ env.TAG }}.md
+          path: |
+            ${{ github.workspace }}/CHANGELOG.md

util/changelog.py

@@ -3,33 +3,39 @@ import copy
 import sys
 import re
 from typing import Tuple
+import subprocess
 
 """
 Changelog generation script, requires PAT with public_repo access, 
 see https://github.com/settings/tokens
 
-usage: changelog [-h] [-e END] [-m {full,incremental}] -p PAT [-r REPO] [-s START] [-t TAG]
+usage: changelog [-h] [-e END] [-m {final,beta}] -p PAT [-r REPO] [-s START] [-t TAG]
 
 Generate Changelogs between tags or commits
 
 optional arguments:
-  -h, --help            show this help message and exit
+  -h, --help            Show this help message and exit
-  -e END, --end END     Ending reference for Changelog(newest)
+  -m {final,beta}, --mode {final,beta}
-  -m {full,incremental}, --mode {full,incremental}
+                        Mode to run changelog for [final, beta]
-                        Mode to run changelog for [full, incremental]
   -p PAT, --pat PAT     Personal Access Token
   -r REPO, --repo REPO  <org/repo> to generate logs for
+  -e END, --end END     Ending reference for Changelog(newest)
   -s START, --start START
                         Starting reference for Changelog(oldest)
-  -t TAG, --tag TAG     Tag to use for changelog generation
+  -t TAG, --tag TAG
+                        Tag to use for changelog generation
+  --start-tag START_TAG 
+                        Tag to use as start reference (instead of -s)
+  -v, --verbose         Verbose mode
 """
 
 
-full = re.compile(r"^(V(\d)+.(\d)+.?(\d)?)$")
+final = re.compile(r"^(V(\d)+.(\d)+)$")
-incremental = re.compile(r"^(V(\d)+.(\d)+.?(\d)?(RC(\d)+)?(DB(\d)+)?)$")
+beta = re.compile(r"^(V(\d)+.(\d)+((RC(\d)+)|(DB(\d)+))?)$")
 
 try:
     from github import Github, UnknownObjectException
+    from github.Label import Label
     from mdutils import MdUtils
 except BaseException:
     sys.exit("Error: run 'pip install PyGithub mdutils'")
@@ -96,16 +102,15 @@ SECTIONS = {
 }
 
 
-class cliArgs():
+class CliArgs:
     def __init__(self) -> dict:
 
-        changelog_choices = ["full", "incremental"]
+        changelog_choices = ["final", "beta"]
 
         parse = argparse.ArgumentParser(
             prog="changelog",
             description="Generate Changelogs between tags or commits"
         )
-
         parse.add_argument(
             '-e', '--end',
             help="Ending reference for Changelog(newest)",
@@ -113,9 +118,9 @@ class cliArgs():
         )
         parse.add_argument(
             "-m", "--mode",
-            help="Mode to run changelog for [full, incremental]",
+            help="Mode to run changelog for [final, beta]",
             type=str, action="store",
-            default="incremental",
+            default="beta",
             choices=changelog_choices
         )
         parse.add_argument(
@@ -140,6 +145,17 @@ class cliArgs():
             help="Tag to use for changelog generation",
             type=str, action="store"
         )
+        parse.add_argument(
+            '--start-tag',
+            dest='start_tag',
+            help="Tag for start reference",
+            type=str, action="store"
+        )
+        parse.add_argument(
+            '-v', '--verbose',
+            help="Verbose mode",
+            action="store_true"
+        )
         options = parse.parse_args()
         self.end = options.end
         self.mode = options.mode
@@ -147,28 +163,73 @@ class cliArgs():
         self.repo = options.repo.rstrip("/")
         self.start = options.start
         self.tag = options.tag
+        self.verbose = options.verbose
+        self.start_tag = options.start_tag
 
 
-class generateTree:
+def validate_sha(hash_value: str) -> bool:
+    if len(hash_value) != 40:
+        return False
+    try:
+        sha_int = int(hash_value, 16)
+    except ValueError:
+        return False
+    return True
+
+
+class GenerateTree:
     def __init__(self, args):
         github = Github(args.pat)
         self.name = args.repo
         self.repo = github.get_repo(self.name)
+        self.args = args
         if args.tag:
             self.tag = args.tag
             self.end = self.repo.get_commit(args.tag).sha
-        elif args.end:
+            if args.end:
+                print("error: set either --end or --tag")
+                exit(1)
+        if args.end:
+            if not validate_sha(args.end):
+                print("error: --end argument is not a valid hash")
+                exit(1)
             self.end = self.repo.get_commit(args.end).sha
-        else:
+            if not args.start:
-            print("need end or tag")
+                print("error: --end argument requires --start")
+                exit(1)
+        if not args.end and not args.tag:
+            print("error: need either --end or --tag")
+            exit(1)
+        if args.start and args.start_tag:
+            print("error: set either --start or --start-tag")
             exit(1)
         if args.start:
+            if not validate_sha(args.start):
+                print("error: --start argument is not a valid hash")
+                exit(1)
             self.start = self.repo.get_commit(args.start).sha
+        elif args.start_tag:
+            self.start = self.select_start_ref(args.start_tag)
         else:
-            self.start = self.get_common(args.mode)
+            assert args.tag
+            self.start = self.get_common_by_tag(args.mode)
+
         self.commits = {}
         self.other_commits = []
+        self.excluded = []
         commits = self.repo.get_commits(sha=self.end)
+
+        # Check if the common ancestor exists in the commit list.
+        found_common_ancestor = False
+        for commit in commits:
+            if commit.sha == self.start:
+                found_common_ancestor = True
+                break
+        if not found_common_ancestor:
+            print("error: the common ancestor was not found")
+            exit(1)
+
+        # Retrieve the complementary information for each commit.
         for commit in commits:
             if commit.sha == self.start:
                 break
@@ -182,35 +243,114 @@ class generateTree:
                     pr_number = p[0].number
                     pull = self.repo.get_pull(pr_number)
                 else:
-                    print(
+                    if args.verbose:
-                        f"Commit has no associated PR {commit.sha}: \"{m}\"")
+                        print(f"info: commit has no associated PR {commit.sha}: \"{m}\"")
                     self.other_commits.append((commit.sha, m))
                     continue
 
+            if pull.state == 'open':
+                if args.verbose:
+                    print(f"info: commit is in tree but only associated with open PR {pr_number}: \"{pull.title}\"")
+                self.other_commits.append((commit.sha, m))
+                continue
+
+            if self.excluded_from_changelog(pull.labels):
+                if args.verbose:
+                    print(f"info: the PR {pr_number}: \"{pull.title}\" was excluded from the changelog")
+                self.excluded.append((commit.sha, m))
+                continue
+
             labels = []
             for label in pull.labels:
                 labels.append(label.name)
+
             self.commits[pull.number] = {
                 "Title": pull.title,
                 "Url": pull.html_url,
                 "labels": labels
             }
 
-    def get_common(self, mode) -> str:
+    @staticmethod
+    def excluded_from_changelog(labels: list[Label]) -> bool:
+        for label in labels:
+            if label.name == 'exclude from changelog':
+                return True
+        return False
+
+    def get_common_ancestor(self, start_tag: str) -> str:
+        print("info: will look for the common ancestor by local git repo")
+        cmd = f'''
+        repo_path=/tmp/$(uuid)
+        (
+            mkdir -p "$repo_path"
+            if [[ ! -d $repo_path || ! -z "$(ls -A $repo_path)" ]]; then
+                exit 1
+            fi
+            pushd "$repo_path"
+            git clone https://github.com/{self.name} .
+            develop_head=$(git show-ref -s origin/develop)
+            common_ancestor=$(git merge-base --octopus "$develop_head" "{start_tag}")
+            echo "$common_ancestor" > "$repo_path/output_file"
+            popd
+        ) > /dev/null 2>&1
+        cat "$repo_path/output_file"
+        rm -rf "$repo_path"
+        '''
+        common_ancestor = subprocess.check_output(f"echo '{cmd}' | /bin/bash", shell=True, text=True).rstrip()
+        if self.args.verbose:
+            print("info: found common ancestor: " + common_ancestor)
+        return common_ancestor
+
+    def get_common_by_tag(self, mode) -> str:
         tags = []
+        found_end_tag = False
         for tag in self.repo.get_tags():
-            if mode == "full":
+            if not found_end_tag and tag.name == self.tag:
-                found = full.match(tag.name)
+                found_end_tag = True
-            else:
+            if found_end_tag:
-                found = incremental.match(tag.name)
+                if mode == "final":
-            if found:
+                    matched_tag = final.match(tag.name)
-                tags.append(tag)
+                else:
-        tree = self.repo.compare(self.end, tags[1].commit.sha)
+                    matched_tag = beta.match(tag.name)
-        return tree.merge_base_commit.sha
+                if matched_tag:
+                    tags.append(tag)
+
+        if len(tags) < 2:
+            return None
+
+        selected_tag = None
+        if self.major_version_match(tags[0].name, self.tag):
+            selected_tag = tags[1]
+        else:
+            selected_tag = tags[0]
+
+        if self.args.verbose:
+            print(f"info: selected start tag {selected_tag.name}: {selected_tag.commit.sha}")
+
+        return self.select_start_ref(selected_tag.name)
+
+    @staticmethod
+    def major_version_match(first_tag: str, second_tag: str) -> bool:
+        major_version_tag_pattern = r"(\d)+."
+        first_tag_major = re.search(major_version_tag_pattern, first_tag)
+        second_tag_major = re.search(major_version_tag_pattern, second_tag)
+        if first_tag_major and second_tag_major and first_tag_major.group(0) == second_tag_major.group(0):
+            return True
+        return False
+
+    def select_start_ref(self, start_tag: str) -> str:
+        if self.major_version_match(start_tag, self.tag):
+            start_commit = self.repo.get_commit(start_tag).sha
+            if self.args.verbose:
+                print(f"info: selected start tag {start_tag} (commit: {start_commit}) "
+                      f"has the same major version of the end tag ({self.tag})")
+            return start_commit
+
+        return self.get_common_ancestor(start_tag)
 
 
-class generateMarkdown():
+class GenerateMarkdown:
-    def __init__(self, repo: generateTree):
+    def __init__(self, repo: GenerateTree):
         self.mdFile = MdUtils(
             file_name='CHANGELOG', title='CHANGELOG'
         )
@@ -225,16 +365,16 @@ class generateMarkdown():
                              f"/compare/{repo.start}...{repo.end})", wrap_width=0)
         sort = self.pull_to_section(repo.commits)
         for section, prs in sort.items():
-            self.write_header_PR(section)
+            self.write_header_pr(section)
             for pr in prs:
-                self.write_PR(pr, repo.commits[pr[0]])
+                self.write_pr(pr, repo.commits[pr[0]])
         if repo.other_commits:
-            self.write_header_no_PR()
+            self.write_header_no_pr()
             for sha, message in repo.other_commits:
-                self.write_no_PR(repo, sha, message)
+                self.write_no_pr(repo, sha, message)
         self.mdFile.create_md_file()
 
-    def write_header_PR(self, section):
+    def write_header_pr(self, section):
         self.mdFile.new_line("---")
         self.mdFile.new_header(level=3, title=section,
                                add_table_of_contents='n')
@@ -242,25 +382,26 @@ class generateMarkdown():
             "|Pull Request|Title")
         self.mdFile.new_line("|:-:|:--")
 
-    def write_header_no_PR(self):
+    def write_header_no_pr(self):
         self.mdFile.new_line()
         self.mdFile.new_line(
             "|Commit|Title")
         self.mdFile.new_line("|:-:|:--")
 
-    def write_PR(self, pr, info):
+    def write_pr(self, pr, info):
         imp = ""
         if pr[1]:
             imp = "**BREAKING** "
         self.mdFile.new_line(
             f"|[#{pr[0]}]({info['Url']})|{imp}{info['Title']}", wrap_width=0)
 
-    def write_no_PR(self, repo, sha, message):
+    def write_no_pr(self, repo, sha, message):
         url = f"https://github.com/{repo.name}/commit/{sha}"
         self.mdFile.new_line(
             f"|[{sha[:8]}]({url})|{message}", wrap_width=0)
 
-    def handle_labels(self, labels) -> Tuple[str, bool]:
+    @staticmethod
+    def handle_labels(labels) -> Tuple[str, bool]:
         for section, values in SECTIONS.items():
             for label in labels:
                 if label in values:
@@ -290,6 +431,6 @@ class generateMarkdown():
         return result
 
 
-arg = cliArgs()
+arg = CliArgs()
-trees = generateTree(arg)
+trees = GenerateTree(arg)
-generateMarkdown(trees)
+GenerateMarkdown(trees)