From b7d746915b82404a34180b6da3fc2ab75537bf0d Mon Sep 17 00:00:00 2001 From: marcelmmc Date: Sun, 17 Sep 2023 07:07:01 -0500 Subject: [PATCH 1/4] feat: automatic release workflow added --- .github/workflows/helper.py | 48 ++++++++++-------------- .github/workflows/version_release.yml | 54 +++++++++++---------------- 2 files changed, 40 insertions(+), 62 deletions(-) diff --git a/.github/workflows/helper.py b/.github/workflows/helper.py index f6105ad..0adb3df 100644 --- a/.github/workflows/helper.py +++ b/.github/workflows/helper.py @@ -1,14 +1,13 @@ -import json import argparse import re import os def create_new_tag(tag): - if not (re.match(r'^v\d', tag)): + if not (re.match(r'^v\d+\.\d+$', tag)): raise Exception(f'tag: {tag} is not giving in the correct format e.i v0.0') - # Notice that this will make tag version of three digits become two digits + # Notice that this could make a tag version of three digits become two digits # e.i 3.2.1 -> 3.3 - digits_tags = (re.match(r'^v\d+.\d+', tag)).group()[1::].split('.') + digits_tags = (re.match(r'^v\d+\.\d+', tag)).group()[1::].split('.') if len(digits_tags) != 2: raise Exception(f'tag: {tag} must contain two version digits') @@ -19,33 +18,23 @@ def store_tag(tag): with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: print(f'new_tag={tag}', file=fh) -def update_files_with_tag(old_tag, new_tag): - # This function needs to take care of updating - # .zenodo.json and CITATION.cff - # TO-DO: make zenodo.json robust to search for version tags - if os.path.isfile(".zenodo.json"): +#TO-DO: decide between regex or replace with old tag +# if regex remove extra param old_tag +# if replace func then remove regex instruction +def update_file_with_tag(f_name, old_tag, new_tag): + if os.path.isfile(f_name): try: - with open(".zenodo.json", "r") as f: - data = json.load(f) - - data["version"] = data["version"].replace(old_tag,new_tag) - data["title"] = data["title"].replace(old_tag,new_tag) - - with open(".zenodo.json", "w") as f: - json.dump(data, f) - - except Exception as e: - print(e) - - if os.path.isfile("CITATION.cff"): - try: - with open("CITATION.cff", "r", encoding="utf-8") as file: - citation = file.read() - modified_citation = citation.replace(old_tag, new_tag) - with open("CITATION.cff", "w", encoding="utf-8") as file: - file.write(modified_citation) + with open(f_name, "r",encoding="utf-8") as f: + data = f.read() + # data = data.replace(old_tag, new_tag) + # data = re.sub(r'v\d+((\.\d+)+)', new_tag,data) + data = re.sub(r'v\d+\.(\d+\.\d+|\d+)', new_tag,data) + with open(f_name, "w",encoding="utf-8") as f: + f.write(data) except Exception as e: print(e) + else: + print(f"Warning: {f_name} doest exist at the current path {os.getcwd()}") def main(args): tag = args.tag @@ -55,7 +44,8 @@ def main(args): else: new_tag = create_new_tag(tag) print(f"Repository with tag: {tag}, creating a new tag with: {new_tag}") - update_files_with_tag(tag,new_tag) + update_file_with_tag(".zenodo.json", tag, new_tag) + update_file_with_tag("CITATION.cff", tag, new_tag) store_tag(new_tag) def run(): diff --git a/.github/workflows/version_release.yml b/.github/workflows/version_release.yml index 66b5478..834a5b0 100644 --- a/.github/workflows/version_release.yml +++ b/.github/workflows/version_release.yml @@ -13,8 +13,7 @@ jobs: uses: actions/checkout@v3 with: token: ${{ secrets.MS3_BOT_TOKEN }} - - # this step could be replaced with dcml_docker action + - name: "Get ms3 package & transform" id: tag continue-on-error: true @@ -24,45 +23,34 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.MS3_BOT_TOKEN }} - # this step could be replaced with dcml_docker action - - name: Setup Github credentials - run: | - git config --global user.name "ms3-bot" - git config --global user.email dcml.annotators@epfl.ch - - name: "Generate new tag version" id: generate_tag run: | - ls -a python .github/workflows/helper.py --tag "${{ steps.tag.outputs.tag_version }}" - ls -a - cat .zenodo.json - # this step could be replaced with dcml_docker action - - name: "Get ms3 package & transform" + - name: Setup Github credentials & push zenodo and citation changes + continue-on-error: true + run: | + git config --global user.name "ms3-bot" + git config --global user.email dcml.annotators@epfl.ch + git add .zenodo.json CITATION.cff + git commit -m 'chore: zenodo and citation files updated with tag: ${{ steps.generate_tag.outputs.new_tag }}' + git push + + # TO-DO: decide whether results of ms3 transform needs to be pushed to repository + - name: "Get ms3 package & apply transform" run: | pip install --upgrade pip pip install ms3 - ls -a ms3 transform -M -N -X -F -D - ls -a - - name: "Create release" - uses: "marvinpinto/action-automatic-releases@latest" + - uses: ncipollo/release-action@v1 with: - repo_token: "${{ secrets.MS3_BOT_TOKEN }}" - automatic_release_tag: "${{ steps.generate_tag.outputs.new_tag }}" - prerelease: false - title: "${{ github.event.pull_request.title }}" - files: | - .zenodo.json - CITATION.cff - ${{ github.event.repository.name }}.zip - ${{ github.event.repository.name }}.datapackage.json - abc.datapackage.json.errors - - - name: "Update release body" - run: | - gh release edit "${{ steps.generate_tag.outputs.new_tag }}" --notes "${{ github.event.pull_request.body }}" --repo "${{ github.event.repository.full_name }}" - env: - GITHUB_TOKEN: ${{ secrets.MS3_BOT_TOKEN }} \ No newline at end of file + artifacts: "${{ github.event.repository.name }}.zip,\ + ${{ github.event.repository.name }}.datapackage.json,\ + ${{ github.event.repository.name }}.datapackage.errors" + body: "${{ github.event.pull_request.body }}" + name: "${{ github.event.pull_request.title }}" + tag: "${{ steps.generate_tag.outputs.new_tag }}" + makeLatest: "latest" + commit: "${{ github.event.pull_request.base.ref }}" From 68765c7f378565e3e0601b92af6ed6286317ba87 Mon Sep 17 00:00:00 2001 From: marcelmmc Date: Sun, 17 Sep 2023 16:41:52 -0500 Subject: [PATCH 2/4] chore: use exact string matching to replace tag and checkout base ref --- .github/workflows/helper.py | 7 +------ .github/workflows/version_release.yml | 3 +-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/helper.py b/.github/workflows/helper.py index 0adb3df..705b744 100644 --- a/.github/workflows/helper.py +++ b/.github/workflows/helper.py @@ -18,17 +18,12 @@ def store_tag(tag): with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: print(f'new_tag={tag}', file=fh) -#TO-DO: decide between regex or replace with old tag -# if regex remove extra param old_tag -# if replace func then remove regex instruction def update_file_with_tag(f_name, old_tag, new_tag): if os.path.isfile(f_name): try: with open(f_name, "r",encoding="utf-8") as f: data = f.read() - # data = data.replace(old_tag, new_tag) - # data = re.sub(r'v\d+((\.\d+)+)', new_tag,data) - data = re.sub(r'v\d+\.(\d+\.\d+|\d+)', new_tag,data) + data = data.replace(old_tag, new_tag) with open(f_name, "w",encoding="utf-8") as f: f.write(data) except Exception as e: diff --git a/.github/workflows/version_release.yml b/.github/workflows/version_release.yml index 834a5b0..2eec97f 100644 --- a/.github/workflows/version_release.yml +++ b/.github/workflows/version_release.yml @@ -13,7 +13,7 @@ jobs: uses: actions/checkout@v3 with: token: ${{ secrets.MS3_BOT_TOKEN }} - + ref: "${{ github.event.pull_request.base.ref }}" - name: "Get ms3 package & transform" id: tag continue-on-error: true @@ -37,7 +37,6 @@ jobs: git commit -m 'chore: zenodo and citation files updated with tag: ${{ steps.generate_tag.outputs.new_tag }}' git push - # TO-DO: decide whether results of ms3 transform needs to be pushed to repository - name: "Get ms3 package & apply transform" run: | pip install --upgrade pip From 1e642babdb1e5c9b8d80eca20d1765b6409da759 Mon Sep 17 00:00:00 2001 From: marcelmmc Date: Mon, 18 Sep 2023 11:37:20 -0500 Subject: [PATCH 3/4] chore: update tag in README.md and use git add on existing files --- .github/workflows/helper.py | 2 ++ .github/workflows/version_release.yml | 14 +++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/helper.py b/.github/workflows/helper.py index 705b744..88f84b0 100644 --- a/.github/workflows/helper.py +++ b/.github/workflows/helper.py @@ -24,6 +24,7 @@ def update_file_with_tag(f_name, old_tag, new_tag): with open(f_name, "r",encoding="utf-8") as f: data = f.read() data = data.replace(old_tag, new_tag) + with open(f_name, "w",encoding="utf-8") as f: f.write(data) except Exception as e: @@ -41,6 +42,7 @@ def main(args): print(f"Repository with tag: {tag}, creating a new tag with: {new_tag}") update_file_with_tag(".zenodo.json", tag, new_tag) update_file_with_tag("CITATION.cff", tag, new_tag) + update_file_with_tag("README.md", tag, new_tag) store_tag(new_tag) def run(): diff --git a/.github/workflows/version_release.yml b/.github/workflows/version_release.yml index 2eec97f..b7fb00b 100644 --- a/.github/workflows/version_release.yml +++ b/.github/workflows/version_release.yml @@ -28,13 +28,21 @@ jobs: run: | python .github/workflows/helper.py --tag "${{ steps.tag.outputs.tag_version }}" - - name: Setup Github credentials & push zenodo and citation changes + - name: Setup Github credentials & push zenodo, citation and README changes continue-on-error: true run: | git config --global user.name "ms3-bot" git config --global user.email dcml.annotators@epfl.ch - git add .zenodo.json CITATION.cff - git commit -m 'chore: zenodo and citation files updated with tag: ${{ steps.generate_tag.outputs.new_tag }}' + if [[ -f .zenodo.json ]]; then + git add .zenodo.json + fi + if [[ -f CITATION.cff ]]; then + git add CITATION.cff + fi + if [[ -f README.md ]]; then + git add README.md + fi + git commit -m 'chore: files updated with tag: ${{ steps.generate_tag.outputs.new_tag }}' git push - name: "Get ms3 package & apply transform" From d5e1e87fcc07a91522fbbc6b44609383890fcbbe Mon Sep 17 00:00:00 2001 From: marcelmmc Date: Mon, 18 Sep 2023 19:21:08 -0500 Subject: [PATCH 4/4] feat: get latest tag and allow to update major version by attaching a label --- .github/workflows/helper.py | 33 +++++++++++++++++++++++---- .github/workflows/version_release.yml | 12 ++++++---- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.github/workflows/helper.py b/.github/workflows/helper.py index 88f84b0..3698fd6 100644 --- a/.github/workflows/helper.py +++ b/.github/workflows/helper.py @@ -1,7 +1,7 @@ import argparse import re import os -def create_new_tag(tag): +def create_new_tag(tag, update_major): if not (re.match(r'^v\d+\.\d+$', tag)): raise Exception(f'tag: {tag} is not giving in the correct format e.i v0.0') @@ -11,8 +11,15 @@ def create_new_tag(tag): if len(digits_tags) != 2: raise Exception(f'tag: {tag} must contain two version digits') - new_digit = int(digits_tags[1]) + 1 - return "v" + str(digits_tags[0]) + "." + str(new_digit) + major_num = int(digits_tags[0]) + minor_num = int(digits_tags[1]) + if update_major: + print(f"Label detected to update major version") + major_num += 1 + minor_num = 0 + else: + minor_num += 1 + return f"v{major_num}.{minor_num}" def store_tag(tag): with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: @@ -24,7 +31,6 @@ def update_file_with_tag(f_name, old_tag, new_tag): with open(f_name, "r",encoding="utf-8") as f: data = f.read() data = data.replace(old_tag, new_tag) - with open(f_name, "w",encoding="utf-8") as f: f.write(data) except Exception as e: @@ -38,7 +44,7 @@ def main(args): if not tag: print(f"Warning: a latest release with a tag does not exist in current repository, starting from {new_tag}") else: - new_tag = create_new_tag(tag) + new_tag = create_new_tag(tag,args.update_major_ver) print(f"Repository with tag: {tag}, creating a new tag with: {new_tag}") update_file_with_tag(".zenodo.json", tag, new_tag) update_file_with_tag("CITATION.cff", tag, new_tag) @@ -49,6 +55,18 @@ def run(): args = parser.parse_args() main(args) + +def str_to_bool(value): + if value.lower() == "true": + return True + elif value.lower() == "false": + return False + else: + raise Exception( + f"Error: value {value} as argument is not accepted\n" + f"retry with true or false" + ) + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( @@ -56,4 +74,9 @@ def run(): help="Require: latest tag", required=True ) + parser.add_argument( + "--update_major_ver", type=str_to_bool, + help="Require: boolean to update the major tag number", + required=True + ) run() \ No newline at end of file diff --git a/.github/workflows/version_release.yml b/.github/workflows/version_release.yml index b7fb00b..28b27ff 100644 --- a/.github/workflows/version_release.yml +++ b/.github/workflows/version_release.yml @@ -12,21 +12,24 @@ jobs: - name: Checkout corpus repository uses: actions/checkout@v3 with: + fetch-depth: 0 token: ${{ secrets.MS3_BOT_TOKEN }} ref: "${{ github.event.pull_request.base.ref }}" - - name: "Get ms3 package & transform" + + - name: "Get latest tag version" id: tag continue-on-error: true run: | - res=$(gh api -H "Accept: application/vnd.github+json" repos/${{ github.event.repository.full_name }}/releases/latest --jq '.tag_name') + res=$(git tag -l --sort=-v:refname | grep --invert-match '\^' | head -n 1) echo "tag_version=${res}" >> $GITHUB_OUTPUT env: GITHUB_TOKEN: ${{ secrets.MS3_BOT_TOKEN }} - - name: "Generate new tag version" + - name: "Generate a new tag version" id: generate_tag run: | - python .github/workflows/helper.py --tag "${{ steps.tag.outputs.tag_version }}" + major_in_PR="${{ contains(github.event.pull_request.labels.*.name, 'major_version')}}" + python .github/workflows/helper.py --tag "${{ steps.tag.outputs.tag_version }}" --update_major_ver "$major_in_PR" - name: Setup Github credentials & push zenodo, citation and README changes continue-on-error: true @@ -46,6 +49,7 @@ jobs: git push - name: "Get ms3 package & apply transform" + continue-on-error: true run: | pip install --upgrade pip pip install ms3