Merge branch 'main' into NDT-751-When-uploading-Template-9-from-RFI-c…

…ancel-still-uploads

.bin/check-db-changes.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# Check if there are changes in the db folder
+if git diff --quiet HEAD^ HEAD -- db; then
+  echo "No changes in the db folder. Skipping sqitch tag."
+else
+  echo "Changes detected in the db folder. Running sqitch tag."
+  docker pull sqitch/sqitch
+  curl -L https://git.io/JJKCn -o sqitch && chmod +x sqitch
+  ./sqitch --chdir db tag "$1" -m "release v$1"
+fi

.github/actions/dev-env-setup/action.yml

@@ -3,33 +3,7 @@ description: 'Sets up asdf, perl and configures the cache'
 runs:
   using: composite
   steps:
-    - name: asdf setup
-      uses: asdf-vm/actions/setup@v2
-    - uses: shogo82148/actions-setup-perl@v1
-    - name: install pg perl library
-      run: sudo apt-get install -y libpq-dev libdbd-pg-perl libreadline-dev uuid-dev
-      shell: bash
-    - name: set perl env variables
-      shell: bash
-      run: | # pragma: allowlist secret
-        echo "PERL5LIB=/home/runner/perl5/lib/perl5" >> $GITHUB_ENV
-        echo "PERL_LOCAL_LIB_ROOT=/home/runner/perl5" >> $GITHUB_ENV
-        echo "PERL_MB_OPT=--install_base '/home/runner/perl5'" >> $GITHUB_ENV
-        echo "PERL_MM_OPT=INSTALL_BASE=/home/runner/perl5" >> $GITHUB_ENV
-        echo "/home/runner/perl5/bin" >> $GITHUB_PATH
-    - uses: actions/cache@v3
-      id: asdf-cache
-      with:
-        path: |
-          ~/.asdf
-        key: ${{ runner.os }}-asdf-cache-${{ hashFiles('.tool-versions') }}-v2
-    - uses: actions/cache@v3
-      id: perl-cache
-      with:
-        path: |
-          ~/perl5
-        key: ${{ runner.os }}-perl-cache-${{ hashFiles('cpanfile') }}-v2
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       id: yarn-cache
       with:
         path: |

.github/actions/local-app-run/action.yaml

@@ -5,21 +5,44 @@ runs:
   steps:
     - name: start postgres and create database
       shell: bash
-      run: make start_pg && make drop_db && make create_db
+      env:
+        PGPASSWORD: mysecretpassword
+      run: |
+        docker run --rm -p 5432:5432 -e POSTGRES_PASSWORD=mysecretpassword -d postgres:15
+        # Wait for PostgreSQL to be ready
+        until docker exec $(docker ps -q -f ancestor=postgres:15) pg_isready -U postgres; do
+          echo "Waiting for PostgreSQL to be ready..."
+          sleep 2
+        done
+        # Additional wait to ensure PostgreSQL is fully ready
+        sleep 5
+        # Check PostgreSQL readiness from the host one more time
+        until pg_isready -h localhost -p 5432 -U postgres; do
+          echo "Waiting for PostgreSQL to be ready from the host..."
+          sleep 2
+        done
+        make drop_db PSQL="psql -h localhost -U postgres"
+        make create_db PSQL="psql -h localhost -U postgres"
     - name: deploy migrations
       shell: bash
+      env:
+        PGUSER: postgres
+        PGPASSWORD: mysecretpassword
       run: |
         cd db
         docker pull sqitch/sqitch
         curl -L https://git.io/JJKCn -o sqitch && chmod +x sqitch
         ./sqitch deploy
     - name: deploy mock schema
       shell: bash
+      env:
+        PGUSER: postgres
+        PGPASSWORD: mysecretpassword
       run: |
         cd mocks_schema
         docker pull sqitch/sqitch
         curl -L https://git.io/JJKCn -o sqitch && chmod +x sqitch
         ./sqitch deploy
     - name: start app
       shell: bash
-      run: docker run -d --network=host -e "PGUSER=postgres" -e "ENABLE_MOCK_AUTH=true" -e "ENABLE_MOCK_COOKIES=true" -e "NEXT_PUBLIC_GROWTHBOOK_API_KEY=dev_MIjxKii1kycPLz7CSjBYui0uERkFRSn7AXbu6oROCRQ" -p 3000:3000 ghcr.io/bcgov/conn-ccbc-portal/ccbc-app:sha-${{ github.sha }}
+      run: docker run -d --network=host -e "PGUSER=postgres" -e "PGPASSWORD=mysecretpassword" -e "ENABLE_MOCK_AUTH=true" -e "ENABLE_MOCK_COOKIES=true" -e "NEXT_PUBLIC_GROWTHBOOK_API_KEY=dev_MIjxKii1kycPLz7CSjBYui0uERkFRSn7AXbu6oROCRQ" -p 3000:3000 ghcr.io/bcgov/conn-ccbc-portal/ccbc-app:sha-${{ github.sha }}

.github/workflows/build.yaml

@@ -20,22 +20,33 @@ jobs:
     strategy:
       matrix:
         include:
+          # build app image on every push
           - image: ghcr.io/bcgov/conn-ccbc-portal/ccbc-app
             context: .
             dockerfile: app/Dockerfile
             name: app
+            build_only_on_main: false
+          # build db image only on main, helper image, does not change
+          # will use cache from registry
           - image: ghcr.io/bcgov/conn-ccbc-portal/ccbc-db
             context: .
             dockerfile: db/Dockerfile
             name: db
-          - image: ghcr.io/bcgov/conn-ccbc-portal/ccbc-cron-sp
-            context: .
-            dockerfile: cron/sp/Dockerfile
-            name: cron-sp
+            build_only_on_main: true
+          # build shp image only on main, cron image, does not change
+          # will use cache from registry
           - image: ghcr.io/bcgov/conn-ccbc-portal/ccbc-cron-shp
             context: .
             dockerfile: cron/shp/Dockerfile
             name: cron-shp
+            build_only_on_main: true
+          # build unit test db only on main, testing image, does not change
+          # will use cache from registry
+          - image: ghcr.io/bcgov/conn-ccbc-portal/ccbc-unit-test-db
+            context: .
+            dockerfile: local_setup/unit_tests/Dockerfile
+            name: db-unit-test
+            build_only_on_main: true
     permissions:
       contents: read
       packages: write
@@ -63,6 +74,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push Docker image
+        if: ${{ matrix.build_only_on_main != true || github.ref == 'refs/heads/main' || matrix.name == 'app' }}
         uses: docker/build-push-action@v6
         with:
           context: ${{ matrix.context }}

.github/workflows/deploy_feature.yaml

@@ -47,7 +47,6 @@ on:
     types: [submitted]
 
 env:
-  TAG: sha-${{ github.sha }}
   FEATURE_NAME: ${{ github.event.pull_request.head.ref }}
 
 jobs:
@@ -81,6 +80,14 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Set TAG based on event type
+        id: set-tag
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request_review" ]]; then
+            echo "TAG=sha-${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          else
+            echo "TAG=sha-${{ github.sha }}" >> $GITHUB_ENV
+          fi
       - name: Deploy
         uses: ./.github/actions/feature
         with:
@@ -208,7 +215,7 @@ jobs:
 
           echo "::set-output name=status::$status"
       - name: Transition Issue
-        if: steps.pr_approval.outputs.result == 'true' && steps.get_status.outputs.status != 'SPRINT Done' && steps.get_status.outputs.status != 'Closed'
+        if: steps.pr_approval.outputs.result == 'true' && steps.get_status.outputs.status != 'SPRINT Done' && steps.get_status.outputs.status != 'Closed' && steps.get_status.outputs.status != 'In Progress (DRAFT PR)'
         run: |
           curl -X POST \
             -H "Authorization: Basic ${{ secrets.JIRA_AUTH }}" \

.github/workflows/install-env.yaml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: dev env setup
         uses: ./.github/actions/dev-env-setup
-      - run: make install_dev_tools
-      - run: make stop_pg # cleanly stop the database, otherwise it will complain in the subsequent jobs
+      # - run: make install_dev_tools
+      # - run: make stop_pg # cleanly stop the database, otherwise it will complain in the subsequent jobs
       - run: yarn install --frozen-lockfile --check-files
         working-directory: ./app

.github/workflows/jira-feat.yaml

@@ -81,7 +81,7 @@ jobs:
 
           echo "::set-output name=status::$status"
       - name: Transition Issue
-        if: github.event.review.state == 'approved' && steps.get_status.outputs.status != 'SPRINT Done' && steps.get_status.outputs.status != 'Closed'
+        if: github.event.review.state == 'approved' && steps.get_status.outputs.status != 'SPRINT Done' && steps.get_status.outputs.status != 'Closed' && steps.get_status.outputs.status != 'In Progress (DRAFT PR)'
         run: |
           curl -X POST \
             -H "Authorization: Basic ${{ secrets.JIRA_AUTH }}" \

.github/workflows/main.yaml

@@ -27,6 +27,10 @@ jobs:
     uses: ./.github/workflows/test-code.yaml
     secrets: inherit
 
+  test-db:
+    uses: ./.github/workflows/test-db.yaml
+    secrets: inherit
+
   test-checks:
     uses: ./.github/workflows/test-checks.yaml
     secrets: inherit
@@ -41,7 +45,7 @@ jobs:
       RENOVATE_PRIVATE_KEY: ${{ secrets.RENOVATE_PRIVATE_KEY }}
 
   test-zap:
-    needs: [build, install-env]
+    needs: [build]
     uses: ./.github/workflows/test-zap.yaml
 
   test-e2e:
@@ -155,17 +159,18 @@ jobs:
         run: |
           echo "tagVersion=$( git tag --merged ${{ github.sha  }} --no-merged ${{ github.event.before }} | grep v*)" >>$GITHUB_OUTPUT
 
-  ensure-sqitch-plan-ends-with-tag:
-    runs-on: ubuntu-latest
-    needs: [is-tagged-release]
-    if: contains(needs.is-tagged-release.outputs.tagVersion, 'v')
-    steps:
-      - uses: actions/checkout@v4
-      - run: ./.bin/sqitch-last-change-is-tag.sh db
+  # sqitch plans will no longer always end in tags, so this is no longer necessary
+  # ensure-sqitch-plan-ends-with-tag:
+  #   runs-on: ubuntu-latest
+  #   needs: [is-tagged-release]
+  #   if: contains(needs.is-tagged-release.outputs.tagVersion, 'v')
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - run: ./.bin/sqitch-last-change-is-tag.sh db
 
   deploy:
     if: github.event.ref == 'refs/heads/main'
-    needs: [test-code, test-containers]
+    needs: [build]
     uses: ./.github/workflows/deploy.yaml
     secrets:
       OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }}

.github/workflows/release-process.yaml

@@ -97,8 +97,10 @@ jobs:
       - name: Setup Sqitch User
         if: steps.checkbox.outputs.result == 'true' && steps.up_to_date.outputs.result == 'true' && steps.pr_approval.outputs.result == 'true' && !github.event.pull_request.draft
         run: |
-          sqitch config --user user.name 'CCBC Service Account'
-          sqitch config --user user.email '[email protected]'
+          docker pull sqitch/sqitch
+          curl -L https://git.io/JJKCn -o sqitch && chmod +x sqitch
+          ./sqitch config --user user.name 'CCBC Service Account'
+          ./sqitch config --user user.email '[email protected]'
       - name: Make Release
         if: steps.checkbox.outputs.result == 'true' && steps.up_to_date.outputs.result == 'true' && steps.pr_approval.outputs.result == 'true' && !github.event.pull_request.draft
         run: |

.github/workflows/test-code.yaml

@@ -2,7 +2,7 @@
 name: test code
 
 env:
-  PGPASSWORD: postgres
+  PGPASSWORD: mysecretpassword
   PGUSER: postgres
   PGDATABASE: ccbc
 
@@ -31,7 +31,7 @@ jobs:
           attempt_limit: 2
           attempt_delay: 2000
       - name: SonarCloud Scan
-        uses: sonarsource/sonarcloud-github-action@master
+        uses: SonarSource/sonarqube-scan-action@v4.2.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
@@ -73,36 +73,27 @@ jobs:
       - name: dev env setup
         uses: ./.github/actions/dev-env-setup
       - name: deploy db migrations
-        run: make start_pg && make drop_db && make deploy_db_migrations
+        run: |
+          docker run --rm -p 5432:5432 -e POSTGRES_PASSWORD=mysecretpassword -d postgres:15
+          # Wait for PostgreSQL to be ready
+          until docker exec $(docker ps -q -f ancestor=postgres:15) pg_isready -U postgres; do
+            echo "Waiting for PostgreSQL to be ready..."
+            sleep 2
+          done
+          # Additional wait to ensure PostgreSQL is fully ready
+          sleep 5
+          # Check PostgreSQL readiness from the host one more time
+          until pg_isready -h localhost -p 5432 -U postgres; do
+            echo "Waiting for PostgreSQL to be ready from the host..."
+            sleep 2
+          done
+          make drop_db PSQL="psql -h localhost -U postgres"
+          make create_db PSQL="psql -h localhost -U postgres"
+          # Deploy migrations
+          docker pull sqitch/sqitch
+          curl -L https://git.io/JJKCn -o sqitch && chmod +x sqitch
+          ./sqitch --chdir db deploy
       - run: yarn build:schema
         working-directory: ./app
       - name: Check for inconsistency between graphql schema and commit schema
         run: git diff --exit-code || (echo 'The contents of the `schema` folder are out of sync with `app/schema/schema.graphql`. Did you forget to \"yarn build:schema\"?' && false)
-
-  reverts:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: dev env setup
-        uses: ./.github/actions/dev-env-setup
-      - name: deploy db migrations
-        run: make start_pg && make drop_db && make deploy_db_migrations
-      - name: revert db migrations
-        run: make revert_db_migrations
-      - name: re-deploy db migrations
-        run: make deploy_db_migrations
-
-  pgtap:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-      - name: dev env setup
-        uses: ./.github/actions/dev-env-setup
-      - name: Run Database Unit Tests
-        run: make db_unit_tests
-      - name: Run Database Style Tests
-        run: make db_style_tests

.github/workflows/test-containers.yaml

@@ -45,6 +45,7 @@ jobs:
         run: echo "results-length=$(cat trivy-results.sarif | jq '.runs[0].results | length')" >> $GITHUB_OUTPUT
 
   trivy-scan-db:
+    if: github.event.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     outputs:
       results-length: ${{ steps.scan-results-length.outputs.results-length }}
@@ -81,7 +82,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Self-hosted Renovate
-        uses: renovatebot/[email protected].12
+        uses: renovatebot/[email protected].13
         with:
           configurationFile: ./.github/renovate.json
           token: ${{ secrets.RENOVATE_GITHUB_TOKEN }}