From ade434f362b26c4d4374d5bb47879047c592474a Mon Sep 17 00:00:00 2001 From: Ndibe Raymond Olisaemeka Date: Tue, 19 Dec 2023 01:38:48 +0100 Subject: [PATCH] refactor github actions * use matrix strategy for similar jobs to save time * update version of third-party actions * reuse workflows Issue: #1030 Signed-off-by: Ndibe Raymond Olisaemeka --- .github/workflows/.checkout.yml | 20 ++++++ .github/workflows/.docker_build_and_push.yml | 44 ++++++++++++ .github/workflows/.scp_action.yml | 23 ++++++ .github/workflows/.ssh_action.yml | 19 +++++ .github/workflows/build_deploy_backend.yml | 70 +++++-------------- .github/workflows/build_deploy_docs.yml | 4 +- .github/workflows/build_deploy_frontend.yml | 12 ++-- .github/workflows/build_locust.yml | 10 +-- .github/workflows/create_destroy_test_vm.yaml | 12 ++-- zubhub_backend/docker-compose.prod.yml | 28 ++++---- 10 files changed, 155 insertions(+), 87 deletions(-) create mode 100644 .github/workflows/.checkout.yml create mode 100644 .github/workflows/.docker_build_and_push.yml create mode 100644 .github/workflows/.scp_action.yml create mode 100644 .github/workflows/.ssh_action.yml diff --git a/.github/workflows/.checkout.yml b/.github/workflows/.checkout.yml new file mode 100644 index 000000000..0fbcae001 --- /dev/null +++ b/.github/workflows/.checkout.yml @@ -0,0 +1,20 @@ +name: Checkout files workflow +on: + workflow_call: + inputs: + ref: + required: false + type: string + repository: + required: false + type: string + +jobs: + checkout: + runs-on: ubuntu-latest + steps: + - name: Checkout files + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + repository: ${{ inputs.repository }} diff --git a/.github/workflows/.docker_build_and_push.yml b/.github/workflows/.docker_build_and_push.yml new file mode 100644 index 000000000..4fb97183f --- /dev/null +++ b/.github/workflows/.docker_build_and_push.yml @@ -0,0 +1,44 @@ +name: Docker build and push workflow +on: + workflow_call: + inputs: + context: + required: true + type: string + file: + required: true + type: string + tags: + required: true + type: string + push: + required: true + type: boolean + +jobs: + docker_build_and_push: + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v5 + with: + context: ${{ inputs.context }}} + file: ${{ inputs.file }} + push: ${{ inputs.push }} + tags: ${{ inputs.tags }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/.scp_action.yml b/.github/workflows/.scp_action.yml new file mode 100644 index 000000000..1143d5e6d --- /dev/null +++ b/.github/workflows/.scp_action.yml @@ -0,0 +1,23 @@ +name: Copy files workflow +on: + workflow_call: + inputs: + source: + required: true + type: string + target: + required: true + type: string + +jobs: + copy_files: + runs-on: ubuntu-latest + steps: + - name: Copy file via scp + uses: appleboy/scp-action@master + with: + host: ${{ secrets.DO_BACKEND_HOST }} + username: ${{ secrets.DO_BACKEND_USERNAME }} + key: ${{ secrets.DO_SSHKEY }} + source: ${{ inputs.source }} + target: ${{ inputs.target }} diff --git a/.github/workflows/.ssh_action.yml b/.github/workflows/.ssh_action.yml new file mode 100644 index 000000000..14deb7a72 --- /dev/null +++ b/.github/workflows/.ssh_action.yml @@ -0,0 +1,19 @@ +name: Execute remote command workflow +on: + workflow_call: + inputs: + script: + required: true + type: string + +jobs: + execute_command: + runs-on: ubuntu-latest + steps: + - name: Executing remote command + uses: appleboy/ssh-action@master + with: + host: ${{ secrets.DO_BACKEND_HOST }} + username: ${{ secrets.DO_BACKEND_USERNAME }} + key: ${{ secrets.DO_SSHKEY }} + script: ${{ inputs.script }} diff --git a/.github/workflows/build_deploy_backend.yml b/.github/workflows/build_deploy_backend.yml index f7383c444..97fea4654 100644 --- a/.github/workflows/build_deploy_backend.yml +++ b/.github/workflows/build_deploy_backend.yml @@ -13,77 +13,38 @@ on: workflow_dispatch: jobs: - build: + build_and_push: runs-on: ubuntu-latest + strategy: + matrix: + service: ['web', 'celery', 'media'] steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Checkout files - uses: actions/checkout@v2 - - - name: Build and push django api - id: docker_build_web - uses: docker/build-push-action@v2 - with: - context: ./zubhub_backend/ - file: ./zubhub_backend/compose/web/prod/Dockerfile - push: true - tags: unstructuredstudio/zubhub-services_web:latest - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - - - name: Build and push celery worker - id: docker_build_celery - uses: docker/build-push-action@v2 - with: - context: ./zubhub_backend/ - file: ./zubhub_backend/compose/celery/prod/Dockerfile - push: true - tags: unstructuredstudio/zubhub-services_celery:latest - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} + - uses: unstructuredstudio/zubhub/.github/workflows/.checkout.yml@master - - name: Build and push media service - id: docker_build_media - uses: docker/build-push-action@v2 + - name: Build and push ${{ matrix.service }} + uses: unstructuredstudio/zubhub/.github/workflows/.docker_build_and_push.yml@master with: context: ./zubhub_backend/ - file: ./zubhub_backend/compose/media/prod/Dockerfile + file: ./zubhub_backend/compose/${{ matrix.service }}/prod/Dockerfile push: true - tags: unstructuredstudio/zubhub-services_media:latest - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} + tags: unstructuredstudio/zubhub-services_${{ matrix.service }}:latest + secrets: inherit deploy: - needs: build + needs: build_and_push runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: unstructuredstudio/zubhub/.github/workflows/.checkout.yml@master - - name: Copy file via scp - uses: appleboy/scp-action@master + - uses: unstructuredstudio/zubhub/.github/workflows/.scp_action.yml@master with: - host: ${{ secrets.DO_BACKEND_HOST }} - username: ${{ secrets.DO_BACKEND_USERNAME }} - key: ${{ secrets.DO_SSHKEY }} source: "." target: "/home/zubhub-services/zubhub" + secrets: inherit - - name: Executing remote command - uses: appleboy/ssh-action@master + - uses: unsctructuredstudio/zubhub/.github/workflows/.ssh_action.yml@master with: - host: ${{ secrets.DO_BACKEND_HOST }} - username: ${{ secrets.DO_BACKEND_USERNAME }} - key: ${{ secrets.DO_SSHKEY }} script: | cp /home/zubhub-services/zubhub/zubhub_backend/compose/deploy_backend.sh /home/zubhub-services/ sudo bash /home/zubhub-services/deploy_backend.sh @@ -91,3 +52,4 @@ jobs: droplets_count=`wc -l < droplets.txt` rm droplets.txt docker service scale zubhub-services_web=$(($droplets_count - 1)) + secrets: inherit diff --git a/.github/workflows/build_deploy_docs.yml b/.github/workflows/build_deploy_docs.yml index fdc5b9674..bdab90f65 100644 --- a/.github/workflows/build_deploy_docs.yml +++ b/.github/workflows/build_deploy_docs.yml @@ -11,8 +11,8 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" - run: | diff --git a/.github/workflows/build_deploy_frontend.yml b/.github/workflows/build_deploy_frontend.yml index ea07d4fc1..ee047492d 100644 --- a/.github/workflows/build_deploy_frontend.yml +++ b/.github/workflows/build_deploy_frontend.yml @@ -17,21 +17,21 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Checkout files - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Build and push id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./zubhub_frontend/zubhub/ file: ./zubhub_frontend/zubhub/Dockerfile.prod @@ -45,7 +45,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Copy file via scp uses: appleboy/scp-action@master diff --git a/.github/workflows/build_locust.yml b/.github/workflows/build_locust.yml index ea70921ed..41ff94655 100644 --- a/.github/workflows/build_locust.yml +++ b/.github/workflows/build_locust.yml @@ -23,21 +23,21 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Checkout files - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Build and push locust id: docker_build_locust - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./locust/ file: ./locust/Dockerfile diff --git a/.github/workflows/create_destroy_test_vm.yaml b/.github/workflows/create_destroy_test_vm.yaml index 9c63b1277..30c92dfa6 100644 --- a/.github/workflows/create_destroy_test_vm.yaml +++ b/.github/workflows/create_destroy_test_vm.yaml @@ -47,7 +47,7 @@ jobs: # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch. - name: Checkout source branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: ${{ env.CHECKOUT_COMMIT_REF }} repository: ${{ env.CHECKOUT_REPO }} @@ -148,7 +148,7 @@ jobs: # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch. - name: Checkout source branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: ${{ env.CHECKOUT_COMMIT_REF }} repository: ${{ env.CHECKOUT_REPO }} @@ -175,20 +175,20 @@ jobs: EOF - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push ${{ matrix.service }} id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: # for frontend, context is ./zubhub_frontend/zubhub/ # for backend services, context is ./zubhub_backend/ diff --git a/zubhub_backend/docker-compose.prod.yml b/zubhub_backend/docker-compose.prod.yml index dbd5c4b3b..6b769d819 100644 --- a/zubhub_backend/docker-compose.prod.yml +++ b/zubhub_backend/docker-compose.prod.yml @@ -95,20 +95,20 @@ services: depends_on: - rabbitmq - prometheus: - image: prom/prometheus - command: - - --config.file=/etc/prometheus/prometheus.yml - deploy: - replicas: 1 - restart_policy: - condition: on-failure - placement: - max_replicas_per_node: 1 - constraints: - - "node.role==manager" - volumes: - - ./compose/prometheus.yml:/etc/prometheus/prometheus.yml:ro + # prometheus: + # image: prom/prometheus + # command: + # - --config.file=/etc/prometheus/prometheus.yml + # deploy: + # replicas: 1 + # restart_policy: + # condition: on-failure + # placement: + # max_replicas_per_node: 1 + # constraints: + # - "node.role==manager" + # volumes: + # - ./compose/prometheus.yml:/etc/prometheus/prometheus.yml:ro secrets: zubhub_services_secrets: