Add more scenarios to system tests

.github/workflows/system-tests.yml

@@ -12,81 +12,292 @@ env:
   REGISTRY: ghcr.io
 
 jobs:
-  system-tests:
+  build-harness:
     strategy:
       fail-fast: false
       matrix:
+        image:
+          - runner
+          - agent
+    runs-on: ubuntu-latest
+    name: Build (${{ matrix.image }})
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+      - name: Pull released image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest system_tests/${{ matrix.image }}:latest
+      - name: Build image
+        run: ./build.sh --images ${{ matrix.image }}
+      - name: List images
+        run: |
+          docker image list
+      - name: Log in to the Container registry
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
+      - name: Tag image for CI run
+        run:
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Push image for CI run
+        run: |
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Tag image for release
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run:
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
+      - name: Push image for release
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run: |
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
+
+  build-apps:
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - weblog
+        library:
+          - ruby
+        app:
+          - rack
+          - sinatra14
+          - sinatra20
+          - sinatra21
+          - rails32
+          - rails40
+          - rails41
+          - rails42
+          - rails50
+          - rails51
+          - rails52
+          - rails60
+          - rails61
+          - rails70
+    runs-on: ubuntu-latest
+    name: Build (${{ matrix.app }})
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+      - name: Checkout dd-trace-rb
+        uses: actions/checkout@v3
+        with:
+          path: 'binaries/dd-trace-rb'
+      - name: Pull released image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest system_tests/${{ matrix.image }}:latest
+      - name: Log in to the Container registry
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
+      - name: Build
+        run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }}
+      - name: Tag image for CI run
+        run:
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Push image for CI run
+        run: |
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Tag image for release
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run:
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest
+      - name: Push image for release
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run: |
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest
+
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        library:
+          - ruby
+        app:
+          - rack
+          - sinatra14
+          - sinatra20
+          - sinatra21
+          - rails32
+          - rails40
+          - rails41
+          - rails42
+          - rails50
+          - rails51
+          - rails52
+          - rails60
+          - rails61
+          - rails70
+        scenario:
+          - DEFAULT
+          - APPSEC_DISABLED
+          - APPSEC_IP_BLOCKING
+          - APPSEC_REQUEST_BLOCKING
         include:
           - library: ruby
-            weblog-variant: rack
-          - library: ruby
-            weblog-variant: sinatra14
-          - library: ruby
-            weblog-variant: sinatra20
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
           - library: ruby
-            weblog-variant: sinatra21
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
           - library: ruby
-            weblog-variant: rails32
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
           - library: ruby
-            weblog-variant: rails40
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
           - library: ruby
-            weblog-variant: rails41
+            app: rack
+            scenario: APPSEC_CUSTOM_RULES
           - library: ruby
-            weblog-variant: rails42
+            app: rack
+            scenario: APPSEC_MISSING_RULES
           - library: ruby
-            weblog-variant: rails50
+            app: rack
+            scenario: APPSEC_CORRUPTED_RULES
           - library: ruby
-            weblog-variant: rails51
+            app: rack
+            scenario: APPSEC_LOW_WAF_TIMEOUT
           - library: ruby
-            weblog-variant: rails52
+            app: rack
+            scenario: APPSEC_CUSTOM_OBFUSCATION
           - library: ruby
-            weblog-variant: rails60
+            app: rack
+            scenario: APPSEC_RATE_LIMITER
           - library: ruby
-            weblog-variant: rails61
+            app: rack
+            scenario: SAMPLING
           - library: ruby
-            weblog-variant: rails70
+            app: rack
+            scenario: PROFILING
     runs-on: ubuntu-latest
-    name: System Tests (${{ matrix.weblog-variant }})
+    needs:
+      - build-harness
+      - build-apps
+    name: Test (${{ matrix.app }}, ${{ matrix.scenario }})
     steps:
       - name: Setup python 3.9
         uses: actions/setup-python@v4
         with:
           python-version: '3.9'
-
       - name: Checkout
         uses: actions/checkout@v3
         with:
           repository: 'DataDog/system-tests'
-
-      - name: Checkout dd-trace-rb
-        uses: actions/checkout@v3
-        with:
-          path: 'binaries/dd-trace-rb'
-
-      - name: Log in to the Container registry
+      - name: Pull agent image
         run: |
-          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
-
-      - name: Build
-        run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.weblog-variant }}
-
-      - name: Run default scenario
-        run: ./run.sh
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-
-      - name: Run APPSEC_CUSTOM_RULES scenario
-        run: ./run.sh APPSEC_CUSTOM_RULES
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }}
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} system_tests/agent:latest
+      - name: Pull runner image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }}
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} system_tests/runner:latest
+      - name: Pull app image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} system_tests/weblog:latest
+      - name: List images
+        run: |
+          docker image list
+      - name: Run scenario
+        run: ./run.sh ${{ matrix.scenario }}
         env:
           DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-
-      - name: Archive logs
+      - name: Archive logs (per scenario)
         uses: actions/upload-artifact@v3
         if: ${{ always() }}
         with:
-          name: system-tests-${{ matrix.library }}-${{ matrix.weblog-variant }}-logs-${{ github.run_id }}-${{ github.sha }}
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario }}-logs-gha${{ github.run_id }}-g${{ github.sha }}
+          path: logs*
+      - name: Archive logs (aggregated)
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-gha${{ github.run_id }}-g${{ github.sha }}
           path: logs*
 
+  aggregate:
+    strategy:
+      fail-fast: false
+      matrix:
+        library:
+          - ruby
+        app:
+          - rack
+          - sinatra14
+          - sinatra20
+          - sinatra21
+          - rails32
+          - rails40
+          - rails41
+          - rails42
+          - rails50
+          - rails51
+          - rails52
+          - rails60
+          - rails61
+          - rails70
+    runs-on: ubuntu-latest
+    needs:
+      - test
+    if: ${{ always() }}
+    name: Aggregate (${{ matrix.app }})
+    steps:
+      - name: Setup python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-gha${{ github.run_id }}-g${{ github.sha }}
+          path: .
       - name: Print fancy log report
-        if: ${{ always() }}
-        run: python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY
+        run: |
+          find logs*
+          python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY
+
+  cleanup:
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - runner
+          - agent
+          - weblog-rack
+          - weblog-sinatra14
+          - weblog-sinatra20
+          - weblog-sinatra21
+          - weblog-rails32
+          - weblog-rails40
+          - weblog-rails41
+          - weblog-rails42
+          - weblog-rails50
+          - weblog-rails51
+          - weblog-rails52
+          - weblog-rails60
+          - weblog-rails61
+          - weblog-rails70
+    runs-on: ubuntu-latest
+    needs:
+      - test
+    if: ${{ always() }}
+    name: Cleanup (${{ matrix.image }})
+    steps:
+      - name: Log in to the Container registry
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
+      - uses: actions/delete-package-versions@v4
+        with:
+          package-version-ids: 'gha${{ github.run_id }}-g${{ github.sha }}'
+          package-name: 'system-tests/${{ matrix.image }}'
+          package-type: 'container'
+        continue-on-error: true