Aggregate test log reports

DataDog · Apr 4, 2023 · 5c23553 · 5c23553
1 parent 0b3a4d8
commit 5c23553
Showing 1 changed file with 124 additions and 3 deletions.
diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
@@ -166,7 +166,7 @@ jobs:
           docker load < runner.tar
           docker load < weblog-${{ matrix.app }}.tar
           docker image list
-      - name: Run default scenario
+      - name: Run scenario
         run: ./run.sh ${{ matrix.scenario }}
         env:
           DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
@@ -176,6 +176,127 @@ jobs:
         with:
           name: system-tests-${{ matrix.library }}-${{ matrix.scenario }}-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: logs*
+
+  aggregate:
+    strategy:
+      fail-fast: false
+      matrix:
+        library:
+          - ruby
+        app:
+          - rack
+          - sinatra14
+          - sinatra20
+          - sinatra21
+          - rails32
+          - rails40
+          - rails41
+          - rails42
+          - rails50
+          - rails51
+          - rails52
+          - rails60
+          - rails61
+          - rails70
+    runs-on: ubuntu-latest
+    needs:
+      - test
+    if: ${{ always() }}
+    name: Aggregate (${{ matrix.app }})
+    steps:
+      - name: Setup python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-DEFAULT-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_MISSING_RULES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_CORRUPTED_RULES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_LOW_WAF_TIMEOUT-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_CUSTOM_OBFUSCATION-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_RATE_LIMITER-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_IP_BLOCKING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_REQUEST_BLOCKING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-SAMPLING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-PROFILING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
       - name: Print fancy log report
-        if: ${{ always() }}
-        run: python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY
+        run: |
+          ls -l logs
+          python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY