From 54e80f1d44c1087853bcfc3d3a0457d202318eb1 Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Mon, 3 Apr 2023 17:24:40 +0200
Subject: [PATCH 01/18] Add more scenarios to system tests

---
 .github/workflows/system-tests.yml | 62 +++++++++++++++++++++++++++++-
 1 file changed, 61 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 974cda45993..3d717ab0ee3 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -74,11 +74,71 @@ jobs:
         run: ./run.sh
         env:
           DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-
       - name: Run APPSEC_CUSTOM_RULES scenario
         run: ./run.sh APPSEC_CUSTOM_RULES
         env:
           DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run APPSEC_CUSTOM_RULES scenario
+        run: ./run.sh APPSEC_CUSTOM_RULES
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD scenario
+        run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE scenario
+        run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING_NOCACHE scenario
+        run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING_NOCACHE
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE scenario
+        run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run APPSEC_MISSING_RULES scenario
+        run: ./run.sh APPSEC_MISSING_RULES
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run APPSEC_CORRUPTED_RULES scenario
+        if: steps.build.outcome == 'success' && github.event.action != 'opened' && !contains(github.event.pull_request.labels.*.name, 'run-default-scenario')
+        run: ./run.sh APPSEC_CORRUPTED_RULES
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run APPSEC_DISABLED scenario
+        run: ./run.sh APPSEC_DISABLED
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run APPSEC_LOW_WAF_TIMEOUT scenario
+        run: ./run.sh APPSEC_LOW_WAF_TIMEOUT
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run APPSEC_CUSTOM_OBFUSCATION scenario
+        run: ./run.sh APPSEC_CUSTOM_OBFUSCATION
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run APPSEC_RATE_LIMITER scenario
+        run: ./run.sh APPSEC_RATE_LIMITER
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run APPSEC_IP_BLOCKING scenario
+        run: ./run.sh APPSEC_IP_BLOCKING
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run APPSEC_REQUEST_BLOCKING scenario
+        run: ./run.sh APPSEC_REQUEST_BLOCKING
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run SAMPLING scenario
+        run: ./run.sh SAMPLING
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
+      - name: Run PROFILING scenario
+        run: ./run.sh PROFILING
+        env:
+          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
 
       - name: Archive logs
         uses: actions/upload-artifact@v3

From c98f7c39209f5933f311968b3bc28603e94f193c Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Mon, 3 Apr 2023 17:32:54 +0200
Subject: [PATCH 02/18] Use a proper matrix for system tests

---
 .github/workflows/system-tests.yml | 46 +++++++++++-------------------
 1 file changed, 17 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 3d717ab0ee3..e44265841e1 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -16,35 +16,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - library: ruby
-            weblog-variant: rack
-          - library: ruby
-            weblog-variant: sinatra14
-          - library: ruby
-            weblog-variant: sinatra20
-          - library: ruby
-            weblog-variant: sinatra21
-          - library: ruby
-            weblog-variant: rails32
-          - library: ruby
-            weblog-variant: rails40
-          - library: ruby
-            weblog-variant: rails41
-          - library: ruby
-            weblog-variant: rails42
-          - library: ruby
-            weblog-variant: rails50
-          - library: ruby
-            weblog-variant: rails51
-          - library: ruby
-            weblog-variant: rails52
-          - library: ruby
-            weblog-variant: rails60
-          - library: ruby
-            weblog-variant: rails61
-          - library: ruby
-            weblog-variant: rails70
+        library:
+          - ruby
+        weblog-variant:
+          - rack
+          - sinatra14
+          - sinatra20
+          - sinatra21
+          - rails32
+          - rails40
+          - rails41
+          - rails42
+          - rails50
+          - rails51
+          - rails52
+          - rails60
+          - rails61
+          - rails70
     runs-on: ubuntu-latest
     name: System Tests (${{ matrix.weblog-variant }})
     steps:

From f68faf408a327cf5aa89365caa5fe950dcb482e5 Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Mon, 3 Apr 2023 17:38:51 +0200
Subject: [PATCH 03/18] Factor scenarios into matrix

---
 .github/workflows/system-tests.yml | 88 +++++++-----------------------
 1 file changed, 20 insertions(+), 68 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index e44265841e1..299721f6960 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -33,8 +33,25 @@ jobs:
           - rails60
           - rails61
           - rails70
+        scenario:
+          - DEFAULT
+          - APPSEC_CUSTOM_RULES
+          - REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
+          - REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
+          - REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
+          - REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
+          - APPSEC_MISSING_RULES
+          - APPSEC_CORRUPTED_RULES
+          - APPSEC_DISABLED
+          - APPSEC_LOW_WAF_TIMEOUT
+          - APPSEC_CUSTOM_OBFUSCATION
+          - APPSEC_RATE_LIMITER
+          - APPSEC_IP_BLOCKING
+          - APPSEC_REQUEST_BLOCKING
+          - SAMPLING
+          - PROFILING
     runs-on: ubuntu-latest
-    name: System Tests (${{ matrix.weblog-variant }})
+    name: System Tests (${{ matrix.scenario }}, ${{ matrix.weblog-variant }})
     steps:
       - name: Setup python 3.9
         uses: actions/setup-python@v4
@@ -59,72 +76,7 @@ jobs:
         run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.weblog-variant }}
 
       - name: Run default scenario
-        run: ./run.sh
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run APPSEC_CUSTOM_RULES scenario
-        run: ./run.sh APPSEC_CUSTOM_RULES
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run APPSEC_CUSTOM_RULES scenario
-        run: ./run.sh APPSEC_CUSTOM_RULES
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD scenario
-        run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE scenario
-        run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING_NOCACHE scenario
-        run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING_NOCACHE
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE scenario
-        run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run APPSEC_MISSING_RULES scenario
-        run: ./run.sh APPSEC_MISSING_RULES
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run APPSEC_CORRUPTED_RULES scenario
-        if: steps.build.outcome == 'success' && github.event.action != 'opened' && !contains(github.event.pull_request.labels.*.name, 'run-default-scenario')
-        run: ./run.sh APPSEC_CORRUPTED_RULES
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run APPSEC_DISABLED scenario
-        run: ./run.sh APPSEC_DISABLED
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run APPSEC_LOW_WAF_TIMEOUT scenario
-        run: ./run.sh APPSEC_LOW_WAF_TIMEOUT
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run APPSEC_CUSTOM_OBFUSCATION scenario
-        run: ./run.sh APPSEC_CUSTOM_OBFUSCATION
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run APPSEC_RATE_LIMITER scenario
-        run: ./run.sh APPSEC_RATE_LIMITER
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run APPSEC_IP_BLOCKING scenario
-        run: ./run.sh APPSEC_IP_BLOCKING
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run APPSEC_REQUEST_BLOCKING scenario
-        run: ./run.sh APPSEC_REQUEST_BLOCKING
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run SAMPLING scenario
-        run: ./run.sh SAMPLING
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Run PROFILING scenario
-        run: ./run.sh PROFILING
+        run: ./run.sh ${{ matrix.scenario }}
         env:
           DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
 
@@ -132,7 +84,7 @@ jobs:
         uses: actions/upload-artifact@v3
         if: ${{ always() }}
         with:
-          name: system-tests-${{ matrix.library }}-${{ matrix.weblog-variant }}-logs-${{ github.run_id }}-${{ github.sha }}
+          name: system-tests-${{ matrix.library }}-${{ matrix.scenario }}-${{ matrix.weblog-variant }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: logs*
 
       - name: Print fancy log report

From 9e4fcc7ca5a9e0a5bb50cc36e014caaaea5a3be9 Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Tue, 4 Apr 2023 10:29:54 +0200
Subject: [PATCH 04/18] Build images once

---
 .github/workflows/system-tests.yml | 127 ++++++++++++++++++++++++-----
 1 file changed, 108 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 299721f6960..74af40d7acb 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -12,13 +12,92 @@ env:
   REGISTRY: ghcr.io
 
 jobs:
-  system-tests:
+  build-harness:
     strategy:
       fail-fast: false
       matrix:
+        image:
+          - runner
+          - agent
+    runs-on: ubuntu-latest
+    name: Build (${{ matrix.image }})
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+      - name: Build image
+        run: ./build.sh --images ${{ matrix.image }}
+      - name: Export image
+        run: |
+          docker image list
+          mkdir -p images
+          docker save system_tests/${{ matrix.image }} > images/${{ matrix.image }}.tar
+      - name: Save image
+        uses: actions/upload-artifact@v3
+        with:
+          name: docker-image-${{ matrix.image }}
+          path: images/*
+          retention-days: 1
+
+  build-apps:
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - weblog
         library:
           - ruby
-        weblog-variant:
+        app:
+          - rack
+          - sinatra14
+          - sinatra20
+          - sinatra21
+          - rails32
+          - rails40
+          - rails41
+          - rails42
+          - rails50
+          - rails51
+          - rails52
+          - rails60
+          - rails61
+          - rails70
+    runs-on: ubuntu-latest
+    name: Build (${{ matrix.app }})
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+      - name: Checkout dd-trace-rb
+        uses: actions/checkout@v3
+        with:
+          path: 'binaries/dd-trace-rb'
+      - name: Log in to the Container registry
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
+      - name: Build
+        run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }}
+      - name: Export image
+        run: |
+          docker image list
+          mkdir -p images
+          docker save system_tests/${{ matrix.image }} > images/${{ matrix.image }}-${{ matrix.app }}.tar
+      - name: Save image
+        uses: actions/upload-artifact@v3
+        with:
+          name: docker-image-${{ matrix.image }}-${{ matrix.app }}
+          path: images/*
+          retention-days: 1
+
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        library:
+          - ruby
+        app:
           - rack
           - sinatra14
           - sinatra20
@@ -35,11 +114,11 @@ jobs:
           - rails70
         scenario:
           - DEFAULT
-          - APPSEC_CUSTOM_RULES
           - REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
           - REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
           - REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
           - REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
+          - APPSEC_CUSTOM_RULES
           - APPSEC_MISSING_RULES
           - APPSEC_CORRUPTED_RULES
           - APPSEC_DISABLED
@@ -51,42 +130,52 @@ jobs:
           - SAMPLING
           - PROFILING
     runs-on: ubuntu-latest
-    name: System Tests (${{ matrix.scenario }}, ${{ matrix.weblog-variant }})
+    needs:
+      - build-harness
+      - build-apps
+    name: Test (${{ matrix.app }}, ${{ matrix.scenario }})
     steps:
       - name: Setup python 3.9
         uses: actions/setup-python@v4
         with:
           python-version: '3.9'
-
       - name: Checkout
         uses: actions/checkout@v3
         with:
           repository: 'DataDog/system-tests'
-
-      - name: Checkout dd-trace-rb
-        uses: actions/checkout@v3
+      - name: Retrieve agent image
+        uses: actions/download-artifact@v3
         with:
-          path: 'binaries/dd-trace-rb'
-
-      - name: Log in to the Container registry
+          name: docker-image-agent
+          path: images
+      - name: Retrieve runner image
+        uses: actions/download-artifact@v3
+        with:
+          name: docker-image-runner
+          path: images
+      - name: Retrieve app image
+        uses: actions/download-artifact@v3
+        with:
+          name: docker-image-weblog-${{ matrix.app }}
+          path: images
+      - name: Load images
         run: |
-          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
-
-      - name: Build
-        run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.weblog-variant }}
-
+          cd images
+          ls -l
+          docker load < agent.tar
+          docker load < runner.tar
+          docker load < weblog-${{ matrix.app }}.tar
+          docker image list
       - name: Run default scenario
         run: ./run.sh ${{ matrix.scenario }}
         env:
           DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-
       - name: Archive logs
         uses: actions/upload-artifact@v3
         if: ${{ always() }}
         with:
-          name: system-tests-${{ matrix.library }}-${{ matrix.scenario }}-${{ matrix.weblog-variant }}-logs-${{ github.run_id }}-${{ github.sha }}
+          name: system-tests-${{ matrix.library }}-${{ matrix.scenario }}-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: logs*
-
       - name: Print fancy log report
         if: ${{ always() }}
         run: python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY

From 34018681c7f3a0f665f8ef4e805c4bc7bc0f1b61 Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Tue, 4 Apr 2023 13:39:35 +0200
Subject: [PATCH 05/18] Aggregate test log reports

---
 .github/workflows/system-tests.yml | 127 ++++++++++++++++++++++++++++-
 1 file changed, 124 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 74af40d7acb..541366984c5 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -166,7 +166,7 @@ jobs:
           docker load < runner.tar
           docker load < weblog-${{ matrix.app }}.tar
           docker image list
-      - name: Run default scenario
+      - name: Run scenario
         run: ./run.sh ${{ matrix.scenario }}
         env:
           DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
@@ -176,6 +176,127 @@ jobs:
         with:
           name: system-tests-${{ matrix.library }}-${{ matrix.scenario }}-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: logs*
+
+  aggregate:
+    strategy:
+      fail-fast: false
+      matrix:
+        library:
+          - ruby
+        app:
+          - rack
+          - sinatra14
+          - sinatra20
+          - sinatra21
+          - rails32
+          - rails40
+          - rails41
+          - rails42
+          - rails50
+          - rails51
+          - rails52
+          - rails60
+          - rails61
+          - rails70
+    runs-on: ubuntu-latest
+    needs:
+      - test
+    if: ${{ always() }}
+    name: Aggregate (${{ matrix.app }})
+    steps:
+      - name: Setup python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-DEFAULT-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_MISSING_RULES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_CORRUPTED_RULES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_LOW_WAF_TIMEOUT-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_CUSTOM_OBFUSCATION-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_RATE_LIMITER-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_IP_BLOCKING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-APPSEC_REQUEST_BLOCKING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-SAMPLING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-PROFILING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: .
+        continue-on-error: true
       - name: Print fancy log report
-        if: ${{ always() }}
-        run: python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY
+        run: |
+          ls -l logs
+          python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY

From 795d62c6c7e3b6d2fac7d4931a7d13c36e20cf1e Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Tue, 4 Apr 2023 14:34:25 +0200
Subject: [PATCH 06/18] Add CI details to image artifacts

---
 .github/workflows/system-tests.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 541366984c5..fc0ad6fc711 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -36,7 +36,7 @@ jobs:
       - name: Save image
         uses: actions/upload-artifact@v3
         with:
-          name: docker-image-${{ matrix.image }}
+          name: docker-image-${{ matrix.image }}-${{ github.run_id }}-${{ github.sha }}
           path: images/*
           retention-days: 1
 
@@ -87,7 +87,7 @@ jobs:
       - name: Save image
         uses: actions/upload-artifact@v3
         with:
-          name: docker-image-${{ matrix.image }}-${{ matrix.app }}
+          name: docker-image-${{ matrix.image }}-${{ matrix.app }}-${{ github.run_id }}-${{ github.sha }}
           path: images/*
           retention-days: 1
 
@@ -146,17 +146,17 @@ jobs:
       - name: Retrieve agent image
         uses: actions/download-artifact@v3
         with:
-          name: docker-image-agent
+          name: docker-image-agent-${{ github.run_id }}-${{ github.sha }}
           path: images
       - name: Retrieve runner image
         uses: actions/download-artifact@v3
         with:
-          name: docker-image-runner
+          name: docker-image-runner-${{ github.run_id }}-${{ github.sha }}
           path: images
       - name: Retrieve app image
         uses: actions/download-artifact@v3
         with:
-          name: docker-image-weblog-${{ matrix.app }}
+          name: docker-image-weblog-${{ matrix.app }}-${{ github.run_id }}-${{ github.sha }}
           path: images
       - name: Load images
         run: |

From 9cba7d933c41d218686f892a9d523b0b645f30c4 Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Tue, 4 Apr 2023 14:38:24 +0200
Subject: [PATCH 07/18] Use find instead of ls

---
 .github/workflows/system-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index fc0ad6fc711..210f17f07a2 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -160,8 +160,8 @@ jobs:
           path: images
       - name: Load images
         run: |
+          find images
           cd images
-          ls -l
           docker load < agent.tar
           docker load < runner.tar
           docker load < weblog-${{ matrix.app }}.tar
@@ -298,5 +298,5 @@ jobs:
         continue-on-error: true
       - name: Print fancy log report
         run: |
-          ls -l logs
+          find logs*
           python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY

From 4869638e15ce569e0262bb507e79c5d28c40e50c Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Tue, 4 Apr 2023 14:43:37 +0200
Subject: [PATCH 08/18] Reduce matrix for tests that do not depend on the
 framework

---
 .github/workflows/system-tests.yml | 49 ++++++++++++++++++++++--------
 1 file changed, 37 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 210f17f07a2..34de01a86d9 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -114,21 +114,46 @@ jobs:
           - rails70
         scenario:
           - DEFAULT
-          - REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
-          - REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
-          - REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
-          - REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
-          - APPSEC_CUSTOM_RULES
-          - APPSEC_MISSING_RULES
-          - APPSEC_CORRUPTED_RULES
           - APPSEC_DISABLED
-          - APPSEC_LOW_WAF_TIMEOUT
-          - APPSEC_CUSTOM_OBFUSCATION
-          - APPSEC_RATE_LIMITER
           - APPSEC_IP_BLOCKING
           - APPSEC_REQUEST_BLOCKING
-          - SAMPLING
-          - PROFILING
+        include:
+          - library: ruby
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
+          - library: ruby
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
+          - library: ruby
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
+          - library: ruby
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
+          - library: ruby
+            app: rack
+            scenario: APPSEC_CUSTOM_RULES
+          - library: ruby
+            app: rack
+            scenario: APPSEC_MISSING_RULES
+          - library: ruby
+            app: rack
+            scenario: APPSEC_CORRUPTED_RULES
+          - library: ruby
+            app: rack
+            scenario: APPSEC_LOW_WAF_TIMEOUT
+          - library: ruby
+            app: rack
+            scenario: APPSEC_CUSTOM_OBFUSCATION
+          - library: ruby
+            app: rack
+            scenario: APPSEC_RATE_LIMITER
+          - library: ruby
+            app: rack
+            scenario: SAMPLING
+          - library: ruby
+            app: rack
+            scenario: PROFILING
     runs-on: ubuntu-latest
     needs:
       - build-harness

From db70d6715d9912ce61723653afcaa4c0677a490e Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Tue, 4 Apr 2023 15:17:01 +0200
Subject: [PATCH 09/18] Fix artifact fetching warnings

---
 .github/workflows/system-tests.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 34de01a86d9..6e344690291 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -249,54 +249,63 @@ jobs:
           name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
+        if: ${{ matrix.app == 'rack' }}
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
           name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
+        if: ${{ matrix.app == 'rack' }}
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
           name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
+        if: ${{ matrix.app == 'rack' }}
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
           name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
+        if: ${{ matrix.app == 'rack' }}
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
           name: system-tests-${{ matrix.library }}-APPSEC_MISSING_RULES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
+        if: ${{ matrix.app == 'rack' }}
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
           name: system-tests-${{ matrix.library }}-APPSEC_CORRUPTED_RULES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
+        if: ${{ matrix.app == 'rack' }}
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
           name: system-tests-${{ matrix.library }}-APPSEC_LOW_WAF_TIMEOUT-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
+        if: ${{ matrix.app == 'rack' }}
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
           name: system-tests-${{ matrix.library }}-APPSEC_CUSTOM_OBFUSCATION-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
+        if: ${{ matrix.app == 'rack' }}
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
           name: system-tests-${{ matrix.library }}-APPSEC_RATE_LIMITER-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
+        if: ${{ matrix.app == 'rack' }}
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
@@ -315,12 +324,14 @@ jobs:
           name: system-tests-${{ matrix.library }}-SAMPLING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
+        if: ${{ matrix.app == 'rack' }}
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
           name: system-tests-${{ matrix.library }}-PROFILING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
+        if: ${{ matrix.app == 'rack' }}
       - name: Print fancy log report
         run: |
           find logs*

From c9b0fdc4689f7df05d348a25247c34bde8119220 Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Tue, 4 Apr 2023 15:24:02 +0200
Subject: [PATCH 10/18] Use artifact aggregation

---
 .github/workflows/system-tests.yml | 101 +++--------------------------
 1 file changed, 9 insertions(+), 92 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 6e344690291..ec49d235eb9 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -195,11 +195,17 @@ jobs:
         run: ./run.sh ${{ matrix.scenario }}
         env:
           DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-      - name: Archive logs
+      - name: Archive logs (per scenario)
         uses: actions/upload-artifact@v3
         if: ${{ always() }}
         with:
-          name: system-tests-${{ matrix.library }}-${{ matrix.scenario }}-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario }}-logs-${{ github.run_id }}-${{ github.sha }}
+          path: logs*
+      - name: Archive logs (aggregated)
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: logs*
 
   aggregate:
@@ -240,98 +246,9 @@ jobs:
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
-          name: system-tests-${{ matrix.library }}-DEFAULT-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-        if: ${{ matrix.app == 'rack' }}
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-        if: ${{ matrix.app == 'rack' }}
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-        if: ${{ matrix.app == 'rack' }}
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-        if: ${{ matrix.app == 'rack' }}
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-APPSEC_MISSING_RULES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-        if: ${{ matrix.app == 'rack' }}
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-APPSEC_CORRUPTED_RULES-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-        if: ${{ matrix.app == 'rack' }}
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-APPSEC_LOW_WAF_TIMEOUT-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-        if: ${{ matrix.app == 'rack' }}
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-APPSEC_CUSTOM_OBFUSCATION-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-        if: ${{ matrix.app == 'rack' }}
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-APPSEC_RATE_LIMITER-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-        if: ${{ matrix.app == 'rack' }}
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-APPSEC_IP_BLOCKING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-APPSEC_REQUEST_BLOCKING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-SAMPLING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
-          path: .
-        continue-on-error: true
-        if: ${{ matrix.app == 'rack' }}
-      - name: Retrieve logs
-        uses: actions/download-artifact@v3
-        with:
-          name: system-tests-${{ matrix.library }}-PROFILING-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
         continue-on-error: true
-        if: ${{ matrix.app == 'rack' }}
       - name: Print fancy log report
         run: |
           find logs*

From 2316c3fd0f644700d9799af0e01c80e4f1113fa1 Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Tue, 4 Apr 2023 15:45:31 +0200
Subject: [PATCH 11/18] Push and pull to ghcr.io to hasten builds

---
 .github/workflows/system-tests.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index ec49d235eb9..57deae761a1 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -26,6 +26,10 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: 'DataDog/system-tests'
+      - name: Pull image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest system_tests/${{ matrix.image }}:latest
       - name: Build image
         run: ./build.sh --images ${{ matrix.image }}
       - name: Export image
@@ -39,6 +43,15 @@ jobs:
           name: docker-image-${{ matrix.image }}-${{ github.run_id }}-${{ github.sha }}
           path: images/*
           retention-days: 1
+      - name: Log in to the Container registry
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
+      - name: Push image
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run: |
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
 
   build-apps:
     strategy:
@@ -74,6 +87,10 @@ jobs:
         uses: actions/checkout@v3
         with:
           path: 'binaries/dd-trace-rb'
+      - name: Pull image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:${{ matrix.app }}
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:${{ matrix.app }} system_tests/${{ matrix.image }}
       - name: Log in to the Container registry
         run: |
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
@@ -90,6 +107,11 @@ jobs:
           name: docker-image-${{ matrix.image }}-${{ matrix.app }}-${{ github.run_id }}-${{ github.sha }}
           path: images/*
           retention-days: 1
+      - name: Push image
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run: |
+          docker tag system_tests/${{ matrix.image }} ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:${{ matrix.app }}
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:${{ matrix.app }}
 
   test:
     strategy:

From 5e88652be0eb06417bde4918a81575eee6c30d8c Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Tue, 4 Apr 2023 16:53:29 +0200
Subject: [PATCH 12/18] Enforce cache source

---
 .github/workflows/system-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 57deae761a1..aed0b9f2ee0 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -31,7 +31,7 @@ jobs:
           docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
           docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest system_tests/${{ matrix.image }}:latest
       - name: Build image
-        run: ./build.sh --images ${{ matrix.image }}
+        run: ./build.sh --images ${{ matrix.image }} --extra-docker-args --cache-from=ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
       - name: Export image
         run: |
           docker image list
@@ -95,7 +95,7 @@ jobs:
         run: |
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
       - name: Build
-        run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }}
+        run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }} --extra-docker-args --cache-from=ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:${{ matrix.app }}
       - name: Export image
         run: |
           docker image list

From 8c8623ca6aa23b025ce3efc2ce8d6ecbd2b54b7e Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Wed, 5 Apr 2023 09:48:52 +0200
Subject: [PATCH 13/18] Revert "Enforce cache source"

This reverts commit be5e053e04fa43cdea1f857daa11eaf5cc1e6aa6.
---
 .github/workflows/system-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index aed0b9f2ee0..57deae761a1 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -31,7 +31,7 @@ jobs:
           docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
           docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest system_tests/${{ matrix.image }}:latest
       - name: Build image
-        run: ./build.sh --images ${{ matrix.image }} --extra-docker-args --cache-from=ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
+        run: ./build.sh --images ${{ matrix.image }}
       - name: Export image
         run: |
           docker image list
@@ -95,7 +95,7 @@ jobs:
         run: |
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
       - name: Build
-        run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }} --extra-docker-args --cache-from=ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:${{ matrix.app }}
+        run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }}
       - name: Export image
         run: |
           docker image list

From 8ebd4f4a6b3d2cd33207672ab1b32620f193e111 Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Wed, 5 Apr 2023 15:54:52 +0200
Subject: [PATCH 14/18] Fail if aggregated log artifact is missing

---
 .github/workflows/system-tests.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 57deae761a1..d8a82320bc2 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -270,7 +270,6 @@ jobs:
         with:
           name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
           path: .
-        continue-on-error: true
       - name: Print fancy log report
         run: |
           find logs*

From 853f5675b2c88b6f08ae74686c15112a0d9b83ce Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Wed, 5 Apr 2023 17:00:19 +0200
Subject: [PATCH 15/18] Use GitHub packages to carry images between jobs

---
 .github/workflows/system-tests.yml | 90 +++++++++++++-----------------
 1 file changed, 40 insertions(+), 50 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index d8a82320bc2..6f59242a5de 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -26,31 +26,31 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: 'DataDog/system-tests'
-      - name: Pull image
+      - name: Pull released image
         run: |
           docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
           docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest system_tests/${{ matrix.image }}:latest
       - name: Build image
         run: ./build.sh --images ${{ matrix.image }}
-      - name: Export image
+      - name: List images
         run: |
           docker image list
-          mkdir -p images
-          docker save system_tests/${{ matrix.image }} > images/${{ matrix.image }}.tar
-      - name: Save image
-        uses: actions/upload-artifact@v3
-        with:
-          name: docker-image-${{ matrix.image }}-${{ github.run_id }}-${{ github.sha }}
-          path: images/*
-          retention-days: 1
       - name: Log in to the Container registry
-        if: ${{ github.ref == 'refs/heads/master' }}
         run: |
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
-      - name: Push image
-        if: ${{ github.ref == 'refs/heads/master' }}
+      - name: Tag image for CI run
+        run:
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Push image for CI run
         run: |
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Tag image for release
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run:
           docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
+      - name: Push image for release
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run: |
           docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
 
   build-apps:
@@ -87,31 +87,29 @@ jobs:
         uses: actions/checkout@v3
         with:
           path: 'binaries/dd-trace-rb'
-      - name: Pull image
+      - name: Pull released image
         run: |
-          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:${{ matrix.app }}
-          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:${{ matrix.app }} system_tests/${{ matrix.image }}
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest system_tests/${{ matrix.image }}:latest
       - name: Log in to the Container registry
         run: |
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
       - name: Build
         run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }}
-      - name: Export image
+      - name: Tag image for CI run
+        run:
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Push image for CI run
         run: |
-          docker image list
-          mkdir -p images
-          docker save system_tests/${{ matrix.image }} > images/${{ matrix.image }}-${{ matrix.app }}.tar
-      - name: Save image
-        uses: actions/upload-artifact@v3
-        with:
-          name: docker-image-${{ matrix.image }}-${{ matrix.app }}-${{ github.run_id }}-${{ github.sha }}
-          path: images/*
-          retention-days: 1
-      - name: Push image
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Tag image for release
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run:
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest
+      - name: Push image for release
         if: ${{ github.ref == 'refs/heads/master' }}
         run: |
-          docker tag system_tests/${{ matrix.image }} ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:${{ matrix.app }}
-          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:${{ matrix.app }}
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest
 
   test:
     strategy:
@@ -190,28 +188,20 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: 'DataDog/system-tests'
-      - name: Retrieve agent image
-        uses: actions/download-artifact@v3
-        with:
-          name: docker-image-agent-${{ github.run_id }}-${{ github.sha }}
-          path: images
-      - name: Retrieve runner image
-        uses: actions/download-artifact@v3
-        with:
-          name: docker-image-runner-${{ github.run_id }}-${{ github.sha }}
-          path: images
-      - name: Retrieve app image
-        uses: actions/download-artifact@v3
-        with:
-          name: docker-image-weblog-${{ matrix.app }}-${{ github.run_id }}-${{ github.sha }}
-          path: images
-      - name: Load images
+      - name: Pull agent image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }}
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} system_tests/agent:latest
+      - name: Pull runner image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }}
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} system_tests/runner:latest
+      - name: Pull app image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} system_tests/weblog:latest
+      - name: List images
         run: |
-          find images
-          cd images
-          docker load < agent.tar
-          docker load < runner.tar
-          docker load < weblog-${{ matrix.app }}.tar
           docker image list
       - name: Run scenario
         run: ./run.sh ${{ matrix.scenario }}

From 0cd950e87948734f463cb7596b22f66a89f2e7f5 Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Wed, 5 Apr 2023 17:34:10 +0200
Subject: [PATCH 16/18] Clean up transient images

---
 .github/workflows/system-tests.yml | 37 ++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 6f59242a5de..26b7ff886f3 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -264,3 +264,40 @@ jobs:
         run: |
           find logs*
           python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY
+
+  cleanup:
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - runner
+          - agent
+          - weblog-rack
+          - weblog-sinatra14
+          - weblog-sinatra20
+          - weblog-sinatra21
+          - weblog-rails32
+          - weblog-rails40
+          - weblog-rails41
+          - weblog-rails42
+          - weblog-rails50
+          - weblog-rails51
+          - weblog-rails52
+          - weblog-rails60
+          - weblog-rails61
+          - weblog-rails70
+    runs-on: ubuntu-latest
+    needs:
+      - test
+    if: ${{ always() }}
+    name: Cleanup (${{ matrix.image }})
+    steps:
+      - name: Log in to the Container registry
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
+      - uses: actions/delete-package-versions@v4
+        with:
+          package-version-ids: gha${{ github.run_id }}-g${{ github.sha }}
+          package-name: system-tests/${{ matrix.image }}
+          package-type: 'container'
+        continue-on-error: true

From b5ee9cf95b9a9140ed34b96a651799cf8ce456d3 Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Wed, 5 Apr 2023 22:19:22 +0200
Subject: [PATCH 17/18] Unify log artifact name

---
 .github/workflows/system-tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 26b7ff886f3..1e6f802a81c 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -211,13 +211,13 @@ jobs:
         uses: actions/upload-artifact@v3
         if: ${{ always() }}
         with:
-          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario }}-logs-${{ github.run_id }}-${{ github.sha }}
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario }}-logs-gha${{ github.run_id }}-g${{ github.sha }}
           path: logs*
       - name: Archive logs (aggregated)
         uses: actions/upload-artifact@v3
         if: ${{ always() }}
         with:
-          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-gha${{ github.run_id }}-g${{ github.sha }}
           path: logs*
 
   aggregate:
@@ -258,7 +258,7 @@ jobs:
       - name: Retrieve logs
         uses: actions/download-artifact@v3
         with:
-          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-${{ github.run_id }}-${{ github.sha }}
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-gha${{ github.run_id }}-g${{ github.sha }}
           path: .
       - name: Print fancy log report
         run: |

From 05730cea1886ba348be0f217715ded4bd9f08638 Mon Sep 17 00:00:00 2001
From: Loic Nageleisen <loic.nageleisen@datadoghq.com>
Date: Wed, 5 Apr 2023 22:48:06 +0200
Subject: [PATCH 18/18] Attempt fix for version_id deletion error

---
 .github/workflows/system-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
index 1e6f802a81c..d9b96520c0e 100644
--- a/.github/workflows/system-tests.yml
+++ b/.github/workflows/system-tests.yml
@@ -297,7 +297,7 @@ jobs:
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
       - uses: actions/delete-package-versions@v4
         with:
-          package-version-ids: gha${{ github.run_id }}-g${{ github.sha }}
-          package-name: system-tests/${{ matrix.image }}
+          package-version-ids: 'gha${{ github.run_id }}-g${{ github.sha }}'
+          package-name: 'system-tests/${{ matrix.image }}'
           package-type: 'container'
         continue-on-error: true