Avoid running tests in parallel which modify shared resources (#825)

Makefile

@@ -207,7 +207,7 @@ GEN_API_DOCS_VERSION ?= v0.4.0
 ENVTEST_VERSION ?= latest
 OPERATOR_SDK_VERSION ?= 1.27.0
 CERTMANAGER_VERSION ?= 1.9.1
-CHAINSAW_VERSION ?= v0.1.6
+CHAINSAW_VERSION ?= v0.1.7
 
 ## Tool Binaries
 KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION)

tests/Dockerfile

@@ -16,7 +16,7 @@ RUN curl -LO https://github.com/kudobuilder/kuttl/releases/download/v0.15.0/kube
     && mv kubectl-kuttl_0.15.0_linux_x86_64 /usr/local/bin/kuttl
 
 # Install chainsaw
-RUN go install github.com/kyverno/[email protected].6
+RUN go install github.com/kyverno/[email protected].7
 
 # Install kubectl and oc
 RUN curl -L -o oc.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest/openshift-client-linux.tar.gz \

tests/e2e-openshift/monitoring/chainsaw-test.yaml

@@ -5,6 +5,8 @@ metadata:
   creationTimestamp: null
   name: monitoring
 spec:
+  # Avoid running this test case in parallel to prevent the deletion of shared resources used by multiple tests, specifically in the context of OpenShift user workload monitoring.
+  concurrent: false
   steps:
   - name: step-00
     try:
@@ -39,4 +41,5 @@ spec:
   - name: step-05
     try:
     - script:
+        timeout: 5m
         content: ./check_metrics.sh

tests/e2e-openshift/red-metrics/chainsaw-test.yaml

@@ -5,6 +5,8 @@ metadata:
   creationTimestamp: null
   name: red-metrics
 spec:
+  # Avoid running this test case in parallel to prevent the deletion of shared resources used by multiple tests, specifically in the context of OpenShift user workload monitoring.
+  concurrent: false
   steps:
   - name: step-00
     try:
@@ -16,7 +18,7 @@ spec:
     try:
     - apply:
         file: 01-install-workload-monitoring.yaml
-    - apply:
+    - assert:
         file: 01-assert.yaml
   - name: step-02
     try:
@@ -31,6 +33,7 @@ spec:
     - assert:
         file: 03-assert.yaml
     - script:
+        timeout: 5m
         content: /bin/sh -c "kubectl get --namespace $NAMESPACE tempo redmetrics -o
           jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}' | grep True"
   - name: step-04
@@ -52,8 +55,10 @@ spec:
     - assert:
         file: 06-assert.yaml
     - script:
+        timeout: 5m
         content: ./check_alert.sh
   - name: step-07
     try:
     - script:
+        timeout: 5m
         content: ./check_metrics.sh

tests/e2e/compatibility/chainsaw-test.yaml

@@ -21,6 +21,7 @@ spec:
   - name: step-02
     try:
     - script:
+        timeout: 5m
         content: kubectl get --namespace $NAMESPACE tempo simplest -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' | grep True
   - name: step-03
     try:

tests/e2e/gateway/chainsaw-test.yaml

@@ -29,5 +29,5 @@ spec:
     - apply:
         file: 03-install-disable-jaeger-query.yaml
     - script:
-        timeout: 2m
+        timeout: 5m
         content: "while true; do container_names=$(kubectl get deployment/tempo-foo-query-frontend -n $NAMESPACE -o=jsonpath='{.spec.template.spec.containers[*].name}'); if [[ -n $(echo \"$container_names\" | grep \"tempo-query\") ]]; then echo \"tempo-query container still exists. Retrying...\"; else echo \"tempo-query container no longer exists. Test passed.\"; exit 0; fi; sleep 5; done"

tests/e2e/generate/chainsaw-test.yaml

@@ -13,6 +13,7 @@ spec:
   - name: step-01
     try:
     - command:
+        timeout: 60s
         args:
         - -c
         - RELATED_IMAGE_TEMPO=docker.io/grafana/tempo:2.2.1 RELATED_IMAGE_TEMPO_QUERY=docker.io/grafana/tempo-query:2.2.1
@@ -22,6 +23,7 @@ spec:
           generated.yaml
         entrypoint: /bin/sh
     - command:
+        timeout: 60s
         args:
         - apply
         - -n

tests/e2e/reconcile/chainsaw-test.yaml

@@ -21,6 +21,7 @@ spec:
   - name: step-02
     try:
     - command:
+        timeout: 60s
         args:
         - delete
         - service
@@ -39,6 +40,7 @@ spec:
   - name: step-04
     try:
     - command:
+        timeout: 60s
         args:
         - -c
         - 'kubectl get --namespace $NAMESPACE configmap tempo-simplest -o jsonpath="{.data.tempo\\.yaml}"
@@ -53,6 +55,7 @@ spec:
   - name: step-06
     try:
     - command:
+        timeout: 60s
         args:
         - -c
         - '! kubectl get --namespace $NAMESPACE configmap tempo-simplest -o jsonpath=''{.data.tempo\.yaml}''