From b93ffd871383972de1ed2904a8b36f61c251b480 Mon Sep 17 00:00:00 2001 From: Isitha Subasinghe Date: Mon, 6 Jan 2025 10:48:14 +1100 Subject: [PATCH 1/9] fix: search and replace null bytes for postgres only.Fixes #13711 (#14030) Signed-off-by: isubasinghe --- persist/sqldb/workflow_archive.go | 13 +++++++++-- test/e2e/argo_server_test.go | 36 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/persist/sqldb/workflow_archive.go b/persist/sqldb/workflow_archive.go index f6d40f4a6e1e..9882487f9392 100644 --- a/persist/sqldb/workflow_archive.go +++ b/persist/sqldb/workflow_archive.go @@ -1,8 +1,10 @@ package sqldb import ( + "bytes" "encoding/json" "fmt" + "strings" "time" log "github.com/sirupsen/logrus" @@ -20,8 +22,9 @@ import ( ) const ( - archiveTableName = "argo_archived_workflows" - archiveLabelsTableName = archiveTableName + "_labels" + archiveTableName = "argo_archived_workflows" + archiveLabelsTableName = archiveTableName + "_labels" + postgresNullReplacement = "ARGO_POSTGRES_NULL_REPLACEMENT" ) type archivedWorkflowMetadata struct { @@ -103,6 +106,9 @@ func (r *workflowArchive) ArchiveWorkflow(wf *wfv1.Workflow) error { if err != nil { return err } + if r.dbType == Postgres { + workflow = bytes.ReplaceAll(workflow, []byte("\\u0000"), []byte(postgresNullReplacement)) + } return r.session.Tx(func(sess db.Session) error { _, err := sess.SQL(). DeleteFrom(archiveTableName). @@ -387,6 +393,9 @@ func (r *workflowArchive) GetWorkflow(uid string, namespace string, name string) return nil, err } var wf *wfv1.Workflow + if r.dbType == Postgres { + archivedWf.Workflow = strings.ReplaceAll(archivedWf.Workflow, postgresNullReplacement, "\\u0000") + } err = json.Unmarshal([]byte(archivedWf.Workflow), &wf) if err != nil { return nil, err diff --git a/test/e2e/argo_server_test.go b/test/e2e/argo_server_test.go index d49c3d5f3713..d451a46628b5 100644 --- a/test/e2e/argo_server_test.go +++ b/test/e2e/argo_server_test.go @@ -2135,6 +2135,42 @@ func (s *ArgoServerSuite) TestRateLimitHeader() { }) } +func (s *ArgoServerSuite) TestPostgresNullBytes() { + // only meaningful for postgres, but shouldn't fail for mysql. + var uid types.UID + _ = uid + + s.Given(). + Workflow(` +metadata: + generateName: archie- + labels: + foo: 1 +spec: + entrypoint: run-archie + templates: + - name: run-archie + container: + image: argoproj/argosay:v2 + args: [echo, "hello \u0000"]`). + When(). + SubmitWorkflow(). + WaitForWorkflow(fixtures.ToBeArchived). + Then(). + ExpectWorkflow(func(t *testing.T, metadata *metav1.ObjectMeta, status *wfv1.WorkflowStatus) { + uid = metadata.UID + }) + + j := s.e().GET("/api/v1/archived-workflows/{uid}", uid). + Expect(). + Status(200). + JSON() + j. + Path("$.spec.templates[0].container.args[1]"). + IsEqual("hello \u0000") + +} + func TestArgoServerSuite(t *testing.T) { suite.Run(t, new(ArgoServerSuite)) } From 683377ce329a430827132691efcec351f748315a Mon Sep 17 00:00:00 2001 From: Mason Malone <651224+MasonM@users.noreply.github.com> Date: Sun, 5 Jan 2025 21:34:03 -0500 Subject: [PATCH 2/9] build: minor upgrades/refactoring to support full CRDs (#14048) Signed-off-by: Mason Malone <651224+MasonM@users.noreply.github.com> --- .devcontainer/pre-build.sh | 4 ++-- Makefile | 8 ++++++-- manifests/base/crds/kustomization.yaml | 5 ----- manifests/base/kustomization.yaml | 7 ------- manifests/cluster-install/kustomization.yaml | 3 ++- .../mysql/argo-mysql-config-secret.yaml | 0 manifests/components/mysql/kustomization.yaml | 10 ++++++++++ .../mysql/mysql-deployment.yaml | 0 .../mysql/mysql-service.yaml | 0 .../mysql/overlays/workflow-controller-configmap.yaml | 0 .../postgres/argo-postgres-config-secret.yaml | 0 manifests/components/postgres/kustomization.yaml | 10 ++++++++++ .../overlays/workflow-controller-configmap.yaml | 0 .../postgres/postgres-deployment.yaml | 0 .../postgres/postgres-service.yaml | 0 .../{quick-start => components}/sso/dex/dev-svc.yaml | 0 .../{quick-start => components}/sso/dex/dex-cm.yaml | 0 .../sso/dex/dex-deploy.yaml | 0 .../{quick-start => components}/sso/dex/dex-rb.yaml | 0 .../{quick-start => components}/sso/dex/dex-role.yaml | 0 .../{quick-start => components}/sso/dex/dex-sa.yaml | 0 .../sso/dex/kustomization.yaml | 5 ----- manifests/components/sso/kustomization.yaml | 9 +++++++++ .../sso/overlays/argo-server-sa.yaml | 0 .../sso/overlays/workflow-controller-configmap.yaml | 0 manifests/namespace-install/kustomization.yaml | 4 +++- manifests/quick-start/minimal/kustomization.yaml | 1 + manifests/quick-start/mysql/kustomization.yaml | 8 +++----- manifests/quick-start/postgres/kustomization.yaml | 8 +++----- manifests/quick-start/sso/kustomization.yaml | 7 +++---- test/e2e/manifests/components/base/kustomization.yaml | 7 ++----- test/e2e/manifests/events/kustomization.yaml | 5 ++--- test/e2e/manifests/minimal/kustomization.yaml | 1 - test/e2e/manifests/mysql/kustomization.yaml | 2 +- test/e2e/manifests/plugins/kustomization.yaml | 4 ---- test/e2e/manifests/postgres/kustomization.yaml | 2 +- test/e2e/manifests/prometheus/kustomization.yaml | 1 - test/e2e/manifests/sso/kustomization.yaml | 4 +--- test/e2e/manifests/stress/kustomization.yaml | 7 +------ 39 files changed, 60 insertions(+), 62 deletions(-) delete mode 100644 manifests/base/crds/kustomization.yaml delete mode 100644 manifests/base/kustomization.yaml rename manifests/{quick-start => components}/mysql/argo-mysql-config-secret.yaml (100%) create mode 100644 manifests/components/mysql/kustomization.yaml rename manifests/{quick-start => components}/mysql/mysql-deployment.yaml (100%) rename manifests/{quick-start => components}/mysql/mysql-service.yaml (100%) rename manifests/{quick-start => components}/mysql/overlays/workflow-controller-configmap.yaml (100%) rename manifests/{quick-start => components}/postgres/argo-postgres-config-secret.yaml (100%) create mode 100644 manifests/components/postgres/kustomization.yaml rename manifests/{quick-start => components}/postgres/overlays/workflow-controller-configmap.yaml (100%) rename manifests/{quick-start => components}/postgres/postgres-deployment.yaml (100%) rename manifests/{quick-start => components}/postgres/postgres-service.yaml (100%) rename manifests/{quick-start => components}/sso/dex/dev-svc.yaml (100%) rename manifests/{quick-start => components}/sso/dex/dex-cm.yaml (100%) rename manifests/{quick-start => components}/sso/dex/dex-deploy.yaml (100%) rename manifests/{quick-start => components}/sso/dex/dex-rb.yaml (100%) rename manifests/{quick-start => components}/sso/dex/dex-role.yaml (100%) rename manifests/{quick-start => components}/sso/dex/dex-sa.yaml (100%) rename manifests/{quick-start => components}/sso/dex/kustomization.yaml (68%) create mode 100644 manifests/components/sso/kustomization.yaml rename manifests/{quick-start => components}/sso/overlays/argo-server-sa.yaml (100%) rename manifests/{quick-start => components}/sso/overlays/workflow-controller-configmap.yaml (100%) diff --git a/.devcontainer/pre-build.sh b/.devcontainer/pre-build.sh index 16727647fa36..252dfa0fe909 100755 --- a/.devcontainer/pre-build.sh +++ b/.devcontainer/pre-build.sh @@ -3,11 +3,11 @@ set -eux # install kubernetes wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash -k3d cluster get k3s-default || k3d cluster create --image rancher/k3s:v1.27.3-k3s1 --wait +k3d cluster get k3s-default || k3d cluster create --image rancher/k3s:v1.29.10-k3s1 --wait k3d kubeconfig merge --kubeconfig-merge-default # install kubectl -curl -LO https://dl.k8s.io/release/v1.27.3/bin/linux/$(go env GOARCH)/kubectl +curl -LO https://dl.k8s.io/release/v1.29.10/bin/linux/$(go env GOARCH)/kubectl chmod +x ./kubectl sudo mv ./kubectl /usr/local/bin/kubectl kubectl cluster-info diff --git a/Makefile b/Makefile index ee5b266209f4..4934001a320c 100644 --- a/Makefile +++ b/Makefile @@ -480,7 +480,10 @@ install: githooks kubectl get ns $(KUBE_NAMESPACE) || kubectl create ns $(KUBE_NAMESPACE) kubectl config set-context --current --namespace=$(KUBE_NAMESPACE) @echo "installing PROFILE=$(PROFILE)" - kubectl kustomize --load-restrictor=LoadRestrictionsNone test/e2e/manifests/$(PROFILE) | sed 's|quay.io/argoproj/|$(IMAGE_NAMESPACE)/|' | sed 's/namespace: argo/namespace: $(KUBE_NAMESPACE)/' | kubectl -n $(KUBE_NAMESPACE) apply --prune -l app.kubernetes.io/part-of=argo -f - + kubectl kustomize --load-restrictor=LoadRestrictionsNone test/e2e/manifests/$(PROFILE) \ + | sed 's|quay.io/argoproj/|$(IMAGE_NAMESPACE)/|' \ + | sed 's/namespace: argo/namespace: $(KUBE_NAMESPACE)/' \ + | KUBECTL_APPLYSET=true kubectl -n $(KUBE_NAMESPACE) apply --applyset=configmaps/install --server-side --prune -f - ifeq ($(PROFILE),stress) kubectl -n $(KUBE_NAMESPACE) apply -f test/stress/massive-workflow.yaml endif @@ -530,7 +533,8 @@ ifeq ($(shell uname),Darwin) brew tap kitproj/kit --custom-remote https://github.com/kitproj/kit brew install kit else - curl -q https://raw.githubusercontent.com/kitproj/kit/main/install.sh | tag=v0.1.8 sh + @echo "Downloading Kit" + curl -fsL --retry 99 "https://github.com/kitproj/kit/releases/download/v0.1.8/kit_0.1.8_$$(uname)_$$(uname -m | sed 's/aarch64/arm64/').tar.gz" | sudo tar -C /usr/local/bin -xzf - kit endif endif diff --git a/manifests/base/crds/kustomization.yaml b/manifests/base/crds/kustomization.yaml deleted file mode 100644 index 3ccdade18fcc..000000000000 --- a/manifests/base/crds/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - minimal diff --git a/manifests/base/kustomization.yaml b/manifests/base/kustomization.yaml deleted file mode 100644 index 0b9a73341d74..000000000000 --- a/manifests/base/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- crds -- workflow-controller -- argo-server diff --git a/manifests/cluster-install/kustomization.yaml b/manifests/cluster-install/kustomization.yaml index 986293cd64b0..9dfb96b1be31 100644 --- a/manifests/cluster-install/kustomization.yaml +++ b/manifests/cluster-install/kustomization.yaml @@ -2,7 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../base +- ../base/workflow-controller +- ../base/argo-server - ./workflow-controller-rbac - ./argo-server-rbac diff --git a/manifests/quick-start/mysql/argo-mysql-config-secret.yaml b/manifests/components/mysql/argo-mysql-config-secret.yaml similarity index 100% rename from manifests/quick-start/mysql/argo-mysql-config-secret.yaml rename to manifests/components/mysql/argo-mysql-config-secret.yaml diff --git a/manifests/components/mysql/kustomization.yaml b/manifests/components/mysql/kustomization.yaml new file mode 100644 index 000000000000..6a5807166ec7 --- /dev/null +++ b/manifests/components/mysql/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - argo-mysql-config-secret.yaml + - mysql-deployment.yaml + - mysql-service.yaml + +patches: + - path: overlays/workflow-controller-configmap.yaml diff --git a/manifests/quick-start/mysql/mysql-deployment.yaml b/manifests/components/mysql/mysql-deployment.yaml similarity index 100% rename from manifests/quick-start/mysql/mysql-deployment.yaml rename to manifests/components/mysql/mysql-deployment.yaml diff --git a/manifests/quick-start/mysql/mysql-service.yaml b/manifests/components/mysql/mysql-service.yaml similarity index 100% rename from manifests/quick-start/mysql/mysql-service.yaml rename to manifests/components/mysql/mysql-service.yaml diff --git a/manifests/quick-start/mysql/overlays/workflow-controller-configmap.yaml b/manifests/components/mysql/overlays/workflow-controller-configmap.yaml similarity index 100% rename from manifests/quick-start/mysql/overlays/workflow-controller-configmap.yaml rename to manifests/components/mysql/overlays/workflow-controller-configmap.yaml diff --git a/manifests/quick-start/postgres/argo-postgres-config-secret.yaml b/manifests/components/postgres/argo-postgres-config-secret.yaml similarity index 100% rename from manifests/quick-start/postgres/argo-postgres-config-secret.yaml rename to manifests/components/postgres/argo-postgres-config-secret.yaml diff --git a/manifests/components/postgres/kustomization.yaml b/manifests/components/postgres/kustomization.yaml new file mode 100644 index 000000000000..b1a2eb799490 --- /dev/null +++ b/manifests/components/postgres/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - argo-postgres-config-secret.yaml + - postgres-deployment.yaml + - postgres-service.yaml + +patches: + - path: overlays/workflow-controller-configmap.yaml diff --git a/manifests/quick-start/postgres/overlays/workflow-controller-configmap.yaml b/manifests/components/postgres/overlays/workflow-controller-configmap.yaml similarity index 100% rename from manifests/quick-start/postgres/overlays/workflow-controller-configmap.yaml rename to manifests/components/postgres/overlays/workflow-controller-configmap.yaml diff --git a/manifests/quick-start/postgres/postgres-deployment.yaml b/manifests/components/postgres/postgres-deployment.yaml similarity index 100% rename from manifests/quick-start/postgres/postgres-deployment.yaml rename to manifests/components/postgres/postgres-deployment.yaml diff --git a/manifests/quick-start/postgres/postgres-service.yaml b/manifests/components/postgres/postgres-service.yaml similarity index 100% rename from manifests/quick-start/postgres/postgres-service.yaml rename to manifests/components/postgres/postgres-service.yaml diff --git a/manifests/quick-start/sso/dex/dev-svc.yaml b/manifests/components/sso/dex/dev-svc.yaml similarity index 100% rename from manifests/quick-start/sso/dex/dev-svc.yaml rename to manifests/components/sso/dex/dev-svc.yaml diff --git a/manifests/quick-start/sso/dex/dex-cm.yaml b/manifests/components/sso/dex/dex-cm.yaml similarity index 100% rename from manifests/quick-start/sso/dex/dex-cm.yaml rename to manifests/components/sso/dex/dex-cm.yaml diff --git a/manifests/quick-start/sso/dex/dex-deploy.yaml b/manifests/components/sso/dex/dex-deploy.yaml similarity index 100% rename from manifests/quick-start/sso/dex/dex-deploy.yaml rename to manifests/components/sso/dex/dex-deploy.yaml diff --git a/manifests/quick-start/sso/dex/dex-rb.yaml b/manifests/components/sso/dex/dex-rb.yaml similarity index 100% rename from manifests/quick-start/sso/dex/dex-rb.yaml rename to manifests/components/sso/dex/dex-rb.yaml diff --git a/manifests/quick-start/sso/dex/dex-role.yaml b/manifests/components/sso/dex/dex-role.yaml similarity index 100% rename from manifests/quick-start/sso/dex/dex-role.yaml rename to manifests/components/sso/dex/dex-role.yaml diff --git a/manifests/quick-start/sso/dex/dex-sa.yaml b/manifests/components/sso/dex/dex-sa.yaml similarity index 100% rename from manifests/quick-start/sso/dex/dex-sa.yaml rename to manifests/components/sso/dex/dex-sa.yaml diff --git a/manifests/quick-start/sso/dex/kustomization.yaml b/manifests/components/sso/dex/kustomization.yaml similarity index 68% rename from manifests/quick-start/sso/dex/kustomization.yaml rename to manifests/components/sso/dex/kustomization.yaml index 636bb3370bec..1a9c0a56518d 100644 --- a/manifests/quick-start/sso/dex/kustomization.yaml +++ b/manifests/components/sso/dex/kustomization.yaml @@ -1,11 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -labels: - - includeSelectors: true - pairs: - app.kubernetes.io/part-of: dex - resources: - dex-cm.yaml - dex-role.yaml diff --git a/manifests/components/sso/kustomization.yaml b/manifests/components/sso/kustomization.yaml new file mode 100644 index 000000000000..5d85d7c8f874 --- /dev/null +++ b/manifests/components/sso/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - dex + +patches: + - path: overlays/workflow-controller-configmap.yaml + - path: overlays/argo-server-sa.yaml diff --git a/manifests/quick-start/sso/overlays/argo-server-sa.yaml b/manifests/components/sso/overlays/argo-server-sa.yaml similarity index 100% rename from manifests/quick-start/sso/overlays/argo-server-sa.yaml rename to manifests/components/sso/overlays/argo-server-sa.yaml diff --git a/manifests/quick-start/sso/overlays/workflow-controller-configmap.yaml b/manifests/components/sso/overlays/workflow-controller-configmap.yaml similarity index 100% rename from manifests/quick-start/sso/overlays/workflow-controller-configmap.yaml rename to manifests/components/sso/overlays/workflow-controller-configmap.yaml diff --git a/manifests/namespace-install/kustomization.yaml b/manifests/namespace-install/kustomization.yaml index aa9de5444113..23010a2aca85 100644 --- a/manifests/namespace-install/kustomization.yaml +++ b/manifests/namespace-install/kustomization.yaml @@ -2,7 +2,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../base + - ../base/crds/minimal + - ../base/workflow-controller + - ../base/argo-server - ./argo-server-rbac - ./workflow-controller-rbac diff --git a/manifests/quick-start/minimal/kustomization.yaml b/manifests/quick-start/minimal/kustomization.yaml index 56c1551bd250..34b94565b39e 100644 --- a/manifests/quick-start/minimal/kustomization.yaml +++ b/manifests/quick-start/minimal/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization resources: - ../base + - ../../base/crds/minimal patches: - path: overlays/workflow-controller-configmap.yaml diff --git a/manifests/quick-start/mysql/kustomization.yaml b/manifests/quick-start/mysql/kustomization.yaml index ffc58889aafd..a3113cf08792 100644 --- a/manifests/quick-start/mysql/kustomization.yaml +++ b/manifests/quick-start/mysql/kustomization.yaml @@ -3,9 +3,7 @@ kind: Kustomization resources: - ../base - - argo-mysql-config-secret.yaml - - mysql-deployment.yaml - - mysql-service.yaml + - ../../base/crds/minimal -patches: - - path: overlays/workflow-controller-configmap.yaml +components: + - ../../components/mysql diff --git a/manifests/quick-start/postgres/kustomization.yaml b/manifests/quick-start/postgres/kustomization.yaml index 2640ed4c0c84..27ded47e1689 100644 --- a/manifests/quick-start/postgres/kustomization.yaml +++ b/manifests/quick-start/postgres/kustomization.yaml @@ -3,9 +3,7 @@ kind: Kustomization resources: - ../base - - argo-postgres-config-secret.yaml - - postgres-deployment.yaml - - postgres-service.yaml + - ../../base/crds/minimal -patches: - - path: overlays/workflow-controller-configmap.yaml +components: + - ../../components/postgres diff --git a/manifests/quick-start/sso/kustomization.yaml b/manifests/quick-start/sso/kustomization.yaml index d70185d43bd4..6c61b26ee012 100644 --- a/manifests/quick-start/sso/kustomization.yaml +++ b/manifests/quick-start/sso/kustomization.yaml @@ -3,8 +3,7 @@ kind: Kustomization resources: - ../base - - dex + - ../../base/crds/minimal -patches: - - path: overlays/workflow-controller-configmap.yaml - - path: overlays/argo-server-sa.yaml +components: + - ../../components/sso diff --git a/test/e2e/manifests/components/base/kustomization.yaml b/test/e2e/manifests/components/base/kustomization.yaml index c0703f2ab210..77d1388745b3 100644 --- a/test/e2e/manifests/components/base/kustomization.yaml +++ b/test/e2e/manifests/components/base/kustomization.yaml @@ -2,6 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component resources: +- ../../../../../manifests/base/crds/minimal +- ../../../../../manifests/quick-start/base - argo-server.service-account-token-secret.yaml patches: @@ -10,8 +12,3 @@ patches: - path: workflow-controller-deployment.yaml - path: workflow-controller-cluster-workflow-template-rbac.yaml - path: minio-deployment.yaml - -labels: -- includeSelectors: true - pairs: - app.kubernetes.io/part-of: argo diff --git a/test/e2e/manifests/events/kustomization.yaml b/test/e2e/manifests/events/kustomization.yaml index 7582055efe74..93ff2ed29fba 100644 --- a/test/e2e/manifests/events/kustomization.yaml +++ b/test/e2e/manifests/events/kustomization.yaml @@ -2,9 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../../manifests/quick-start/minimal -- https://raw.githubusercontent.com/argoproj/argo-events/v1.2.0/manifests/install.yaml -- https://raw.githubusercontent.com/argoproj/argo-events/v1.2.0/examples/eventbus/native.yaml +- https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/install.yaml +- https://raw.githubusercontent.com/argoproj/argo-events/stable/examples/eventbus/native.yaml components: - ../components/base diff --git a/test/e2e/manifests/minimal/kustomization.yaml b/test/e2e/manifests/minimal/kustomization.yaml index 64a3e5ea9963..49835596f127 100644 --- a/test/e2e/manifests/minimal/kustomization.yaml +++ b/test/e2e/manifests/minimal/kustomization.yaml @@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../../manifests/quick-start/minimal - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_eventbus.yaml - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_eventsources.yaml - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_sensors.yaml diff --git a/test/e2e/manifests/mysql/kustomization.yaml b/test/e2e/manifests/mysql/kustomization.yaml index 3506c9f6aac5..23a05fae5ea8 100644 --- a/test/e2e/manifests/mysql/kustomization.yaml +++ b/test/e2e/manifests/mysql/kustomization.yaml @@ -2,13 +2,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../../manifests/quick-start/mysql - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_eventbus.yaml - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_eventsources.yaml - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_sensors.yaml components: - ../components/base +- ../../../../manifests/components/mysql - ../components/local-argo namespace: argo diff --git a/test/e2e/manifests/plugins/kustomization.yaml b/test/e2e/manifests/plugins/kustomization.yaml index 18250ac970b2..e218c7bb095b 100644 --- a/test/e2e/manifests/plugins/kustomization.yaml +++ b/test/e2e/manifests/plugins/kustomization.yaml @@ -7,8 +7,4 @@ resources: - hello-executor-plugin.service-account-token-secret.yaml - hello-executor-plugin-configmap.yaml -labels: - - includeSelectors: true - pairs: - app.kubernetes.io/part-of: argo namespace: argo diff --git a/test/e2e/manifests/postgres/kustomization.yaml b/test/e2e/manifests/postgres/kustomization.yaml index b496a9a68518..ee8408cef576 100644 --- a/test/e2e/manifests/postgres/kustomization.yaml +++ b/test/e2e/manifests/postgres/kustomization.yaml @@ -2,13 +2,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../../../../manifests/quick-start/postgres - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_eventbus.yaml - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_eventsources.yaml - https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/base/crds/argoproj.io_sensors.yaml components: - ../components/base +- ../../../../manifests/components/postgres - ../components/local-argo namespace: argo \ No newline at end of file diff --git a/test/e2e/manifests/prometheus/kustomization.yaml b/test/e2e/manifests/prometheus/kustomization.yaml index 8b5970f01db5..b3a7b69bac0e 100644 --- a/test/e2e/manifests/prometheus/kustomization.yaml +++ b/test/e2e/manifests/prometheus/kustomization.yaml @@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../../../../manifests/quick-start/minimal - ../../../../manifests/quick-start/base/prometheus components: diff --git a/test/e2e/manifests/sso/kustomization.yaml b/test/e2e/manifests/sso/kustomization.yaml index 8aba99ee23d3..86834313d241 100644 --- a/test/e2e/manifests/sso/kustomization.yaml +++ b/test/e2e/manifests/sso/kustomization.yaml @@ -1,11 +1,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: -- ../../../../manifests/quick-start/sso - components: - ../components/base +- ../../../../manifests/components/sso - ../components/local-argo namespace: argo \ No newline at end of file diff --git a/test/e2e/manifests/stress/kustomization.yaml b/test/e2e/manifests/stress/kustomization.yaml index a391023381ed..af7f54b6fece 100644 --- a/test/e2e/manifests/stress/kustomization.yaml +++ b/test/e2e/manifests/stress/kustomization.yaml @@ -12,9 +12,4 @@ patches: - path: workflow-controller-configmap.yaml - path: workflow-controller-deployment.yaml - path: argo-server-deployment.yaml - - path: minio-deployment.yaml - -labels: - - includeSelectors: true - pairs: - app.kubernetes.io/part-of: argo + - path: minio-deployment.yaml \ No newline at end of file From a91bd843b54c7511121da3561776e41c46f28809 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:47:39 +0000 Subject: [PATCH 3/9] chore(deps): bump github.com/go-git/go-git/v5 from 5.11.0 to 5.13.1 in the go_modules group (#14055) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 20 ++++++++++---------- go.sum | 57 ++++++++++++++++++++++++++------------------------------- 2 files changed, 36 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index 573f763584d9..c0009a9eaa39 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/evanphx/json-patch v5.8.0+incompatible github.com/expr-lang/expr v1.16.9 github.com/gavv/httpexpect/v2 v2.16.0 - github.com/go-git/go-git/v5 v5.11.0 + github.com/go-git/go-git/v5 v5.13.1 github.com/go-jose/go-jose/v3 v3.0.3 github.com/go-openapi/jsonreference v0.20.4 github.com/go-sql-driver/mysql v1.7.1 @@ -49,7 +49,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/tidwall/gjson v1.17.0 github.com/upper/db/v4 v4.7.0 github.com/valyala/fasttemplate v1.2.2 @@ -62,7 +62,7 @@ require ( go.opentelemetry.io/otel/sdk v1.32.0 go.opentelemetry.io/otel/sdk/metric v1.32.0 golang.org/x/crypto v0.31.0 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.10.0 golang.org/x/time v0.5.0 @@ -93,7 +93,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/cyphar/filepath-securejoin v0.3.6 // indirect github.com/distribution/reference v0.5.0 // indirect github.com/evilmonkeyinc/jsonpath v0.8.1 // indirect github.com/fatih/color v1.15.0 // indirect @@ -132,7 +132,7 @@ require ( github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/fasthash v1.0.3 // indirect - github.com/skeema/knownhosts v1.2.1 // indirect + github.com/skeema/knownhosts v1.3.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/vbatts/tar-split v0.11.3 // indirect github.com/x448/float16 v0.8.4 // indirect @@ -141,8 +141,8 @@ require ( go.opentelemetry.io/otel/trace v1.32.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/tools v0.23.0 // indirect google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect @@ -174,7 +174,7 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect + github.com/ProtonMail/go-crypto v1.1.3 // indirect github.com/ajg/form v1.5.1 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/awalterschulze/gographviz v0.0.0-20200901124122-0eecad45bd71 // indirect @@ -215,7 +215,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-git/go-billy/v5 v5.6.1 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/swag v0.22.6 // indirect @@ -278,7 +278,7 @@ require ( github.com/rs/xid v1.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sanity-io/litter v1.5.5 // indirect - github.com/sergi/go-diff v1.2.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect diff --git a/go.sum b/go.sum index 0151f8eb9814..82a733427887 100644 --- a/go.sum +++ b/go.sum @@ -77,8 +77,8 @@ github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v1.1.3 h1:nRBOetoydLeUb4nHajyO2bKqMLfWQ/ZPwkXqXxPxCFk= +github.com/ProtonMail/go-crypto v1.1.3/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/TwiN/go-color v1.4.1 h1:mqG0P/KBgHKVqmtL5ye7K0/Gr4l6hTksPgTgMk3mUzc= @@ -162,7 +162,6 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y github.com/blushft/go-diagrams v0.0.0-20201006005127-c78c821223d9 h1:mV+hh0rMjzrhg7Jc/GKwpa+y/0BMHGOHdM9yY1GYyFI= github.com/blushft/go-diagrams v0.0.0-20201006005127-c78c821223d9/go.mod h1:nDeXEIaeDV+mAK1gBD3/RJH67DYPC0GdaznWN7sB07s= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= -github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -176,7 +175,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -198,8 +196,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/cyphar/filepath-securejoin v0.3.6 h1:4d9N5ykBnSp5Xn2JkhocYDkOpURL/18CYMpo6xB9uWM= +github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -232,8 +230,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= -github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy v1.2.3 h1:xwIyKHbaP5yfT6O9KIeYJR5549MXRQkoQMRXGztz8YQ= +github.com/elazarl/goproxy v1.2.3/go.mod h1:YfEbZtqP4AetfO6d40vWchF3znWX7C7Vd6ZMfdL8z64= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -275,18 +273,18 @@ github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNll github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gizak/termui/v3 v3.1.0/go.mod h1:bXQEBkJpzxUAKf0+xq9MSWAvWZlE7c+aidmyFlkYTrY= -github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= -github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= -github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-billy/v5 v5.6.1 h1:u+dcrgaguSSkbjzHwelEjc0Yj300NUevrrPphk/SoRA= +github.com/go-git/go-billy/v5 v5.6.1/go.mod h1:0AsLr1z2+Uksi4NlElmMblP5rPcDZNRCD8ujZCRR2BE= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= -github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= +github.com/go-git/go-git/v5 v5.13.1 h1:DAQ9APonnlvSWpvolXWIuV6Q6zXy2wHbN4cVlNR5Q+M= +github.com/go-git/go-git/v5 v5.13.1/go.mod h1:qryJB4cSBoq3FRoBRf5A77joojuBcmPJ0qu3XXXVixc= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= @@ -681,8 +679,8 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= @@ -746,8 +744,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sethvargo/go-limiter v0.7.2 h1:FgC4N7RMpV5gMrUdda15FaFTkQ/L4fEqM7seXMs4oO8= github.com/sethvargo/go-limiter v0.7.2/go.mod h1:C0kbSFbiriE5k2FFOe18M1YZbAR2Fiwf72uGu0CXCcU= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -760,8 +758,8 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= -github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY= +github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -803,8 +801,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= @@ -914,9 +913,7 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= @@ -931,8 +928,8 @@ golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20200908183739-ae8ad444f925/go.mod h1:1phAWC201xIgDyaFpmDeZkgf70Q4Pd/CNqfRtVPtxNw= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -952,8 +949,9 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -982,7 +980,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= @@ -1056,7 +1053,6 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1077,7 +1073,6 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= @@ -1099,7 +1094,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -1139,8 +1133,9 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 2719e064de5e5b7635c59d155b1b4c1f81c05066 Mon Sep 17 00:00:00 2001 From: shuangkun tian <72060326+shuangkun@users.noreply.github.com> Date: Tue, 7 Jan 2025 05:19:30 +0800 Subject: [PATCH 4/9] fix: add workflow template level pod annotations and labels to template. Fixes: #12945 (#12987) Signed-off-by: shuangkun --- .../cluster_workflow_template_types.go | 5 ++ pkg/apis/workflow/v1alpha1/common.go | 1 + .../v1alpha1/workflow_template_types.go | 5 ++ pkg/apis/workflow/v1alpha1/workflow_types.go | 5 ++ .../operator_workflow_template_ref_test.go | 68 +++++++++++++++++++ workflow/templateresolution/context.go | 25 +++++++ 6 files changed, 109 insertions(+) diff --git a/pkg/apis/workflow/v1alpha1/cluster_workflow_template_types.go b/pkg/apis/workflow/v1alpha1/cluster_workflow_template_types.go index a9c27f620e8d..2bb5dc112f9c 100644 --- a/pkg/apis/workflow/v1alpha1/cluster_workflow_template_types.go +++ b/pkg/apis/workflow/v1alpha1/cluster_workflow_template_types.go @@ -57,6 +57,11 @@ func (cwftmpl *ClusterWorkflowTemplate) GetResourceScope() ResourceScope { return ResourceScopeCluster } +// GetPodMetadata returns the PodMetadata of cluster workflow template. +func (cwftmpl *ClusterWorkflowTemplate) GetPodMetadata() *Metadata { + return cwftmpl.Spec.PodMetadata +} + // GetWorkflowSpec returns the WorkflowSpec of cluster workflow template. func (cwftmpl *ClusterWorkflowTemplate) GetWorkflowSpec() *WorkflowSpec { return &cwftmpl.Spec diff --git a/pkg/apis/workflow/v1alpha1/common.go b/pkg/apis/workflow/v1alpha1/common.go index 6a7c584b4601..daba916e849c 100644 --- a/pkg/apis/workflow/v1alpha1/common.go +++ b/pkg/apis/workflow/v1alpha1/common.go @@ -20,6 +20,7 @@ type TemplateHolder interface { GroupVersionKind() schema.GroupVersionKind GetTemplateByName(name string) *Template GetResourceScope() ResourceScope + GetPodMetadata() *Metadata } // WorkflowSpecHolder is an object that holds a WorkflowSpec; e.g., WorkflowTemplate, and ClusterWorkflowTemplate diff --git a/pkg/apis/workflow/v1alpha1/workflow_template_types.go b/pkg/apis/workflow/v1alpha1/workflow_template_types.go index 1317fc18b2a5..707128f56097 100644 --- a/pkg/apis/workflow/v1alpha1/workflow_template_types.go +++ b/pkg/apis/workflow/v1alpha1/workflow_template_types.go @@ -56,6 +56,11 @@ func (wftmpl *WorkflowTemplate) GetResourceScope() ResourceScope { return ResourceScopeNamespaced } +// GetPodMetadata returns the PodMetadata of workflow template. +func (wftmpl *WorkflowTemplate) GetPodMetadata() *Metadata { + return wftmpl.Spec.PodMetadata +} + // GetWorkflowSpec returns the WorkflowSpec of workflow template. func (wftmpl *WorkflowTemplate) GetWorkflowSpec() *WorkflowSpec { return &wftmpl.Spec diff --git a/pkg/apis/workflow/v1alpha1/workflow_types.go b/pkg/apis/workflow/v1alpha1/workflow_types.go index 3b340fa692cc..9c28a0113d95 100644 --- a/pkg/apis/workflow/v1alpha1/workflow_types.go +++ b/pkg/apis/workflow/v1alpha1/workflow_types.go @@ -3410,6 +3410,11 @@ func (wf *Workflow) GetResourceScope() ResourceScope { return ResourceScopeLocal } +// GetPodMetadata returns the PodMetadata of a workflow. +func (wf *Workflow) GetPodMetadata() *Metadata { + return wf.Spec.PodMetadata +} + // GetWorkflowSpec returns the Spec of a workflow. func (wf *Workflow) GetWorkflowSpec() WorkflowSpec { return wf.Spec diff --git a/workflow/controller/operator_workflow_template_ref_test.go b/workflow/controller/operator_workflow_template_ref_test.go index 8fbf54d85f7d..353ad9cf8aed 100644 --- a/workflow/controller/operator_workflow_template_ref_test.go +++ b/workflow/controller/operator_workflow_template_ref_test.go @@ -633,3 +633,71 @@ func TestWorkflowTemplateWithDynamicRef(t *testing.T) { woc.operate(ctx) assert.Equal(t, wfv1.WorkflowSucceeded, woc.wf.Status.Phase) } + +const wfTemplateWithPodMetadata = ` +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: workflow-template +spec: + entrypoint: whalesay-template + podMetadata: + labels: + workflow-template-label: hello + annotations: + all-pods-should-have-this: value + arguments: + parameters: + - name: message + value: hello world + + templates: + - name: whalesay-template + inputs: + parameters: + - name: message + container: + image: docker/whalesay + command: [cowsay] + args: ["{{inputs.parameters.message}}"]` + +const wfWithTemplateRef = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + name: test-workflow + namespace: argo-workflows-system +spec: + podMetadata: + labels: + caller-label: hello + entrypoint: start + templates: + - name: start + steps: + - - name: hello + templateRef: + name: workflow-template + template: whalesay-template + clusterScope: true + arguments: + parameters: + - name: message + value: Hello Bug` + +func TestWorkflowTemplateWithPodMetadata(t *testing.T) { + cancel, controller := newController(wfv1.MustUnmarshalWorkflow(wfWithTemplateRef), wfv1.MustUnmarshalClusterWorkflowTemplate(wfTemplateWithPodMetadata)) + defer cancel() + + ctx := context.Background() + woc := newWorkflowOperationCtx(wfv1.MustUnmarshalWorkflow(wfWithTemplateRef), controller) + woc.operate(ctx) + assert.Equal(t, wfv1.WorkflowRunning, woc.wf.Status.Phase) + pods, err := listPods(woc) + require.NoError(t, err) + assert.NotEmpty(t, len(pods.Items) > 0, "pod was not created successfully") + pod := pods.Items[0] + assert.Contains(t, pod.Labels, "caller-label") + assert.Contains(t, pod.Labels, "workflow-template-label") + assert.Contains(t, pod.Annotations, "all-pods-should-have-this") +} diff --git a/workflow/templateresolution/context.go b/workflow/templateresolution/context.go index 131864685887..8921da725816 100644 --- a/workflow/templateresolution/context.go +++ b/workflow/templateresolution/context.go @@ -104,6 +104,10 @@ func (ctx *Context) GetTemplateByName(name string) (*wfv1.Template, error) { ctx.log.Debugf("Getting the template by name: %s", name) tmpl := ctx.tmplBase.GetTemplateByName(name) + + podMetadata := ctx.tmplBase.GetPodMetadata() + ctx.addPodMetadata(podMetadata, tmpl) + if tmpl == nil { return nil, errors.Errorf(errors.CodeNotFound, "template %s not found", name) } @@ -138,6 +142,9 @@ func (ctx *Context) GetTemplateFromRef(tmplRef *wfv1.TemplateRef) (*wfv1.Templat template = wftmpl.GetTemplateByName(tmplRef.Template) + podMetadata := wftmpl.GetPodMetadata() + ctx.addPodMetadata(podMetadata, template) + if template == nil { return nil, errors.Errorf(errors.CodeNotFound, "template %s not found in workflow template %s", tmplRef.Template, tmplRef.Name) } @@ -268,3 +275,21 @@ func (ctx *Context) WithClusterWorkflowTemplate(name string) (*Context, error) { } return ctx.WithTemplateBase(cwftmpl), nil } + +// addPodMetadata add podMetadata in workflow template level to template +func (ctx *Context) addPodMetadata(podMetadata *wfv1.Metadata, tmpl *wfv1.Template) { + if podMetadata != nil { + if tmpl.Metadata.Annotations == nil { + tmpl.Metadata.Annotations = make(map[string]string) + } + for k, v := range podMetadata.Annotations { + tmpl.Metadata.Annotations[k] = v + } + if tmpl.Metadata.Labels == nil { + tmpl.Metadata.Labels = make(map[string]string) + } + for k, v := range podMetadata.Labels { + tmpl.Metadata.Labels[k] = v + } + } +} From 82537a7737028a0a2ee579f7d8d1e54dabcf6481 Mon Sep 17 00:00:00 2001 From: chengjoey <30427474+chengjoey@users.noreply.github.com> Date: Tue, 7 Jan 2025 06:13:50 +0800 Subject: [PATCH 5/9] fix(controller): validation failed when dynamic templateRef is used in nested template (#14053) Signed-off-by: joey --- workflow/validate/validate.go | 15 ++++++- workflow/validate/validate_test.go | 69 ++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/workflow/validate/validate.go b/workflow/validate/validate.go index 9b5e51fd4549..386e23c5f903 100644 --- a/workflow/validate/validate.go +++ b/workflow/validate/validate.go @@ -962,7 +962,12 @@ func (ctx *templateValidationCtx) validateSteps(scope map[string]interface{}, tm if err != nil { return err } - resolvedTmpl, err := ctx.validateTemplateHolder(&step, tmplCtx, &FakeArguments{}, workflowTemplateValidation) + var args wfv1.ArgumentsProvider + args = &FakeArguments{} + if step.TemplateRef != nil { + args = &step.Arguments + } + resolvedTmpl, err := ctx.validateTemplateHolder(&step, tmplCtx, args, workflowTemplateValidation) if err != nil { return errors.Errorf(errors.CodeBadRequest, "templates.%s.steps[%d].%s %s", tmpl.Name, i, step.Name, err.Error()) } @@ -1355,7 +1360,13 @@ func (ctx *templateValidationCtx) validateDAG(scope map[string]interface{}, tmpl return errors.Errorf(errors.CodeBadRequest, "templates.%s cannot use 'continueOn' when using 'depends'. Instead use 'dep-task.Failed'/'dep-task.Errored'", tmpl.Name) } - resolvedTmpl, err := ctx.validateTemplateHolder(&task, tmplCtx, &FakeArguments{}, workflowTemplateValidation) + var args wfv1.ArgumentsProvider + args = &FakeArguments{} + if task.TemplateRef != nil { + args = &task.Arguments + } + + resolvedTmpl, err := ctx.validateTemplateHolder(&task, tmplCtx, args, workflowTemplateValidation) if err != nil { return errors.Errorf(errors.CodeBadRequest, "templates.%s.tasks.%s %s", tmpl.Name, task.Name, err.Error()) diff --git a/workflow/validate/validate_test.go b/workflow/validate/validate_test.go index ccbded208598..abe8695eef99 100644 --- a/workflow/validate/validate_test.go +++ b/workflow/validate/validate_test.go @@ -3304,3 +3304,72 @@ func TestShouldCheckValidationToSpacedParameters(t *testing.T) { // Do not allow leading or trailing spaces in parameters require.ErrorContains(t, err, "failed to resolve {{ workflow.thisdoesnotexist }}") } + +var dynamicWorkflowTemplateARefB = ` +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: workflow-template-a +spec: + templates: + - name: template-a + inputs: + parameters: + - name: message + steps: + - - name: step-a + templateRef: + name: workflow-template-b + template: "{{ inputs.parameters.message }}" +` + +var dynamicWorkflowTemplateRefB = ` +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: workflow-template-b +spec: + templates: + - name: template-b + container: + image: docker/whalesay + command: [cowsay] + args: ["hello from template"] +` + +var dynamicTemplateRefWorkflow = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: dynamic-workflow- +spec: + entrypoint: whalesay + templates: + - name: whalesay + steps: + - - name: whalesay + templateRef: + name: workflow-template-a + template: template-a + arguments: + parameters: + - name: message + value: "template-b" +` + +func TestDynamicWorkflowTemplateRef(t *testing.T) { + wf := wfv1.MustUnmarshalWorkflow(dynamicTemplateRefWorkflow) + wftmplA := wfv1.MustUnmarshalWorkflowTemplate(dynamicWorkflowTemplateARefB) + wftmplB := wfv1.MustUnmarshalWorkflowTemplate(dynamicWorkflowTemplateRefB) + + err := createWorkflowTemplate(wftmplA) + require.NoError(t, err) + err = createWorkflowTemplate(wftmplB) + require.NoError(t, err) + + err = ValidateWorkflow(wftmplGetter, cwftmplGetter, wf, nil, ValidateOpts{}) + require.NoError(t, err) + + _ = deleteWorkflowTemplate(wftmplA.Name) + _ = deleteWorkflowTemplate(wftmplB.Name) +} From 2ece85c23646574434518a3ffb2236f2162f58b1 Mon Sep 17 00:00:00 2001 From: Tianchu Zhao Date: Tue, 7 Jan 2025 15:50:31 +1100 Subject: [PATCH 6/9] fix: validate template of the same name. Fixes #13763 (#14043) Signed-off-by: Tianchu Zhao --- workflow/validate/validate.go | 5 +-- workflow/validate/validate_test.go | 50 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/workflow/validate/validate.go b/workflow/validate/validate.go index 386e23c5f903..8dd841798628 100644 --- a/workflow/validate/validate.go +++ b/workflow/validate/validate.go @@ -479,13 +479,14 @@ func (ctx *templateValidationCtx) validateTemplate(tmpl *wfv1.Template, tmplCtx } + templateScope := tmplCtx.GetTemplateScope() tmplID := getTemplateID(tmpl) - _, ok := ctx.results[tmplID] + _, ok := ctx.results[templateScope+tmplID] if ok { // we can skip the rest since it has been validated. return nil } - ctx.results[tmplID] = true + ctx.results[templateScope+tmplID] = true for globalVar, val := range ctx.globalParams { scope[globalVar] = val diff --git a/workflow/validate/validate_test.go b/workflow/validate/validate_test.go index abe8695eef99..331ee6c8cab9 100644 --- a/workflow/validate/validate_test.go +++ b/workflow/validate/validate_test.go @@ -1538,6 +1538,56 @@ func TestNestedTemplateRef(t *testing.T) { require.NoError(t, err) } +var templateRefTargetWithInput = ` +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: template-ref-target-with-input +spec: + templates: + - name: A + inputs: + parameters: + - name: message + container: + image: alpine:3.11 + command: [sh, -c] + args: ["echo {{inputs.parameters.message}}"] +` + +var nestedTemplateRefWithError = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: template-ref- +spec: + entrypoint: main + templates: + - name: main + steps: + - - name: call-A + templateRef: + name: template-ref-target + template: A + - - name: call-A-input + template: A + - name: A + steps: + - - name: call-B + templateRef: + name: template-ref-target-with-input + template: A +` + +func TestNestedTemplateRefWithError(t *testing.T) { + err := createWorkflowTemplateFromSpec(templateRefTarget) + require.NoError(t, err) + err = createWorkflowTemplateFromSpec(templateRefTargetWithInput) + require.NoError(t, err) + err = validate(nestedTemplateRefWithError) + require.EqualError(t, err, "templates.main.steps[1].call-A-input templates.A.steps[0].call-B templates.A inputs.parameters.message was not supplied") +} + var undefinedTemplateRef = ` apiVersion: argoproj.io/v1alpha1 kind: Workflow From e088cfc66e6bf263d273b20d15a7e6722ff4e3ea Mon Sep 17 00:00:00 2001 From: Unperceivable Date: Wed, 8 Jan 2025 07:02:20 +0100 Subject: [PATCH 7/9] feat: Fixes #8646 visualize wf before submitting (#14034) Signed-off-by: Unperceivable --- .../cron-workflows/cron-workflow-editor.tsx | 6 + .../components/editors/graph-viewer.tsx | 384 ++++++++++++++++++ ui/src/shared/models/workflows.ts | 15 +- .../workflow-template-editor.tsx | 6 + .../workflows/components/workflow-editor.tsx | 6 + 5 files changed, 412 insertions(+), 5 deletions(-) create mode 100644 ui/src/shared/components/editors/graph-viewer.tsx diff --git a/ui/src/cron-workflows/cron-workflow-editor.tsx b/ui/src/cron-workflows/cron-workflow-editor.tsx index fcad11038366..4ee19e5cf639 100644 --- a/ui/src/cron-workflows/cron-workflow-editor.tsx +++ b/ui/src/cron-workflows/cron-workflow-editor.tsx @@ -1,6 +1,7 @@ import {Tabs} from 'argo-ui/src/components/tabs/tabs'; import * as React from 'react'; +import {GraphViewer} from '../shared/components/editors/graph-viewer'; import {LabelsAndAnnotationsEditor} from '../shared/components/editors/labels-and-annotations-editor'; import {MetadataEditor} from '../shared/components/editors/metadata-editor'; import {WorkflowParametersEditor} from '../shared/components/editors/workflow-parameters-editor'; @@ -94,6 +95,11 @@ export function CronWorkflowEditor({ } /> ) + }, + { + key: 'graph', + title: 'Graph', + content: } ]} /> diff --git a/ui/src/shared/components/editors/graph-viewer.tsx b/ui/src/shared/components/editors/graph-viewer.tsx new file mode 100644 index 000000000000..5f4f363a3b0a --- /dev/null +++ b/ui/src/shared/components/editors/graph-viewer.tsx @@ -0,0 +1,384 @@ +import * as React from 'react'; +import {useEffect, useState} from 'react'; + +import {genres} from '../../../workflows/components/workflow-dag/genres'; +import {WorkflowDagRenderOptions} from '../../../workflows/components/workflow-dag/workflow-dag'; +import {WorkflowDagRenderOptionsPanel} from '../../../workflows/components/workflow-dag/workflow-dag-render-options-panel'; +import {ClusterWorkflowTemplate, CronWorkflow, DAGTask, Template, Workflow, WorkflowStep, WorkflowTemplate, WorkflowTemplateRef} from '../../models'; +import {services} from '../../services'; +import {GraphPanel} from '../graph/graph-panel'; +import {Graph} from '../graph/types'; +import {Icon} from '../icon'; + +export function GraphViewer({workflowDefinition}: {workflowDefinition: Workflow | WorkflowTemplate | ClusterWorkflowTemplate | CronWorkflow}) { + const [workflow, setWorkflow] = useState(workflowDefinition); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(); + const [state, saveOptions] = useState({ + expandNodes: new Set(), + showArtifacts: false, + showInvokingTemplateName: false, + showTemplateRefsGrouping: false + }); + const defaultNodeParams: { + classNames: string; + progress: number; + icon: Icon; + } = { + classNames: 'Skipped', + progress: 0, + icon: 'clock' + }; + + useEffect(() => { + if ('workflowTemplateRef' in workflowDefinition.spec && isLoading) { + setWorkflowFromRefrence(workflowDefinition.spec.workflowTemplateRef) + .then(() => { + setError(null); + setIsLoading(false); + }) + .catch(err => { + setError(err); + }); + } else if ('workflowSpec' in workflowDefinition.spec && isLoading) { + const convertedCronWorkflow = convertFromCronWorkflow(workflowDefinition as CronWorkflow); + setWorkflow(convertedCronWorkflow); + setIsLoading(false); + } else { + setIsLoading(false); + } + }, [workflow]); + + if (isLoading) { + const currentState = error ? `${error.name}: ${error.message}` : 'Loading...'; + return
{currentState}
; + } + + const name = workflow.metadata.name ? `${workflow.metadata.name}-${generateNamePostfix(5)}` : `${workflow.metadata.generateName}${generateNamePostfix(5)}`; + const graph = populateGraphFromWorkflow(workflow, name); + + return ( + saveOptions(workflowDagRenderOptions)} />} + /> + ); + + function setWorkflowFromRefrence(workflowTemplateRef: WorkflowTemplateRef): Promise { + const referenceName = workflowTemplateRef.name; + if ('clusterScope' in workflowTemplateRef && workflowTemplateRef.clusterScope == true) { + return services.clusterWorkflowTemplate + .get(referenceName) + .then(clusterWorkflowTemplate => { + setWorkflow(clusterWorkflowTemplate); + }) + .catch(err => { + const explicitError = new Error(`${err.message}, no workflowTemplateRef "${referenceName}" found in clusterWorkflowTemplates`); + explicitError.stack = err.stack; + throw explicitError; + }); + } else { + return services.workflowTemplate + .get(referenceName, workflowDefinition.metadata.namespace) + .then(workflowTemplate => { + setWorkflow(workflowTemplate); + }) + .catch(err => { + const explicitError = new Error(`${err.message}, no workflowTemplateRef "${referenceName}" found in workflowTemplates`); + explicitError.stack = err.stack; + throw explicitError; + }); + } + } + + function convertFromCronWorkflow(cronWorkflow: CronWorkflow): Workflow { + return { + apiVersion: 'argoproj.io/v1alpha1', + kind: 'Workflow', + metadata: cronWorkflow.metadata, + spec: cronWorkflow.spec.workflowSpec + }; + } + + function populateGraphFromWorkflow(workflow: Workflow, name: string): Graph { + const graph = new Graph(); + + const templates = workflow.spec.templates; + const templateMap = new Map(); + const templateLeafMap = new Map(); + let previousSteps: string[] = []; + + templates.forEach((template: Template) => { + templateMap.set(template.name, template); + templateLeafMap.set(template.name, getLeafNodes(template)); + }); + + const entrypoint = workflow.spec.entrypoint; + + function processTemplate(templateName: string, parentNodeName: string = null): Graph { + const template = templateMap.get(templateName); + + if (!template) { + console.error(`Template "${templateName}" not found.`); + return; + } + if (template.dag) { + processDAGTemplate(template, parentNodeName); + } else if (template.steps) { + processStepsTemplate(template, parentNodeName); + } else { + processPodTemplate(templateName, parentNodeName); + } + } + function processDAGTemplate(template: Template, parentNodeName: string) { + createNode(parentNodeName, getStringAfterDelimiter(parentNodeName), 'DAG'); + template.dag.tasks.forEach((task: DAGTask) => { + let nodeLabel = task.name; + const nodeName = `${parentNodeName}.${task.name}`; + const retryNodeName = `${nodeName}.retry`; + const taskGroupName = `${nodeName}.TaskGroup`; + const retryStrategy = getRetryStrategy(task); + const executionStrategy = getExecutionStrategy(task); + + if (retryStrategy) { + createNode(retryNodeName, nodeLabel, 'Retry'); + } + if (executionStrategy) { + nodeLabel = `${nodeLabel}${executionStrategy}`; + createNode(taskGroupName, nodeLabel, 'TaskGroup'); + } + + createNode(nodeName, nodeLabel, getTaskGenre(task)); + + if (task.depends || task.dependencies) { + const dependencies = task.dependencies ? task.dependencies : parseDepends(task.depends); + const dependencyLabel = task.depends ? task.depends : task.dependencies.join(' && '); + dependencies.forEach((dep: string) => { + let dependancyName = `${parentNodeName}.${dep}`; + if (graph.nodes.get(dependancyName).genre == 'Pod') { + if (retryStrategy) { + createEdge(dependancyName, retryNodeName); + dependancyName = retryNodeName; + } + if (executionStrategy) { + createEdge(dependancyName, taskGroupName); + dependancyName = taskGroupName; + } + createEdge(dependancyName, nodeName, dependencyLabel); + } else { + const depTemplate = getTemplateNameFromTask(template.dag, dep); + const templateLeafNodes = templateLeafMap.get(depTemplate); + if (templateLeafNodes) { + templateLeafNodes.forEach((leaf: string) => { + let leafNode = `${dependancyName}.${leaf}`; + if (retryStrategy) { + createEdge(parentNodeName, retryNodeName); + leafNode = retryNodeName; + } + if (executionStrategy) { + createEdge(parentNodeName, taskGroupName); + leafNode = taskGroupName; + } + createEdge(leafNode, nodeName, dependencyLabel); + }); + } + } + }); + } else { + let newParentTaskName = parentNodeName; + if (retryStrategy) { + createEdge(parentNodeName, retryNodeName); + newParentTaskName = retryNodeName; + } + if (executionStrategy) { + createEdge(parentNodeName, taskGroupName); + newParentTaskName = taskGroupName; + } + createEdge(newParentTaskName, nodeName); + } + + if (isTemplateNested(task)) { + processTemplate(task.template, nodeName); + } + previousSteps = []; + }); + } + + function processStepsTemplate(template: Template, parentNodeName: string) { + if (!graph.nodes.has(parentNodeName)) { + createNode(parentNodeName, parentNodeName, 'Steps'); + } + createEdge(parentNodeName, `${parentNodeName}.0`); + + template.steps.forEach((stepGroup: WorkflowStep[], stepGroupIndex: number) => { + let groupName = `${parentNodeName}.${stepGroupIndex}`; + createNode(groupName, `[${stepGroupIndex}]`, 'StepGroup'); + previousSteps.forEach((prevStep: string) => { + createEdge(prevStep, groupName); + }); + previousSteps = []; + + stepGroup.forEach((step: WorkflowStep) => { + const nodeName = `${groupName}.${step.name}`; + const nodeLabel = `${step.name}${getExecutionStrategy(step)}`; + const retryStrategy = getRetryStrategy(step); + if (retryStrategy) { + const retryNodeName = `${nodeName}.retry`; + createNode(retryNodeName, nodeLabel, 'Retry'); + createEdge(groupName, retryNodeName); + groupName = retryNodeName; + } + + createNode(nodeName, nodeLabel, getTaskGenre(step)); + createEdge(groupName, nodeName); + previousSteps.push(nodeName); + + if (isTemplateNested(step)) { + processTemplate(step.template, nodeName); + } + }); + }); + } + + function processPodTemplate(templateName: string, parentNodeName: string) { + if (templateMap.size === 1) { + templateName = parentNodeName; + parentNodeName = ''; + } + createEdge(parentNodeName, templateName); + createNode(templateName, templateName, 'Pod'); + } + function getTaskGenre(task: DAGTask | WorkflowStep): 'Steps' | 'DAG' | 'Pod' { + if (templateMap.has(task.template)) { + const template = templateMap.get(task.template); + if ('steps' in template) { + return 'Steps'; + } else if ('dag' in template) { + return 'DAG'; + } + } + return 'Pod'; + } + + function isTemplateNested(node: DAGTask | WorkflowStep): boolean { + if (templateMap.has(node.template)) { + const template = templateMap.get(node.template); + return 'dag' in template || 'steps' in template; + } + return false; + } + + function getRetryStrategy(node: DAGTask | WorkflowStep): string { + if (templateMap.has(node.template)) { + const template = templateMap.get(node.template); + if ('retryStrategy' in template) { + return `\n{retryStrategy: ${String(template.retryStrategy)}}`; + } + } + return ''; + } + + const createNode = (name: string, label: string, genre: string, params = defaultNodeParams) => { + graph.nodes.set(name, { + label, + genre, + ...params + }); + }; + + const createEdge = (from: string, to: string, label?: string) => { + const edgeData = label ? {label: label} : {}; + graph.edges.set({v: from, w: to}, edgeData); + }; + + processTemplate(entrypoint, name); + + return graph; + } + function parseDepends(dependsString: string): string[] { + const taskNameRegex = /([a-zA-Z0-9-_]+)(?:\.[a-zA-Z]+)?/g; + const taskNames = new Set(); + let match; + while ((match = taskNameRegex.exec(dependsString)) !== null) { + taskNames.add(match[1]); + } + + return Array.from(taskNames); + } + + function generateNamePostfix(len: number): string { + const chars = 'abcdefghijklmnopqrstuvwxyz0123456789'; + let result = ''; + for (let i = 0; i < len; i++) { + const randomIndex = Math.floor(Math.random() * chars.length); + result += chars[randomIndex]; + } + return result; + } + + function getLeafNodes(template: Template) { + const allTaskNames = new Set(); + const dependentTaskNames = new Set(); + let independentTasks: string[] = []; + + if (template.dag) { + template.dag.tasks.forEach((task: DAGTask) => { + allTaskNames.add(task.name); + if (task.depends) { + const dependencies = parseDepends(task.depends); + dependencies.forEach((dep: string) => { + dependentTaskNames.add(dep); + }); + } + }); + independentTasks = Array.from(allTaskNames).filter(nodeName => !dependentTaskNames.has(nodeName)); + } else if (template.steps) { + const lastStepGroupIdx = template.steps.length - 1; + const lastStepGroup = template.steps[lastStepGroupIdx]; + + lastStepGroup.forEach((step: WorkflowStep) => { + independentTasks.push(`${lastStepGroupIdx}.${step.name}`); + }); + } + + return independentTasks ? independentTasks : null; + } + + function getTemplateNameFromTask(dag: {tasks: {name: string; template: string}[]}, nodeName: string): string | null { + const task = dag.tasks.find(t => t.name === nodeName); + return task ? task.template : null; + } + + function getExecutionStrategy(task: DAGTask | WorkflowStep) { + let executionStrategy = ''; + + if (task.withItems) { + executionStrategy += `\n{withItems: ${String(task.withItems)}}`; + } else if (task.withParam) { + executionStrategy += `\n{withParam: ${String(task.withParam)}}`; + } else if (task.withSequence) { + executionStrategy += `\n{withSequence: ${String(task.withSequence)}}`; + } + + return executionStrategy; + } + + function getStringAfterDelimiter(input: string, delimiter: string = '.'): string { + const lastIndex = input.lastIndexOf(delimiter); + if (lastIndex === -1) { + return input; + } + return input.substring(lastIndex + 1); + } +} diff --git a/ui/src/shared/models/workflows.ts b/ui/src/shared/models/workflows.ts index b44876bc8405..08a83725174a 100644 --- a/ui/src/shared/models/workflows.ts +++ b/ui/src/shared/models/workflows.ts @@ -920,7 +920,7 @@ export interface Sequence { count?: number; } -export interface DAGTask { +export interface BaseDAGTask { name: string; /** @@ -938,16 +938,21 @@ export interface DAGTask { */ arguments?: Arguments; - /** - * Dependencies are name of other targets which this depends on - */ - dependencies?: string[]; onExit?: string; withItems?: any[]; withParam?: string; withSequence?: Sequence; } +/** + * DAGTask interface allows either `depends` or `dependencies`, but not both. + */ +export type DAGTask = BaseDAGTask & + /** + * Dependencies are name of other targets which this depends on + */ + ({depends?: string; dependencies?: never} | {depends?: never; dependencies?: string[]}); + /** * WorkflowStep is a reference to a template to execute in a series of step */ diff --git a/ui/src/workflow-templates/workflow-template-editor.tsx b/ui/src/workflow-templates/workflow-template-editor.tsx index 83a23e64450d..e2b369cdec52 100644 --- a/ui/src/workflow-templates/workflow-template-editor.tsx +++ b/ui/src/workflow-templates/workflow-template-editor.tsx @@ -1,6 +1,7 @@ import {Tabs} from 'argo-ui/src/components/tabs/tabs'; import * as React from 'react'; +import {GraphViewer} from '../shared/components/editors/graph-viewer'; import {LabelsAndAnnotationsEditor} from '../shared/components/editors/labels-and-annotations-editor'; import {MetadataEditor} from '../shared/components/editors/metadata-editor'; import {WorkflowParametersEditor} from '../shared/components/editors/workflow-parameters-editor'; @@ -67,6 +68,11 @@ export function WorkflowTemplateEditor({ onChange={workflowMetadata => onChange({...template, spec: {...template.spec, workflowMetadata}})} /> ) + }, + { + key: 'graph', + title: 'Graph', + content: } ]} /> diff --git a/ui/src/workflows/components/workflow-editor.tsx b/ui/src/workflows/components/workflow-editor.tsx index 9f0524335f35..bbcaac0bfb07 100644 --- a/ui/src/workflows/components/workflow-editor.tsx +++ b/ui/src/workflows/components/workflow-editor.tsx @@ -1,6 +1,7 @@ import {Tabs} from 'argo-ui/src/components/tabs/tabs'; import * as React from 'react'; +import {GraphViewer} from '../../shared/components/editors/graph-viewer'; import {MetadataEditor} from '../../shared/components/editors/metadata-editor'; import {WorkflowParametersEditor} from '../../shared/components/editors/workflow-parameters-editor'; import {ObjectEditor} from '../../shared/components/object-editor'; @@ -56,6 +57,11 @@ export function WorkflowEditor({ key: 'metadata', title: 'MetaData', content: onChange({...workflow, metadata})} /> + }, + { + key: 'graph', + title: 'Graph', + content: } ]} /> From 09d5ee75ed1cf480e4a8c2663256a4b12db720da Mon Sep 17 00:00:00 2001 From: Isitha Subasinghe Date: Fri, 10 Jan 2025 10:26:50 +1100 Subject: [PATCH 8/9] fix: ensure namespace parallelism and parallelism work together. Fixes #10985 (#14039) Signed-off-by: isubasinghe --- docs/parallelism.md | 2 + workflow/controller/controller.go | 8 +- workflow/controller/controller_test.go | 1 + workflow/sync/chain_throttler.go | 41 ----- workflow/sync/chain_throttler_test.go | 24 --- workflow/sync/multi_throttler.go | 241 +++++++++++++++++++++++++ workflow/sync/multi_throttler_test.go | 204 +++++++++++++++++++++ workflow/sync/throttler.go | 211 ---------------------- workflow/sync/throttler_test.go | 153 ---------------- 9 files changed, 451 insertions(+), 434 deletions(-) delete mode 100644 workflow/sync/chain_throttler.go delete mode 100644 workflow/sync/chain_throttler_test.go create mode 100644 workflow/sync/multi_throttler.go create mode 100644 workflow/sync/multi_throttler_test.go delete mode 100644 workflow/sync/throttler.go delete mode 100644 workflow/sync/throttler_test.go diff --git a/docs/parallelism.md b/docs/parallelism.md index bae14e3c45fe..f94b68fccf14 100644 --- a/docs/parallelism.md +++ b/docs/parallelism.md @@ -18,6 +18,8 @@ data: namespaceParallelism: "4" ``` +When namespace parallelism is enabled, it is plausible for a workflow with a lower priority to be run first if a namespace is at its namespace parallelism limits. + !!! Note Workflows that are executing but restricted from running more nodes due to other mechanisms will still count toward parallelism limits. diff --git a/workflow/controller/controller.go b/workflow/controller/controller.go index d396115a582a..fc36c5e4e393 100644 --- a/workflow/controller/controller.go +++ b/workflow/controller/controller.go @@ -242,11 +242,8 @@ func NewWorkflowController(ctx context.Context, restConfig *rest.Config, kubecli } func (wfc *WorkflowController) newThrottler() sync.Throttler { - f := func(key string) { wfc.wfQueue.AddRateLimited(key) } - return sync.ChainThrottler{ - sync.NewThrottler(wfc.Config.Parallelism, sync.SingleBucket, f), - sync.NewThrottler(wfc.Config.NamespaceParallelism, sync.NamespaceBucket, f), - } + f := func(key string) { wfc.wfQueue.Add(key) } + return sync.NewMultiThrottler(wfc.Config.Parallelism, wfc.Config.NamespaceParallelism, f) } // runGCcontroller runs the workflow garbage collector controller @@ -482,6 +479,7 @@ func (wfc *WorkflowController) notifySemaphoreConfigUpdate(cm *apiv1.ConfigMap) log.Warnf("received object from indexer %s is not an unstructured", indexes.SemaphoreConfigIndexName) continue } + log.Infof("Adding workflow %s/%s", un.GetNamespace(), un.GetName()) wfc.wfQueue.AddRateLimited(fmt.Sprintf("%s/%s", un.GetNamespace(), un.GetName())) } } diff --git a/workflow/controller/controller_test.go b/workflow/controller/controller_test.go index d9756765b858..61073963b3c0 100644 --- a/workflow/controller/controller_test.go +++ b/workflow/controller/controller_test.go @@ -953,6 +953,7 @@ func TestNotifySemaphoreConfigUpdate(t *testing.T) { for i := 0; i < 3; i++ { key, _ := controller.wfQueue.Get() controller.wfQueue.Done(key) + controller.wfQueue.Forget(key) } assert.Equal(0, controller.wfQueue.Len()) diff --git a/workflow/sync/chain_throttler.go b/workflow/sync/chain_throttler.go deleted file mode 100644 index 3ea22759e238..000000000000 --- a/workflow/sync/chain_throttler.go +++ /dev/null @@ -1,41 +0,0 @@ -package sync - -import ( - "time" - - wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" -) - -type ChainThrottler []Throttler - -func (c ChainThrottler) Init(wfs []wfv1.Workflow) error { - for _, t := range c { - if err := t.Init(wfs); err != nil { - return err - } - } - return nil -} - -func (c ChainThrottler) Add(key Key, priority int32, creationTime time.Time) { - for _, t := range c { - t.Add(key, priority, creationTime) - } -} - -func (c ChainThrottler) Admit(key Key) bool { - for _, t := range c { - if !t.Admit(key) { - return false - } - } - return true -} - -func (c ChainThrottler) Remove(key Key) { - for _, t := range c { - t.Remove(key) - } -} - -var _ Throttler = ChainThrottler{} diff --git a/workflow/sync/chain_throttler_test.go b/workflow/sync/chain_throttler_test.go deleted file mode 100644 index c4ed1868ec26..000000000000 --- a/workflow/sync/chain_throttler_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package sync - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/argoproj/argo-workflows/v3/workflow/sync/mocks" -) - -func TestChainThrottler(t *testing.T) { - m := &mocks.Throttler{} - m.On("Add", "foo", int32(1), time.Time{}).Return() - m.On("Admit", "foo").Return(false) - m.On("Remove", "foo").Return() - - c := ChainThrottler{m} - c.Add("foo", 1, time.Time{}) - assert.False(t, c.Admit("foo")) - c.Remove("foo") - - assert.True(t, ChainThrottler{}.Admit("foo")) -} diff --git a/workflow/sync/multi_throttler.go b/workflow/sync/multi_throttler.go new file mode 100644 index 000000000000..434201e148e8 --- /dev/null +++ b/workflow/sync/multi_throttler.go @@ -0,0 +1,241 @@ +package sync + +import ( + "container/heap" + "sync" + "time" + + "k8s.io/client-go/tools/cache" + + wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" +) + +//go:generate mockery --name=Throttler + +// Throttler allows the controller to limit number of items it is processing in parallel. +// Items are processed in priority order, and one processing starts, other items (including higher-priority items) +// will be kept pending until the processing is complete. +// Implementations should be idempotent. +type Throttler interface { + Init(wfs []wfv1.Workflow) error + Add(key Key, priority int32, creationTime time.Time) + // Admit returns if the item should be processed. + Admit(key Key) bool + // Remove notifies throttler that item processing is no longer needed + Remove(key Key) +} + +type Key = string +type QueueFunc func(Key) + +// NewMultiThrottler creates a new multi throttler for throttling both namespace and global parallelism, a parallelism value of zero disables throttling +func NewMultiThrottler(parallelism int, namespaceParallelismLimit int, queue QueueFunc) Throttler { + namespaceParallelism := make(map[string]int) + return &multiThrottler{ + queue: queue, + namespaceParallelism: namespaceParallelism, + namespaceParallelismDefault: namespaceParallelismLimit, + totalParallelism: parallelism, + running: make(map[Key]bool), + pending: make(map[string]*priorityQueue), + lock: &sync.Mutex{}, + } +} + +type multiThrottler struct { + queue QueueFunc + namespaceParallelism map[string]int + namespaceParallelismDefault int + totalParallelism int + running map[Key]bool + pending map[string]*priorityQueue + lock *sync.Mutex +} + +func (m *multiThrottler) Init(wfs []wfv1.Workflow) error { + m.lock.Lock() + defer m.lock.Unlock() + + keys := []Key{} + for _, wf := range wfs { + if wf.Status.Phase != wfv1.WorkflowRunning { + continue + } + key, err := cache.MetaNamespaceKeyFunc(&wf) + if err != nil { + return err + } + keys = append(keys, key) + } + + for _, key := range keys { + m.running[key] = true + } + return nil +} + +func (m *multiThrottler) namespaceCount(namespace string) (int, int) { + setLimit, has := m.namespaceParallelism[namespace] + if !has { + m.namespaceParallelism[namespace] = m.namespaceParallelismDefault + setLimit = m.namespaceParallelismDefault + } + if setLimit == 0 { + // return count is no longer accurate, but preserves behaviour + return 0, 0 + } + count := 0 + for key := range m.running { + ns, _, _ := cache.SplitMetaNamespaceKey(key) + if ns == namespace { + count++ + } + } + return count, setLimit +} + +func (m *multiThrottler) namespaceAllows(namespace string) bool { + count, limit := m.namespaceCount(namespace) + return count < limit || limit == 0 +} + +func (m *multiThrottler) Add(key Key, priority int32, creationTime time.Time) { + m.lock.Lock() + defer m.lock.Unlock() + namespace, _, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + return + } + _, ok := m.pending[namespace] + if !ok { + m.pending[namespace] = &priorityQueue{itemByKey: make(map[string]*item)} + } + + m.pending[namespace].add(key, priority, creationTime) + m.queueThrottled() +} + +func (m *multiThrottler) Admit(key Key) bool { + m.lock.Lock() + defer m.lock.Unlock() + + _, ok := m.running[key] + if ok { + return true + } + m.queueThrottled() + return false +} + +func (m *multiThrottler) Remove(key Key) { + m.lock.Lock() + defer m.lock.Unlock() + + namespace, _, _ := cache.SplitMetaNamespaceKey(key) + delete(m.running, key) + m.pending[namespace].remove(key) + m.queueThrottled() +} + +func (m *multiThrottler) queueThrottled() { + if m.totalParallelism != 0 && len(m.running) >= m.totalParallelism { + return + } + + minPq := &priorityQueue{itemByKey: make(map[string]*item)} + + for _, pq := range m.pending { + if len(pq.items) == 0 { + continue + } + currItem := pq.peek() + + namespace, _, err := cache.SplitMetaNamespaceKey(currItem.key) + if err != nil { + return + } + if !m.namespaceAllows(namespace) { + continue + } + + minPq.add(currItem.key, currItem.priority, currItem.creationTime) + } + if len(minPq.items) > 0 { + bestItem := minPq.pop() + bestNamespace, _, _ := cache.SplitMetaNamespaceKey(bestItem.key) + m.pending[bestNamespace].pop() + m.running[bestItem.key] = true + m.queue(bestItem.key) + } +} + +type item struct { + key string + creationTime time.Time + priority int32 + index int +} + +type priorityQueue struct { + items []*item + itemByKey map[string]*item +} + +func (pq *priorityQueue) pop() *item { + return heap.Pop(pq).(*item) +} + +func (pq *priorityQueue) peek() *item { + return pq.items[0] +} + +func (pq *priorityQueue) add(key Key, priority int32, creationTime time.Time) { + if res, ok := pq.itemByKey[key]; ok { + if res.priority != priority { + res.priority = priority + heap.Fix(pq, res.index) + } + } else { + heap.Push(pq, &item{key: key, priority: priority, creationTime: creationTime}) + } +} + +func (pq *priorityQueue) remove(key Key) { + if item, ok := pq.itemByKey[key]; ok { + heap.Remove(pq, item.index) + delete(pq.itemByKey, key) + } +} + +func (pq priorityQueue) Len() int { return len(pq.items) } + +func (pq priorityQueue) Less(i, j int) bool { + if pq.items[i].priority == pq.items[j].priority { + return pq.items[i].creationTime.Before(pq.items[j].creationTime) + } + return pq.items[i].priority > pq.items[j].priority +} + +func (pq priorityQueue) Swap(i, j int) { + pq.items[i], pq.items[j] = pq.items[j], pq.items[i] + pq.items[i].index = i + pq.items[j].index = j +} + +func (pq *priorityQueue) Push(x interface{}) { + n := len(pq.items) + item := x.(*item) + item.index = n + pq.items = append(pq.items, item) + pq.itemByKey[item.key] = item +} + +func (pq *priorityQueue) Pop() interface{} { + old := pq.items + n := len(old) + item := old[n-1] + item.index = -1 + pq.items = old[0 : n-1] + delete(pq.itemByKey, item.key) + return item +} diff --git a/workflow/sync/multi_throttler_test.go b/workflow/sync/multi_throttler_test.go new file mode 100644 index 000000000000..d321c31c0766 --- /dev/null +++ b/workflow/sync/multi_throttler_test.go @@ -0,0 +1,204 @@ +package sync + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + fakewfclientset "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/fake" +) + +func TestMultiNoParallelismSamePriority(t *testing.T) { + throttler := NewMultiThrottler(0, 0, func(Key) {}) + + throttler.Add("default/c", 0, time.Now().Add(2*time.Hour)) + throttler.Add("default/b", 0, time.Now().Add(1*time.Hour)) + throttler.Add("default/a", 0, time.Now()) + + assert.True(t, throttler.Admit("default/a")) + assert.True(t, throttler.Admit("default/b")) + assert.True(t, throttler.Admit("default/c")) +} + +func TestMultiNoParallelismMultipleBuckets(t *testing.T) { + throttler := NewMultiThrottler(1, 1, func(Key) {}) + throttler.Add("a/0", 0, time.Now()) + throttler.Add("a/1", 0, time.Now().Add(-1*time.Second)) + throttler.Add("b/0", 0, time.Now().Add(-2*time.Second)) + throttler.Add("b/1", 0, time.Now().Add(-3*time.Second)) + + assert.True(t, throttler.Admit("a/0")) + assert.False(t, throttler.Admit("a/1")) + assert.False(t, throttler.Admit("b/0")) + assert.False(t, throttler.Admit("b/1")) + throttler.Remove("a/0") + assert.True(t, throttler.Admit("b/1")) +} + +func TestMultiWithParallelismLimitAndPriority(t *testing.T) { + queuedKey := "" + throttler := NewMultiThrottler(2, 0, func(key string) { queuedKey = key }) + + throttler.Add("default/a", 1, time.Now()) + throttler.Add("default/b", 2, time.Now()) + throttler.Add("default/c", 3, time.Now()) + throttler.Add("default/d", 4, time.Now()) + + assert.True(t, throttler.Admit("default/a"), "is started, even though low priority") + assert.True(t, throttler.Admit("default/b"), "is started, even though low priority") + assert.False(t, throttler.Admit("default/c"), "cannot start") + assert.False(t, throttler.Admit("default/d"), "cannot start") + assert.Equal(t, "default/b", queuedKey) + queuedKey = "" + + throttler.Remove("default/a") + assert.True(t, throttler.Admit("default/b"), "stays running") + assert.True(t, throttler.Admit("default/d"), "top priority") + assert.False(t, throttler.Admit("default/c")) + assert.Equal(t, "default/d", queuedKey) + queuedKey = "" + + throttler.Remove("default/b") + assert.True(t, throttler.Admit("default/d"), "top priority") + assert.True(t, throttler.Admit("default/c"), "now running too") + assert.Equal(t, "default/c", queuedKey) +} + +func TestMultiInitWithWorkflows(t *testing.T) { + queuedKey := "" + throttler := NewMultiThrottler(1, 1, func(key string) { queuedKey = key }) + ctx := context.Background() + + wfclientset := fakewfclientset.NewSimpleClientset( + wfv1.MustUnmarshalWorkflow(` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + labels: + workflows.argoproj.io/phase: Running + name: a + namespace: default +spec: + entrypoint: whalesay + templates: + - name: whalesay + container: + image: docker/whalesay:latest + command: [cowsay] + args: ["hello world"] +status: + phase: Running + startedAt: "2020-06-19T17:37:05Z" +`), + wfv1.MustUnmarshalWorkflow(` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + labels: + workflows.argoproj.io/phase: Running + name: b + namespace: default +spec: + entrypoint: whalesay + templates: + - name: whalesay + container: + image: docker/whalesay:latest + command: [cowsay] + args: ["hello world"] +status: + phase: Running + startedAt: "2020-06-19T17:37:05Z" +`)) + wfList, err := wfclientset.ArgoprojV1alpha1().Workflows("default").List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + err = throttler.Init(wfList.Items) + require.NoError(t, err) + assert.True(t, throttler.Admit("default/a")) + assert.True(t, throttler.Admit("default/b")) + + throttler.Add("default/c", 0, time.Now()) + throttler.Add("default/d", 0, time.Now()) + assert.False(t, throttler.Admit("default/c")) + assert.False(t, throttler.Admit("default/d")) + + throttler.Remove("default/a") + assert.Equal(t, "", queuedKey) + assert.False(t, throttler.Admit("default/c")) + assert.False(t, throttler.Admit("default/d")) + + queuedKey = "" + throttler.Remove("default/b") + assert.Equal(t, "default/c", queuedKey) + assert.True(t, throttler.Admit("default/c")) + assert.False(t, throttler.Admit("default/d")) + + queuedKey = "" + throttler.Remove("default/c") + assert.Equal(t, "default/d", queuedKey) + assert.True(t, throttler.Admit("default/d")) +} + +func TestTotalAllowNamespaceLimit(t *testing.T) { + namespaceLimits := make(map[string]int) + namespaceLimits["a"] = 2 + namespaceLimits["b"] = 1 + throttler := &multiThrottler{ + queue: func(key Key) {}, + namespaceParallelism: namespaceLimits, + namespaceParallelismDefault: 6, + totalParallelism: 4, + running: make(map[Key]bool), + pending: make(map[string]*priorityQueue), + lock: &sync.Mutex{}, + } + throttler.Add("a/0", 1, time.Now()) + throttler.Add("b/0", 2, time.Now()) + throttler.Add("a/1", 3, time.Now()) + throttler.Add("a/2", 4, time.Now()) + throttler.Add("a/3", 5, time.Now()) + throttler.Add("a/4", 6, time.Now()) + throttler.Add("b/1", 7, time.Now()) + + assert.True(t, throttler.Admit("a/0")) + assert.True(t, throttler.Admit("b/0")) + assert.True(t, throttler.Admit("a/1")) + + assert.False(t, throttler.Admit("a/2")) + assert.False(t, throttler.Admit("a/3")) + assert.False(t, throttler.Admit("a/4")) + assert.False(t, throttler.Admit("b/1")) + + throttler.Add("c/0", 8, time.Now()) + assert.True(t, throttler.Admit("c/0")) +} + +func TestPriorityAcrossNamespaces(t *testing.T) { + throttler := NewMultiThrottler(3, 1, func(Key) {}) + throttler.Add("a/0", 0, time.Now()) + throttler.Add("a/1", 0, time.Now()) + throttler.Add("a/2", 0, time.Now()) + throttler.Add("b/0", 1, time.Now()) + throttler.Add("b/1", 1, time.Now()) + throttler.Add("b/2", 1, time.Now()) + + assert.True(t, throttler.Admit("a/0")) + assert.True(t, throttler.Admit("b/0")) + assert.False(t, throttler.Admit("a/1")) + assert.False(t, throttler.Admit("a/2")) + assert.True(t, throttler.Admit("b/0")) + assert.False(t, throttler.Admit("b/1")) + assert.False(t, throttler.Admit("b/2")) + throttler.Remove("a/0") + assert.False(t, throttler.Admit("b/1")) + assert.True(t, throttler.Admit("a/1")) + throttler.Remove("b/0") + assert.True(t, throttler.Admit("b/1")) + assert.False(t, throttler.Admit("a/2")) +} diff --git a/workflow/sync/throttler.go b/workflow/sync/throttler.go deleted file mode 100644 index 674cff254908..000000000000 --- a/workflow/sync/throttler.go +++ /dev/null @@ -1,211 +0,0 @@ -package sync - -import ( - "container/heap" - "sync" - "time" - - "k8s.io/client-go/tools/cache" - - wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" -) - -//go:generate mockery --name=Throttler - -// Throttler allows the controller to limit number of items it is processing in parallel. -// Items are processed in priority order, and one processing starts, other items (including higher-priority items) -// will be kept pending until the processing is complete. -// Implementations should be idempotent. -type Throttler interface { - Init(wfs []wfv1.Workflow) error - Add(key Key, priority int32, creationTime time.Time) - // Admit returns if the item should be processed. - Admit(key Key) bool - // Remove notifies throttler that item processing is no longer needed - Remove(key Key) -} - -type Key = string -type QueueFunc func(Key) - -type BucketKey = string -type BucketFunc func(Key) BucketKey - -var SingleBucket BucketFunc = func(key Key) BucketKey { return "" } -var NamespaceBucket BucketFunc = func(key Key) BucketKey { - namespace, _, _ := cache.SplitMetaNamespaceKey(key) - return namespace -} - -type throttler struct { - queue QueueFunc - bucketFunc BucketFunc - inProgress buckets - pending map[BucketKey]*priorityQueue - lock *sync.Mutex - parallelism int -} - -type bucket map[Key]bool -type buckets map[BucketKey]bucket - -// NewThrottler returns a throttle that only runs `parallelism` items at once. When an item may need processing, -// `queue` is invoked. -func NewThrottler(parallelism int, bucketFunc BucketFunc, queue QueueFunc) Throttler { - return &throttler{ - queue: queue, - bucketFunc: bucketFunc, - inProgress: make(buckets), - pending: make(map[BucketKey]*priorityQueue), - lock: &sync.Mutex{}, - parallelism: parallelism, - } -} - -func (t *throttler) Init(wfs []wfv1.Workflow) error { - t.lock.Lock() - defer t.lock.Unlock() - if t.parallelism == 0 { - return nil - } - - for _, wf := range wfs { - key, err := cache.MetaNamespaceKeyFunc(&wf) - if err != nil { - return err - } - if wf.Status.Phase == wfv1.WorkflowRunning { - bucketKey := t.bucketFunc(key) - if _, ok := t.inProgress[bucketKey]; !ok { - t.inProgress[bucketKey] = make(bucket) - } - t.inProgress[bucketKey][key] = true - } - } - return nil -} - -func (t *throttler) Add(key Key, priority int32, creationTime time.Time) { - t.lock.Lock() - defer t.lock.Unlock() - if t.parallelism == 0 { - return - } - bucketKey := t.bucketFunc(key) - if _, ok := t.pending[bucketKey]; !ok { - t.pending[bucketKey] = &priorityQueue{itemByKey: make(map[string]*item)} - } - t.pending[bucketKey].add(key, priority, creationTime) - t.queueThrottled(bucketKey) -} - -func (t *throttler) Admit(key Key) bool { - t.lock.Lock() - defer t.lock.Unlock() - if t.parallelism == 0 { - return true - } - bucketKey := t.bucketFunc(key) - if x, ok := t.inProgress[bucketKey]; ok && x[key] { - return true - } - t.queueThrottled(bucketKey) - return false -} - -func (t *throttler) Remove(key Key) { - t.lock.Lock() - defer t.lock.Unlock() - bucketKey := t.bucketFunc(key) - if x, ok := t.inProgress[bucketKey]; ok { - delete(x, key) - } - if x, ok := t.pending[bucketKey]; ok { - x.remove(key) - } - t.queueThrottled(bucketKey) -} - -func (t *throttler) queueThrottled(bucketKey BucketKey) { - if _, ok := t.inProgress[bucketKey]; !ok { - t.inProgress[bucketKey] = make(bucket) - } - inProgress := t.inProgress[bucketKey] - pending, ok := t.pending[bucketKey] - for ok && pending.Len() > 0 && t.parallelism > len(inProgress) { - key := pending.pop().key - inProgress[key] = true - t.queue(key) - } -} - -type item struct { - key string - creationTime time.Time - priority int32 - index int -} - -type priorityQueue struct { - items []*item - itemByKey map[string]*item -} - -func (pq *priorityQueue) pop() *item { - return heap.Pop(pq).(*item) -} - -func (pq *priorityQueue) peek() *item { - return pq.items[0] -} - -func (pq *priorityQueue) add(key Key, priority int32, creationTime time.Time) { - if res, ok := pq.itemByKey[key]; ok { - if res.priority != priority { - res.priority = priority - heap.Fix(pq, res.index) - } - } else { - heap.Push(pq, &item{key: key, priority: priority, creationTime: creationTime}) - } -} - -func (pq *priorityQueue) remove(key Key) { - if item, ok := pq.itemByKey[key]; ok { - heap.Remove(pq, item.index) - delete(pq.itemByKey, key) - } -} - -func (pq priorityQueue) Len() int { return len(pq.items) } - -func (pq priorityQueue) Less(i, j int) bool { - if pq.items[i].priority == pq.items[j].priority { - return pq.items[i].creationTime.Before(pq.items[j].creationTime) - } - return pq.items[i].priority > pq.items[j].priority -} - -func (pq priorityQueue) Swap(i, j int) { - pq.items[i], pq.items[j] = pq.items[j], pq.items[i] - pq.items[i].index = i - pq.items[j].index = j -} - -func (pq *priorityQueue) Push(x interface{}) { - n := len(pq.items) - item := x.(*item) - item.index = n - pq.items = append(pq.items, item) - pq.itemByKey[item.key] = item -} - -func (pq *priorityQueue) Pop() interface{} { - old := pq.items - n := len(old) - item := old[n-1] - item.index = -1 - pq.items = old[0 : n-1] - delete(pq.itemByKey, item.key) - return item -} diff --git a/workflow/sync/throttler_test.go b/workflow/sync/throttler_test.go deleted file mode 100644 index 4337eb8bdf7c..000000000000 --- a/workflow/sync/throttler_test.go +++ /dev/null @@ -1,153 +0,0 @@ -package sync - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/cache" - - wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - fakewfclientset "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/fake" -) - -func Test_NamespaceBucket(t *testing.T) { - assert.Equal(t, "a", NamespaceBucket("a/b")) -} - -func TestNoParallelismSamePriority(t *testing.T) { - throttler := NewThrottler(0, SingleBucket, nil) - - throttler.Add("c", 0, time.Now().Add(2*time.Hour)) - throttler.Add("b", 0, time.Now().Add(1*time.Hour)) - throttler.Add("a", 0, time.Now()) - - assert.True(t, throttler.Admit("a")) - assert.True(t, throttler.Admit("b")) - assert.True(t, throttler.Admit("c")) -} - -func TestNoParallelismMultipleBuckets(t *testing.T) { - throttler := NewThrottler(1, func(key Key) BucketKey { - namespace, _, _ := cache.SplitMetaNamespaceKey(key) - return namespace - }, func(key string) {}) - - throttler.Add("a/0", 0, time.Now()) - throttler.Add("a/1", 0, time.Now()) - throttler.Add("b/0", 0, time.Now()) - throttler.Add("b/1", 0, time.Now()) - - assert.True(t, throttler.Admit("a/0")) - assert.False(t, throttler.Admit("a/1")) - assert.True(t, throttler.Admit("b/0")) - throttler.Remove("a/0") - assert.True(t, throttler.Admit("a/1")) -} - -func TestWithParallelismLimitAndPriority(t *testing.T) { - queuedKey := "" - throttler := NewThrottler(2, SingleBucket, func(key string) { queuedKey = key }) - - throttler.Add("a", 1, time.Now()) - throttler.Add("b", 2, time.Now()) - throttler.Add("c", 3, time.Now()) - throttler.Add("d", 4, time.Now()) - - assert.True(t, throttler.Admit("a"), "is started, even though low priority") - assert.True(t, throttler.Admit("b"), "is started, even though low priority") - assert.False(t, throttler.Admit("c"), "cannot start") - assert.False(t, throttler.Admit("d"), "cannot start") - assert.Equal(t, "b", queuedKey) - queuedKey = "" - - throttler.Remove("a") - assert.True(t, throttler.Admit("b"), "stays running") - assert.True(t, throttler.Admit("d"), "top priority") - assert.False(t, throttler.Admit("c")) - assert.Equal(t, "d", queuedKey) - queuedKey = "" - - throttler.Remove("b") - assert.True(t, throttler.Admit("d"), "top priority") - assert.True(t, throttler.Admit("c"), "now running too") - assert.Equal(t, "c", queuedKey) -} - -func TestInitWithWorkflows(t *testing.T) { - queuedKey := "" - throttler := NewThrottler(1, SingleBucket, func(key string) { queuedKey = key }) - ctx := context.Background() - - wfclientset := fakewfclientset.NewSimpleClientset( - wfv1.MustUnmarshalWorkflow(` -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - labels: - workflows.argoproj.io/phase: Running - name: a - namespace: default -spec: - entrypoint: whalesay - templates: - - name: whalesay - container: - image: docker/whalesay:latest - command: [cowsay] - args: ["hello world"] -status: - phase: Running - startedAt: "2020-06-19T17:37:05Z" -`), - wfv1.MustUnmarshalWorkflow(` -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - labels: - workflows.argoproj.io/phase: Running - name: b - namespace: default -spec: - entrypoint: whalesay - templates: - - name: whalesay - container: - image: docker/whalesay:latest - command: [cowsay] - args: ["hello world"] -status: - phase: Running - startedAt: "2020-06-19T17:37:05Z" -`)) - wfList, err := wfclientset.ArgoprojV1alpha1().Workflows("default").List(ctx, metav1.ListOptions{}) - require.NoError(t, err) - err = throttler.Init(wfList.Items) - require.NoError(t, err) - assert.True(t, throttler.Admit("default/a")) - assert.True(t, throttler.Admit("default/b")) - - throttler.Add("default/c", 0, time.Now()) - throttler.Add("default/d", 0, time.Now()) - assert.False(t, throttler.Admit("default/c")) - assert.False(t, throttler.Admit("default/d")) - - throttler.Remove("default/a") - assert.Equal(t, "", queuedKey) - assert.False(t, throttler.Admit("default/c")) - assert.False(t, throttler.Admit("default/d")) - - queuedKey = "" - throttler.Remove("default/b") - assert.Equal(t, "default/c", queuedKey) - assert.True(t, throttler.Admit("default/c")) - assert.False(t, throttler.Admit("default/d")) - - queuedKey = "" - throttler.Remove("default/c") - assert.Equal(t, "default/d", queuedKey) - assert.True(t, throttler.Admit("default/d")) -} From a7a72e7ca3c2bbf127b65fdcf1b5b85fbd2aad19 Mon Sep 17 00:00:00 2001 From: Alan Clucas Date: Fri, 10 Jan 2025 10:28:07 +0000 Subject: [PATCH 9/9] fix: update upload-artifact and download-artifact (#14070) Signed-off-by: Alan Clucas --- .github/dependabot.yml | 4 ---- .github/workflows/ci-build.yaml | 4 ++-- .github/workflows/docs.yaml | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1b590db438fd..40a43b0f02da 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -64,10 +64,6 @@ updates: schedule: interval: "weekly" day: "saturday" - ignore: - # temporarily ignore until https://github.com/actions/download-artifact/issues/249 is resolved - - dependency-name: "actions/download-artifact" - - dependency-name: "actions/upload-artifact" # ignore all non-security updates: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#open-pull-requests-limit open-pull-requests-limit: 0 labels: diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index 9d1d23dae820..e185e91e9a12 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -205,7 +205,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - name: Upload - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: ${{matrix.image}}_image.tar path: /tmp/${{matrix.image}}_image.tar @@ -323,7 +323,7 @@ jobs: echo " token: xxxxxx" >> $KUBECONFIG until kubectl cluster-info ; do sleep 10s ; done - name: Download images - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: pattern: '*_image.tar' path: /tmp diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 54157f925cd4..b16e1a62bbd2 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -40,7 +40,7 @@ jobs: run: git diff --exit-code # Upload the site so reviewers see it. - name: Upload Docs Site - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: docs path: site