From a1337fd3338ea7ee08107604f5fb4084bde7a375 Mon Sep 17 00:00:00 2001 From: Tao He Date: Fri, 3 Feb 2023 09:57:16 +0800 Subject: [PATCH] Use k3d to replace kind for faster CI and less disk usage Signed-off-by: Tao He --- .github/workflows/vineyard-operator.yaml | 18 +++++- k8s/Makefile | 62 +++++-------------- .../k8s/v1alpha1/zz_generated.deepcopy.go | 16 +++++ k8s/hack/k3d.yaml | 30 +++++++++ k8s/test/e2e/Makefile | 29 +++------ .../assembly-demo/assembly-job-config.yaml | 4 +- k8s/test/e2e/assembly-demo/assembly-job.yaml | 2 +- .../assembly/distributed-assembly-e2e.yaml | 4 +- k8s/test/e2e/assembly/local-assembly-e2e.yaml | 4 +- .../e2e/autogenerated-helm-chart/e2e.yaml | 9 +-- .../failover-demo/failover-job-config.yaml | 10 +-- k8s/test/e2e/failover-demo/failover-job.yaml | 2 +- k8s/test/e2e/failover/e2e.yaml | 6 +- k8s/test/e2e/kind-with-local-registry.yaml | 15 ----- k8s/test/e2e/kind.yaml | 26 -------- .../repartition-dask-helm-values.yaml | 3 +- .../e2e/repartition-demo/repartition-job.yaml | 2 +- .../e2e/repartition/dask-repartition-e2e.yaml | 9 +-- .../e2e/scheduling-outside-cluster/e2e.yaml | 7 ++- .../e2e/serialize-demo/serialize-demo.yaml | 2 +- k8s/test/e2e/serialize/e2e.yaml | 6 +- .../sidecar-with-custom-sidecar.yaml | 2 +- .../sidecar-with-default-sidecar.yaml | 2 +- k8s/test/e2e/sidecar/e2e.yaml | 4 +- k8s/test/e2e/spill/e2e.yaml | 4 +- k8s/test/e2e/vineyardd.yaml | 12 ++-- k8s/test/e2e/workflow-demo/workflow-job1.yaml | 2 +- k8s/test/e2e/workflow-demo/workflow-job2.yaml | 2 +- k8s/test/e2e/workflow/e2e.yaml | 4 +- .../build-kind-cluster-with-local-registry.sh | 56 ----------------- ...delete-kind-cluster-with-local-registry.sh | 19 ------ 31 files changed, 136 insertions(+), 237 deletions(-) create mode 100644 k8s/hack/k3d.yaml delete mode 100644 k8s/test/e2e/kind-with-local-registry.yaml delete mode 100644 k8s/test/e2e/kind.yaml delete mode 100644 k8s/test/hack/build-kind-cluster-with-local-registry.sh delete mode 100644 k8s/test/hack/delete-kind-cluster-with-local-registry.sh diff --git a/.github/workflows/vineyard-operator.yaml b/.github/workflows/vineyard-operator.yaml index f8b0bc28b..0c872e211 100644 --- a/.github/workflows/vineyard-operator.yaml +++ b/.github/workflows/vineyard-operator.yaml @@ -100,6 +100,9 @@ jobs: exit -1 fi + # prepare golangci-lint for faster installation + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.49.0 + function ec() { [[ "$1" == "-h" ]] && { shift && eval $* > /dev/null 2>&1; ec=$?; echo $ec; } || eval $*; ec=$?; } ec make -C k8s golint @@ -122,7 +125,7 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - + - name: Build docker image run: | make -C k8s docker-build @@ -144,6 +147,12 @@ jobs: docker tag ${IMG} ${LATEST_IMG} docker push ${LATEST_IMG} + - name: Install k3d + if: ${{ matrix.job != 'release' }} + run: | + curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + k3d cluster create -c k8s/hack/k3d.yaml + - name: Generate the python image for tests if: ${{ matrix.job != 'release' }} run: | @@ -202,4 +211,9 @@ jobs: - name: e2e-tests-workflow if: ${{ matrix.job == 'e2e-tests-workflow' }} run: | - make -C k8s e2e-tests-workflow \ No newline at end of file + make -C k8s e2e-tests-workflow + + - name: Uninstall k3d + if: ${{ always() }} + run: | + k3d cluster delete -c k8s/hack/k3d.yaml || true diff --git a/k8s/Makefile b/k8s/Makefile index bf8f97daa..2c2355764 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -54,20 +54,6 @@ all: manager check: lint e2e-test .PHONY: check -# Build a 4-nodes(1 master and 3 workers) kind cluster with local registry -build-local-cluster: - @touch /tmp/e2e-k8s.config - @echo "the kubeconfig path is /tmp/e2e-k8s.config" - @echo "Creating the kind cluster with local registry" - @bash test/hack/build-kind-cluster-with-local-registry.sh -.PHONY: build-local-cluster - -delete-local-cluster: - @rm /tmp/e2e-k8s.config - @echo "Deleting the kind cluster with local registry" - @bash test/hack/delete-kind-cluster-with-local-registry.sh -.PHONY: delete-local-cluster - prepare-e2e-test: e2e @echo "Install e2e test dependencies..." @sudo bash test/hack/prepare-e2e.sh @@ -78,118 +64,104 @@ prepare-e2e-test: e2e # install the cert-manager and wait for ready install-cert-manager: @echo "Installing cert-manager..." - @kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml --kubeconfig=/tmp/e2e-k8s.config - @kubectl wait pods -n cert-manager --all --for condition=Ready --timeout=300s --kubeconfig=/tmp/e2e-k8s.config + @kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml + @kubectl wait pods -n cert-manager --all --for condition=Ready --timeout=300s @echo "Cert-Manager ready." # install the vineyard-operator and wait for ready install-vineyard-operator: @echo "Installing vineyard-operator..." - @docker tag vineyardcloudnative/vineyard-operator:latest localhost:5001/vineyard-operator:latest - @docker push localhost:5001/vineyard-operator:latest - @export IMG=localhost:5001/vineyard-operator:latest KUBECONFIG=/tmp/e2e-k8s.config && make deploy - @kubectl wait deployment/vineyard-controller-manager --for condition=Available -n vineyard-system --timeout=300s --kubeconfig=/tmp/e2e-k8s.config + @make deploy + @kubectl wait deployment/vineyard-controller-manager --for condition=Available -n vineyard-system --timeout=300s @echo "Vineyard-Operator Ready" # install the vineyard cluster and wait for ready install-vineyard-cluster: @echo "Installing vineyard cluster..." - @docker tag vineyardcloudnative/vineyardd:alpine-latest localhost:5001/vineyardd:alpine-latest - @docker push localhost:5001/vineyardd:alpine-latest - @kubectl apply -f test/e2e/vineyardd.yaml --kubeconfig=/tmp/e2e-k8s.config - @kubectl wait vineyardd/vineyardd-sample --for condition=Available -n vineyard-system --timeout=300s --kubeconfig=/tmp/e2e-k8s.config + @kubectl apply -f test/e2e/vineyardd.yaml + @kubectl wait vineyardd/vineyardd-sample --for condition=Available -n vineyard-system --timeout=300s @echo "Vineyard cluster Ready" -install-vineyard: build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster +install-vineyard: install-cert-manager install-vineyard-operator install-vineyard-cluster sleep 6000 -e2e-tests-assembly-local: kustomize prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster +e2e-tests-assembly-local: kustomize prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster @echo "Running local assembly e2e test..." @cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/assembly/local-assembly-e2e.yaml @echo "local assembly e2e test passed." @# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator) @cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - @make delete-local-cluster .PHONY: e2e-tests-assembly-local -e2e-tests-assembly-distributed: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster +e2e-tests-assembly-distributed: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster @echo "Running distributed assembly e2e test..." @cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/assembly/distributed-assembly-e2e.yaml @echo "distributed assembly e2e test passed." @# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator) @cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - @make delete-local-cluster .PHONY: e2e-tests-assembly-distributed -e2e-tests-autogenerated-helm-chart: prepare-e2e-test build-local-cluster install-cert-manager +e2e-tests-autogenerated-helm-chart: prepare-e2e-test install-cert-manager @echo "Running autogenerated helm chart e2e test..." @cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/autogenerated-helm-chart/e2e.yaml @echo "autogenerated helm chart e2e test passed." @# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator) @cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - @make delete-local-cluster .PHONY: e2e-tests-autogenerated-helm-chart -e2e-tests-failover: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster +e2e-tests-failover: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster @echo "Running failover e2e test..." @cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/failover/e2e.yaml @echo "failover e2e test passed." @# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator) @cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - @make delete-local-cluster .PHONY: e2e-tests-failover -e2e-tests-repartition-dask: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster +e2e-tests-repartition-dask: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster @echo "Running repartition dask e2e test..." @cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/repartition/dask-repartition-e2e.yaml @echo "repartition dask e2e test passed." @# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator) @cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - @make delete-local-cluster .PHONY: e2e-tests-repartition-dask -e2e-tests-scheduler-outside-cluster: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster +e2e-tests-scheduler-outside-cluster: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster @echo "Running scheduler outside cluster e2e test..." @cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/scheduling-outside-cluster/e2e.yaml @echo "scheduler outside cluster e2e test passed." @# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator) @cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - @make delete-local-cluster .PHONY: e2e-tests-scheduler-outside-cluster -e2e-tests-serialize: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster +e2e-tests-serialize: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster @echo "Running serialize e2e test..." @cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/serialize/e2e.yaml @echo "serialize e2e test passed." @# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator) @cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - @make delete-local-cluster .PHONY: e2e-tests-serialize -e2e-tests-sidecar: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator +e2e-tests-sidecar: prepare-e2e-test install-cert-manager install-vineyard-operator @echo "Running sidecar e2e test..." @cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/sidecar/e2e.yaml @echo "sidecar e2e test passed." @# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator) @cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - @make delete-local-cluster .PHONY: e2e-tests-sidecar -e2e-tests-spill: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator +e2e-tests-spill: prepare-e2e-test install-cert-manager install-vineyard-operator @echo "Running spill e2e test..." @cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/spill/e2e.yaml @echo "spill e2e test passed." @# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator) @cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - @make delete-local-cluster .PHONY: e2e-tests-spill -e2e-tests-workflow: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster +e2e-tests-workflow: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster @echo "Running workflow e2e test..." @cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/workflow/e2e.yaml @echo "workflow e2e test passed." @# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator) @cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - @make delete-local-cluster .PHONY: e2e-tests-workflow # Build manager binary diff --git a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go index 38ecca866..fe7431195 100644 --- a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go +++ b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go @@ -421,6 +421,21 @@ func (in *OperationStatus) DeepCopy() *OperationStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PluginImageConfig) DeepCopyInto(out *PluginImageConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PluginImageConfig. +func (in *PluginImageConfig) DeepCopy() *PluginImageConfig { + if in == nil { + return nil + } + out := new(PluginImageConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Recover) DeepCopyInto(out *Recover) { *out = *in @@ -730,6 +745,7 @@ func (in *VineyarddSpec) DeepCopyInto(out *VineyarddSpec) { out.Service = in.Service out.Etcd = in.Etcd in.VineyardConfig.DeepCopyInto(&out.VineyardConfig) + out.PluginConfig = in.PluginConfig out.MetricConfig = in.MetricConfig out.Volume = in.Volume } diff --git a/k8s/hack/k3d.yaml b/k8s/hack/k3d.yaml new file mode 100644 index 000000000..d7d8a0f81 --- /dev/null +++ b/k8s/hack/k3d.yaml @@ -0,0 +1,30 @@ +apiVersion: k3d.io/v1alpha3 +kind: Simple +name: cluster +servers: 1 +agents: 3 +image: rancher/k3s:latest +volumes: # repeatable flags are represented as YAML lists + - volume: /var/run/docker.sock:/var/run/docker.sock + nodeFilters: + - server:* + - agent:* +options: + k3d: + wait: true + timeout: "600s" + disableLoadbalancer: false + disableImageVolume: false + disableRollback: false + loadbalancer: + configOverrides: + - settings.workerConnections=2048 + k3s: + extraArgs: + - arg: --docker + nodeFilters: + - server:* + - agent:* + kubeconfig: + updateDefaultKubeconfig: true + switchCurrentContext: true diff --git a/k8s/test/e2e/Makefile b/k8s/test/e2e/Makefile index 4d5ac9fd6..6c4e22eb9 100644 --- a/k8s/test/e2e/Makefile +++ b/k8s/test/e2e/Makefile @@ -15,7 +15,6 @@ push-%: % local-assembly-images: local-assembly assembly-job1 assembly-job2 publish-local-assembly-images: push-local-assembly push-assembly-job1 push-assembly-job2 -e2e-test-assembly: push-local-assembly push-assembly-job1 push-assembly-job2 local-assembly: docker build assembly-demo/ -f Dockerfile \ @@ -39,7 +38,6 @@ assembly-job2: distributed-assembly-images: distributed-assembly distributed-assembly-job1 distributed-assembly-job2 distributed-assembly-job3 publish-distributed-assembly-images: push-distributed-assembly push-distributed-assembly-job1 push-distributed-assembly-job2 push-distributed-assembly-job3 -e2e-test-distributed-assembly: push-local-assembly push-distributed-assembly push-distributed-assembly-job1 push-distributed-assembly-job2 push-distributed-assembly-job3 distributed-assembly: docker build assembly-demo/ -f Dockerfile \ @@ -71,7 +69,6 @@ distributed-assembly-job3: serialize-images: serialize-job publish-serialize-images: push-serialize-job -e2e-test-serialize: push-serialize-job serialize-job: docker build . -f Dockerfile \ @@ -83,7 +80,6 @@ serialize-job: failover-images: backup-job recover-job build-local-object build-distributed-object-step1 build-distributed-object-step2 get-local-object get-distributed-object publish-failover-images: push-backup-job push-recover-job push-build-local-object push-build-distributed-object-step1 push-build-distributed-object-step2 push-get-local-object push-get-distributed-object -e2e-test-failover: push-backup-job push-recover-job push-build-local-object push-build-distributed-object-step1 push-build-distributed-object-step2 push-get-local-object push-get-distributed-object backup-job: docker build failover-demo/ -f Dockerfile \ @@ -131,7 +127,6 @@ get-distributed-object: repartition-images: dask-repartition dask-repartition-job1 dask-repartition-job2 dask-worker-with-vineyard publish-repartition-images: push-dask-repartition push-dask-repartition-job1 push-dask-repartition-job2 push-dask-worker-with-vineyard -e2e-test-repartition: push-dask-repartition push-dask-repartition-job1 push-dask-repartition-job2 push-dask-worker-with-vineyard dask-repartition: docker build repartition-demo/ -f Dockerfile \ @@ -163,7 +158,6 @@ dask-worker-with-vineyard: sidecar-images: sidecar-job publish-sidecar-images: push-sidecar-job -e2e-test-sidecar: push-sidecar-job sidecar-job: docker build sidecar-demo/ -f Dockerfile \ @@ -175,7 +169,6 @@ sidecar-job: workflow-images: workflow-job1 workflow-job2 publish-workflow-images: push-workflow-job1 push-workflow-job2 -e2e-test-workflow: push-workflow-job1 push-workflow-job2 workflow-job1: docker build workflow-demo/ -f Dockerfile \ @@ -189,15 +182,13 @@ workflow-job2: -t $(REGISTRY)/workflow-job2:$(TAG) .PHONY: workflow-job2 -# publish all images to ghcr registry -ALL-IMAGES ?= \ -publish-local-assembly-images \ -publish-distributed-assembly-images \ -publish-serialize-images \ -publish-failover-images \ -publish-repartition-images \ -publish-sidecar-images \ -publish-workflow-images - -publish-all-images: $(ALL-IMAGES) -.PHONY: publish-all-images \ No newline at end of file +############# publish all images to ghcr registry ############################# + +publish-all-images: publish-local-assembly-images \ + publish-distributed-assembly-images \ + publish-serialize-images \ + publish-failover-images \ + publish-repartition-images \ + publish-sidecar-images \ + publish-workflow-images +.PHONY: publish-all-images diff --git a/k8s/test/e2e/assembly-demo/assembly-job-config.yaml b/k8s/test/e2e/assembly-demo/assembly-job-config.yaml index ec5b0112c..a6c0434e7 100644 --- a/k8s/test/e2e/assembly-demo/assembly-job-config.yaml +++ b/k8s/test/e2e/assembly-demo/assembly-job-config.yaml @@ -44,7 +44,7 @@ distributedJob1: jobName: none dataName: "" schedulerName: "" - nodeName: kind-worker + nodeName: k3d-cluster-agent-0 distributedJob2: @@ -56,7 +56,7 @@ distributedJob2: jobName: none dataName: "" schedulerName: "" - nodeName: kind-worker2 + nodeName: k3d-cluster-agent-1 distributedJob3: jobName: distributed-assembly-job3 diff --git a/k8s/test/e2e/assembly-demo/assembly-job.yaml b/k8s/test/e2e/assembly-demo/assembly-job.yaml index 416de2d25..da6358dce 100644 --- a/k8s/test/e2e/assembly-demo/assembly-job.yaml +++ b/k8s/test/e2e/assembly-demo/assembly-job.yaml @@ -49,7 +49,7 @@ spec: {{- end }} containers: - name: {{ (datasource "config").$job.jobName }} - image: localhost:5001/{{ (datasource "config").$job.jobName }} + image: ghcr.io/v6d-io/v6d/{{ (datasource "config").$job.jobName }} env: - name: JOB_NAME value: {{ (datasource "config").$job.jobName }} diff --git a/k8s/test/e2e/assembly/distributed-assembly-e2e.yaml b/k8s/test/e2e/assembly/distributed-assembly-e2e.yaml index eb3f60bf3..56bac91dc 100644 --- a/k8s/test/e2e/assembly/distributed-assembly-e2e.yaml +++ b/k8s/test/e2e/assembly/distributed-assembly-e2e.yaml @@ -16,11 +16,11 @@ # Test assembly operation in different node setup: env: kind - kubeconfig: /tmp/e2e-k8s.config + kubeconfig: ~/.kube/config steps: - name: download all distributed assembly images into kind cluster command: | - make -C k8s/test/e2e e2e-test-distributed-assembly REGISTRY=localhost:5001 + make -C k8s/test/e2e distributed-assembly-images - name: install job1 and job2 command: | kubectl create namespace vineyard-job diff --git a/k8s/test/e2e/assembly/local-assembly-e2e.yaml b/k8s/test/e2e/assembly/local-assembly-e2e.yaml index 45396cb75..33fb2d381 100644 --- a/k8s/test/e2e/assembly/local-assembly-e2e.yaml +++ b/k8s/test/e2e/assembly/local-assembly-e2e.yaml @@ -16,11 +16,11 @@ # Test assembly operation in the same node setup: env: kind - kubeconfig: /tmp/e2e-k8s.config + kubeconfig: ~/.kube/config steps: - name: download all local assembly images into kind cluster command: | - make -C k8s/test/e2e e2e-test-assembly REGISTRY=localhost:5001 + make -C k8s/test/e2e local-assembly-images - name: install job1 command: | kubectl create namespace vineyard-job diff --git a/k8s/test/e2e/autogenerated-helm-chart/e2e.yaml b/k8s/test/e2e/autogenerated-helm-chart/e2e.yaml index 1db48d1a6..ee1f761b0 100644 --- a/k8s/test/e2e/autogenerated-helm-chart/e2e.yaml +++ b/k8s/test/e2e/autogenerated-helm-chart/e2e.yaml @@ -15,17 +15,12 @@ setup: env: kind - kubeconfig: /tmp/e2e-k8s.config + kubeconfig: ~/.kube/config steps: - - name: download the vineyard operator image - command: | - docker tag vineyardcloudnative/vineyard-operator:latest localhost:5001/vineyard-operator:latest - docker push localhost:5001/vineyard-operator:latest - name: use helm chart to install the operator command: | helm install vineyard-operator charts/vineyard-operator \ --namespace vineyard-system \ - --set controllerManager.manager.image.repository=localhost:5001/vineyard-operator \ --create-namespace wait: - namespace: vineyard-system @@ -33,7 +28,7 @@ setup: for: condition=Available - name: download the local assembly images into kind cluster command: | - make -C k8s/test/e2e e2e-test-assembly REGISTRY=localhost:5001 + make -C k8s/test/e2e local-assembly-images - name: install vineyardd command: | kubectl apply -f k8s/test/e2e/vineyardd.yaml diff --git a/k8s/test/e2e/failover-demo/failover-job-config.yaml b/k8s/test/e2e/failover-demo/failover-job-config.yaml index a796a533c..75e5ec356 100644 --- a/k8s/test/e2e/failover-demo/failover-job-config.yaml +++ b/k8s/test/e2e/failover-demo/failover-job-config.yaml @@ -15,20 +15,20 @@ localObject: jobName: build-local-object - nodeName: "kind-worker2" + nodeName: "k3d-cluster-agent-1" distributedObject1: jobName: build-distributed-object-step1 - nodeName: "kind-worker" + nodeName: "k3d-cluster-agent-0" distributedObject2: jobName: build-distributed-object-step2 - nodeName: "kind-worker3" + nodeName: "k3d-cluster-agent-2" getLocalObject: jobName: get-local-object - nodeName: "kind-worker2" + nodeName: "k3d-cluster-agent-1" getDistributedObject: jobName: get-distributed-object - nodeName: "kind-worker2" + nodeName: "k3d-cluster-agent-1" diff --git a/k8s/test/e2e/failover-demo/failover-job.yaml b/k8s/test/e2e/failover-demo/failover-job.yaml index 3d5081335..6c5d74bae 100644 --- a/k8s/test/e2e/failover-demo/failover-job.yaml +++ b/k8s/test/e2e/failover-demo/failover-job.yaml @@ -34,7 +34,7 @@ spec: {{- end }} containers: - name: {{ (datasource "config").$job.jobName }} - image: localhost:5001/{{ (datasource "config").$job.jobName }} + image: ghcr.io/v6d-io/v6d/{{ (datasource "config").$job.jobName }} env: - name: JOB_NAME value: {{ (datasource "config").$job.jobName }} diff --git a/k8s/test/e2e/failover/e2e.yaml b/k8s/test/e2e/failover/e2e.yaml index 8fb31b94a..5386c9255 100644 --- a/k8s/test/e2e/failover/e2e.yaml +++ b/k8s/test/e2e/failover/e2e.yaml @@ -15,13 +15,13 @@ setup: env: kind - kubeconfig: /tmp/e2e-k8s.config + kubeconfig: ~/.kube/config steps: - name: setup the backup path - command: worker=($(docker ps | grep kind-worker | awk -F ' ' '{print $1}')); for c in ${worker[@]}; do docker exec $c mkdir -p /var/vineyard/dump; done + command: worker=($(docker ps | grep k3d-cluster-agent | awk -F ' ' '{print $1}')); for c in ${worker[@]}; do docker exec $c mkdir -p /var/vineyard/dump; done - name: download all failover images into kind cluster command: | - make -C k8s/test/e2e e2e-test-failover REGISTRY=localhost:5001 + make -C k8s/test/e2e failover-images - name: install local-object command: | kubectl create namespace vineyard-job diff --git a/k8s/test/e2e/kind-with-local-registry.yaml b/k8s/test/e2e/kind-with-local-registry.yaml deleted file mode 100644 index 037ecbdf1..000000000 --- a/k8s/test/e2e/kind-with-local-registry.yaml +++ /dev/null @@ -1,15 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -containerdConfigPatches: -- |- - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5001"] - endpoint = ["http://kind-registry:5000"] -nodes: -- role: control-plane - image: kindest/node:v1.24.0 -- role: worker - image: kindest/node:v1.24.0 -- role: worker - image: kindest/node:v1.24.0 -- role: worker - image: kindest/node:v1.24.0 \ No newline at end of file diff --git a/k8s/test/e2e/kind.yaml b/k8s/test/e2e/kind.yaml deleted file mode 100644 index 86c5f1433..000000000 --- a/k8s/test/e2e/kind.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2020-2023 Alibaba Group Holding Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane - image: kindest/node:v1.24.0 -- role: worker - image: kindest/node:v1.24.0 -- role: worker - image: kindest/node:v1.24.0 -- role: worker - image: kindest/node:v1.24.0 diff --git a/k8s/test/e2e/repartition-demo/repartition-dask-helm-values.yaml b/k8s/test/e2e/repartition-demo/repartition-dask-helm-values.yaml index d7580dba6..8fa3cc526 100644 --- a/k8s/test/e2e/repartition-demo/repartition-dask-helm-values.yaml +++ b/k8s/test/e2e/repartition-demo/repartition-dask-helm-values.yaml @@ -15,7 +15,6 @@ scheduler: image: - repository: localhost:5001/dask tag: "2022.8.1" jupyter: @@ -25,7 +24,7 @@ worker: # worker numbers replicas: 3 image: - repository: localhost:5001/dask-worker-with-vineyard + repository: ghcr.io/v6d-io/v6d/dask-worker-with-vineyard tag: latest env: - name: VINEYARD_IPC_SOCKET diff --git a/k8s/test/e2e/repartition-demo/repartition-job.yaml b/k8s/test/e2e/repartition-demo/repartition-job.yaml index 56771891f..f64eb9f6b 100644 --- a/k8s/test/e2e/repartition-demo/repartition-job.yaml +++ b/k8s/test/e2e/repartition-demo/repartition-job.yaml @@ -52,7 +52,7 @@ spec: schedulerName: vineyard-scheduler containers: - name: {{ (datasource "config").$job.jobName }} - image: localhost:5001/{{ (datasource "config").$job.jobName }} + image: ghcr.io/v6d-io/v6d/{{ (datasource "config").$job.jobName }} imagePullPolicy: IfNotPresent env: - name: JOB_NAME diff --git a/k8s/test/e2e/repartition/dask-repartition-e2e.yaml b/k8s/test/e2e/repartition/dask-repartition-e2e.yaml index ca1732b7a..dd2f0bd78 100644 --- a/k8s/test/e2e/repartition/dask-repartition-e2e.yaml +++ b/k8s/test/e2e/repartition/dask-repartition-e2e.yaml @@ -15,16 +15,11 @@ setup: env: kind - kubeconfig: /tmp/e2e-k8s.config + kubeconfig: ~/.kube/config steps: - - name: download dask image for syncing the dask worker and scheduler. - command: | - docker pull ghcr.io/dask/dask:2022.8.1 - docker tag ghcr.io/dask/dask:2022.8.1 localhost:5001/dask:2022.8.1 - docker push localhost:5001/dask:2022.8.1 - name: download all dask repartition images into kind cluster command: | - make -C k8s/test/e2e e2e-test-repartition REGISTRY=localhost:5001 + make -C k8s/test/e2e repartition-images - name: install dask-scheduler and dask-worker command: | helm repo add dask https://helm.dask.org/ diff --git a/k8s/test/e2e/scheduling-outside-cluster/e2e.yaml b/k8s/test/e2e/scheduling-outside-cluster/e2e.yaml index 2c679eaad..57a886744 100644 --- a/k8s/test/e2e/scheduling-outside-cluster/e2e.yaml +++ b/k8s/test/e2e/scheduling-outside-cluster/e2e.yaml @@ -15,12 +15,15 @@ setup: env: kind - kubeconfig: /tmp/e2e-k8s.config + kubeconfig: ~/.kube/config steps: - name: scheduling workload command: | kubectl create ns vineyard-job - go run k8s/cmd/main.go k8s/cmd/scheduler.go /tmp/e2e-k8s.config k8s/test/e2e/scheduling-outside-cluster-demo/test.yaml + go run k8s/cmd/main.go \ + k8s/cmd/scheduler.go \ + ~/.kube/config \ + k8s/test/e2e/scheduling-outside-cluster-demo/test.yaml - name: wait job completed command: | kubectl wait --for=condition=complete --timeout=5m job/v6d-workflow-demo-job2 -n vineyard-job diff --git a/k8s/test/e2e/serialize-demo/serialize-demo.yaml b/k8s/test/e2e/serialize-demo/serialize-demo.yaml index ac1d874f8..05cbaaa89 100644 --- a/k8s/test/e2e/serialize-demo/serialize-demo.yaml +++ b/k8s/test/e2e/serialize-demo/serialize-demo.yaml @@ -39,7 +39,7 @@ spec: schedulerName: vineyard-scheduler containers: - name: serialize-demo - image: localhost:5001/serialize-job + image: ghcr.io/v6d-io/v6d/serialize-job imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/run diff --git a/k8s/test/e2e/serialize/e2e.yaml b/k8s/test/e2e/serialize/e2e.yaml index ff667e3db..390ae4650 100644 --- a/k8s/test/e2e/serialize/e2e.yaml +++ b/k8s/test/e2e/serialize/e2e.yaml @@ -15,13 +15,13 @@ setup: env: kind - kubeconfig: /tmp/e2e-k8s.config + kubeconfig: ~/.kube/config steps: - name: setup the serialize path - command: worker=($(docker ps | grep kind-worker | awk -F ' ' '{print $1}')); for c in ${worker[@]}; do docker exec $c mkdir -p /var/vineyard/serialize; done + command: worker=($(docker ps | grep k3d-cluster-agent | awk -F ' ' '{print $1}')); for c in ${worker[@]}; do docker exec $c mkdir -p /var/vineyard/serialize; done - name: download all serialize images into kind cluster command: | - make -C k8s/test/e2e e2e-test-serialize REGISTRY=localhost:5001 + make -C k8s/test/e2e serialize-images - name: install serialize-demo command: | kubectl apply -f k8s/test/e2e/serialize-demo/serialize-pv.yaml diff --git a/k8s/test/e2e/sidecar-demo/sidecar-with-custom-sidecar.yaml b/k8s/test/e2e/sidecar-demo/sidecar-with-custom-sidecar.yaml index b8d871eca..10721f70f 100644 --- a/k8s/test/e2e/sidecar-demo/sidecar-with-custom-sidecar.yaml +++ b/k8s/test/e2e/sidecar-demo/sidecar-with-custom-sidecar.yaml @@ -44,7 +44,7 @@ spec: spec: containers: - name: job - image: localhost:5001/sidecar-job + image: ghcr.io/v6d-io/v6d/sidecar-job imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c", "python3 /job.py"] env: diff --git a/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml b/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml index eccbdd39d..eb3c5eadc 100644 --- a/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml +++ b/k8s/test/e2e/sidecar-demo/sidecar-with-default-sidecar.yaml @@ -33,7 +33,7 @@ spec: spec: containers: - name: job - image: localhost:5001/sidecar-job + image: ghcr.io/v6d-io/v6d/sidecar-job imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c", "python3 /job.py"] env: diff --git a/k8s/test/e2e/sidecar/e2e.yaml b/k8s/test/e2e/sidecar/e2e.yaml index 6d924d171..dc8c9a68f 100644 --- a/k8s/test/e2e/sidecar/e2e.yaml +++ b/k8s/test/e2e/sidecar/e2e.yaml @@ -16,11 +16,11 @@ # Test assembly operation in the same node setup: env: kind - kubeconfig: /tmp/e2e-k8s.config + kubeconfig: ~/.kube/config steps: - name: download all sidecar images into kind cluster command: | - make -C k8s/test/e2e e2e-test-sidecar REGISTRY=localhost:5001 + make -C k8s/test/e2e sidecar-images - name: install app with default sidecar command: | kubectl create namespace vineyard-job diff --git a/k8s/test/e2e/spill/e2e.yaml b/k8s/test/e2e/spill/e2e.yaml index 80c8f269f..d441d9922 100644 --- a/k8s/test/e2e/spill/e2e.yaml +++ b/k8s/test/e2e/spill/e2e.yaml @@ -15,10 +15,10 @@ setup: env: kind - kubeconfig: /tmp/e2e-k8s.config + kubeconfig: ~/.kube/config steps: - name: setup the serialize path - command: worker=($(docker ps | grep kind-worker | awk -F ' ' '{print $1}')); for c in ${worker[@]}; do docker exec $c mkdir -p /var/vineyard/spill; done + command: worker=($(docker ps | grep k3d-cluster-agent | awk -F ' ' '{print $1}')); for c in ${worker[@]}; do docker exec $c mkdir -p /var/vineyard/spill; done - name: install vineyardd command: | kubectl apply -f k8s/test/e2e/spill-demo/vineyardd-with-spill.yaml diff --git a/k8s/test/e2e/vineyardd.yaml b/k8s/test/e2e/vineyardd.yaml index a219a6a73..e6bbd3c0c 100644 --- a/k8s/test/e2e/vineyardd.yaml +++ b/k8s/test/e2e/vineyardd.yaml @@ -27,13 +27,13 @@ spec: type: ClusterIP port: 9600 vineyardConfig: - image: localhost:5001/vineyardd:alpine-latest + image: vineyardcloudnative/vineyardd:alpine-latest imagePullPolicy: IfNotPresent # Users can define their own plugin image here, # the next images are only for kubernetes CI test. pluginConfig: - backupImage: localhost:5001/backup-job - recoverImage: localhost:5001/recover-job - daskRepartitionImage: localhost:5001/dask-repartition - localAssemblyImage: localhost:5001/local-assembly - distributedAssemblyImage: localhost:5001/distributed-assembly \ No newline at end of file + backupImage: ghcr.io/v6d-io/v6d/backup-job + recoverImage: ghcr.io/v6d-io/v6d/recover-job + daskRepartitionImage: ghcr.io/v6d-io/v6d/dask-repartition + localAssemblyImage: ghcr.io/v6d-io/v6d/local-assembly + distributedAssemblyImage: ghcr.io/v6d-io/v6d/distributed-assembly diff --git a/k8s/test/e2e/workflow-demo/workflow-job1.yaml b/k8s/test/e2e/workflow-demo/workflow-job1.yaml index b59a12620..b45cb501d 100644 --- a/k8s/test/e2e/workflow-demo/workflow-job1.yaml +++ b/k8s/test/e2e/workflow-demo/workflow-job1.yaml @@ -37,7 +37,7 @@ spec: schedulerName: vineyard-scheduler containers: - name: job1 - image: localhost:5001/workflow-job1 + image: ghcr.io/v6d-io/v6d/workflow-job1 env: - name: JOB_NAME value: v6d-workflow-demo-job1 diff --git a/k8s/test/e2e/workflow-demo/workflow-job2.yaml b/k8s/test/e2e/workflow-demo/workflow-job2.yaml index e18b6170c..8ad0d9145 100644 --- a/k8s/test/e2e/workflow-demo/workflow-job2.yaml +++ b/k8s/test/e2e/workflow-demo/workflow-job2.yaml @@ -38,7 +38,7 @@ spec: schedulerName: vineyard-scheduler containers: - name: job2 - image: localhost:5001/workflow-job2 + image: ghcr.io/v6d-io/v6d/workflow-job2 imagePullPolicy: IfNotPresent env: - name: JOB_NAME diff --git a/k8s/test/e2e/workflow/e2e.yaml b/k8s/test/e2e/workflow/e2e.yaml index cd33ceedc..bdeeb9fc8 100644 --- a/k8s/test/e2e/workflow/e2e.yaml +++ b/k8s/test/e2e/workflow/e2e.yaml @@ -15,11 +15,11 @@ setup: env: kind - kubeconfig: /tmp/e2e-k8s.config + kubeconfig: ~/.kube/config steps: - name: download all workflow images into kind cluster command: | - make -C k8s/test/e2e e2e-test-workflow REGISTRY=localhost:5001 + make -C k8s/test/e2e workflow-images - name: install job1 command: | kubectl create namespace vineyard-job diff --git a/k8s/test/hack/build-kind-cluster-with-local-registry.sh b/k8s/test/hack/build-kind-cluster-with-local-registry.sh deleted file mode 100644 index 8380fe7da..000000000 --- a/k8s/test/hack/build-kind-cluster-with-local-registry.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -set -o errexit - -kind_name=kind -kubeconfig_path=/tmp/e2e-k8s.config - -if [ ! -z "$1" ] ; then - kind_name=$1 -fi -# create registry container unless it already exists -reg_name='kind-registry' -reg_port='5001' -if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then - docker run \ - -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \ - registry:2 -fi - -# create a cluster with the local registry enabled in containerd -cat </dev/null || true)" == 'true' ]; then - docker stop ${reg_name} - docker rm ${reg_name} -fi - -# delete the kind cluster -kind delete cluster --name=${kind_name} -