From 2205a044084cc313022083005772ff56415160da Mon Sep 17 00:00:00 2001 From: lou-lan Date: Fri, 26 Apr 2024 11:57:27 +0800 Subject: [PATCH] enhance e2e for restart api-server case Signed-off-by: lou-lan --- test/Makefile | 53 ++++++++++++-------- test/e2e/common/ds.go | 32 +++++++++++- test/e2e/common/egw.go | 12 ++++- test/e2e/common/log.go | 44 ++++++++++++++++ test/e2e/egressgateway/egressgateway_test.go | 4 +- test/e2e/egresspolicy/egresspolicy_test.go | 6 +-- test/e2e/reliability/reliability_test.go | 2 +- 7 files changed, 124 insertions(+), 29 deletions(-) create mode 100644 test/e2e/common/log.go diff --git a/test/Makefile b/test/Makefile index 1690e4df3..1b92bf631 100644 --- a/test/Makefile +++ b/test/Makefile @@ -261,26 +261,37 @@ deploy_project: done ; \ fi - helm --kubeconfig=$(KIND_KUBECONFIG) uninstall -n $(E2E_NAMESPACE) project || true - HELM_OPTION="" ; \ - if [ -n "$(PYROSCOPE_LOCAL_PORT)" ] ; then \ - echo "add env" ; \ - ServerAddress=$$(docker network inspect kind -f {{\(index\ $$.IPAM.Config\ 0\).Gateway}}) ; \ - HELM_OPTION+=" --set agent.debug.pyroscopeServerAddr=http://$${ServerAddress}:$(PYROSCOPE_LOCAL_PORT) " ; \ - HELM_OPTION+=" --set controller.debug.pyroscopeServerAddr=http://$${ServerAddress}:$(PYROSCOPE_LOCAL_PORT) " ; \ - fi ; \ - HELM_OPTION+=" --set agent.prometheus.enabled=true --set controller.prometheus.enabled=true " ; \ - HELM_OPTION+=" --set agent.debug.logLevel=debug --set controller.debug.logLevel=debug " ; \ - HELM_OPTION+=" --set controller.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key=node-role.kubernetes.io/control-plane " ; \ - HELM_OPTION+=" --set controller.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator=Exists " ; \ - if [ "$(E2E_IP_FAMILY)" == "dual" ] ; then HELM_OPTION+=" --set feature.enableIPv4=true --set feature.enableIPv6=true" ; fi ; \ - if [ "$(E2E_IP_FAMILY)" == "ipv4" ] ; then HELM_OPTION+=" --set feature.enableIPv4=true --set feature.enableIPv6=false" ; fi ; \ - if [ "$(E2E_IP_FAMILY)" == "ipv6" ] ; then HELM_OPTION+=" --set feature.enableIPv4=false --set feature.enableIPv6=true" ; fi ; \ - helm --kubeconfig=$(KIND_KUBECONFIG) install project $(ROOT_DIR)/charts \ - -n $(E2E_NAMESPACE) --create-namespace --wait --debug \ - --set global.imageTagOverride=$(PROJECT_IMAGE_VERSION) \ - $${HELM_OPTION} \ - || { KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) ./scripts/debugCluster.sh $(KIND_KUBECONFIG) "detail" $(E2E_NAMESPACE) ; exit 1 ; } ; \ - exit 0 + @echo "" > config.yaml + @if [ -z '$(PYROSCOPE_LOCAL_PORT)' ]; then \ + echo "PYROSCOPE_LOCAL_PORT is not set. Skipping setting pyroscopeServerAddr."; \ + else \ + yq e '.agent.debug.pyroscopeServerAddr = "http://${ServerAddress}:$(PYROSCOPE_LOCAL_PORT)"' -i config.yaml; \ + yq e '.controller.debug.pyroscopeServerAddr = "http://${ServerAddress}:$(PYROSCOPE_LOCAL_PORT)"' -i config.yaml; \ + fi + yq e '.agent.prometheus.enabled = "true"' -i config.yaml + yq e '.controller.prometheus.enabled = "true"' -i config.yaml + yq e '.agent.debug.logLevel = "debug"' -i config.yaml + yq e '.controller.debug.logLevel = "debug"' -i config.yaml + @if [ "$(E2E_IP_FAMILY)" = "dual" ]; then \ + yq e '.feature.enableIPv4 = true' -i config.yaml; \ + yq e '.feature.enableIPv6 = true' -i config.yaml; \ + elif [ "$(E2E_IP_FAMILY)" = "ipv4" ]; then \ + yq e '.feature.enableIPv4 = true' -i config.yaml; \ + yq e '.feature.enableIPv6 = false' -i config.yaml; \ + elif [ "$(E2E_IP_FAMILY)" = "ipv6" ]; then \ + yq e '.feature.enableIPv4 = false' -i config.yaml; \ + yq e '.feature.enableIPv6 = true' -i config.yaml; \ + else \ + echo "Unknown IP family $(E2E_IP_FAMILY)"; \ + exit 1; \ + fi + yq e '.controller.nodeSelector."kubernetes.io/hostname" = "$(KIND_CLUSTER_NAME)-control-plane"' -i config.yaml + HELM_OPTION+=" --set controller.nodeSelector."kubernetes.io/hostname"=$(KIND_CLUSTER_NAME)-control-plane" ; \ + helm --kubeconfig=$(KIND_KUBECONFIG) install project $(ROOT_DIR)/charts \ + -n $(E2E_NAMESPACE) --create-namespace --wait --debug \ + --set global.imageTagOverride=$(PROJECT_IMAGE_VERSION) --values config.yaml \ + || { KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) ./scripts/debugCluster.sh $(KIND_KUBECONFIG) "detail" $(E2E_NAMESPACE) ; exit 1 ; } ; \ + exit 0 .PHONY: uninstall_egress uninstall_egress: KIND_KUBECONFIG ?= $(E2E_KIND_KUBECONFIG_PATH) @@ -289,7 +300,7 @@ uninstall_egress: @echo -e "\033[35m [helm uninstall egress] \033[0m" helm uninstall project --wait --debug -n $(E2E_NAMESPACE) \ --kubeconfig=$(KIND_KUBECONFIG) || { $(KIND_CLUSTER_NAME) ./scripts/debugCluster.sh $(KIND_KUBECONFIG) "detail" $(E2E_NAMESPACE) ; exit 1 ; } ; \ - kubectl --kubeconfig=$(KIND_KUBECONFIG) get all --all-namespaces | grep -q "egressgateway"` && { echo "error: found egressgateway resources" ; exit 1 ; } + kubectl --kubeconfig=$(KIND_KUBECONFIG) get all --all-namespaces | grep -q "egressgateway" && { echo "error: found egressgateway resources" ; exit 1 ; } # test kind is ok .PHONY: install_example_app diff --git a/test/e2e/common/ds.go b/test/e2e/common/ds.go index 57b0b8402..d8a7fb930 100644 --- a/test/e2e/common/ds.go +++ b/test/e2e/common/ds.go @@ -5,6 +5,7 @@ package common import ( "context" + "encoding/json" "fmt" "time" @@ -44,6 +45,8 @@ func CreateDaemonSet(ctx context.Context, cli client.Client, name string, image }, } + log := NewLogger() + err := cli.Create(ctx, res) if err != nil { return nil, err @@ -53,7 +56,8 @@ func CreateDaemonSet(ctx context.Context, cli client.Client, name string, image select { case <-ctx.Done(): _ = DeleteObj(context.Background(), cli, res) - return nil, fmt.Errorf("create DaemonSet time out") + log.Log("create DaemonSet time out") + return nil, fmt.Errorf(log.Save()) default: err := cli.Get(ctx, types.NamespacedName{Namespace: res.Namespace, Name: res.Name}, res) if err != nil { @@ -68,6 +72,32 @@ func CreateDaemonSet(ctx context.Context, cli client.Client, name string, image return res, nil } + log.Log("--------") + log.Log(fmt.Sprintf("CurrentNumberScheduled=%v\nDesiredNumberScheduled=%v\nNumberAvailable=%v", a, b, c)) + + nodes := new(corev1.NodeList) + err = cli.List(ctx, nodes) + if err != nil { + return nil, err + } + for _, node := range nodes.Items { + t := "node " + node.Name + " --- " + for _, condition := range node.Status.Conditions { + t = t + fmt.Sprintf("%v=%v ", condition.Type, condition.Status) + } + log.Log(t) + } + pods := new(corev1.PodList) + err = cli.List(ctx, pods) + if err != nil { + return nil, err + } + for _, pod := range pods.Items { + t := "pod " + pod.Name + " --- " + raw, _ := json.Marshal(pod.Status) + t += string(raw) + log.Log(t) + } time.Sleep(time.Second / 2) } } diff --git a/test/e2e/common/egw.go b/test/e2e/common/egw.go index 8565fea2d..129d0be18 100644 --- a/test/e2e/common/egw.go +++ b/test/e2e/common/egw.go @@ -5,6 +5,7 @@ package common import ( "context" + "encoding/json" "fmt" "reflect" "sort" @@ -270,15 +271,24 @@ func DeleteEgressGateway(ctx context.Context, cli client.Client, egw *egressv1.E return err } + log := NewLogger() + for { select { case <-ctx.Done(): - return e2eerr.ErrTimeout + log.Log("check delete egress gateway object timeout") + return fmt.Errorf(log.Save()) default: err = cli.Get(ctx, types.NamespacedName{Name: egw.Name}, egw) if apierrors.IsNotFound(err) { return nil } + if err != nil { + log.Log("--------------------------") + log.Log(err.Error()) + } + raw, _ := json.Marshal(egw) + log.Log(string(raw)) time.Sleep(time.Second / 2) } } diff --git a/test/e2e/common/log.go b/test/e2e/common/log.go new file mode 100644 index 000000000..c35f9fe03 --- /dev/null +++ b/test/e2e/common/log.go @@ -0,0 +1,44 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package common + +import ( + "fmt" + "strings" + "time" +) + +type Logger struct { + logs []string + startTime time.Time + endTime time.Time +} + +func NewLogger() *Logger { + return &Logger{ + startTime: time.Now(), + } +} + +func (l *Logger) Log(message string) { + l.logs = append(l.logs, message) +} + +func (l *Logger) Save() string { + endTime := time.Now() + duration := endTime.Sub(l.startTime) + + var sb strings.Builder + + // Append all log entries + for _, log := range l.logs { + sb.WriteString(log) + sb.WriteString("\n") + } + + // Append the total time spent in seconds + sb.WriteString(fmt.Sprintf("Total duration: %.2f seconds\n", duration.Seconds())) + + return sb.String() +} diff --git a/test/e2e/egressgateway/egressgateway_test.go b/test/e2e/egressgateway/egressgateway_test.go index 11a069aa4..51f8c2673 100644 --- a/test/e2e/egressgateway/egressgateway_test.go +++ b/test/e2e/egressgateway/egressgateway_test.go @@ -179,7 +179,7 @@ var _ = Describe("Operate EgressGateway", Label("EgressGateway"), Ordered, func( ctx = context.Background() // create DaemonSet - ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+faker.Word(), config.Image, time.Minute/2) + ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+faker.Word(), config.Image, time.Minute*2) Expect(err).NotTo(HaveOccurred()) GinkgoWriter.Printf("succeeded to create DaemonSet: %s\n", ds.Name) podLabelSelector = &metav1.LabelSelector{MatchLabels: ds.Labels} @@ -457,7 +457,7 @@ var _ = Describe("Operate EgressGateway", Label("EgressGateway"), Ordered, func( var err error // create ds for eip test - ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+faker.Word(), config.Image, time.Minute/2) + ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+faker.Word(), config.Image, time.Minute*2) Expect(err).NotTo(HaveOccurred()) GinkgoWriter.Printf("Create DaemonSet: %s\n", ds.Name) diff --git a/test/e2e/egresspolicy/egresspolicy_test.go b/test/e2e/egresspolicy/egresspolicy_test.go index e41b01c64..360213e39 100644 --- a/test/e2e/egresspolicy/egresspolicy_test.go +++ b/test/e2e/egresspolicy/egresspolicy_test.go @@ -82,11 +82,11 @@ var _ = Describe("EgressPolicy", Serial, func() { ctx := context.Background() var err error // create DaemonSet-A DaemonSet-B for A/B test - dsA, err = common.CreateDaemonSet(ctx, cli, "ds-a-"+faker.Word(), config.Image, time.Minute/2) + dsA, err = common.CreateDaemonSet(ctx, cli, "ds-a-"+faker.Word(), config.Image, time.Minute*2) Expect(err).NotTo(HaveOccurred()) GinkgoWriter.Printf("Create DaemonSet A: %s\n", dsA.Name) - dsB, err = common.CreateDaemonSet(ctx, cli, "ds-b-"+faker.Word(), config.Image, time.Minute/2) + dsB, err = common.CreateDaemonSet(ctx, cli, "ds-b-"+faker.Word(), config.Image, time.Minute*2) Expect(err).NotTo(HaveOccurred()) GinkgoWriter.Printf("Create DaemonSet B: %s\n", dsB.Name) @@ -526,7 +526,7 @@ var _ = Describe("EgressPolicy", Serial, func() { ctx = context.Background() // create DaemonSet - ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+uuid.NewString(), config.Image, time.Minute/2) + ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+uuid.NewString(), config.Image, time.Minute*2) Expect(err).NotTo(HaveOccurred()) GinkgoWriter.Printf("succeeded to create DaemonSet: %s\n", ds.Name) podLabelSelector = &metav1.LabelSelector{MatchLabels: ds.Labels} diff --git a/test/e2e/reliability/reliability_test.go b/test/e2e/reliability/reliability_test.go index 495811da8..00d7c6899 100644 --- a/test/e2e/reliability/reliability_test.go +++ b/test/e2e/reliability/reliability_test.go @@ -66,7 +66,7 @@ var _ = Describe("Reliability", Serial, Label("Reliability"), func() { GinkgoWriter.Printf("v4DefaultEip: %s, v6DefaultEip: %s\n", v4DefaultEip, v6DefaultEip) // daemonSet - daemonSet, err = common.CreateDaemonSet(ctx, cli, "ds-reliability-"+uuid.NewString(), config.Image, time.Minute/2) + daemonSet, err = common.CreateDaemonSet(ctx, cli, "ds-reliability-"+uuid.NewString(), config.Image, time.Minute*2) Expect(err).NotTo(HaveOccurred()) GinkgoWriter.Printf("succeeded to create DaemonSet: %s\n", daemonSet.Name)