Merge pull request #1335 from spidernet-io/enhance-e2e-for-restart-ap…

…iserver

enhance e2e for restart api-server case

test/Makefile

@@ -261,26 +261,37 @@ deploy_project:
 		done ; \
 	fi
 	- helm --kubeconfig=$(KIND_KUBECONFIG) uninstall -n $(E2E_NAMESPACE) project || true
-	HELM_OPTION="" ; \
-    	if [ -n "$(PYROSCOPE_LOCAL_PORT)" ] ; then \
-			echo "add env" ; \
-			ServerAddress=$$(docker network inspect kind -f {{\(index\ $$.IPAM.Config\ 0\).Gateway}}) ; \
-			HELM_OPTION+=" --set agent.debug.pyroscopeServerAddr=http://$${ServerAddress}:$(PYROSCOPE_LOCAL_PORT) " ; \
-			HELM_OPTION+=" --set controller.debug.pyroscopeServerAddr=http://$${ServerAddress}:$(PYROSCOPE_LOCAL_PORT) " ; \
-		fi ; \
-		HELM_OPTION+=" --set agent.prometheus.enabled=true --set controller.prometheus.enabled=true  " ; \
-		HELM_OPTION+=" --set agent.debug.logLevel=debug --set controller.debug.logLevel=debug " ; \
-		HELM_OPTION+=" --set controller.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key=node-role.kubernetes.io/control-plane  " ; \
-		HELM_OPTION+=" --set controller.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator=Exists  " ; \
-		if [ "$(E2E_IP_FAMILY)" == "dual" ] ; then HELM_OPTION+=" --set feature.enableIPv4=true --set feature.enableIPv6=true" ; fi ; \
-		if [ "$(E2E_IP_FAMILY)" == "ipv4" ] ; then HELM_OPTION+=" --set feature.enableIPv4=true --set feature.enableIPv6=false" ; fi ; \
-		if [ "$(E2E_IP_FAMILY)" == "ipv6" ] ; then HELM_OPTION+=" --set feature.enableIPv4=false --set feature.enableIPv6=true" ; fi ; \
-		helm --kubeconfig=$(KIND_KUBECONFIG) install project $(ROOT_DIR)/charts \
-				-n $(E2E_NAMESPACE) --create-namespace --wait --debug \
-				--set global.imageTagOverride=$(PROJECT_IMAGE_VERSION) \
-				$${HELM_OPTION} \
-				|| { KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) ./scripts/debugCluster.sh $(KIND_KUBECONFIG) "detail"  $(E2E_NAMESPACE) ; exit 1 ; } ; \
-		exit 0
+	@echo "" > config.yaml
+	@if [ -z '$(PYROSCOPE_LOCAL_PORT)' ]; then \
+		echo "PYROSCOPE_LOCAL_PORT is not set. Skipping setting pyroscopeServerAddr."; \
+	else \
+		yq e '.agent.debug.pyroscopeServerAddr = "http://${ServerAddress}:$(PYROSCOPE_LOCAL_PORT)"' -i config.yaml; \
+		yq e '.controller.debug.pyroscopeServerAddr = "http://${ServerAddress}:$(PYROSCOPE_LOCAL_PORT)"' -i config.yaml; \
+	fi
+	yq e '.agent.prometheus.enabled = "true"' -i config.yaml
+	yq e '.controller.prometheus.enabled = "true"' -i config.yaml
+	yq e '.agent.debug.logLevel = "debug"' -i config.yaml
+	yq e '.controller.debug.logLevel = "debug"' -i config.yaml
+	@if [ "$(E2E_IP_FAMILY)" = "dual" ]; then \
+		yq e '.feature.enableIPv4 = true' -i config.yaml; \
+		yq e '.feature.enableIPv6 = true' -i config.yaml; \
+	elif [ "$(E2E_IP_FAMILY)" = "ipv4" ]; then \
+		yq e '.feature.enableIPv4 = true' -i config.yaml; \
+		yq e '.feature.enableIPv6 = false' -i config.yaml; \
+	elif [ "$(E2E_IP_FAMILY)" = "ipv6" ]; then \
+		yq e '.feature.enableIPv4 = false' -i config.yaml; \
+		yq e '.feature.enableIPv6 = true' -i config.yaml; \
+	else \
+		echo "Unknown IP family $(E2E_IP_FAMILY)"; \
+		exit 1; \
+	fi
+	yq e '.controller.nodeSelector."kubernetes.io/hostname" = "$(KIND_CLUSTER_NAME)-control-plane"' -i config.yaml
+	HELM_OPTION+=" --set controller.nodeSelector."kubernetes.io/hostname"=$(KIND_CLUSTER_NAME)-control-plane" ; \
+	helm --kubeconfig=$(KIND_KUBECONFIG) install project $(ROOT_DIR)/charts \
+			-n $(E2E_NAMESPACE) --create-namespace --wait --debug \
+			--set global.imageTagOverride=$(PROJECT_IMAGE_VERSION) --values config.yaml \
+			|| { KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) ./scripts/debugCluster.sh $(KIND_KUBECONFIG) "detail"  $(E2E_NAMESPACE) ; exit 1 ; } ; \
+	exit 0
 
 .PHONY: uninstall_egress
 uninstall_egress: KIND_KUBECONFIG ?= $(E2E_KIND_KUBECONFIG_PATH)
@@ -289,7 +300,7 @@ uninstall_egress:
 	@echo -e "\033[35m [helm uninstall egress] \033[0m"
 	helm uninstall project --wait --debug -n $(E2E_NAMESPACE) \
 	     --kubeconfig=$(KIND_KUBECONFIG)  || { $(KIND_CLUSTER_NAME) ./scripts/debugCluster.sh $(KIND_KUBECONFIG) "detail" $(E2E_NAMESPACE) ; exit 1 ; } ; \
-	kubectl --kubeconfig=$(KIND_KUBECONFIG) get all --all-namespaces  | grep -q "egressgateway"` && { echo "error: found egressgateway resources" ; exit 1 ; }
+	kubectl --kubeconfig=$(KIND_KUBECONFIG) get all --all-namespaces  | grep -q "egressgateway" && { echo "error: found egressgateway resources" ; exit 1 ; }
 
 # test kind is ok
 .PHONY: install_example_app

test/e2e/common/ds.go

@@ -5,6 +5,7 @@ package common
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"time"
 
@@ -44,6 +45,8 @@ func CreateDaemonSet(ctx context.Context, cli client.Client, name string, image
 		},
 	}
 
+	log := NewLogger()
+
 	err := cli.Create(ctx, res)
 	if err != nil {
 		return nil, err
@@ -53,7 +56,8 @@ func CreateDaemonSet(ctx context.Context, cli client.Client, name string, image
 		select {
 		case <-ctx.Done():
 			_ = DeleteObj(context.Background(), cli, res)
-			return nil, fmt.Errorf("create DaemonSet time out")
+			log.Log("create DaemonSet time out")
+			return nil, fmt.Errorf(log.Save())
 		default:
 			err := cli.Get(ctx, types.NamespacedName{Namespace: res.Namespace, Name: res.Name}, res)
 			if err != nil {
@@ -68,6 +72,32 @@ func CreateDaemonSet(ctx context.Context, cli client.Client, name string, image
 				return res, nil
 			}
 
+			log.Log("--------")
+			log.Log(fmt.Sprintf("CurrentNumberScheduled=%v\nDesiredNumberScheduled=%v\nNumberAvailable=%v", a, b, c))
+
+			nodes := new(corev1.NodeList)
+			err = cli.List(ctx, nodes)
+			if err != nil {
+				return nil, err
+			}
+			for _, node := range nodes.Items {
+				t := "node " + node.Name + " --- "
+				for _, condition := range node.Status.Conditions {
+					t = t + fmt.Sprintf("%v=%v ", condition.Type, condition.Status)
+				}
+				log.Log(t)
+			}
+			pods := new(corev1.PodList)
+			err = cli.List(ctx, pods)
+			if err != nil {
+				return nil, err
+			}
+			for _, pod := range pods.Items {
+				t := "pod " + pod.Name + " --- "
+				raw, _ := json.Marshal(pod.Status)
+				t += string(raw)
+				log.Log(t)
+			}
 			time.Sleep(time.Second / 2)
 		}
 	}

test/e2e/common/egw.go

@@ -5,6 +5,7 @@ package common
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"reflect"
 	"sort"
@@ -270,15 +271,24 @@ func DeleteEgressGateway(ctx context.Context, cli client.Client, egw *egressv1.E
 		return err
 	}
 
+	log := NewLogger()
+
 	for {
 		select {
 		case <-ctx.Done():
-			return e2eerr.ErrTimeout
+			log.Log("check delete egress gateway object timeout")
+			return fmt.Errorf(log.Save())
 		default:
 			err = cli.Get(ctx, types.NamespacedName{Name: egw.Name}, egw)
 			if apierrors.IsNotFound(err) {
 				return nil
 			}
+			if err != nil {
+				log.Log("--------------------------")
+				log.Log(err.Error())
+			}
+			raw, _ := json.Marshal(egw)
+			log.Log(string(raw))
 			time.Sleep(time.Second / 2)
 		}
 	}

test/e2e/common/log.go

@@ -0,0 +1,44 @@
+// Copyright 2022 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package common
+
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
+type Logger struct {
+	logs      []string
+	startTime time.Time
+	endTime   time.Time
+}
+
+func NewLogger() *Logger {
+	return &Logger{
+		startTime: time.Now(),
+	}
+}
+
+func (l *Logger) Log(message string) {
+	l.logs = append(l.logs, message)
+}
+
+func (l *Logger) Save() string {
+	endTime := time.Now()
+	duration := endTime.Sub(l.startTime)
+
+	var sb strings.Builder
+
+	// Append all log entries
+	for _, log := range l.logs {
+		sb.WriteString(log)
+		sb.WriteString("\n")
+	}
+
+	// Append the total time spent in seconds
+	sb.WriteString(fmt.Sprintf("Total duration: %.2f seconds\n", duration.Seconds()))
+
+	return sb.String()
+}

test/e2e/egressgateway/egressgateway_test.go

@@ -179,7 +179,7 @@ var _ = Describe("Operate EgressGateway", Label("EgressGateway"), Ordered, func(
 			ctx = context.Background()
 
 			// create DaemonSet
-			ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+faker.Word(), config.Image, time.Minute/2)
+			ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+faker.Word(), config.Image, time.Minute*2)
 			Expect(err).NotTo(HaveOccurred())
 			GinkgoWriter.Printf("succeeded to create DaemonSet: %s\n", ds.Name)
 			podLabelSelector = &metav1.LabelSelector{MatchLabels: ds.Labels}
@@ -457,7 +457,7 @@ var _ = Describe("Operate EgressGateway", Label("EgressGateway"), Ordered, func(
 			var err error
 
 			// create ds for eip test
-			ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+faker.Word(), config.Image, time.Minute/2)
+			ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+faker.Word(), config.Image, time.Minute*2)
 			Expect(err).NotTo(HaveOccurred())
 			GinkgoWriter.Printf("Create DaemonSet: %s\n", ds.Name)
 

test/e2e/egresspolicy/egresspolicy_test.go

@@ -82,11 +82,11 @@ var _ = Describe("EgressPolicy", Serial, func() {
 			ctx := context.Background()
 			var err error
 			// create DaemonSet-A DaemonSet-B for A/B test
-			dsA, err = common.CreateDaemonSet(ctx, cli, "ds-a-"+faker.Word(), config.Image, time.Minute/2)
+			dsA, err = common.CreateDaemonSet(ctx, cli, "ds-a-"+faker.Word(), config.Image, time.Minute*2)
 			Expect(err).NotTo(HaveOccurred())
 			GinkgoWriter.Printf("Create DaemonSet A: %s\n", dsA.Name)
 
-			dsB, err = common.CreateDaemonSet(ctx, cli, "ds-b-"+faker.Word(), config.Image, time.Minute/2)
+			dsB, err = common.CreateDaemonSet(ctx, cli, "ds-b-"+faker.Word(), config.Image, time.Minute*2)
 			Expect(err).NotTo(HaveOccurred())
 			GinkgoWriter.Printf("Create DaemonSet B: %s\n", dsB.Name)
 
@@ -526,7 +526,7 @@ var _ = Describe("EgressPolicy", Serial, func() {
 				ctx = context.Background()
 
 				// create DaemonSet
-				ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+uuid.NewString(), config.Image, time.Minute/2)
+				ds, err = common.CreateDaemonSet(ctx, cli, "ds-"+uuid.NewString(), config.Image, time.Minute*2)
 				Expect(err).NotTo(HaveOccurred())
 				GinkgoWriter.Printf("succeeded to create DaemonSet: %s\n", ds.Name)
 				podLabelSelector = &metav1.LabelSelector{MatchLabels: ds.Labels}

test/e2e/reliability/reliability_test.go

@@ -66,7 +66,7 @@ var _ = Describe("Reliability", Serial, Label("Reliability"), func() {
 			GinkgoWriter.Printf("v4DefaultEip: %s, v6DefaultEip: %s\n", v4DefaultEip, v6DefaultEip)
 
 			// daemonSet
-			daemonSet, err = common.CreateDaemonSet(ctx, cli, "ds-reliability-"+uuid.NewString(), config.Image, time.Minute/2)
+			daemonSet, err = common.CreateDaemonSet(ctx, cli, "ds-reliability-"+uuid.NewString(), config.Image, time.Minute*2)
 			Expect(err).NotTo(HaveOccurred())
 			GinkgoWriter.Printf("succeeded to create DaemonSet: %s\n", daemonSet.Name)