diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b4c9ced..5913b6a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -18,10 +18,12 @@ jobs: - uses: actions/setup-go@v4 with: go-version: '1.21' - - name: Create k8s Kind Cluster - uses: helm/kind-action@v1.7.0 - with: - install_only: true + - name: Install Minikube + run: | + wget https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + sudo cp minikube-linux-amd64 /usr/local/bin/minikube + sudo chmod +x /usr/local/bin/minikube + - uses: azure/setup-helm@v3 - name: Run e2e run: | diff --git a/Dockerfile b/Dockerfile index 95b2857..2630b60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.21 as builder +FROM --platform=${BUILDPLATFORM:-linux/amd64} docker.io/golang:1.21 as builder ARG TARGETPLATFORM ARG BUILDPLATFORM diff --git a/DockerfileDebug b/DockerfileDebug index 3b9dfcd..22d514a 100644 --- a/DockerfileDebug +++ b/DockerfileDebug @@ -1,4 +1,4 @@ -FROM golang:1.21 as builder +FROM docker.io/golang:1.21 as builder WORKDIR /code @@ -8,7 +8,7 @@ RUN go mod download RUN go install github.com/go-delve/delve/cmd/dlv@latest RUN go build -gcflags="all=-N -l" -o /app main.go -FROM nicolaka/netshoot:latest +FROM docker.io/ubuntu:22.04 ENV GOTRACEBACK=crash WORKDIR / COPY --from=builder /app . diff --git a/DockerfileTestGrow b/DockerfileTestGrow index 6ae21c9..96eaf3e 100644 --- a/DockerfileTestGrow +++ b/DockerfileTestGrow @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM docker.io/ubuntu:22.04 RUN apt-get update -y && apt-get install watch LABEL org.opencontainers.image.source="https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics" diff --git a/DockerfileTestShrink b/DockerfileTestShrink index bb71987..56af879 100644 --- a/DockerfileTestShrink +++ b/DockerfileTestShrink @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM docker.io/ubuntu:22.04 RUN apt-get update -y && apt-get install watch LABEL org.opencontainers.image.source="https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics" diff --git a/Makefile b/Makefile index 2a1fac1..9f9b6d2 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,10 @@ $(LOCALBIN): ginkgo: - test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo@latest + test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo@v2.13.1 + +crane: + test -s $(LOCALBIN)/crane || GOBIN=$(LOCALBIN) go install github.com/google/go-containerregistry/cmd/crane@latest .PHONY: fmt fmt: ## Run go fmt against code. @@ -27,8 +30,14 @@ helm-docs: $(LOCALBIN)/helm-docs --template-files "${GITROOT}/chart/README.md.gotmpl" cat "${GITROOT}/Header.md" "${GITROOT}/chart/README.md" > "${GITROOT}/README.md" -new_kind: - ./scripts/create_kind.sh +minikube_new: + ./scripts/create-minikube.sh + +minikube_scale_up: + minikube node add + +minikube_scale_down: + minikube node delete m02 init: fmt vet @@ -41,7 +50,10 @@ deploy_e2e_debug: init deploy_local: init ./scripts/deploy.sh -deploy_e2e: init ginkgo new_kind +deploy_e2e: init ginkgo crane minikube_new + ENV='e2e' ./scripts/deploy.sh + +deploy_e2e_dirty: init ENV='e2e' ./scripts/deploy.sh release-docker: diff --git a/chart/templates/test_deployments.yaml b/chart/templates/test_deployments.yaml index cb505d7..ff8e018 100644 --- a/chart/templates/test_deployments.yaml +++ b/chart/templates/test_deployments.yaml @@ -15,8 +15,8 @@ spec: name: shrink-test spec: containers: - - image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest - imagePullPolicy: "{{ .Values.dev.image.imagePullPolicy }}" + - image: "{{ .Values.dev.shrink.image }}" + imagePullPolicy: "{{ .Values.dev.shrink.imagePullPolicy }}" name: shrink-test resources: requests: @@ -54,8 +54,8 @@ spec: name: grow-test spec: containers: - - image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest - imagePullPolicy: "{{ .Values.dev.image.imagePullPolicy }}" + - image: "{{ .Values.dev.grow.image }}" + imagePullPolicy: "{{ .Values.dev.grow.imagePullPolicy }}" name: grow-test resources: requests: diff --git a/chart/values.yaml b/chart/values.yaml index 786fb98..1803877 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -36,10 +36,14 @@ prometheus: pprof: false -# -- For local development of kind and/or deploy grow and shrink test pods +# -- For local development or testing that will deploy grow and shrink pods and debug service dev: enabled: false - image: + shrink: + image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest + imagePullPolicy: IfNotPresent + grow: + image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest imagePullPolicy: IfNotPresent podAnnotations: {} diff --git a/scripts/create-minikube.sh b/scripts/create-minikube.sh new file mode 100755 index 0000000..3b211aa --- /dev/null +++ b/scripts/create-minikube.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +minikube delete || true +c=$(docker ps -q) && [[ $c ]] && docker kill $c +docker network prune -f +minikube start \ + --kubernetes-version="${K8S_VERSION}" \ + --insecure-registry "10.0.0.0/24" +minikube addons enable registry + +# Add Service Monitor CRD +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 2dd0300..91be204 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -5,15 +5,52 @@ set -e -source scripts/helpers.sh +CURRENT_DIR="$(realpath $(dirname ${BASH_SOURCE[0]}))" +cd "${CURRENT_DIR}" || true +source helpers.sh function main() { local image_tag local dockerfile + local registry + local image + local common_set_values + local common_set_values_arr + local grow_repo_image + local shrink_repo_image + local e2e_values_arr trap 'trap_func' EXIT ERR + while [ "$(kubectl get pods -n kube-system -l kubernetes.io/minikube-addons=registry -o=jsonpath='{.items[*].status.phase}')" != "Running Running" ]; do + echo "waiting for minikube registry and proxy pod to start. Sleep 10" && sleep 10 + done + # Need both. External to push and internal for pods to pull from registry in cluster + external_registry="$(minikube ip):5000" + internal_registry="$(kubectl get service -n kube-system registry --template='{{.spec.clusterIP}}')" + + ### Build and Load Images ### + + grow_repo_image="k8s-ephemeral-storage-grow-test:latest" + + docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f ../DockerfileTestGrow \ + -t "${external_registry}/${grow_repo_image}" -t "${internal_registry}/${grow_repo_image}" ../. + + docker save "${external_registry}/${grow_repo_image}" > /tmp/image.tar + ${LOCALBIN}/crane push --insecure /tmp/image.tar "${external_registry}/${grow_repo_image}" + rm /tmp/image.tar + + + shrink_repo_image="k8s-ephemeral-storage-shrink-test:latest" + + docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f ../DockerfileTestShrink \ + -t "${external_registry}/${shrink_repo_image}" -t "${internal_registry}/${shrink_repo_image}" ../. + + docker save "${external_registry}/${shrink_repo_image}" > /tmp/image.tar + ${LOCALBIN}/crane push --insecure /tmp/image.tar "${external_registry}/${shrink_repo_image}" + rm /tmp/image.tar + if [[ $ENV == "debug" ]]; then image_tag="debug-latest" dockerfile="DockerfileDebug" @@ -22,25 +59,31 @@ function main() { dockerfile="Dockerfile" fi - common_set_values_arr=( - "image.repository=local.io/local/$DEPLOYMENT_NAME", - "image.tag=${image_tag}", - "image.imagePullPolicy=Never", - "deploy_type=Deployment", - "log_level=debug" - "dev.enabled=true", - "dev.image.imagePullPolicy=Never" - "metrics.adjusted_polling_rate=true" - "pprof=true" - ) + # Main image + main_repo_image="${DEPLOYMENT_NAME}:${image_tag}" + docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f ../${dockerfile} \ + -t "${external_registry}/${main_repo_image}" -t "${internal_registry}/${main_repo_image}" ../. - docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f DockerfileTestGrow -t ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest . - kind load docker-image -v 9 --name "${DEPLOYMENT_NAME}-cluster" --nodes "${DEPLOYMENT_NAME}-cluster-worker" "ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest" + docker save "${external_registry}/${main_repo_image}" > /tmp/image.tar + ${LOCALBIN}/crane push --insecure /tmp/image.tar "${external_registry}/${main_repo_image}" + rm /tmp/image.tar - docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f DockerfileTestShrink -t ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest . - kind load docker-image -v 9 --name "${DEPLOYMENT_NAME}-cluster" --nodes "${DEPLOYMENT_NAME}-cluster-worker" "ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest" - if [[ $ENV =~ "e2e" ]]; then + ### Install Chart ### + + common_set_values_arr=( + "image.repository=${internal_registry}/$DEPLOYMENT_NAME" + "image.tag=${image_tag}" + "deploy_type=Deployment" + "log_level=debug" + "dev.enabled=true" + "dev.shrink.image=${internal_registry}/${shrink_repo_image}" + "dev.grow.image=${internal_registry}/${grow_repo_image}" + "metrics.adjusted_polling_rate=true" + "pprof=true" + ) + + if [[ $ENV =~ "e2e" ]]; then e2e_values_arr=("interval=5") common_set_values_arr+=("${e2e_values_arr[@]}") @@ -48,27 +91,21 @@ function main() { common_set_values=$(echo ${common_set_values_arr[*]} | sed 's/, /,/g' | sed 's/ /,/g') - docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f ${dockerfile} -t local.io/local/$DEPLOYMENT_NAME:$image_tag . - kind load docker-image -v 9 --name "${DEPLOYMENT_NAME}-cluster" --nodes "${DEPLOYMENT_NAME}-cluster-worker" "local.io/local/${DEPLOYMENT_NAME}:${image_tag}" - - - - # Install Par Chart - helm upgrade --install $DEPLOYMENT_NAME ./chart \ + helm upgrade --install "${DEPLOYMENT_NAME}" ../chart \ --set "${common_set_values}" \ --create-namespace \ - --namespace $DEPLOYMENT_NAME + --namespace "${DEPLOYMENT_NAME}" - # Patch deploy so Kind image upload to work. + + # Patch deploy so minikube image upload works. if [[ $ENV == "debug" ]]; then # Disable for Debugging of Delve. - kubectl patch deployments.apps -n $DEPLOYMENT_NAME k8s-ephemeral-storage-metrics -p \ + kubectl patch deployments.apps -n "${DEPLOYMENT_NAME}" k8s-ephemeral-storage-metrics -p \ '{ "spec": {"template": { "spec":{"securityContext": null, "containers":[{"name":"metrics", "livenessProbe": null, "readinessProbe": null, "securityContext": null, "command": null, "args": null }]}}}}' fi - - # kill dangling port forwards if found. - # Exporter Porter + # Kill dangling port forwards if found. + # Main Exporter Port sudo ss -aK '( dport = :9100 or sport = :9100 )' | true # Prometheus Port sudo ss -aK '( dport = :9090 or sport = :9090 )' | true @@ -81,31 +118,22 @@ function main() { printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/k8s-ephemeral-storage-metrics 9100:9100 || sleep 5; done ) & - # Start Prometheus Port Forward - ( - sleep 10 - printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/prometheus-operated 9090:9090 || sleep 5; done - ) & - # Start Pprof Forward - ( - sleep 10 - printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/pprof 6060:6060 || sleep 5; done - ) & + # Wait until main pod comes up + while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do + echo "waiting for k8s-ephemeral-storage-metrics pod to start. Sleep 10" && sleep 10 + done if [[ $ENV == "debug" ]]; then # Background log following for manager ( sleep 10 - printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done + printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done ) & - while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do - echo "waiting for k8s-ephemeral-storage-metrics pod to start. Sleep 10" && sleep 10 - done kubectl port-forward -n $DEPLOYMENT_NAME services/debug 30002:30002 elif [[ $ENV == "e2e" ]]; then - ${LOCALBIN}/ginkgo -v -r ./tests/e2e/... + ${LOCALBIN}/ginkgo -v -r ../tests/e2e/... elif [[ $ENV == "e2e-debug" ]]; then sleep infinity else diff --git a/scripts/kind.yaml b/scripts/kind.yaml deleted file mode 100755 index cebee6d..0000000 --- a/scripts/kind.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: - - role: control-plane - - role: worker diff --git a/tests/e2e/deployment_test.go b/tests/e2e/deployment_test.go index 23cd463..2cf9fc7 100644 --- a/tests/e2e/deployment_test.go +++ b/tests/e2e/deployment_test.go @@ -109,7 +109,7 @@ func checkPrometheus(checkSlice []string) { func WatchContainerPercentage() { status := 0 - re := regexp.MustCompile(`ephemeral_storage_container_limit_percentage{container="grow-test",node_name="ephemeral-metrics-cluster-worker".+,pod_namespace="ephemeral-metrics"}\s+(.+)`) + re := regexp.MustCompile(`ephemeral_storage_container_limit_percentage{container="grow-test",node_name="minikube".+,pod_namespace="ephemeral-metrics"}\s+(.+)`) output := requestPrometheusString() match := re.FindAllStringSubmatch(output, -1) floatValue, _ := strconv.ParseFloat(match[0][1], 64) @@ -132,7 +132,7 @@ func WatchDeadPod() { func WatchNodePercentage() { status := 0 - re := regexp.MustCompile(`ephemeral_storage_node_percentage\{node_name="ephemeral-metrics-cluster-control-plane"}\s+(.+)`) + re := regexp.MustCompile(`ephemeral_storage_node_percentage\{node_name="minikube"}\s+(.+)`) output := requestPrometheusString() match := re.FindAllStringSubmatch(output, -1) floatValue, _ := strconv.ParseFloat(match[0][1], 64) @@ -145,7 +145,7 @@ func WatchNodePercentage() { func WatchPollingRate(pollRateUpper float64, pollingRateLower float64, timeout time.Duration) { status := 0 startTime := time.Now() - re := regexp.MustCompile(`ephemeral_storage_adjusted_polling_rate\{node_name="ephemeral-metrics-cluster-control-plane"}\s+(.+)`) + re := regexp.MustCompile(`ephemeral_storage_adjusted_polling_rate\{node_name="minikube"}\s+(.+)`) for { elapsed := time.Since(startTime) if elapsed >= timeout { @@ -211,14 +211,14 @@ func WatchEphemeralPodSize(podname string, sizeChange float64, timeout time.Dura var _ = ginkgo.Describe("Test Metrics\n", func() { ginkgo.Context("Observe labels\n", func() { - ginkgo.Specify("\nReturn A Record IP addresses and Proxy IP address", func() { + ginkgo.Specify("\nMake sure all metrics are in the exporter", func() { var checkSlice []string checkSlice = append(checkSlice, "ephemeral_storage_pod_usage", "ephemeral_storage_node_available", "ephemeral_storage_node_capacity", "ephemeral_storage_node_percentage", "pod_name=\"k8s-ephemeral-storage", "ephemeral_storage_adjusted_polling_rate", - "node_name=\"ephemeral-metrics-cluster-worker", "node_name=\"ephemeral-metrics-cluster-control-plane", + "node_name=\"minikube", "ephemeral_storage_container_limit_percentage") checkPrometheus(checkSlice) })