From 3b4f9feef86fe5dfc6d77b8f52cf0a29607ae508 Mon Sep 17 00:00:00 2001 From: gitlayzer Date: Mon, 22 Apr 2024 18:43:32 +0800 Subject: [PATCH] =?UTF-8?q?Upgrade=20=E9=A1=B9=E7=9B=AE=E5=88=9B=E5=BB=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 3 + .golangci.yml | 40 ++++ Dockerfile | 33 +++ Makefile | 198 +++++++++++++++++ PROJECT | 19 ++ README.md | 76 +++++++ api/v1alpha1/groupversion_info.go | 36 +++ api/v1alpha1/upgrade_types.go | 76 +++++++ api/v1alpha1/zz_generated.deepcopy.go | 135 ++++++++++++ cmd/main.go | 148 +++++++++++++ .../devops-engineer.com.cn_upgrades.yaml | 106 +++++++++ config/crd/kustomization.yaml | 23 ++ config/crd/kustomizeconfig.yaml | 19 ++ config/default/kustomization.yaml | 142 ++++++++++++ config/default/manager_auth_proxy_patch.yaml | 39 ++++ config/default/manager_config_patch.yaml | 10 + config/manager/kustomization.yaml | 2 + config/manager/manager.yaml | 102 +++++++++ config/prometheus/kustomization.yaml | 2 + config/prometheus/monitor.yaml | 25 +++ .../rbac/auth_proxy_client_clusterrole.yaml | 16 ++ config/rbac/auth_proxy_role.yaml | 24 ++ config/rbac/auth_proxy_role_binding.yaml | 19 ++ config/rbac/auth_proxy_service.yaml | 21 ++ config/rbac/kustomization.yaml | 18 ++ config/rbac/leader_election_role.yaml | 44 ++++ config/rbac/leader_election_role_binding.yaml | 19 ++ config/rbac/role.yaml | 32 +++ config/rbac/role_binding.yaml | 19 ++ config/rbac/service_account.yaml | 8 + config/rbac/upgrade_editor_role.yaml | 31 +++ config/rbac/upgrade_viewer_role.yaml | 27 +++ config/samples/_v1alpha1_upgrade.yaml | 12 + config/samples/kustomization.yaml | 4 + config/samples/sample-nginx.yaml | 20 ++ go.mod | 74 +++++++ go.sum | 205 ++++++++++++++++++ hack/boilerplate.go.txt | 15 ++ internal/controller/config.go | 26 +++ internal/controller/suite_test.go | 90 ++++++++ internal/controller/upgrade.go | 61 ++++++ internal/controller/upgrade_controller.go | 165 ++++++++++++++ .../controller/upgrade_controller_test.go | 84 +++++++ internal/utils/resources.go | 76 +++++++ logo/Upgrade.png | Bin 0 -> 34642 bytes test/e2e/e2e_suite_test.go | 32 +++ test/e2e/e2e_test.go | 122 +++++++++++ test/utils/utils.go | 140 ++++++++++++ 48 files changed, 2638 insertions(+) create mode 100644 .dockerignore create mode 100644 .golangci.yml create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 PROJECT create mode 100644 api/v1alpha1/groupversion_info.go create mode 100644 api/v1alpha1/upgrade_types.go create mode 100644 api/v1alpha1/zz_generated.deepcopy.go create mode 100644 cmd/main.go create mode 100644 config/crd/bases/devops-engineer.com.cn_upgrades.yaml create mode 100644 config/crd/kustomization.yaml create mode 100644 config/crd/kustomizeconfig.yaml create mode 100644 config/default/kustomization.yaml create mode 100644 config/default/manager_auth_proxy_patch.yaml create mode 100644 config/default/manager_config_patch.yaml create mode 100644 config/manager/kustomization.yaml create mode 100644 config/manager/manager.yaml create mode 100644 config/prometheus/kustomization.yaml create mode 100644 config/prometheus/monitor.yaml create mode 100644 config/rbac/auth_proxy_client_clusterrole.yaml create mode 100644 config/rbac/auth_proxy_role.yaml create mode 100644 config/rbac/auth_proxy_role_binding.yaml create mode 100644 config/rbac/auth_proxy_service.yaml create mode 100644 config/rbac/kustomization.yaml create mode 100644 config/rbac/leader_election_role.yaml create mode 100644 config/rbac/leader_election_role_binding.yaml create mode 100644 config/rbac/role.yaml create mode 100644 config/rbac/role_binding.yaml create mode 100644 config/rbac/service_account.yaml create mode 100644 config/rbac/upgrade_editor_role.yaml create mode 100644 config/rbac/upgrade_viewer_role.yaml create mode 100644 config/samples/_v1alpha1_upgrade.yaml create mode 100644 config/samples/kustomization.yaml create mode 100644 config/samples/sample-nginx.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hack/boilerplate.go.txt create mode 100644 internal/controller/config.go create mode 100644 internal/controller/suite_test.go create mode 100644 internal/controller/upgrade.go create mode 100644 internal/controller/upgrade_controller.go create mode 100644 internal/controller/upgrade_controller_test.go create mode 100644 internal/utils/resources.go create mode 100644 logo/Upgrade.png create mode 100644 test/e2e/e2e_suite_test.go create mode 100644 test/e2e/e2e_test.go create mode 100644 test/utils/utils.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a3aab7a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Ignore build and test binaries. +bin/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..aed8644 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,40 @@ +run: + deadline: 5m + allow-parallel-runners: true + +issues: + # don't skip warning about doc comments + # don't exclude the default set of lint + exclude-use-default: false + # restore some of the defaults + # (fill in the rest as needed) + exclude-rules: + - path: "api/*" + linters: + - lll + - path: "internal/*" + linters: + - dupl + - lll +linters: + disable-all: true + enable: + - dupl + - errcheck + - exportloopref + - goconst + - gocyclo + - gofmt + - goimports + - gosimple + - govet + - ineffassign + - lll + - misspell + - nakedret + - prealloc + - staticcheck + - typecheck + - unconvert + - unparam + - unused diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..aca26f9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +# Build the manager binary +FROM golang:1.21 AS builder +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY cmd/main.go cmd/main.go +COPY api/ api/ +COPY internal/controller/ internal/controller/ + +# Build +# the GOARCH has not a default value to allow the binary be built according to the host where the command +# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO +# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, +# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b397beb --- /dev/null +++ b/Makefile @@ -0,0 +1,198 @@ +# Image URL to use all building/pushing image targets +IMG ?= controller:latest +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.29.0 + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# CONTAINER_TOOL defines the container tool to be used for building images. +# Be aware that the target commands are only tested with Docker which is +# scaffolded by default. However, you might want to replace it to use other +# tools. (i.e. podman) +CONTAINER_TOOL ?= docker + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk command is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +.PHONY: test +test: manifests generate fmt vet envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + +# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. +.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. +test-e2e: + go test ./test/e2e/ -v -ginkgo.v + +.PHONY: lint +lint: golangci-lint ## Run golangci-lint linter & yamllint + $(GOLANGCI_LINT) run + +.PHONY: lint-fix +lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes + $(GOLANGCI_LINT) run --fix + +##@ Build + +.PHONY: build +build: manifests generate fmt vet ## Build manager binary. + go build -o bin/manager cmd/main.go + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run ./cmd/main.go + +# If you wish to build the manager image targeting other platforms you can use the --platform flag. +# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. +# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +.PHONY: docker-build +docker-build: ## Build docker image with the manager. + $(CONTAINER_TOOL) build -t ${IMG} . + +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + $(CONTAINER_TOOL) push ${IMG} + +# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple +# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: +# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ +# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) +# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. +PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le +.PHONY: docker-buildx +docker-buildx: ## Build and push docker image for the manager for cross-platform support + # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + - $(CONTAINER_TOOL) buildx create --name project-v3-builder + $(CONTAINER_TOOL) buildx use project-v3-builder + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx rm project-v3-builder + rm Dockerfile.cross + +.PHONY: build-installer +build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. + mkdir -p dist + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default > dist/install.yaml + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: deploy +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - + +.PHONY: undeploy +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + +##@ Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Binaries +KUBECTL ?= kubectl +KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION) +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION) +ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION) +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) + +## Tool Versions +KUSTOMIZE_VERSION ?= v5.3.0 +CONTROLLER_TOOLS_VERSION ?= v0.14.0 +ENVTEST_VERSION ?= release-0.17 +GOLANGCI_LINT_VERSION ?= v1.54.2 + +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. +$(ENVTEST): $(LOCALBIN) + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +$(GOLANGCI_LINT): $(LOCALBIN) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,${GOLANGCI_LINT_VERSION}) + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary (ideally with version) +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f $(1) ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +GOBIN=$(LOCALBIN) go install $${package} ;\ +mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\ +} +endef diff --git a/PROJECT b/PROJECT new file mode 100644 index 0000000..19b127a --- /dev/null +++ b/PROJECT @@ -0,0 +1,19 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html +domain: devops-engineer.com.cn +layout: +- go.kubebuilder.io/v4 +projectName: upgrade-controller +repo: github.com/gitlayzer/upgrade-controller +resources: +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: devops-engineer.com.cn + kind: UpGrade + path: github.com/gitlayzer/upgrade-controller/api/v1alpha1 + version: v1alpha1 +version: "3" diff --git a/README.md b/README.md index d73b8e2..0eb3b9d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,78 @@ # upgrade-controller This is a controller that can upgrade Pods in a Deployment in place. + +![](./logo/Upgrade.png) + +Upgrade 是一个用于原地升级 K8S Deployment 中的 Pod 的控制器 +### Prerequisites + +- go version v1.21.0+ +- docker version 17.03+. +- kubectl version v1.11.3+. +- Access to a Kubernetes v1.11.3+ cluster. + +### Quick Start + +```shell +# Clone the repository +git clone https://github.com/gitlayzer/upgrade-controller.git + +# Go to the controller directory +cd upgrade-controller + +# Install the CRD +make install + +# Run the controller +make run + +# Deploy an example Deployment +kubectl apply -f config/samples/samplle-nginx.yaml + +# Deploy an example Deployment +kubectl apply -f config/samples/_v1alpha1_upgrade.yaml + +# Check Pods status +root@ubuntu:~# kubectl get pod -o custom-columns=POD_NAME:.spec.containers[*].name,POD_IMAGE:.spec.containers[*].image,PODIP:.status.podIP,START_TIME:.status.startTime +POD_NAME POD_IMAGE PODIP START_TIME +nginx nginx:alpine 10.244.1.34 2024-04-22T10:16:29Z +nginx nginx:alpine 10.244.1.33 2024-04-22T10:16:29Z +nginx nginx:alpine 10.244.1.32 2024-04-22T10:16:29Z + +# Check Upgraded Pods status +root@ubuntu:~# kubectl get pod -o custom-columns=POD_NAME:.spec.containers[*].name,POD_IMAGE:.spec.containers[*].image,PODIP:.status.podIP,START_TIME:.status.startTime +POD_NAME POD_IMAGE PODIP START_TIME +nginx nginx:1.25.4 10.244.1.34 2024-04-22T10:16:29Z +nginx nginx:1.25.4 10.244.1.33 2024-04-22T10:16:29Z +nginx nginx:1.25.4 10.244.1.32 2024-04-22T10:16:29Z + +# Check Upgrade status +root@ubuntu:~# kubectl get upgrades upgrade-nginx +NAME UPGRADERREPLICAS UPGRADERDEPLOYMENT UPGRADERTYPE UPGRADERSTATUS AGE +upgrade-nginx 3 nginx upgrade Successful 2m6s + +# Check Controller logs +root@ubuntu:~/upgrade-controller# make run +/root/upgrade-controller/bin/controller-gen-v0.14.0 rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases +/root/upgrade-controller/bin/controller-gen-v0.14.0 object:headerFile="hack/boilerplate.go.txt" paths="./..." +go fmt ./... +go vet ./... +go run ./cmd/main.go +2024-04-22T18:24:51+08:00 INFO setup starting manager +2024-04-22T18:24:51+08:00 INFO controller-runtime.metrics Starting metrics server +2024-04-22T18:24:51+08:00 INFO starting server {"kind": "health probe", "addr": "[::]:8081"} +2024-04-22T18:24:51+08:00 INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} +2024-04-22T18:24:51+08:00 INFO Starting EventSource {"controller": "upgrade", "controllerGroup": "devops-engineer.com.cn", "controllerKind": "UpGrade", "source": "kind source: *v1alpha1.UpGrade"} +2024-04-22T18:24:51+08:00 INFO Starting Controller {"controller": "upgrade", "controllerGroup": "devops-engineer.com.cn", "controllerKind": "UpGrade"} +2024-04-22T18:24:51+08:00 INFO Starting workers {"controller": "upgrade", "controllerGroup": "devops-engineer.com.cn", "controllerKind": "UpGrade", "worker count": 1} +I0422 18:37:15.116166 3991587 upgrade_controller.go:79] UpGrade type is upgrade +I0422 18:37:15.132634 3991587 upgrade.go:24] Start to upgrade pod nginx-7494f5f88c-869w7 +I0422 18:37:15.141965 3991587 upgrade.go:58] Upgrade pod success +I0422 18:37:15.142722 3991587 upgrade.go:24] Start to upgrade pod nginx-7494f5f88c-bbdnc +I0422 18:37:15.149261 3991587 upgrade.go:58] Upgrade pod success +I0422 18:37:15.149824 3991587 upgrade.go:24] Start to upgrade pod nginx-7494f5f88c-xcdsl +I0422 18:37:15.158018 3991587 upgrade.go:58] Upgrade pod success +I0422 18:37:15.169933 3991587 upgrade_controller.go:74] UpGrade pod is successful +I0422 18:37:15.170140 3991587 upgrade_controller.go:74] UpGrade pod is successful +I0422 18:39:16.421543 3991587 upgrade_controller.go:74] UpGrade pod is successful +``` \ No newline at end of file diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..8531fbd --- /dev/null +++ b/api/v1alpha1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2024 gitlayzer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=devops-engineer.com.cn +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "devops-engineer.com.cn", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1alpha1/upgrade_types.go b/api/v1alpha1/upgrade_types.go new file mode 100644 index 0000000..50808cc --- /dev/null +++ b/api/v1alpha1/upgrade_types.go @@ -0,0 +1,76 @@ +/* +Copyright 2024 gitlayzer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// UpGradeSpec defines the desired state of UpGrade +type UpGradeSpec struct { + Type string `json:"type"` + DeploymentRef DeploymentRef `json:"deploymentRef"` + UpgradeReplicas int `json:"upgradeReplicas"` + Images []images `json:"images"` +} + +type DeploymentRef struct { + Name string `json:"name"` + Namespace string `json:"namespace"` +} + +type images struct { + Image string `json:"image"` +} + +// UpGradeStatus defines the observed state of UpGrade +type UpGradeStatus struct { + // Type 类型主要是 upgrade + Type string `json:"type"` + // Status 状态 + Status string `json:"status"` + // UpGradeReplicas 升级的副本数 + UpGradeReplicas int `json:"upgradeReplicas"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// UpGrade is the Schema for the upgrades API +type UpGrade struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec UpGradeSpec `json:"spec,omitempty"` + Status UpGradeStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// UpGradeList contains a list of UpGrade +type UpGradeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []UpGrade `json:"items"` +} + +func init() { + SchemeBuilder.Register(&UpGrade{}, &UpGradeList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..3e6fbf9 --- /dev/null +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,135 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2024 gitlayzer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentRef) DeepCopyInto(out *DeploymentRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentRef. +func (in *DeploymentRef) DeepCopy() *DeploymentRef { + if in == nil { + return nil + } + out := new(DeploymentRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpGrade) DeepCopyInto(out *UpGrade) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpGrade. +func (in *UpGrade) DeepCopy() *UpGrade { + if in == nil { + return nil + } + out := new(UpGrade) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *UpGrade) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpGradeList) DeepCopyInto(out *UpGradeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]UpGrade, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpGradeList. +func (in *UpGradeList) DeepCopy() *UpGradeList { + if in == nil { + return nil + } + out := new(UpGradeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *UpGradeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpGradeSpec) DeepCopyInto(out *UpGradeSpec) { + *out = *in + out.DeploymentRef = in.DeploymentRef + if in.Images != nil { + in, out := &in.Images, &out.Images + *out = make([]images, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpGradeSpec. +func (in *UpGradeSpec) DeepCopy() *UpGradeSpec { + if in == nil { + return nil + } + out := new(UpGradeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpGradeStatus) DeepCopyInto(out *UpGradeStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpGradeStatus. +func (in *UpGradeStatus) DeepCopy() *UpGradeStatus { + if in == nil { + return nil + } + out := new(UpGradeStatus) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..060e8c0 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,148 @@ +/* +Copyright 2024 gitlayzer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/tls" + "flag" + "os" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + devopsengineercomcnv1alpha1 "github.com/gitlayzer/upgrade-controller/api/v1alpha1" + "github.com/gitlayzer/upgrade-controller/internal/controller" + //+kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(devopsengineercomcnv1alpha1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme +} + +func main() { + var metricsAddr string + var enableLeaderElection bool + var probeAddr string + var secureMetrics bool + var enableHTTP2 bool + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", false, + "If set the metrics endpoint is served securely") + flag.BoolVar(&enableHTTP2, "enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + + tlsOpts := []func(*tls.Config){} + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + webhookServer := webhook.NewServer(webhook.Options{ + TLSOpts: tlsOpts, + }) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + }, + WebhookServer: webhookServer, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "4bc71969.devops-engineer.com.cn", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + if err = (&controller.UpGradeReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "UpGrade") + os.Exit(1) + } + //+kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/config/crd/bases/devops-engineer.com.cn_upgrades.yaml b/config/crd/bases/devops-engineer.com.cn_upgrades.yaml new file mode 100644 index 0000000..fd806c1 --- /dev/null +++ b/config/crd/bases/devops-engineer.com.cn_upgrades.yaml @@ -0,0 +1,106 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: upgrades.devops-engineer.com.cn +spec: + group: devops-engineer.com.cn + names: + kind: UpGrade + listKind: UpGradeList + plural: upgrades + singular: upgrade + shortNames: + - upgrade + categories: + - all + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: This is the API to update the Deployment's upgrade + type: object + properties: + spec: + description: UpGradeSpec defines the desired state of UpGrade + type: object + properties: + type: + description: Type is the type of the UpGrade + type: string + deploymentRef: + description: DeploymentRef is the reference to the deployment that needs to be upgraded + type: object + properties: + name: + description: Name is the name of the deployment + type: string + namespace: + description: Namespace is the namespace of the deployment + type: string + required: + - name + - namespace + images: + description: Images is the list of images that needs to be upgraded + type: array + items: + type: object + properties: + image: + description: Define the image that needs to be upgraded + type: string + required: + - image + upgradeReplicas: + description: Define the number of replicas for the UpGrade + type: integer + required: + - deploymentRef + - images + - type + - upgradeReplicas + status: + description: UpGradeStatus defines the observed state of UpGrade + type: object + properties: + status: + description: Status is the status of the upgrade + type: string + type: + description: Type is the type of the UpGrader + type: string + upgradeReplicas: + description: UpgradeReplicas is the number of replicas for the UpGrade + type: integer + subresources: + status: {} + scale: + specReplicasPath: .spec.upgradeReplicas + statusReplicasPath: .status.UpGradeReplicas + additionalPrinterColumns: + - name: UpGraderReplicas + type: integer + description: UpGraderReplicas is the number of replicas for the UpGrader + jsonPath: .spec.upgradeReplicas + - name: UpGraderDeployment + type: string + description: UpGraderDeployment is the deployment name for the UpGrader + jsonPath: .spec.deploymentRef.name + - name: UpGraderType + type: string + description: UpGraderType is the type of the UpGrader + jsonPath: .spec.type + - name: UpGraderStatus + type: string + description: UpGraderStatus is the status of the UpGrader + jsonPath: .status.status + - name: Age + type: date + description: Age is the time when the UpGrader was created + jsonPath: .metadata.creationTimestamp diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 0000000..9786102 --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,23 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/devops-engineer.com.cn_upgrades.yaml +#+kubebuilder:scaffold:crdkustomizeresource + +patches: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- path: patches/webhook_in_upgrades.yaml +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- path: patches/cainjection_in_upgrades.yaml +#+kubebuilder:scaffold:crdkustomizecainjectionpatch + +# [WEBHOOK] To enable webhook, uncomment the following section +# the following config is for teaching kustomize how to do kustomization for CRDs. + +#configurations: +#- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 0000000..ec5c150 --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 0000000..5124604 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,142 @@ +# Adds namespace to all resources. +namespace: upgrade-controller-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: upgrade-controller- + +# Labels to add to all resources and selectors. +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue + +resources: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus + +patches: +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +- path: manager_auth_proxy_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- path: manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +#- path: webhookcainjection_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +#replacements: +# - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # namespace of the certificate CR +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - select: +# kind: CustomResourceDefinition +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - select: +# kind: CustomResourceDefinition +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# - source: # Add cert-manager annotation to the webhook Service +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name # namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace # namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 0000000..70c3437 --- /dev/null +++ b/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,39 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=0" + ports: + - containerPort: 8443 + protocol: TCP + name: https + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml new file mode 100644 index 0000000..f6f5891 --- /dev/null +++ b/config/default/manager_config_patch.yaml @@ -0,0 +1,10 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 0000000..5c5f0b8 --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- manager.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 0000000..c88ed4b --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,102 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: namespace + app.kubernetes.io/instance: system + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager + app.kubernetes.io/name: deployment + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: + # TODO(user): Uncomment the following code to configure the nodeAffinity expression + # according to the platforms which are supported by your solution. + # It is considered best practice to support multiple architectures. You can + # build your manager image using the makefile target docker-buildx. + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/arch + # operator: In + # values: + # - amd64 + # - arm64 + # - ppc64le + # - s390x + # - key: kubernetes.io/os + # operator: In + # values: + # - linux + securityContext: + runAsNonRoot: true + # TODO(user): For common cases that do not require escalating privileges + # it is recommended to ensure that all your Pods/Containers are restrictive. + # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + # Please uncomment the following code if your project does NOT have to work on old Kubernetes + # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). + # seccompProfile: + # type: RuntimeDefault + containers: + - command: + - /manager + args: + - --leader-elect + image: controller:latest + name: manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 0000000..ed13716 --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 0000000..ff7f601 --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,25 @@ +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: servicemonitor + app.kubernetes.io/instance: controller-manager-metrics-monitor + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml new file mode 100644 index 0000000..8db6aa4 --- /dev/null +++ b/config/rbac/auth_proxy_client_clusterrole.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: metrics-reader + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml new file mode 100644 index 0000000..deb6140 --- /dev/null +++ b/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,24 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: proxy-role + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 0000000..bc65296 --- /dev/null +++ b/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/instance: proxy-rolebinding + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml new file mode 100644 index 0000000..93bbb62 --- /dev/null +++ b/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: service + app.kubernetes.io/instance: controller-manager-metrics-service + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 0000000..731832a --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,18 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 4 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 0000000..dd58839 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,44 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: role + app.kubernetes.io/instance: leader-election-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000..e0bfc42 --- /dev/null +++ b/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: rolebinding + app.kubernetes.io/instance: leader-election-rolebinding + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 0000000..b8ae3c1 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manager-role +rules: +- apiGroups: + - devops-engineer.com.cn + resources: + - upgrades + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - devops-engineer.com.cn + resources: + - upgrades/finalizers + verbs: + - update +- apiGroups: + - devops-engineer.com.cn + resources: + - upgrades/status + verbs: + - get + - patch + - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml new file mode 100644 index 0000000..a9a6e3d --- /dev/null +++ b/config/rbac/role_binding.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/instance: manager-rolebinding + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml new file mode 100644 index 0000000..04062b4 --- /dev/null +++ b/config/rbac/service_account.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: controller-manager + namespace: system diff --git a/config/rbac/upgrade_editor_role.yaml b/config/rbac/upgrade_editor_role.yaml new file mode 100644 index 0000000..c9b0cb9 --- /dev/null +++ b/config/rbac/upgrade_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit upgrades. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: upgrade-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: upgrade-editor-role +rules: +- apiGroups: + - devops-engineer.com.cn + resources: + - upgrades + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - devops-engineer.com.cn + resources: + - upgrades/status + verbs: + - get diff --git a/config/rbac/upgrade_viewer_role.yaml b/config/rbac/upgrade_viewer_role.yaml new file mode 100644 index 0000000..32cac49 --- /dev/null +++ b/config/rbac/upgrade_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view upgrades. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: upgrade-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: upgrade-controller + app.kubernetes.io/part-of: upgrade-controller + app.kubernetes.io/managed-by: kustomize + name: upgrade-viewer-role +rules: +- apiGroups: + - devops-engineer.com.cn + resources: + - upgrades + verbs: + - get + - list + - watch +- apiGroups: + - devops-engineer.com.cn + resources: + - upgrades/status + verbs: + - get diff --git a/config/samples/_v1alpha1_upgrade.yaml b/config/samples/_v1alpha1_upgrade.yaml new file mode 100644 index 0000000..178a55d --- /dev/null +++ b/config/samples/_v1alpha1_upgrade.yaml @@ -0,0 +1,12 @@ +apiVersion: devops-engineer.com.cn/v1alpha1 +kind: UpGrade +metadata: + name: upgrade-nginx +spec: + upgradeReplicas: 3 + type: "upgrade" + deploymentRef: + name: nginx + namespace: default + images: + - image: nginx:1.25.4 diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml new file mode 100644 index 0000000..6f107d1 --- /dev/null +++ b/config/samples/kustomization.yaml @@ -0,0 +1,4 @@ +## Append samples of your project ## +resources: +- _v1alpha1_upgrade.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/sample-nginx.yaml b/config/samples/sample-nginx.yaml new file mode 100644 index 0000000..25265de --- /dev/null +++ b/config/samples/sample-nginx.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - name: http + containerPort: 80 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d128953 --- /dev/null +++ b/go.mod @@ -0,0 +1,74 @@ +module github.com/gitlayzer/upgrade-controller + +go 1.21 + +require ( + github.com/evanphx/json-patch v4.12.0+incompatible + github.com/onsi/ginkgo/v2 v2.14.0 + github.com/onsi/gomega v1.30.0 + k8s.io/api v0.29.0 + k8s.io/apimachinery v0.29.0 + k8s.io/client-go v0.29.0 + k8s.io/klog/v2 v2.110.1 + sigs.k8s.io/controller-runtime v0.17.2 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.26.0 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.16.1 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.29.0 // indirect + k8s.io/component-base v0.29.0 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..6963f89 --- /dev/null +++ b/go.sum @@ -0,0 +1,205 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= +github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= +github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= +k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= +k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= +k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= +k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= +k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= +k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= +k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= +k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= +k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0= +sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 0000000..e50b515 --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2024 gitlayzer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/internal/controller/config.go b/internal/controller/config.go new file mode 100644 index 0000000..1a6ecef --- /dev/null +++ b/internal/controller/config.go @@ -0,0 +1,26 @@ +package controller + +import ( + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" + "path/filepath" +) + +// GetClientSet 获取一个 clientSet +func GetClientSet() *kubernetes.Clientset { + // 从本地文件加载 kubeConfig + conf, err := clientcmd.BuildConfigFromFlags("", filepath.Join(homedir.HomeDir(), ".kube", "config")) + if err != nil { + panic(err) + } + + // 使用获取到的 kubeConfig 创建一个 clientSet + clientSet, err := kubernetes.NewForConfig(conf) + if err != nil { + panic(err) + } + + // 返回 clientSet + return clientSet +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go new file mode 100644 index 0000000..ee1041c --- /dev/null +++ b/internal/controller/suite_test.go @@ -0,0 +1,90 @@ +/* +Copyright 2024 gitlayzer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + "path/filepath" + "runtime" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + devopsengineercomcnv1alpha1 "github.com/gitlayzer/upgrade-controller/api/v1alpha1" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + + // The BinaryAssetsDirectory is only required if you want to run the tests directly + // without call the makefile target test. If not informed it will look for the + // default path defined in controller-runtime which is /usr/local/kubebuilder/. + // Note that you must have the required binaries setup under the bin directory to perform + // the tests directly. When we run make test it will be setup and used automatically. + BinaryAssetsDirectory: filepath.Join("..", "..", "bin", "k8s", + fmt.Sprintf("1.29.0-%s-%s", runtime.GOOS, runtime.GOARCH)), + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = devopsengineercomcnv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/internal/controller/upgrade.go b/internal/controller/upgrade.go new file mode 100644 index 0000000..7e3f921 --- /dev/null +++ b/internal/controller/upgrade.go @@ -0,0 +1,61 @@ +package controller + +import ( + "context" + "fmt" + jsonpath "github.com/evanphx/json-patch" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" +) + +type patchUpgradeOperation struct { + Op string `json:"op"` + Path string `json:"path"` + + Value interface{} `json:"value,omitempty"` +} + +// UpgradePodByImages 更新 Pod 的镜像 +func UpgradePodByImages(ctx context.Context, pod *corev1.Pod, image []string, clientSet *kubernetes.Clientset) error { + klog.Infof("Start to upgrade pod %s", pod.Name) + patchList := make([]patchUpgradeOperation, 0) + for k, imageName := range image { + p := patchUpgradeOperation{ + Op: "replace", + Path: fmt.Sprintf("/spec/containers/%v/image", k), + Value: imageName, + } + patchList = append(patchList, p) + } + patchBytes, err := json.Marshal(patchList) + if err != nil { + klog.Error(err) + return err + } + + jsonPatch, err := jsonpath.DecodePatch(patchBytes) + if err != nil { + klog.Error("DecodePatch error: ", err) + return err + } + + jsonPatchBytes, err := json.Marshal(jsonPatch) + if err != nil { + klog.Error("json Marshal error: ", err) + return err + } + + _, err = clientSet.CoreV1().Pods(pod.Namespace).Patch(ctx, pod.Name, types.JSONPatchType, jsonPatchBytes, metav1.PatchOptions{}) + if err != nil { + klog.Error("Patch pod error: ", err) + return err + } + + klog.Info("Upgrade pod success") + + return nil +} diff --git a/internal/controller/upgrade_controller.go b/internal/controller/upgrade_controller.go new file mode 100644 index 0000000..254a95f --- /dev/null +++ b/internal/controller/upgrade_controller.go @@ -0,0 +1,165 @@ +/* +Copyright 2024 gitlayzer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "github.com/gitlayzer/upgrade-controller/internal/utils" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + devopsengineercomcnv1alpha1 "github.com/gitlayzer/upgrade-controller/api/v1alpha1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + PodStatusSuccess = "Successful" + PodStatusRunning = "Running" + PodStatusFailed = "Failure" + + Upgrade = "upgrade" +) + +// UpGradeReconciler reconciles a UpGrade object +type UpGradeReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +//+kubebuilder:rbac:groups=devops-engineer.com.cn,resources=upgrades,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=devops-engineer.com.cn,resources=upgrades/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=devops-engineer.com.cn,resources=upgrades/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the UpGrade object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.17.2/pkg/reconcile +func (r *UpGradeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // 打印控制器启动的日志 + _ = ctrl.Log.WithName("UpGrader").WithValues("UpGrader", req.NamespacedName) + + UpGrade := &devopsengineercomcnv1alpha1.UpGrade{} + + // 获取 UpGrade 对象 + err := r.Get(ctx, req.NamespacedName, UpGrade) + if err != nil { + if client.IgnoreNotFound(err) != nil { + klog.Error("Failed to get UpGrader: ", err) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + + // 如果 UpGrade 的 Status.Status 为 Successful，则直接返回不再执行后续操作 + if UpGrade.Status.Status == PodStatusSuccess { + klog.Info("UpGrade pod is successful") + return reconcile.Result{}, nil + } + + // 如果 UpGrade 的 Spec.Type 为 upgrade，则将 UpGrade 的 Status.Type 设置为 upgrade + if UpGrade.Spec.Type == Upgrade { + klog.Info("UpGrade type is upgrade") + UpGrade.Status.Type = Upgrade + } + + // 如果 UpGrade 的 Status.Status 为 Running，则直接返回不再执行后续操作 + UpGrade.Status.Status = PodStatusRunning + err = r.Client.Status().Update(ctx, UpGrade) + if err != nil { + klog.Error("Failed to update UpGrade status: ", err) + return reconcile.Result{}, err + } + + // 获取 Deployment 的 pod 列表 + podList := utils.GetPodsByDeployment(UpGrade.Spec.DeploymentRef.Name, UpGrade.Spec.DeploymentRef.Namespace, GetClientSet()) + if len(podList) == 0 { + klog.Error("Failed to get pod list") + return reconcile.Result{}, nil + } + + // 获取升级副本数 + var replicaCount = UpGrade.Spec.UpgradeReplicas + + // 如果 UpgradeReplicas 指定为 0，则使用 pod 数量作为升级副本数 + if UpGrade.Spec.UpgradeReplicas == 0 { + // 则去获取 Deployment 的副本数 + utils.GetDeploymentReplicas(UpGrade.Spec.DeploymentRef.Name, UpGrade.Spec.DeploymentRef.Namespace, GetClientSet()) + + // 将 Deployment 的副本数作为升级副本数 + UpGrade.Spec.UpgradeReplicas = replicaCount + } + + // 如果升级副本数大于 pod 数量，则将升级副本数设置为 pod 数量 + if len(podList) < UpGrade.Spec.UpgradeReplicas { + replicaCount = len(podList) + } + + // 获取升级镜像列表 + imageList := make([]string, 0) + if UpGrade.Spec.Type == Upgrade { + for _, v := range UpGrade.Spec.Images { + imageList = append(imageList, v.Image) + } + } + + // 升级 pod + for i := 0; i < replicaCount; i++ { + err = UpgradePodByImages(ctx, &podList[i], imageList, GetClientSet()) + if err != nil { + klog.Error("Failed to upgrade pod: ", err) + UpGrade.Status.Status = PodStatusFailed + err = r.Client.Status().Update(ctx, UpGrade) + if err != nil { + klog.Error("Failed to update UpGrader status: ", err) + return reconcile.Result{}, err + } + return reconcile.Result{Requeue: false}, err + } + } + + // 升级成功, 更新 UpGrader 的 Status + UpGrade.Status.Status = PodStatusSuccess + err = r.Client.Status().Update(ctx, UpGrade) + if err != nil { + klog.Error("Failed to update UpGrade status: ", err) + return reconcile.Result{}, err + } + + // 将更新的副本数写入到 UpGrader 的 Status 中 + UpGrade.Status.UpGradeReplicas = replicaCount + err = r.Client.Status().Update(ctx, UpGrade) + if err != nil { + klog.Error("Failed to update UpGrade status: ", err) + return reconcile.Result{}, err + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *UpGradeReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&devopsengineercomcnv1alpha1.UpGrade{}). + Complete(r) +} diff --git a/internal/controller/upgrade_controller_test.go b/internal/controller/upgrade_controller_test.go new file mode 100644 index 0000000..4e00b89 --- /dev/null +++ b/internal/controller/upgrade_controller_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2024 gitlayzer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + devopsengineercomcnv1alpha1 "github.com/gitlayzer/upgrade-controller/api/v1alpha1" +) + +var _ = Describe("UpGrade Controller", func() { + Context("When reconciling a resource", func() { + const resourceName = "test-resource" + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceName, + Namespace: "default", // TODO(user):Modify as needed + } + upgrade := &devopsengineercomcnv1alpha1.UpGrade{} + + BeforeEach(func() { + By("creating the custom resource for the Kind UpGrade") + err := k8sClient.Get(ctx, typeNamespacedName, upgrade) + if err != nil && errors.IsNotFound(err) { + resource := &devopsengineercomcnv1alpha1.UpGrade{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + // TODO(user): Specify other spec details if needed. + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + } + }) + + AfterEach(func() { + // TODO(user): Cleanup logic after each test, like removing the resource instance. + resource := &devopsengineercomcnv1alpha1.UpGrade{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + By("Cleanup the specific resource instance UpGrade") + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + }) + It("should successfully reconcile the resource", func() { + By("Reconciling the created resource") + controllerReconciler := &UpGradeReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + + _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + // TODO(user): Add more specific assertions depending on your controller's reconciliation logic. + // Example: If you expect a certain status condition after reconciliation, verify it here. + }) + }) +}) diff --git a/internal/utils/resources.go b/internal/utils/resources.go new file mode 100644 index 0000000..e7a0ce4 --- /dev/null +++ b/internal/utils/resources.go @@ -0,0 +1,76 @@ +package utils + +import ( + "context" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" +) + +// GetPodsByDeployment 用于根据 Deployment 获取关联的 Pod 列表 +func GetPodsByDeployment(deploymentName, namespace string, client *kubernetes.Clientset) []corev1.Pod { + deployment, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + if err != nil { + klog.Error("Create ClientSet error: ", err) + return nil + } + + rsIds := getRsIdsByDeployment(deployment, client) + podsList := make([]corev1.Pod, 0) + for _, rsId := range rsIds { + pods := getPodsByReplicaSet(&rsId, client, namespace) + podsList = append(podsList, pods...) + } + + return podsList +} + +// getRsIdsByDeployment 用于将 Deployment 关联的 ReplicaSet 列表获取出来 +func getRsIdsByDeployment(deployment *appsv1.Deployment, client *kubernetes.Clientset) []appsv1.ReplicaSet { + rsList, err := client.AppsV1().ReplicaSets(deployment.Namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: labels.Set(deployment.Spec.Selector.MatchLabels).String(), + }) + if err != nil { + klog.Error("List ReplicaSets error: ", err) + return nil + } + + rsIds := make([]appsv1.ReplicaSet, len(rsList.Items)) + + for _, rs := range rsList.Items { + rsIds = append(rsIds, rs) + } + return rsIds +} + +// getPodsByReplicaSet 用于根据 ReplicaSet 获取关联的 Pod 列表 +func getPodsByReplicaSet(replicaSet *appsv1.ReplicaSet, client *kubernetes.Clientset, namespace string) []corev1.Pod { + podList, err := client.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Error("List Pods error: ", err) + return nil + } + + pods := make([]corev1.Pod, 0) + for _, pod := range podList.Items { + // 找到 pod OwnerReferences uid 相同的 pod + if pod.OwnerReferences != nil && len(pod.OwnerReferences) == 1 && pod.OwnerReferences[0].UID == replicaSet.UID { + pods = append(pods, pod) + } + } + return pods +} + +// GetDeploymentReplicas 用于获取 Deployment 的副本数 +func GetDeploymentReplicas(deploymentName, namespace string, client *kubernetes.Clientset) int32 { + deployment, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + if err != nil { + klog.Error("Get Deployment error: ", err) + return 0 + } + + return deployment.Status.Replicas +} diff --git a/logo/Upgrade.png b/logo/Upgrade.png new file mode 100644 index 0000000000000000000000000000000000000000..db1222e0aa71feb5879e381b45da213cc7341d7b GIT binary patch literal 34642 zcmeFY1yh~dvNcRX5}e>}!3pjT!FA!T!5xA-gy8OOL4vzG!QI`1yUW7)p6q+}-8uJr z-&5}oxK*oa)mp&QJ!khEJ$m#cSWZR^0Tu@q0s;a-LR?q@0^+qO1jMTn=y$+390#W8 zz}qW(1u;R0vQfN!2nZqw31I;x7oDSYm^8GWJ8*IH?mVg!r=JL?@F&O+W9I~^PjW3i z)B>ByNnMA8;V!+T?-&Lw7;sPwSa3FxW05|@P!bUod`)X+78a2gc!S=kr#F_IzF(G@ zpI2O;=R9hJIsQma*mWX9IJ4%}bTBt|IjxA(1@-0=#NV$^NaH_W{rv&>mPiJo6FLFn zHPPQMwpY+3Nw1;5Lj3FX39IAXKYvaX4Z$N?0gVLl>R+#~eb8)w|0(c{vw#0>s#kuK zpRxY?*+gIEa4i4Z6^J)TlD-K4ZRjh6pjpM=j|HADE(XCVTJioLQx!0PFKGYx0yzGs z>EJiN7{7`9@i@TIKOXl#9F>3}ihI}f-^+mb?I`5`qobd=UiUM8^ZduuR}kP!-XGh5 z^lzrcNg*PUXwu$wuKV8|6_@)H|>joVCGeH0D=qH8O&_6mD{)j*p%lzMY0_?_rbTldg!Y=GH z;XkH6qXNEA{j-S>e*b0~1?H#0o6tYj4LJHofc}T0F({uX5dvQQlg5Cff21w2qyKkD zeH`9I8N7M;!&EBZi}W9-gy`Q)M~Ooakf^=+CqRIse+KB^9gT_u@|pqup9lbs{+YJ_ z(a~IzuXrT$Q2&^ke*^de`_Cpq=>2KhNBH*vB>p2n1SEg1``;b)$^Cr*_5O&!zaPN= z=qM5Xt1YY!zJE;pBnNy^{9_ZL|IIWh3UB~1{s_>&BL%Rde>zG^1|S9eA8Gs_k>Y{vG;!=?e9$fPe}1Uip2jy+P_fke;mO7g|xqt>HiODi>l)c z9J?ubW{=b^W(}nO+PytFphn||fPWEa!4lUqWUm#H1q#yGjym>YQL$zec|s+kjK2F< zWh(?YkmuV{A#@T4R|PEmBB}Uq!z&R&AEpt}A=5DbD*L~N1Ry&1hHNvdtbfq*QN|hD zjv|1jhK2Ao6nRz7aEv^uD?j{IP=0iL%7UsM7S0MQ&@&?8ZrtXs&kUX2iPQ_5f?A6c0nkso4bYai8rogc(72)_xR zbMB)Xr@$g>H#6t_$hLx3M;kxcU^AUw|ET~zoNa=hC-!rm+}pJ@n|+&<j>8KHS45 zfBhJ;Wu(nouD4g`IlIPZc1A#m-Wk*_t%U|dOUhrzVCNTu#B)BcI>bi%`eU2+<5ELG z%Sxezkul{0>dl0{)192@! zpa81L!^#IDYjlK4pI>Uh4n zm$D&Ug)as3QDUkAgP&@yv2a`cy5wh(W**}q)alC1sV{~AdU>lu0cy|t?bfbFfxpNx zA}_GlLf-GWO5Tz(mAPeE*U*Nw$nVV1OzX)+ijAiI!-U0j1x}0T!kUF8%wLH+-UjfN z#l&4b+nP6eSAr|^+6a%ekrta-EK(fSamE)e93a*=S;Tyy_Lc#f~*1y;THrh7o9|vEi zBNiqKTASKtV=>1ViC=9t1oV58Tmfy*RS%-ZtRXEX%)47AFHH2O3E(P zkkK8-UKSc`ajr$)o75q_Q;$=cJ)iM8BU2l#<;N-87Pe;(0!kv(@AU+70s-oJe7C*G zE4ok8{h@P1Kf}5SY3&|PIm!|~nzzrB$6uJv9OZm-iU{1wo=ECMQK#H4Ek+oo8 zDzP-vet=2k5!P=@Rm1RWBV`p1jUV@8yBBSU(I4*XY+Zq;X{h{9CX2s`Gug-s`$m26Z*wuLa+6E`vde^^!MWuxWrCv|)g~ttDorv4ezeqD zea?<~Cre`H%I*Eiutdqx3;HyCQnUmesEy+1#8HkVxkEI6LRYC8qA7l9D10dL~AejAb6hqeptumN~Xzj7`|) zXECPA;=qmxeTd(97!YaU!&S>_JO#BItrlVU_|5H6n^yc+ z{on`#Y~hQt=d`9|!{$)$PKPZn7}DGpjB3&dnvsST3}=&w8`2W*KifW~R``kQJ6hD| z_uugv*hN?A&>HzJAr>~6PD14GeRELu%MUfu>CJ= zU;cf*zOhA9h8>6=LtiG&onB&yIC`VdN2TeJU-}+xnYjFs>-|Aa z#~de?bJf`yH#mdgH54ZMBD%$5-|ovwO<1DLC(eWf&+AUHxd&-V@0(l#HxlbwfVkPvZ6VCZN zTj*0e{2K*c$il@F#HD76IuGnJY%5D#z>_-L2ZbXtUeP7oyCK3kx6tgA4Mvoy>&@WF z^DT=$&gr7T+B1>~wj%F7sy$Q8FbCycYp74>jaJz{(#&dMcO?w;!B)kt;6BJ^a@^bZ z*AV4FB(`@8b%9);#nAe&x7|G-dI7P}`NHae%Mc~dZm-cBS!fy?fq%nTHeI% z(iA6R%xLEF&FSjAHg)~`NH3_ML$9T@nhcj!fa4>}!zIl>qFkp1XyejarpfJhjPO!~(&rj<)XM zK&b3oy?rQNFiNHl`BmUk`bRAVnE_d>1=RfW-PH?H_ikR%QWo@muJpbdp5hEl^#_%E zgXq(Drvz8go_=YU?u$94lLn-b@uisk+dG&)dhJJKC1<^Q5QWVkor7fE^3TYZ37iT0 zZkdpx#(v(OZ?ezM%v{SGC*)eR#p?Y!cfq;5y!5``NwwjN8o%p4^qT;M{GlJXt zVl$Gzt*Lb>XcONHz$e6Z_epD7yO?s3i0JZhG>ae64CaNB_?f}Rs>L#jsg!hzDAj%n zGqf?1 zH(!>Pl;Cx8@`I*ovT3Z1rSPH|t+4{yO~x{g{4sTsVtU%B*sWz@ln%E>`1gwFXk&b@f!dGTSRCD6)3&-s)!UL%;9(_&N1-YXlQR%ouk#+{ z$kcE4V6FmcW@+9gGUqAZE>{o^@|M8c=jk>3=>4JGog_1r{oUySeV6(1^7pGi#jd2T z0EFg+suEqsUf8NH3lv(hNP#JJq*(A7c0`z8p6_<~pL#HLJv5z;ZrT?a`|#{v=Gty* zbwR0Yr7DecEbj~z%O5|b_<0KaiUP7~=e^gN@8HWbxZR#}m)8)xo&`bDMQ?)TIQlcg zZI_VUD~^2u3ZUT_k9K~9QGVZB|Ezp@jgN3|k9*JpSr~zA5VtmzOeuxtjaikyTc7l; zi9kb&as;J%G_7Mg&2w;&i()I!B=xIeevn@ij;3vp*DJ*`J@ z7yiv_e&euc1{5v|DY`0>yh9BPSpQz*M%~E~(Qq82fG)j3#_Y{W#-4)?BO@bKnKTaD z+1fcBYF<$_{q0Ym4A#0bz8*pKMiRal0%6<^WAw)8>^~uB+&*_v^YVJ`3dNvpdP|QK z_;hTY_|ACAtbS5KucF`_@0Xs%XTsC<_LrIocNrz6HZ~38cnR6)>}b4o0}8|9 znR2tBa7;{;*yh+_Iw|o}YI$KLuM6p#+rB-lw@)Xrd`%CV=>ze6&K5=Zt z(HzA~R?$FopnHM_%h=T4gPIxct#W?}sp z?7H9c-YD&-Ttwj6PAhCuK2ym1r>K+XJ92d^F;a!~SCjIJCg*Q8GskVT*RPK8wLgKX z+{8bQH>iZl>P|Hdc<>BMjU2nvHC-kVDB72N@+4xHgbk}6r2;gGP4Q3xckSCAJa5HM&=wgr)F4X^tAHUg{6548@T0iG?m5iV0y=Q-`~3Yu+P>G$iz7A>Mp1E_4WN3MWvZn z4zcwYSM)&v~Qqi%p8nTNU7B0qFQL-_kNk=2eES&ZVRLOb@^|I zU|ecms}$C{pZ@fE8ZrjyZ*c}w^E5X(xUYSx{b^uiw5^eZW$si~>Ux6=4Qg1&Dec~{tH_LcLKOQc&FvEUI+iz%~F z)4lHZS>11rsO0@gR+zNs=*!E>ykyvVl`ES^1WR%jfa>_=`|9YMm!HZZ9|-Y)h8rsIQh*|RG?}l^W2PNy5;n78GteDAycUh%x#$O^=_WLG$>gRk8Ig%-O5aRfAIz_7)ulAR>`Cu zZ4RJIR;?W0Yq0t`Nzj<5TLvR3f2~bZMV8NQi#D|wg27usr++Qs7$Kc8hO3^XUfn-7 zLg?@1F`N?7j~vyFy>Dm^6juw4S=fT;&_Z z)u~9gN2+-lm3I55FGk!{KF_uuu;ZRWKW)vkT1O7}I5{yzy8;2s*GRZdx$4~dtf8>c zBg1<=y$W9Qez}phk~D&3UvZc^r&cL(>GQO1Uu$RL=YJ7{!=!_k#8o;LvZfz9+jf2F;`n-!Gt6%W@SuEO3M(%2Fi_p=4om^9{N@DNpv*}($mT){XO^YBW}@( z@dRJh=1j-u3MNgqLd%|&v3xG)qK6m}%}NPwl&=itVo3vt@iIt_tz5|j84l?}Rfx%Gc8vV#$ziVzbue@NLCXZqv>2f6 zG4?~XXywbMIrCe+vlGb495-t zaelD}m{RPr3{PYqvk?#pF#pxKP+S-t)UpaOAipj9wwW6Yj=^NT1w;JE+o8UpTbqA> zi@+m+D{(6~*=TQ7_-+Jktk^R3QNwK&r88#Wh$5K5?neHt3GDhF{F%hNd%q6;SQp3g zmBd0_Nk@8Sgw?_$lAQB8Jzq3WT_&;G8nH@IqROVd?9kDrI;$m{l+@HnWj}cfOc(v; zZg5i>RB=&J0B3Jhhrw2rk6)tF04M$&=^AIBDJ0v`r0DAfr&TMS$Cn%H>RB6n%bxi{ zEoOIyvg9-~8U_p?+UaD>CqR1(Zf3ZvAu8#|IYXQ#|&Jt@m!Qs z^XgbvsF>sY8YxpHBcw#Q#Ui~hHaHSj0>|iOrED1;F*af~?0Mk`ofsb(EmhcP)BI4X zDc()JRKM$kN!N&&AZn@*JlVu6qTqme-}b66m8K~xe0{OIY#o*@a>?(`EL$m|xL_Nk zq)6S@6}t_oKXAKU?Y+z`FMAw5n_sQo8lvu2N^w8$H?2B!Ba1z{JwM)ojSaX@c6**% z}l60ynR+mQ1OuJGZ|bbL-g$} zL!;^3-pxScL(McSY++=}bly}_ax`|iGt$hLDl>jvcM_Bq5)vYdZO|GU7ZlUk&)Gh7 zN!0n?`os5U5Mp-`!E@4xf(m{T=owL%F|=!>*`q7ElE$NV z3cLn|-6l*Og|Hcb#a4Jp0-sO6 zUiHo!&cVTfq|bAasYn-{5wMxIEXH^)r9pL1z8~kcJC#FbYB(k9exf($xs4+F5#Edd zT8F$-n%$~-Z?oo!)nqF;iJ*LdR>qc4AM~wXKh1sPj@y>r?9f+nG@zgY=OrjhXEr_I z6>Tykshw;0gnvNdbMSfy~bXX z?fmm~O{1!m_T+fUi7rp(h7wQY88K+G;92$It4ct0b~-XR>Ow%UbO*&@ywgkAsi!d} zC07k0(rb}XG_!2p*@S$Wlv>f^SC|p$um>1V{fdo^DOJeKUr({J}*(P z##zxP1??U?)LrN(DJgmMVsQ^zLypKWKYG3FY;0^S>9}sA%d<>sx3Y&m9AWUBoT&@0 z0wS+Ocf!hAeNaW+nbZ4EqdA?8>t$FO;bPJYxVWBb7#4>NCO17%<-Mu~PnWd%Z~PA(hMW+FpXbS+nO{luL9eyyI~M~nOH zhWA1B)|7d5#my#my_KRUG!bR}8d>D=Xl7vfEvz=>^utDkztncr(K?%>^*+JSsQ zIX0W+Jkb;jzciBo{R(pek|dnJ;5o;4T4QEFJ9a-$KlRw{c+YqHD?p7Vp?umb%l=Vs zqdPb@UnX6WAj|KSe4X>n(SjTJWTmzz6m!bWe6iCXP6W$%GFQS9_kG#{BZW=l0GPON~PxdS}JV=bU!anPk(yhcW1(=|RfU<_ z+*-|kQ^K>|ntzyBnwZcMJZ9y2HVH#^-8`0hSFu27tu4~of1Qx{#b*IJ=az|_Hcx7! zfBLZ`Z}oFA|9oHliJr#%;3hd(A!=;&b!793g!A+aR(mGp^kenE|)6x7U zE-tPaThfwp#szluC4TNunzE{DY(3F#kIgpj@c>0;3kOPYSY5f)tZTN+HF#!brv4*% z^_(Wnbtn04I+r6C1wH-gQdp&aFBU=xSxTO~ra8?}QO+iAzyv!+Xz4uAbjhxIT#k4S z9Bm&}kKM4dvO3yWT3+3ptUROvFm_+7akB&1un0 zn-z4ZtfC_7Sm2RhbUaQuM(-7AHnH*=H05^C+ z-f&zSiJm;wavcljU<)fnvmh zszxZ6RW4N5{2GaL+BlMuk_2ltW>bV*jt7Pb@$tuA042q(Y7E@3Ydq_OlLEpt!HE&N zPew*oegr(>doe@Kls2AiH*zKmE33BKw7i(-Ik35dT2fYzx99w@fnDw4q9U&z9q7|e z^3EnFCzZ=vZx$&XMUB{=wnQ+sr)z=TY^bc1zjWboYxjDxe!QBJwaF4muQu$Il9cT1 z>FH5)KAJ!GV8Pjby1#yK0Cv_&gPU>4UZ43d&XMdpWH2pYBEeI#iJCodoiOmE;$W{j z#PifFicjlg%!hlvc9yrV@*NzM1gE;IMA5gf4}` zxM~8P?T#wkD*t*KMbONOr?7enki-sG01S9psxpFWJp94UKLxRQBr9<(A|i51qtnu$ zfDtgvm@j}wjRZ}zoA$om0!y9(N7iqyW(`PQ#U+%Otb9;vua4*SjO{~@5*^GYKg>Jd z_Jr_!AN}P9TY~mA-PP&oE635hNFWOy7T4+zbf>ScyM-Zpt(ifWvK^nJx}#%v;|6tp z1R4w{F_N+jcCDW|(+p*sZz|A)o@LP*PY-Thy#2PjyX)Hy3xBFP{M3DJp+I^T-NUth^I2s60=&eZa#~kQJ3OM%W z(-}atcL$OUVQ=!sQ0c?XaVi#YZV4sH-HS84xbRm<)NN=;RJ$0+H4>Cj;ol|pIl4)~ zin8g`!r^w412-6KAOi)lf=kGv14v#wCG>^#a3zP!zO2Et+Hfenk38Dr3%`IF^RBpN z)x>fPBih}<50Tw{Ia7@FyfRBVw!W&uy6m$cGz*LGu48VWY3nS=&d&aM3O@Z2!5}ed zpI2`)L3MMy}~g~M1T z6X|%1sr6;&WTlx9Imy_KoeVp-AB#BH+622c$mel zZ^pb%^=7=EFEiJ>zgJaVh}AJjs;IFaw*rS|OFf>r8=uW7g#pMcWHmER?n_{j?h2$YjaCCHgj2w4=aZs1!et>Kin zTntXJTh5<}v)~+;JGFVWdxJ9hJjut>I4=Oy4NG|!t@IyrZW)09S z;sO8;>a%}G(AZuq{Au(WYfT!rBHLb1A3ORZklO0|Oh8 zU-`@0$Qy_Tped(>xVSXJ(4KGM;o)1+aRf3m7O_+%Jrpu&d-C=kF$?6uSFP?~neDW- zG-8`+sK+`z(WND}$c3MplHG(kkY&t*r0SQwy&VaP# z9ttcJAHHG!uWja=nG3^=^CEVsZ~&g zr+mufX}EjXB4=D4G|2)?(U>yZ7#2MxB?ptL9_cIjKwiC0KYrl!X&a@G`&;ALw(y_c z-N{k1P^nXX20+;eSV?NN7I6nQ3!wkrNdkp*7Ko6K7;*GVY!Fwtjlqm)@K# zxjoVAl<5dvw@S*o{rd9|yx zwP|$Av3qvsAqE8Zo1c8~1w_r8w+7PMsnk=prNO8aZe~P?Z(eWJx&<*hNuuPzSR6L` zO9H&w>#`TuX8A`Obh_I?VJr@SE%>25_X8c8>c1M|A?k(Fz)5)$jdB|Xb3I(4j84WF z9(bwj@FY%?5)|k>H}eK@Iq?cdvgusPy7ks8>Zt(lB?uN4hc9{^;U#mAN2Tv;M=8RM zZ7`x!?p)Wmc?V*_Nk0PoKoxSYgaz0tU0_>wT90-wZiOD1}th3kF)@suX-A8Ps zJ6x1+`QdIaPlh(fG2WmU&`1-E{cfg<^YV;cFdDT{o^}KDmRo_OPrhj#xJ8K{xIObE z_Oax$Vz7Yp5uysi8$+%y*-&#CprwdQccUaAVx_r~VCFyP53Zi%lEdn;N$Pnlq1w>1 zU}LES^BK_$!m8!ZmpP3wHXT(Bk(X)KpS(>>CPkGPpb5srz$iOdtUYwK-x*$*VnVE? zsg53f%>7l#i?)_)rEQZp2IVV&^Bq<26^ppCl2ROFZ{61CRR>cQHMRbJTsd9c^5i_K zOSa`~VH)O5ocUOt_7%gy@T|13-1L@Xf7cL3r&`Z&VzF)l_@3HA9YnJFF5aIZEBh}F z+L37J=$?3pfl{fdsXa$I(uDgkM?g_4E7NYyxEk|*whxu|O6+R|JQlTHtl7^pWzmS< zcNqk4X(wYtzgaCPD!M1w_3(4ob*$8z`*ehMaRl_e^svL5mUk6v61^mU5WAIbaa-kbQXH4v159Ac`ob<{E4_XJr84o2j#wx~yU2kkIo=-jPes@bvJ@49jsLFcn?F zJAzBy`2^8P38mI(`SAdYY*;Xab%6c5l0fN@7c_}@2p zqdSQgduqX?ZWZxo1*XU5p<@0VfB_$! z$@0vhm3h52QsD!;p24f`^anC6d23?fIM?E&C?DvNwPVF^>g%P=@6q?I%J179kIqJ&x+u zSuXHPPqlo4eamGnAmGFK`R7-YOt%9n&M26rDCSjQN{mZ8B+32x2QCCc7Y7+iu*YPx zv(4-eDNH~euVVx{s>r$7Sqx*G;g}#?KGX5+O170dHxCvgI((mAdSy z{F7*vsA|pwGkD#f?S6iJ6$iDW+vH#tU3RDhtq9wg3#k#%;cZBXmffMXtm_C-OHCVk z0bN#S%w!{$MFmNL&?K|(56>y6^78z|J(j`StWy_iPPpRz@>G-CBa1eX5~WKrB)YY| z{Z!)p^4y-NcpU4mOa1X9!I%P;1hVK$fKV0^I)l{0IRIf%Kb78=a!aSgtvs_Smd#%S zjg@*wW6m2AEdgmbb~_%FYSV`HMJ~;+fizvK5sd;~9G?Y+U zhsh6WzHjEKU#!#t&T;!ZAco+BKp+^naq|=v$Lkr@$kzS5)D-0`vbSm?BL31lpqa*I zZa2>P3CsE^l~RpsV-$=z=3atYA*6-HM#?5t4UMM5>p5La$Dr=uBWwXwa_J;ZRaIw1 z|KlxOH#V>|k7XKvv)lDSalE?5w^IQ1E`TbIW3?Vr-A6-DZ?gpm5T+P<#jumx+uQ9k zEa3^?`xh+=;Hx?S>4!Ts!D|zD?RE&rsy6&!;h-OAEY?}1yw2P}PjFAC+m^QFdj@4URtWxU6luk(ziM;!`Xn8wSfqv}`P$?Z| zlhHI(*3ma)I@@GZL-wK^WCn?a4(llsZa(-24wHLV}FWF4uo+38%IvT9l& zwlRc6Zj+B0WFB8sM$TD&4!?eQ7kEN5rjA8csG6f|+k@W9f4`Rlvg!fuIaq7Y$!a&+ zk6P6Aqmn1hi{I!lq0%_~^UPGCd^=36 z)&;DbTIA5t(IXfJStunKiOQS5fB$Yh5JUE3&<;;&Vv*ZswWTrbl35F8Wus-EW521* z<7mDTljrOwWLz^@nId9uqLy87eqJ8AGb`*r7WJ87}^fzUzSbyvlNjPODqE_jwOy zVna7n^GVZTFGFvW?dY8SXkjlJ$cQ3ZkpF=NMDpSK%pj8YW><;ca`J{5nq+2^Y>EDv zcO!^|Pp9JZD|Az7XzoR3lC@R(yeSxx%G9B@;9XxpJCNiRZzlDj@0NFU<=i$c?UySK z^2^dvm0uRR$Fm=1lyW**cD>IQ3EdFM*!UKgn3%ZM%uj!O0-t4a?HV&PoIH2vY1I>g zZZcHYl;Ftdb=yVwPNht%k*Uj`-k&DjH{z*Sj*?W;U~{VN4HIw4v3uQ1Rt}juim(Jz z%z_)#hG{?iH=-G#j5N4>VautpS(Q?w2d5K)mfO)vP8brA zHyjk-J2n9INT8GI~ zAvb__lk`!5`%Vs(LWaktF9QF4uYRRKqs!-hgN@5O9kn`3Y_g2d&V&QoKzQtpp`oFi zahy`Ce8k!Pdtp60hp~5vc(;dvgkDK+5=o5rZ%xu{dq)68q&-}i`06$o?t}3@22K38 zasU;pfA`Q*%i!m#A-dFOK*oHmXl&$^&EHP3I&&(JTjej3@j3$TD9uWp=ZEMyTSwa| zqF}stTBDh9+H1xWGV7KiC3HSrecDXa^}6GjG2WX8Hb;ALdYWYtzs|n4>pX!{abw@L zksS^8Nx+ls$5&iHAQYcsNi;$l6Z}jmX1J3-2su!M|k=-D{hwDUoAC7MF+Z*p;-Y_z+Dgui(8y|HI3~U_vPf;&AZWIf3h(7J_K?v_1VPPzpud8`YFu0PR8mY zh)Yx^+W^s5I8|dCjkD(TV4XCqzlYagWLC``!$lecQW6_8yE5ScYpp)9Cwsl49czi5 zB2vDJT?0o6D7;RKhkCX98H~N+Siy)|*d(}tr{WB^_=x+)zU9s;~_AN2j)WeKPj z0l*Z<7bb&}>-|hvGPg4wZOFEaleY#>+uwHqMqCg8*3gD3%X6Lxz@FMX0R&ZFuewkU zLKcgy%)wP8-~Hag82^h$r)Kp=mfWQtQ@{%gND~wOU8Ey|aKtaJm&eOcfTD+b79}b9 z=`5a0Ow~wz>-g0kv*4#~<{E%w4&<78>e>m(r17f!>=?@;oI=+n?Wyj+VhOdl`+ zupAy6HW}PvK9mv6XM2y!7G|OmBC01m*Iox)VybTkl=|{dSVl*6cB?>3Wn#E+%+Put zN+qwmK1P$N_rK8~SN*B=IsngF%#qB?SVbqf)sLo0t_Bkw z^>K-{5?GCEHIP1^QvIX*Hq9>FS`%BYXEiTlS@{$L#XNvBu0brzZ0~wG9U3LhUl>= zuH>SMir5b>QRI}L{I?04_Nby8=;N%K2Y_xvQ&?1#&3-jmB6GUfe|Y_ztTOI(Tsvo~ zBGdUb(+sxEQi=%a`-1HIkLh_V1+*QX0_z#FPLB?W^!4Qh1>fB`NyAX= z{F|Ilp96oh4CDH~uMwQa!%5yOY;1#$)zh#u2fyYIt^lD=78vhD>Uuw2Q^X*&scJjU zOaK$4XGbfmQY~1LB>`dhvzvp9(a-$AjSQ+hv%FNt9G&Vx2H?K7*%Q0_^7z)Ywc`z* z89S>g1+@h;{N||rD5ps;&|(-^*Tu)j0~fMilDi>ef+5i7{cIy?o0%3k&C(7^aYs6T4;HCK9vHD!T8670&Y6mhYf4hZ>1d!e&yQ&@vN6r{~fZFLjr&Za~?8 zi3MaEkk>(B{*C~a(e?aJi+r7E#--28oHK{({h)g!SJ?~iQ^bDnMYx-&R4Ta_Q=SAK zgCKOyEcB%T@{W}+LyT`6R{=aBpNGF$$`-Ng10EG*g+^}ps;D=qzr3dPO+A2?27Ewi z17==JV(ZWp6Y=+8;9oz=u57#?aQ&6LRn4I^0A8g-2V2byO=;6%l;>^#EHvDy^yLbkdp18rLlyzIyd~zY3!Xg37io@aE_VQ4O zJ>P#h&N}!ojs*WMG*-Krm~tDkpzTL4dO#h5G|;$QpHBN(nBfA|>0{(r)uH^rjp0lm z1}3JvnVZW;+2zLF49(v}o>&l&tBer2wx7mhn6B0$5_Q~y^H!vg#qVr39kugWmn5w! z$rHQf)Fmd?wH~6y2wdlW3!st0#1p`~0`^x^Kv1V!0EojW-@{S0Wmz9BJPyE7(TX;l zfD1hFkhA5wFG@5Rxaq#fvFR1xzJ2=p>C=PvQ0>z?+yIl(fMUnp*lX^2ti4QYO9fXP{tjEr8{ z?hZWkeE^gTQtpwE5ZI!2;tSnLEq^W{5fBl%-j++)qF@#<;z9vrzz-4!(?*OUAKEB& zMbz{5Jr6Mh)xT;| z?dklYN50NF(y{&1#S;kRsxE!>S!9LB1RdSRla?? z`$;1zpbijNTUzE;!R+%LIKq|pe|-h1!UtZqud?(r9ow78p%BZNXa-H?o+;%mlW}EM zSVZJx&NVJ?k23!g12$0)J@9FEOn=)rDbYoBvs7+L%M84CuK*% zD92{l>Fv|A${mxhtD#gU zoUG%U{Rn56+&I(8x( zJNpYL8HKJpJkP8d;GxKaG+2yn@=N7S2mfDtZ~YbJ`o;?zsDKCvNJux*B}fZMN_Tfi zcMhSXba#w^bd12j&?t>acMM2LgLK1tv-kJB|HD~l^V_r5Gc!-z_Z6SGC}{(*RE2ru zv2t>0j|QROON__2gISL9YiV~fqk}#+!@0RR!BPCjiF%Bk+Fe@$00{Bt1vHZ`?ZQ*G zrf&<;qyl~bFZy7i+@M(cqXnd&S>TT7V5U(2ApL^}P^G7UTxk8{U9Gq(?=+yVrUq|@ z!(9(R;p2cKf|gxS3TtWDDv;0E{qW&K?4u{7FgZGhOk=|1wM-WUO&zr{(%i2JOkOHf z%EH>tHFMDw^;K2QsB4+%A^Zj$5cxOiaOoU6{4d9s*Zw427QY7fz$$Crtm&OY0J`4> z=oGN6D@-2;OBXXa$A2v@ci&;DS1@dQDjuJ)GgVk%1=aUEU+qSEZa7 z+}ux7nz=szOf%JQa6K;rJvKt{d?jI}zUTZ9x#x$N1mc>TJ|W17-txgvYGJ0cY-b>r zuyl1}CW;D?LIy~!0Bdl#O*7Y)s3j2XcaxsRlpjrZ)xWhu#56Y4> zPScbMQO63k0NmD4ERBA>z)coBFqt8A&q=)(( z5bWNU#&rMm)+obV8)^6a6_4t{Ta}YA1K8>CojvFTpz=I)Xo{GYku0@OCHcF1=wgZ$ zp~9!xVAqTm$fYfLhRn%K^`a!a=SG9|ff%r?TWt}QFvdX8Q*#{oQp_c&p<+uwh-^?% zvrKEBAgS6r3J6CVGY^CIRFU|!e0sKG=BP#Uu~cJTz``RA$jTQE8V!}w$M(MXAfdZw z3!rSSq)xv;noseMR_C|i=~cFNJVd(;GF@Beh(DpRR|Hsfgk^ua1ZnFbGeZ(`9F3sd zI;ws*SZ`?Z6<^#&+h5;VFvp`NOMga4hH#vS_O!z_@OW>-I4|T+rJ%ld)ZDow=_o45 zk$SU?k#i1BV;e~+DYq$NR<)rt z1+xoiv8j+%-I#hE^85Z67)F_VLN>gmTs8b z^%!{y4H|Yi5_9@UMlD4GC_oP#_y^_YCi~9I!g-rxB873i=aMe^nhMzkw%HQ4xb(5| zmiIL5DBiXeO{_aBYOxvl8FWiRf17rbq;B8n;nFtu0>yX+l?&z~XCDhA5XC69&8nCA&gF)>U}Z^~o$Rcs~5cQQJG5;SXk5 zfz;0d1oise{cds(K@O0@7J$$s3}Z{BIIwwp5O~Z|4C|i6gYt6i7m$bEv{#kk=;m1n zF~wNi28-p)Caj+NfcBhR#-@6D?FB-7f8^_GDXS4arJnzi|f@%Yr{>k9tbel`MXo5MDz$ zttc{pzfuIKU=zvLP#tT^^Q2;Kq~4IUFo#MsNC+Gr9JG0Pd1-Wl?LIoSlUzGH z98@nRIFhWD8zlalW5r$|nbN{wTIqDi?&bXsfW4l7hGdw{SIc#`oo=9+aZ3KRp}vnJ zc^O%8%Do}+tDUN;8X-!||Lr5GOJdAbz+faB-<|?hfi|MQ8A<}DO(u1Cky-?Jo)VDK4LH|=lffjSwk(d4D$13YN2Wi zx#R3=Vh1(e(Q2fxR(&!-YJUhi@&JH@sC^$e2M$Up(3w33q8FwNK$~xJyOk8i_sLNV zyCc|Dp5y_7!QD90cWQ1v< zfm{4D3h>Ab2r)yunV7~AfNpJ9^k(uAHiXyJd0v3Lb$*L6UlO{6ttkg_DpqAXk(a-M~g=m<|q{za_*&HS5cwUTbz&V&BXw% z!r;zm;h_Ehh-dOGECll=RP{GBoogw#G-*bj{L& zSo|^8S=YvXZNvQJLxpcoiTdR%X=1;kJ4-#9+qhS+I$qxyZn7wW$W(IQ%<=X#Gq@k@ zLgwcXp>Db|d`Y;hr)rt|IUk#8e@8u|7VU2?NY9NI2Wx@%&3X9iF`XtNZ@kN*d?drAsB?N%a<81|adjM^y z1BUZPzBQVa5&cGK>Eu3#*k_kME#?oOU^Xp+O4N6-)`}kPG-FkGTvS?`VVQ10fizB{ zvF+voqSkC6FXe3k3)3VDt|z=%EpZvg?g?m@Q>*l3id0n+)n)>JpfMAPZ6Zm{gn*!t z831xNk|^w5M><3s9kctXrjpXoayeptW}~+I=TBMm{9z0!^=&%45+)fxQ+7hUlP%1o z#}Dr7gs)0D`Ic+Iq--c0{v&{nD{B4bA;|bq zJN^74`_E|A0o^$E85`Gg?J0%=Z7r&d!S=Q?Rf8a`b@LB_mjqQl4qRM8YoD_#-x=fW zy>k(^Yr|9_lqZ|niuDBP^tGaI6J&7$RpfO6(U@Il0;(#?i|Xx$O&$+o^|8m(+>rfaAeb=rISI1v7}gn9&@ z&EU!mwDAb$KH2lGQj!QiH7REAyq02siTbC#w!trNrXwI62a?s1VF8cbD_?*(B+S_x zRx`dC|0GRnL`59`ug)H!ygi5nOysgxJSe#AAZH1oi`+Ln%EpK zef5s|QQGo-Lco(KCz}r&CE3q-al$Bp;49S=EFYHGh@3yxL-vEie*2o;gn8+r8-g#F zxH+!?$Jzkmv%sse((r)kYRfJEi&3aTX=y2BLP29tgRnTa6v9q8mqn9vZroR8|BYz$ zWcpjRtNw!Dc4c?=H6XTHO2hNO;whiWCBnw0g<8zo%J!vV&R-+Z~-_^fx+;2DS@9?9m)$JWz-ArR=H7*C2{3ts{dU zcpb7)JL~j!U3US_ZsU^VG7#h3PnTzoF|oF~s^99kC?;Fx&TlndEgCp?eY$Y#G2k_` zv9~!}uj4b^(lP^{ddZK#Cxf-jHd8>++0|jffo@~=D@N&TD8>YvnUN74h;lL;jeZ31 zeoalVB?lOJ(_ICft;Ot~qN4G!uA$I9VygzG9%8=kK)fRwI53{(q9B3EwK`YlAQEi; zt|h*}YVc06Emp_HAmfnD5-Hp#=~!=p?cC0;0FY3eJ_-17UWgGO0*Sfh^WL?{@f5Kh zSYPI5Hr(Yiw7fI%CNMg)&cOGBn)5Vro1{z1BDAf5c z!+j-re{wj_^GiJGt1CEUIQ<|$ql^^X4!R#)2C`eG z&%Vs;9vnEQhi88CZoX(%!IHs8Nvx2oA1dR|$`o@Gqn&yv)9@g~)ehdw-{Yh|32Oxy z*^OA|W(UE3TL?Uk@Luuv<(p+ir=Nm0FFBGZ(T?k7S0$Xcive{~R#`c0Yis+<`kPw) zdIQ;^cQaQCn;~nPOZXH)c5nn+i|17#)bBzPU$VFg=M>P6kpkGpUn?ptx$=mVw7nwr z9(+t^+mXN=F>{C>>Qy1(dk8=bauE+8%wK5P?;`GZhtGqZmUgonia=KQ2S*9@M{9}G{YWu;s^hDgZGmzIl~d1M;oL5y?m{s zt1^kBYY9TjK0{wb&4I+@s~cL@RXr%OMqYkAg6ddPsgnP=INjTWzP8bjdy$qlq4o>? z0gtr=#d80>>e}aoRF?Tfo%o|c^HwtSIEuBsO-*5L2d}GOq{y)2omI<622ozFKZU|O zQ?Q>0>m%p$%7WtRV4G0$ggg*q(HE_Yq%pCrpd2ZlDho?l*DGaQx)ejdCDy12mRSeR$xk&S zTNGc0Z)#|^32%opRjgm%u`7Hpqq4K_m*C2dxaogxBop$s6#^Z4MmAXT_UQ$29&*oo z&AdmG3SBT#9_L+~_*<*1kHgfmPd?HMac%<+X1CcM*$@gk!SfT*4lgd|Vv5fEzs$qm zmG$(lW&pdVu-MQGu#~un!h&{+&U@E3o`Tt@fbe(j1ccOiG?Qp{2N}7D-&MoRjLuJp zj2A4y3m*tun`nXOJpd%P(d{lCe>I2MXS{RlxnMYgMD&w{Nv3IZhoFkBMIe5gJlaJH zu;HO6F`#W}Ps1y=R)K5Dp|QH=s$CVCE#tiW%pWOTEe+_`W@`NcoQ<0HGDp zmcK-zy|N=c>&rlhF~aXUqRZ4#t&^xc?;*e&eo*-}Ma`QXm*3*GN!gaSwv$t392BZ( z(=Wf!=)ty{TOX{~mC~lf@YTCyDa$E~d?YoS$k30e zf-4Jn9Y|*m;XDRi_xJt`aOE!Co(<4H%s6hE&A_>@VR-qLe^|>K}qh>OBeQ z-^C!u_@LjJtNeEakD;o%1;j#`QBYy6;`5AtWThXd%i|^<%H%~nVDhWArIugDSIR(%+`jd1I)fM-Va^Pi^fdH}#d|4#On=P~E> zM80zHJTJ&&wJ(T}0Z`#Z*=jSKIJGQjiz&8YRPF3McU~+-AVWA)2QZ3E=3(Ipj6m|- zbwe0!o>qUd)j(bSI658iHHhjMl)=c!p&>_VH`TOgo7p^TsS%fsqNCu+t!7UkQ?dbe zas$&rvObA#w_xgt@(54ytLL)sPXZ{LxxlUYkGNgWX1ciTe^}FYYu%pm4fftM7g_V$ zk`GpD%vJ zgK>MYO--MY@Hi$~Jvf*KEX#_%w;x=;JoNrR+)W=_@WK^8YRO^W#mVB&;d-f1$OT(n zMcyQJj!QwYFHKvI+IDP(C16fhhsjT}0JA8Qq6I;hX)id=zvNFqBRM_pi z0&O%Chrv25{*7nLbCTwMwuSRUfsRZdasB&*Z(?tq_|K_gK z^GHg^S|t-4hv1^3_*o$I|2WgSWRxtu9d&D-uPR$u@LJ1KU6IQJvRbOOt0v~)I4)G1 zf5%5r&3?s92g&~!v`oD5ix;G9nT(mQow3QG((UOUxIBri=RIr zyG4HUx9jzbx)seMX(_H_uBWz0{8H01cMkn0nE z1u?CUIrko_Oh3kyk>-(n{9fIN(T@hN>p>{GM81jp_zFmH zsBscT+Pf&5)F_yrCQdyLpdn+uFNx*aFdBwzL}sBOxrAl~cp;62xSJCjkXwpfcLYRO zsPS;tFKSI)LL|^LlRi|tBlDri8o$900?Ix&E(Hoh@sDd&pKh1mEAnY<8{@$kF#oTF#!QOqGH2XBd73hD^c?*~A0aaG38}Od8+ziL~9a&qOS`nPJ zw6tV!rTwb1u1RkalDEEx@0P`B{j8z#dbIwLMzd$BgF) zA7PRBUbAdA{D{^J4%4o@yIT7Kve&E`!9qqzT+`caH?`sU)noaBiN?tm*2!Ha3U9@Z(^fg9q{FDT z`h}U*H{0s6=g;*yXXu-ou!Ea57Td=wT816;q<)pT3AS$ISN(uY%k84EF|@MRATLk5 zEBO2}gty1O=;}x9boZI5=f^KZ#vdk|+2P;{5bL`ph-3A${w&W<>gB&Psdl%X%-BGY zpcDCJ%OhY8AV~lDg#|Wf@zpx%-(){z7yV4Y@2Zkp7S&^3{ z9#?!S>CCWmu?2(H0E6F)P7~!aX`7Sm6yEyr7NQo;Py2ubntem zxG+2Zl3T)k-?zSl#jxWss3nZE_b#uq*W}&w5!9#wD%dCzPTSdQZl2j5YV~@4cU++e zU@^J;oJZw(doLGgTWM$)xWH_srb<25Q@J$G)ew`)B7CATUn}ryqEb@bft@D%tJ&Gv z3pd-AB9{_68JWQ6hwyRi%VPWd9S7ydmot+kuk2LZm5UY;5|kg`iXc20S;=6r@?9T| z!WzMDMQrB4wAsUVtLa;)8wfzQKm+F9Gua%aH>?@`1}cn63KeRb6GH3bV?A{s;bJzHH{Njt#AH7pp(SP*2~dDIl?0NLJ;F(wm^WP=5zYB#IxK{D9xv?D4DtM1(N4QVL#FQ!p0slmYk^m zX1Z1QdVFP*$c|7lIBbcX(bC{rZ}$bF}K z3;vT&c+Hc9wB*~AkwXUHRk#3(>6e&#MM(^qV+ynO!8x+gBjQ?pX`9(TltKdu7#>q# zA_wEwb>Pw{!3{|=CHh@e6Rn+6F$oY-q+|OtyL!FdynK~nPP-ZS4!AsmON!zoy92@_ z7nyDN_M7MyQ9~wqW`ng;;KbwC#?o%MeuyO)3j~#Ub)=tDGnw7r;1e95d|0Z$c$Srw z)h;eBPJQkpymi@tHcE z0doV%2t#%sBdT}aUuSY(8y1%G)+JgP`enVs9HE@s$sTQlv<3rT) zvm%x}fPP3FqN_Wr_k1SUpEc!aZs+r*L6e+!65EZmZCPtxt+(^1EvG(Jyx)S2z7Du~ z-SRsLPU?Fg;edFB0*%-%QGUE2snqrQo=4@37Kx>lloUH?=dwgH|U7auUXCif1M!g0Ei7=eIigIFHF&Ju`g!2wo@10h8L6pjf*_v8Q8bwEJ=MrL@7) zWrE4z>WcTzwj3moo;7)2N#&rjJS0UjN9K0or7vpqOh@^n`4B7_gCk;n>6*81Hu3LA zdACsQdD@FIktbdRTLNGHLXk$Pha^NY>NuO+&Hd=;$mc4h^(uc|LJ8p2qgSiZ^*0C zmGf9ZtS;rku~51N@@F%aYA{2c=8+JLV~Vn9w)6S%l<2KJ-tk6G4R-q@M2FKD(H8bA zD1jT4)@tpGdkW+_^MkkP#N#~x*fM=>l}ZrW*qn+d7FReDTs$vFi(C`Wt?ek}9vC;P>w6SaHOuN}p6Aa5$PTmSih` zk~i~LTI*f;GGmLjMe6*!(?p+K;<`_JZW^cjg=-!af{^a^$uI{mEAj^80d#?yqo;rZ zuMAM0c_KH%LCf|btbzObt?dL#Z-nwps6OWxiSCR{-LIKT?D2I&aBg~8(!IHQmrhCN zC3m#)rUi+I-NfqQv~8~zU(XK~eX17Le3#8WJkR7Ngxz~@IKZLlr-FYBZ9puUr@0U- zK%3c)PETPBNwOv~zRm_xo0RV~#=+nOxKP7hdeh$xH~ePj3(Vf(r^Tkv79NGw+g$iS zH3E#e5-R7ptR$`dOILQ+XU^3pbu*fhLN8QukQ@4TpS3i{6T71}A5F|7+&+0nHgt&l zgbhZV)f}ICGC|3$plJQSYP`Ee?F8-8PP7a7N`=~&V4d6QV`UPUHw(JwUj)zc$&6t| z!We4{hmw2J)7R~gU5?#vvwzO)1BLdFG>D|IV!BFLZ7*8hEG2c#y(>39o|mum z)~Au^(=S1}9f-{{Vl(bC+D+`!TKJGUjP=dMH>J&B%4z$sh``jRN6$F+we{<|Juk=m z{jt|}K&eRxjp|h@^iAzPH6O)L2^H_g{X4?yJo8BZtcQYx|ipC-Tg8*0M4-$0RCms;W>g&d0JzlHaotw?Y?Rg%?kUm5vu)3qiR zpN5mYd5JRKklh4V`r4*ZO@@4IM{m%|YD70kDfG+gIQpSeu7WqJ`t3k{h3-V_S8mio zw@W-LD2v--qcWsOsccfjEBmVuLK&NQOOIS}ou6;~~ewkyP+_RkVL6>kb6}<43l>*1_ta5Vp z1FtSsnI6uE%^Mjbj|)E&w+Bzl=q@IZTzohV{4$y3)9{Q3%a0enjj$ z5RW1IYKOSBwo3X&6B?=xVH)#(uUj-*6bkoN;C=&*Z+5hzw8JloQ5xn*L%;Jb;>Qte=b)YWhC2n+S7O6py)H{-BnPSbj)LCIF9|I;mp>sdm-zvt4kfMt;9dr$66m&=$g3OTH{t^xZ5n zsbtvwed96gGP=MzW7|@MSReNh2BzylAp{pXF3T|$Ne{;X2@3W%#g9g2Me+mnt4W>I zX(x~q^XLn$-W_Zqse>-cPi(xhk?k?U@%S-pv>M-fBQ%gYf7i9EO6kuL!^Ck#e_yEf zEVVl(X6oYIw1m)+<>Bai?vF9jC+`cWPup-?ZnpSq`>q~|*txywRjQT?H#uGmxdW~G zM6s{b%ro%4<(B@kv$HErHMo_ouM)q;a4q`sZtDDEvQ;?0u(0rI$H5;mR$THRW0U9% zOq_FsPiR(f~En?D!FTmE^EDjIq_LmMWy zKMr}~JV2F`{%FKa*%Bt=wHwZog7L)8W2fI0%D)ZgYe^}ON}+W}At~)N^lbf4xxpu!0VlixYN&@o}t8Hm{V|Sx`fZdfY#r=nb+74A--d1*tP{bZWv7cHe@;*KB@f3C0l4*SwhZf8j4 zwg~rUkk)zmSN)sB!5M#zy)j03q^*x$4rA*a*w2Qop6^$0op7q~K2>ghKRgg)BC0x3 z9Bu^n-BE9*KPYe1eyDJ@w@C#y>y=lMtON#D)aEOqJiN4KMgEPiOrUu`bcDD64MK3e ziOtigb6iCHTs=6D&Bs{?;@vzSPU-1}8qfSqWl#`Q6I5L2dym-+Cb{n*z>935gbtLOA zH@Kt&2-i}GB>z#KCz`*SL7kO6vcsnG+1G83pKr#E+%M-nCj9I!_UYD2Be4$t`oRgS zwzswfCY?y0N2>L$XT_M38pRH-Df)t$TZRts^0u5IJ^*pTZhqI|B0g^@JbR!#v(4vJ zwkh0QgoLks@(cR%Mk`G zcM6w1&sBH{bfK<$XCqvrG#K$=yuoSY5BlqL|1oCn-kk(ESB_sy?#zLY@2-+ufgJ(i zU}H3IkSl2=r8^IUfJ*^pOJlN;*>swu8}{3K_fg^>pin;@L>uOU|J}7aj2ORCyv*9w zrq%q#_p_ECgR_D<-jtcD=W8aKTLurEMsBBnL;-8@Ei9P2CZ5HSlauh-wpKMwWLtf!9y z1P}(CBr6UJ%dSw3qj!VK?sKs5b+n7vRI-|*ta9}+nP-xm92+^sEi{T_JlMp7m`#l* zB}8&9!XH?=pHkj^pAX>!lsqi)V~Uo7O(7l6(=8~IM)mEH!H}ES@D?5fm;5ZA(ylf> z9ak;Gl-v88{n#_}RrK0*iad3zMZNsLDHl7UoX+)rw5do+dP8y#Y%9-KcQ=0@1+(Ga zkM%DJex7V~Mx9J+NzHAh0#{^}XkSS*A_6}PJ1~^AX|>z#Yw0avD2ab?>Op%Tg~a*l zZnh{$?UgXSUQp#G4zWJ9;h>h?@j>#PO{C{H)AwM~(HZ%B(xbs5!KG8MQ-Ws^bcT7I zKrnypPflE{;AQT?-v6pTt6~mCWqw6)4)?tBc#9hL zcK;Ug(ZbqKk-v&+%%d1fpX9;iYZ8Pc8Gm=c=rv{=O%6P+gSKWCkRLYu&pZav0y0(I zEj@N2n#_}k2XV6hyhDniw;zpM3A10C%nx6^fMu#2&+zQI$Ji5GlAJ3{#8})!P8aNx z*F+JulTp9^q$4P{u&To{2)kBb86;Tj8U6D@BI10D&J+?&Zk81;P-yBnna)%gp)1g% z%R&;etw!|E>P7{Tn$#kS8T=>yJdcu;AA`BSn6Wot{*iZw9w+DV8ZIVw;a3V3Q zaice>!1|F&1#g+%#<3>IP@eYU2&pTgTc%Loy0DR)@m6EbwyAHpsCQENyK+uG4}zf* zE6!b`Xf}APoIl|n=o!)F#A z7)|xw8%GK1MR{u<;xj!-9^b9XUQmA}6p0SDx@;}w82x;BEjk${QJ*M6O=l(rdGmOf zB=5s~Y^G!Rh)QaX`UPOV2!n5C2Txm;QW6BLGUNXVuj^xz-?%lu1HT>0gO=-G! zQa=iXDF|Xd0uAy4ax<<;;)3z%LwuGa3CN89 z|J8Sfzn*lvKHQsemgMWX9>i=FptmPqYMgd_SMI(yt;{qi3DYQLxtVm>r!qkI8g z>bs-0P)b>0k-r_0g(_v(z#7?6&++hO(|6aTtxcJ_&bVfH6(UV-r{CaWS z&8$O|DbyRFf0b7+ZOneq zH;`H38(7*pGRtxqH`Vt6C)rE{6lwF^!dU23ETnd-Zg1!#{y{ivBH_U?5-g{BAvhXhw(wDyx7(t$#z$ zzio