From dc9b94932bca523628d1336e1c69a8193e2eccea Mon Sep 17 00:00:00 2001 From: Mulham Raee Date: Mon, 6 Jan 2025 15:46:07 +0100 Subject: [PATCH 1/2] Fix karpenter-operator hard-coded region --- karpenter-operator/controllers/karpenter/manifests.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter-operator/controllers/karpenter/manifests.go b/karpenter-operator/controllers/karpenter/manifests.go index da720c7f57..23c1f542b6 100644 --- a/karpenter-operator/controllers/karpenter/manifests.go +++ b/karpenter-operator/controllers/karpenter/manifests.go @@ -165,7 +165,7 @@ func ReconcileKarpenterDeployment(deployment *appsv1.Deployment, }, { Name: "AWS_REGION", - Value: "us-east-1", + Value: hcp.Spec.Platform.AWS.Region, }, { Name: "AWS_SHARED_CREDENTIALS_FILE", From fdd76ee5a56a9384216e1fa87ab5919f4a59fa19 Mon Sep 17 00:00:00 2001 From: Mulham Raee Date: Wed, 8 Jan 2025 11:43:34 +0100 Subject: [PATCH 2/2] Add karpenter_machine_approver controller --- Dockerfile.dev | 1 + Makefile | 2 +- .../certificates/conditions.go | 7 + .../karpenter/karpenter_controller.go | 8 +- .../controllers/karpenter/machine_approver.go | 233 ++++++++++++++++++ .../karpenter/machine_approver_test.go | 184 ++++++++++++++ karpenter-operator/main.go | 5 + karpenter-operator/manifests/operator.go | 215 ++++++++++------ 8 files changed, 579 insertions(+), 76 deletions(-) create mode 100644 karpenter-operator/controllers/karpenter/machine_approver.go create mode 100644 karpenter-operator/controllers/karpenter/machine_approver_test.go diff --git a/Dockerfile.dev b/Dockerfile.dev index 01f2818a95..8690df5a43 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -12,6 +12,7 @@ FROM registry.access.redhat.com/ubi9:latest COPY --from=builder /hypershift/bin/hypershift \ /hypershift/bin/hcp \ /hypershift/bin/hypershift-operator \ + /hypershift/bin/karpenter-operator \ /hypershift/bin/control-plane-operator \ /hypershift/bin/control-plane-pki-operator \ /usr/bin/ diff --git a/Makefile b/Makefile index 650a7f8fb5..a03101fb00 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ all: build e2e tests pre-commit: all verify test -build: hypershift-operator control-plane-operator control-plane-pki-operator hypershift product-cli +build: hypershift-operator control-plane-operator control-plane-pki-operator karpenter-operator hypershift product-cli .PHONY: update update: workspace-sync api-deps api api-docs deps clients diff --git a/control-plane-pki-operator/certificates/conditions.go b/control-plane-pki-operator/certificates/conditions.go index 6fa22717ec..6ca1d868d9 100644 --- a/control-plane-pki-operator/certificates/conditions.go +++ b/control-plane-pki-operator/certificates/conditions.go @@ -25,6 +25,13 @@ func HasTrueCondition(csr *certificatesv1.CertificateSigningRequest, conditionTy return false } +// IsCertificateRequestPending returns true if a certificate request has no +// "Approved" or "Denied" conditions; false otherwise. +func IsCertificateRequestPending(csr *certificatesv1.CertificateSigningRequest) bool { + approved, denied := GetCertApprovalCondition(&csr.Status) + return !approved && !denied +} + func GetCertApprovalCondition(status *certificatesv1.CertificateSigningRequestStatus) (approved bool, denied bool) { for _, c := range status.Conditions { if c.Type == certificatesv1.CertificateApproved { diff --git a/karpenter-operator/controllers/karpenter/karpenter_controller.go b/karpenter-operator/controllers/karpenter/karpenter_controller.go index 682b247bed..9e58286a8b 100644 --- a/karpenter-operator/controllers/karpenter/karpenter_controller.go +++ b/karpenter-operator/controllers/karpenter/karpenter_controller.go @@ -9,6 +9,7 @@ import ( "github.com/openshift/hypershift/karpenter-operator/controllers/karpenter/assets" supportassets "github.com/openshift/hypershift/support/assets" "github.com/openshift/hypershift/support/upsert" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -17,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" @@ -73,7 +75,8 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, man return []ctrl.Request{{NamespacedName: client.ObjectKey{Namespace: r.Namespace}}} } return nil - }))); err != nil { + }, + ))); err != nil { return fmt.Errorf("failed to watch CRDs: %w", err) } @@ -92,7 +95,8 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, man return nil } return []ctrl.Request{{NamespacedName: client.ObjectKeyFromObject(o)}} - }))); err != nil { + }, + ))); err != nil { return fmt.Errorf("failed to watch Deployment: %w", err) } diff --git a/karpenter-operator/controllers/karpenter/machine_approver.go b/karpenter-operator/controllers/karpenter/machine_approver.go new file mode 100644 index 0000000000..535fba1f4d --- /dev/null +++ b/karpenter-operator/controllers/karpenter/machine_approver.go @@ -0,0 +1,233 @@ +package karpenter + +import ( + "context" + "fmt" + "os" + "strings" + + awsutil "github.com/openshift/hypershift/cmd/infra/aws/util" + "github.com/openshift/hypershift/control-plane-pki-operator/certificates" + + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + certificatesv1client "k8s.io/client-go/kubernetes/typed/certificates/v1" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/source" + + awssdk "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" +) + +const ( + nodeBootstrapperUsername = "system:serviceaccount:openshift-machine-config-operator:node-bootstrapper" +) + +type MachineApproverController struct { + client client.Client + certClient *certificatesv1client.CertificatesV1Client +} + +func (r *MachineApproverController) SetupWithManager(mgr ctrl.Manager) error { + certClient, err := certificatesv1client.NewForConfig(mgr.GetConfig()) + if err != nil { + return err + } + r.certClient = certClient + r.client = mgr.GetClient() + + c, err := controller.New("karpenter_machine_approver", mgr, controller.Options{Reconciler: r}) + if err != nil { + return fmt.Errorf("failed to construct karpenter_machine_approver controller: %w", err) + } + + csrFilterFn := func(csr *certificatesv1.CertificateSigningRequest) bool { + if csr.Spec.SignerName != certificatesv1.KubeAPIServerClientKubeletSignerName { + return false + } + // only reconcile pending CSRs (not approved and not denied). + if !certificates.IsCertificateRequestPending(csr) { + return false + } + // only reconcile kubernetes.io/kube-apiserver-client-kubelet when it is created by the node bootstrapper + if csr.Spec.Username != nodeBootstrapperUsername { + mgr.GetLogger().Info("Ignoring csr because it is not from the node bootstrapper", "csr", csr.Name) + return false + } + return true + } + + if err := c.Watch(source.Kind( + mgr.GetCache(), + &certificatesv1.CertificateSigningRequest{}, + &handler.TypedEnqueueRequestForObject[*certificatesv1.CertificateSigningRequest]{}, + predicate.NewTypedPredicateFuncs(csrFilterFn), + )); err != nil { + return fmt.Errorf("failed to watch CertificateSigningRequest: %v", err) + } + + return nil +} + +func (r *MachineApproverController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + log.Info("Reconciling CSR", "req", req) + + csr := &certificatesv1.CertificateSigningRequest{} + if err := r.client.Get(ctx, req.NamespacedName, csr); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, fmt.Errorf("failed to get csr %s: %v", req.NamespacedName, err) + } + + // Return early if deleted + if !csr.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + // If a CSR is approved/denied after being added to the queue, + // but before we reconcile it, trying to approve it will result in an error and cause a loop. + // Return early if the CSR has been approved/denied externally. + if !certificates.IsCertificateRequestPending(csr) { + log.Info("CSR is already processed ", "csr", csr.Name) + return ctrl.Result{}, nil + } + + ec2Client, err := getEC2Client() + if err != nil { + return ctrl.Result{}, err + } + + authorized, err := r.authorize(ctx, csr, ec2Client) + if err != nil { + return ctrl.Result{}, err + } + + if authorized { + log.Info("Attempting to approve CSR", "csr", csr.Name) + if err := r.approve(ctx, csr); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to approve csr %s: %v", csr.Name, err) + } + } + + return ctrl.Result{}, nil +} + +// TODO: include a creation time window for the nodeclaim, the instance and csr triplets and also ratelimit and short circuit approval based on the number of pending CSRs +func (r *MachineApproverController) authorize(ctx context.Context, csr *certificatesv1.CertificateSigningRequest, ec2Client ec2iface.EC2API) (bool, error) { + x509cr, err := certificates.ParseCSR(csr.Spec.Request) + if err != nil { + return false, err + } + + nodeName := strings.TrimPrefix(x509cr.Subject.CommonName, "system:node:") + if len(nodeName) == 0 { + return false, fmt.Errorf("subject common name does not have a valid node name") + } + + nodeClaims, err := listNodeClaims(ctx, r.client) + if err != nil { + return false, err + } + + dnsNames, err := getEC2InstancesDNSNames(ctx, nodeClaims, ec2Client) + if err != nil { + return false, err + } + + for _, dnsName := range dnsNames { + if nodeName == dnsName { + return true, nil // approve node client cert + } + } + + return false, nil +} + +func getEC2InstancesDNSNames(ctx context.Context, nodeClaims *unstructured.UnstructuredList, ec2Client ec2iface.EC2API) ([]string, error) { + ec2InstanceIDs := []string{} + for _, claim := range nodeClaims.Items { + nodeName := claim.UnstructuredContent()["status"].(map[string]interface{})["nodeName"] + if nodeName != nil { + // skip if a node is already created for this nodeClaim. + continue + } + providerID := claim.UnstructuredContent()["status"].(map[string]interface{})["providerID"].(string) + instanceID := providerID[strings.LastIndex(providerID, "/")+1:] + + ec2InstanceIDs = append(ec2InstanceIDs, instanceID) + } + + if len(ec2InstanceIDs) == 0 { + return nil, nil + } + + output, err := ec2Client.DescribeInstancesWithContext(ctx, &ec2.DescribeInstancesInput{ + InstanceIds: awssdk.StringSlice(ec2InstanceIDs), + }) + if err != nil { + return nil, err + } + + dnsNames := []string{} + for _, reservation := range output.Reservations { + for _, instance := range reservation.Instances { + dnsNames = append(dnsNames, *instance.PrivateDnsName) + } + } + return dnsNames, nil +} + +func (r *MachineApproverController) approve(ctx context.Context, csr *certificatesv1.CertificateSigningRequest) error { + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Reason: "KarpenterCSRApprove", + Message: "Auto approved by karpenter_machine_approver", + Status: corev1.ConditionTrue, + }) + + _, err := r.certClient.CertificateSigningRequests().UpdateApproval(ctx, csr.Name, csr, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("error updating approval for csr: %v", err) + } + + return nil +} + +func getEC2Client() (ec2iface.EC2API, error) { + // AWS_SHARED_CREDENTIALS_FILE and AWS_REGION envvar should be set in operator deployment + // when reconciling an AWS hosted control plane + if os.Getenv("AWS_SHARED_CREDENTIALS_FILE") == "" { + return nil, fmt.Errorf("AWS credentials not set") + } + + awsSession := awsutil.NewSession("karpenter-operator", "", "", "", "") + ec2Client := ec2.New(awsSession, awssdk.NewConfig()) + return ec2Client, nil +} + +func listNodeClaims(ctx context.Context, client client.Client) (*unstructured.UnstructuredList, error) { + nodeClaimList := &unstructured.UnstructuredList{} + nodeClaimList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "karpenter.sh", + Version: "v1", + Kind: "NodeClaim", + }) + err := client.List(ctx, nodeClaimList) + if err != nil { + return nil, fmt.Errorf("failed to list NodeClaims: %w", err) + } + + return nodeClaimList, nil +} diff --git a/karpenter-operator/controllers/karpenter/machine_approver_test.go b/karpenter-operator/controllers/karpenter/machine_approver_test.go new file mode 100644 index 0000000000..ce0c521b56 --- /dev/null +++ b/karpenter-operator/controllers/karpenter/machine_approver_test.go @@ -0,0 +1,184 @@ +package karpenter + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "net" + "testing" + + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + certificatesv1 "k8s.io/api/certificates/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" +) + +type fakeEC2Client struct { + ec2iface.EC2API + + instances []*ec2.Instance +} + +func (fake *fakeEC2Client) DescribeInstancesWithContext(aws.Context, *ec2.DescribeInstancesInput, ...request.Option) (*ec2.DescribeInstancesOutput, error) { + return &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + { + Instances: fake.instances, + }, + }, + }, nil +} + +func TestAuthorize(t *testing.T) { + scheme := runtime.NewScheme() + _ = hyperv1.AddToScheme(scheme) + + // Register the NodeClaim GVK in the scheme + nodeClaimGVK := schema.GroupVersionKind{ + Group: "karpenter.sh", + Version: "v1", + Kind: "NodeClaim", + } + scheme.AddKnownTypeWithName(nodeClaimGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName( + schema.GroupVersionKind{ + Group: "karpenter.sh", + Version: "v1", + Kind: "NodeClaimList", + }, + &unstructured.UnstructuredList{}, + ) + + fakeNodeClaim := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "status": map[string]interface{}{ + "providerID": "aws:///fakeproviderID", + }, + }, + } + fakeNodeClaim.SetGroupVersionKind(nodeClaimGVK) + + testCases := []struct { + name string + instances []*ec2.Instance + x509csr []byte + objects []client.Object + wantErr string + authorize bool + }{ + { + name: "When CSR request is invalid it should error", + x509csr: []byte("-----BEGIN??\n"), + wantErr: "PEM block type must be CERTIFICATE REQUEST", + authorize: false, + }, + { + name: "When CSR common name is invalid node name it should error", + x509csr: createCSR("system:node:"), + wantErr: "subject common name does not have a valid node name", + authorize: false, + }, + { + name: "When there are no nodeClaims it should not be authorized", + x509csr: createCSR("system:node:test1"), + authorize: false, + }, + { + name: "When there are no EC2 instances it should not be authorized", + x509csr: createCSR("system:node:test1"), + objects: []client.Object{fakeNodeClaim}, + authorize: false, + }, + { + name: "When CSR common name does NOT match any EC2 instance PrivateDnsName it should not be authorized", + instances: []*ec2.Instance{ + { + PrivateDnsName: aws.String("test2"), + }, + }, + x509csr: createCSR("system:node:test1"), + objects: []client.Object{fakeNodeClaim}, + authorize: false, + }, + { + name: "When CSR common name matches an EC2 instance PrivateDnsName it should be authorized", + instances: []*ec2.Instance{ + { + PrivateDnsName: aws.String("test1"), + }, + }, + x509csr: createCSR("system:node:test1"), + objects: []client.Object{fakeNodeClaim}, + authorize: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(tc.objects...). + Build() + + r := &MachineApproverController{ + client: fakeClient, + } + fakeEC2Client := &fakeEC2Client{ + instances: tc.instances, + } + + csr := &certificatesv1.CertificateSigningRequest{ + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: tc.x509csr, + }, + } + + authorized, err := r.authorize(context.Background(), csr, fakeEC2Client) + if tc.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tc.wantErr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(authorized).To(Equal(tc.authorize)) + } + + }) + } +} + +func createCSR(commonName string) []byte { + keyBytes, _ := rsa.GenerateKey(rand.Reader, 2048) + subj := pkix.Name{ + Organization: []string{"system:nodes"}, + CommonName: commonName, + } + + template := x509.CertificateRequest{ + Subject: subj, + SignatureAlgorithm: x509.SHA256WithRSA, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"node1", "node1.local"}, + } + csrOut := new(bytes.Buffer) + + csrBytes, _ := x509.CreateCertificateRequest(rand.Reader, &template, keyBytes) + pem.Encode(csrOut, &pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrBytes}) + return csrOut.Bytes() +} diff --git a/karpenter-operator/main.go b/karpenter-operator/main.go index 3a949800c2..953633f289 100644 --- a/karpenter-operator/main.go +++ b/karpenter-operator/main.go @@ -106,6 +106,11 @@ func run(ctx context.Context) error { return fmt.Errorf("failed to setup controller with manager: %w", err) } + mac := karpenter.MachineApproverController{} + if err := mac.SetupWithManager(mgr); err != nil { + return fmt.Errorf("failed to setup controller with manager: %w", err) + } + if err := mgr.Start(ctx); err != nil { return fmt.Errorf("failed to start manager: %w", err) } diff --git a/karpenter-operator/manifests/operator.go b/karpenter-operator/manifests/operator.go index a0d2e5e631..0c79fa0d6c 100644 --- a/karpenter-operator/manifests/operator.go +++ b/karpenter-operator/manifests/operator.go @@ -10,12 +10,15 @@ import ( "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/upsert" "github.com/openshift/hypershift/support/util" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" intstr "k8s.io/apimachinery/pkg/util/intstr" k8sutilspointer "k8s.io/utils/ptr" + client "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -107,87 +110,153 @@ func ReconcileKarpenterOperatorDeployment(deployment *appsv1.Deployment, }, }, }, - { - Name: "serviceaccount-token", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, }, - Containers: []corev1.Container{ - { - Name: name, - Image: hypershiftOperatorImage, - ImagePullPolicy: corev1.PullAlways, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "target-kubeconfig", - MountPath: "/mnt/kubeconfig", - }, - }, - Env: []corev1.EnvVar{ - { - Name: "MY_NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - }, - Command: []string{ - "/usr/bin/karpenter-operator", - }, - Args: []string{ - "--target-kubeconfig=/mnt/kubeconfig/target-kubeconfig", - "--namespace=$(MY_NAMESPACE)", - "--control-plane-operator-image=" + controlPlaneOperatorImage, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/healthz", - Port: intstr.FromString("http"), - Scheme: corev1.URISchemeHTTP, - }, - }, - InitialDelaySeconds: 60, - PeriodSeconds: 60, - SuccessThreshold: 1, - FailureThreshold: 5, - TimeoutSeconds: 5, - }, + }, + }, + } - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/readyz", - Port: intstr.FromString("http"), - Scheme: corev1.URISchemeHTTP, - }, - }, - PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 3, - TimeoutSeconds: 5, - }, - Ports: []corev1.ContainerPort{ - { - Name: "metrics", - ContainerPort: 8000, - }, - { - Name: "http", - ContainerPort: 8081, - Protocol: corev1.ProtocolTCP, - }, - }, + mainContainer := corev1.Container{ + Name: name, + Image: hypershiftOperatorImage, + ImagePullPolicy: corev1.PullAlways, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "target-kubeconfig", + MountPath: "/mnt/kubeconfig", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "MY_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", }, }, }, }, + Command: []string{ + "/usr/bin/karpenter-operator", + }, + Args: []string{ + "--target-kubeconfig=/mnt/kubeconfig/target-kubeconfig", + "--namespace=$(MY_NAMESPACE)", + "--control-plane-operator-image=" + controlPlaneOperatorImage, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 60, + PeriodSeconds: 60, + SuccessThreshold: 1, + FailureThreshold: 5, + TimeoutSeconds: 5, + }, + + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromString("http"), + Scheme: corev1.URISchemeHTTP, + }, + }, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + TimeoutSeconds: 5, + }, + Ports: []corev1.ContainerPort{ + { + Name: "metrics", + ContainerPort: 8000, + }, + { + Name: "http", + ContainerPort: 8081, + Protocol: corev1.ProtocolTCP, + }, + }, + } + + switch hcp.Spec.Platform.Type { + case hyperv1.AWSPlatform: + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: "serviceaccount-token", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + }, + }, + }, + corev1.Volume{ + Name: "provider-creds", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "karpenter-credentials", + }, + }, + }) + + mainContainer.Env = append(mainContainer.Env, + corev1.EnvVar{ + Name: "AWS_SHARED_CREDENTIALS_FILE", + Value: "/etc/provider/credentials", + }, + corev1.EnvVar{ + Name: "AWS_REGION", + Value: hcp.Spec.Platform.AWS.Region, + }, + corev1.EnvVar{ + Name: "AWS_SDK_LOAD_CONFIG", + Value: "true", + }) + + mainContainer.VolumeMounts = append(mainContainer.VolumeMounts, + corev1.VolumeMount{ + Name: "serviceaccount-token", + MountPath: "/var/run/secrets/openshift/serviceaccount", + }, + corev1.VolumeMount{ + Name: "provider-creds", + MountPath: "/etc/provider", + }) + + deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, corev1.Container{ + Name: "token-minter", + Image: controlPlaneOperatorImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/usr/bin/control-plane-operator", "token-minter"}, + Args: []string{ + "--service-account-namespace=kube-system", + "--service-account-name=karpenter", + "--token-file=/var/run/secrets/openshift/serviceaccount/token", + fmt.Sprintf("--kubeconfig-secret-namespace=%s", deployment.Namespace), + "--kubeconfig-secret-name=service-network-admin-kubeconfig", + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "serviceaccount-token", + MountPath: "/var/run/secrets/openshift/serviceaccount", + }, + }, + }) } + deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, mainContainer) + util.AvailabilityProber(kas.InClusterKASReadyURL(hcp.Spec.Platform.Type), controlPlaneOperatorImage, &deployment.Spec.Template.Spec) deploymentConfig := config.DeploymentConfig{ AdditionalLabels: map[string]string{