cilium (#616)

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Enabled auditd on masters, workers and bastions.
 - Add registry mirror setting for containerd.
+- Add CiliumLocalRedirectPolicy for aad pod identity.
 
 ### Changed
 
@@ -31,6 +32,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [Azure] Bump `azuredisk-csi-driver` to `1.19.0-gs1`.
 - [Azure] Bump `azure-cloud-controller-manager` to `1.1.17-gs1`.
 - [Azure] Bump `azure-cloud-node-manager` to `1.1.17-gs1`.
+- Switch from `calico` and `kube-proxy` to `Cilium`.
+
+### Removed
+
+- Don't create priority classes any more (chart-operator does that now).
 
 ### Fixed
 

platforms/aws/giantnetes/main.tf

@@ -157,6 +157,7 @@ locals {
     "MasterMountETCD"              = var.master_instance["volume_etcd"]
     "OIDCIssuerURL"                = "https://${var.oidc_issuer_dns}.${var.base_domain}"
     "PodCIDR"                      = var.aws_cni_cidr_v2
+    "NodePodCIDRSize"              = var.node_pod_cidr_size
     "PodInfraImage"                = var.pod_infra_image
     "Provider"                     = "aws"
     "Users"                        = yamldecode(base64decode(jsondecode(data.http.bastion_users.body).content))

platforms/aws/giantnetes/variables.tf

@@ -207,6 +207,12 @@ variable "aws_cni_subnets_v2" {
   default     = ["100.64.0.0/18", "100.64.64.0/18", "100.64.128.0/18"]
 }
 
+variable "node_pod_cidr_size" {
+  description = "Size of Pod CIDR to be allocated for each node."
+  type        = string
+  default     = "25"
+}
+
 variable "docker_cidr" {
   description = "CIDR for Docker."
   default     = "172.17.0.1/16"

platforms/azure/giantnetes/main.tf

@@ -165,6 +165,7 @@ locals {
     "MasterCount"              = var.master_count
     "OIDCIssuerURL"            = "https://${var.oidc_issuer_dns}.${var.base_domain}"
     "PodCIDR"                  = var.pod_cidr
+    "NodePodCIDRSize"          = var.node_pod_cidr_size
     "PodInfraImage"            = var.pod_infra_image
     "Provider"                 = "azure"
     "Users"                    = yamldecode(base64decode(jsondecode(data.http.bastion_users.body).content))

platforms/azure/giantnetes/variables.tf

@@ -242,6 +242,12 @@ variable "pod_cidr" {
   default     = "10.0.128.0/17"
 }
 
+variable "node_pod_cidr_size" {
+  description = "Size of Pod CIDR to be allocated for each node."
+  type        = string
+  default     = "25"
+}
+
 variable "docker_cidr" {
   description = "CIDR for Docker."
   default     = "172.17.0.1/16"

templates/files/apps/common/cilium-app.yaml

@@ -0,0 +1,99 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cilium-user-values
+  namespace: giantswarm
+data:
+  values: |
+    hubble:
+      relay:
+        enabled: true
+    defaultPolicies:
+      enabled: true
+    egressMasqueradeInterfaces: eth0
+    eni:
+      enabled: false
+    kubeProxyReplacement: strict
+    k8sServiceHost: "{{ .APIInternalDomainName }}"
+    k8sServicePort: "443"
+    ipam:
+      mode: cluster-pool
+      operator:
+        clusterPoolIPv4PodCIDR: "{{ .PodCIDR }}"
+        clusterPoolIPv4MaskSize: {{ .NodePodCIDRSize }}
+    localRedirectPolicy: true
+    tunnel: vxlan
+    nodeinit:
+      enabled: true
+    image:
+      repository: {{.DockerRegistry}}/giantswarm/cilium
+    certgen:
+      image:
+        repository: {{.DockerRegistry}}/giantswarm/cilium-certgen
+    hubble:
+      relay:
+        image:
+          repository: {{.DockerRegistry}}/giantswarm/hubble-relay
+      ui:
+        backend:
+          image:
+            repository: {{.DockerRegistry}}/giantswarm/hubble-ui-backend
+        frontend:
+          image:
+            repository: {{.DockerRegistry}}/giantswarm/hubble-ui
+        proxy:
+          image:
+            repository: {{.DockerRegistry}}/giantswarm/envoy
+    etcd:
+      image:
+        repository: {{.DockerRegistry}}/giantswarm/cilium-etcd-operator
+    operator:
+      image:
+        repository: {{.DockerRegistry}}/giantswarm/cilium-operator
+    nodeinit:
+      image:
+        repository: {{.DockerRegistry}}/giantswarm/cilium-startup-script
+    preflight:
+      image:
+        repository: {{.DockerRegistry}}/giantswarm/cilium
+    clustermesh:
+      apiserver:
+        image:
+          repository: {{.DockerRegistry}}/giantswarm/clustermesh-apiserver
+---
+apiVersion: application.giantswarm.io/v1alpha1
+kind: App
+metadata:
+  labels:
+    app-operator.giantswarm.io/version: 0.0.0
+    app.kubernetes.io/name: cilium
+  name: cilium
+  namespace: giantswarm
+spec:
+  catalog: default
+  config:
+    configMap:
+      name: ""
+      namespace: ""
+    secret:
+      name: ""
+      namespace: ""
+  install: {}
+  kubeConfig:
+    context:
+      name: ""
+    inCluster: true
+    secret:
+      name: ""
+      namespace: ""
+  name: cilium
+  namespace: kube-system
+  namespaceConfig: {}
+  userConfig:
+    configMap:
+      name: "cilium-user-values"
+      namespace: "giantswarm"
+    secret:
+      name: ""
+      namespace: ""
+  version: 0.2.3

templates/files/conf/k8s-addons

@@ -1,29 +1,30 @@
 #!/bin/bash
 set -eu
-KUBECTL="/opt/bin/kubectl --kubeconfig=/etc/kubernetes/kubeconfig/addons.yaml"
+KUBECONFIG=/etc/kubernetes/kubeconfig/addons.yaml
+KUBECTL="/opt/bin/kubectl --kubeconfig=$KUBECONFIG"
 
 while ! curl --output /dev/null --silent --head --insecure "https://{{ .APIDomainName }}"; do sleep 1 && echo 'Waiting for master'; done
 
 # Create giantswarm, global and monitoring namespaces
 NAMESPACES="giantswarm global monitoring"
 for namespace in ${NAMESPACES}
 do
-  while
-      $KUBECTL get namespace ${namespace}
-      [ "$?" -ne "0" ]
+  while true
   do
-      while
-          $KUBECTL create namespace ${namespace}
-          [ "$?" -ne "0" ]
-      do
-          echo "failed to create namespace ${namespace}, retrying in 5 sec"
-          sleep 5s
-      done
+    # If namespace exists there is nothing to do so I exit the inner loop
+    $KUBECTL get namespace ${namespace} && break
+
+    # Create namespace and exit the loop if successful
+    $KUBECTL create namespace ${namespace} && break
+
+    # Something went wrong, try again
+    echo "failed to create namespace ${namespace}, retrying in 5 sec"
+    sleep 5s
   done
 done
 
 # label namespaces (required for network egress policies)
-NAMESPACES="default giantswarm kube-system monitoring global" 
+NAMESPACES="default giantswarm kube-system monitoring global"
 for namespace in ${NAMESPACES}
 do
     if ! $KUBECTL get namespaces -l name=${namespace} | grep ${namespace}; then
@@ -53,71 +54,53 @@ do
     done
 done
 
-# create kube-proxy configmap
-while
-    $KUBECTL create configmap kube-proxy --from-file=kube-proxy.yaml=/srv/kube-proxy-config.yaml -o yaml --dry-run | $KUBECTL apply -n kube-system -f -
-    [ "$?" -ne "0" ]
-do
-    echo "failed to configure kube-proxy from /srv/kube-proxy-confg.yaml, retrying in 5 sec"
-    sleep 5s
-done
+# TODO Begin Remove once all MCs will be running cilium.
+# Install cilium app and wait for it to be running.
+$KUBECTL apply -f /srv/apps/common/cilium-app.yaml
 
-# apply kube-proxy
-KUBE_PROXY_MANIFESTS="kube-proxy-sa.yaml kube-proxy-ds.yaml"
-for manifest in $KUBE_PROXY_MANIFESTS
+# Wait for cilium DS to exist and be satisfied
+set +e
+echo "Waiting for cilium DS to exist..."
+n=0
+exitcode=-1
+until [ "$exitcode" -eq 0 ]
 do
-    while
-        $KUBECTL apply -f /srv/$manifest
-        [ "$?" -ne "0" ]
-    do
-        echo "failed to apply /srv/$manifest, retrying in 5 sec"
-        sleep 5s
-    done
+   $KUBECTL -n kube-system get ds cilium
+   exitcode=$?
+   n=$((n+1))
+   if [ "$n" -ge 20 ]
+   then
+     echo "Failed waiting for cilium DS to exist"
+     exit 1
+   fi
+   sleep 10
 done
+set -e
 
-# restart ds to apply config from configmap
-$KUBECTL delete pods -l k8s-app=kube-proxy -n kube-system
+echo "Waiting for cilium DS to be rolled out..."
+$KUBECTL -n kube-system rollout status daemonset/cilium
 
 {{ if eq .Provider "aws" }}
-## Apply AWS VPC CNI and Calico for ensuring network policies
-CNI_FILE="aws-cni.yaml calico-policy-only.yaml"
-{{ end -}}
-{{ if eq .Provider "azure" }}
-## Apply Calico for ensuring network policies
-CNI_FILE="calico-policy-only.yaml"
-{{ end -}}
-
-{{ if and (ne .Provider "aws") (ne .Provider "azure") }}
-## Apply Calico with all its components
-CNI_FILE="calico-all.yaml"
-{{ end -}}
-for manifest in ${CNI_FILE}
-do
-    while
-        $KUBECTL apply -f /srv/$manifest
-        [ "$?" -ne "0" ]
-    do
-        echo "failed to apply /srv/$manifest, retrying in 5 sec"
-        sleep 5s
-    done
-done
+# Delete legacy CNI.
+$KUBECTL delete -f /srv/aws-cni.yaml --ignore-not-found=true || true
+{{ end }}
 
-echo "Waiting for calico-node to be ready..."
-n=0
-exitcode=-1
-until [ "$n" -ge 20 ] || [ "$exitcode" -eq 0 ]
+# Delete calico
+$KUBECTL delete -f /srv/calico-policy-only.yaml --ignore-not-found=true || true
+
+# Delete kube-proxy.
+for manifest in kube-proxy-sa.yaml kube-proxy-ds.yaml
 do
-   $KUBECTL -n kube-system -l k8s-app=calico-node wait --for=condition=Ready --timeout=30s pods
-   exitcode=$?
-   n=$((n+1))
-   sleep 10
+  $KUBECTL delete -f /srv/$manifest --ignore-not-found=true || true
 done
 
-# delete calico-kube-controllers
-$KUBECTL -n kube-system delete deploy calico-kube-controllers --ignore-not-found=true
+$KUBECTL delete configmap kube-proxy --ignore-not-found=true || true
+# TODO End Remove once all MCs will be running cilium.
 
-# Apply all apps.
-$KUBECTL apply -f /srv/apps/ --recursive
+{{ if eq .Provider "azure" }}
+# Apply CiliumLocalRedirectPolicy for cluster-autoscaler
+$KUBECTL apply -f /srv/azure-ad-pod-identity-lrp.yaml
+{{ end }}
 
 # apply k8s addons
 MANIFESTS="default-storage-class.yaml
@@ -134,4 +117,10 @@ do
     done
 done
 
+# Wait for App CRD to exist.
+while ! $KUBECTL get crd apps.application.giantswarm.io ; do sleep 1 && echo 'Waiting for App CRD to exist'; done
+
+# Apply all apps.
+$KUBECTL apply -f /srv/apps/ --recursive
+
 echo "Addons successfully installed"