External OpenStack Cloud Controller Manager implementation (kubernete…

…s-sigs#5491)

* External OpenStack Cloud Controller Manager implementation

* Adding controller image tag

* Minor fixes

* Restructuring the external cloud controller to work with KubeADM

cluster.yml

@@ -97,6 +97,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kubespray-defaults}
+    - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
     - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }

inventory/sample/group_vars/all/all.yml

@@ -51,10 +51,13 @@ loadbalancer_apiserver_healthcheck_port: 8081
 ## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external'
 ## When openstack is used make sure to source in the openstack credentials
 ## like you would do when using openstack-client before starting the playbook.
-## Note: The 'external' cloud provider is not supported.
-## TODO(riverzhang): https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#running-cloud-controller-manager
 # cloud_provider:
 
+## When cloud_provider is set to 'external', you can set the cloud controller to deploy
+## Supported cloud controllers are: 'openstack'
+## When openstack is used make sure to source in the openstack credentials
+# external_cloud_provider:
+
 ## Set these proxy values in order to update package manager and docker daemon to use proxies
 # http_proxy: ""
 # https_proxy: ""

inventory/sample/group_vars/all/openstack.yml

@@ -15,6 +15,21 @@
 # openstack_lbaas_monitor_timeout: "30s"
 # openstack_lbaas_monitor_max_retries: "3"
 
+## Values for the external OpenStack Cloud Controller
+# external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP"
+# external_openstack_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP"
+# external_openstack_lbaas_floating_network_id: "Neutron network ID to get floating IP from"
+# external_openstack_lbaas_floating_subnet_id: "Neutron subnet ID to get floating IP from"
+# external_openstack_lbaas_use_octavia: true
+# external_openstack_lbaas_method: "ROUND_ROBIN"
+# external_openstack_lbaas_create_monitor: false
+# external_openstack_lbaas_monitor_delay: "1m"
+# external_openstack_lbaas_monitor_timeout: "30s"
+# external_openstack_lbaas_monitor_max_retries: "3"
+
+## The tag of the external OpenStack Cloud Controller image
+# external_openstack_cloud_controller_image_tag: "latest"
+
 ## To use Cinder CSI plugin to provision volumes set this value to true
 ## Make sure to source in the openstack credentials
 # cinder_csi_enabled: true

roles/kubernetes-apps/external_cloud_controller/meta/main.yml

@@ -0,0 +1,12 @@
+---
+dependencies:
+  - role: kubernetes-apps/external_cloud_controller/openstack
+    when:
+      - cloud_provider is defined
+      - cloud_provider == "external"
+      - external_cloud_provider is defined
+      - external_cloud_provider == "openstack"
+      - inventory_hostname == groups['kube-master'][0]
+    tags:
+      - external-cloud-controller
+      - external-openstack

roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+# The external cloud controller will need credentials to access
+# openstack apis. Per default these values will be
+# read from the environment.
+external_openstack_auth_url: "{{ lookup('env','OS_AUTH_URL')  }}"
+external_openstack_username: "{{ lookup('env','OS_USERNAME')  }}"
+external_openstack_password: "{{ lookup('env','OS_PASSWORD')  }}"
+external_openstack_region: "{{ lookup('env','OS_REGION_NAME')  }}"
+external_openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}"
+external_openstack_tenant_name: "{{ lookup('env','OS_TENANT_NAME') }}"
+external_openstack_domain_name: "{{ lookup('env','OS_USER_DOMAIN_NAME') }}"
+external_openstack_domain_id: "{{ lookup('env','OS_USER_DOMAIN_ID') }}"
+external_openstack_cacert: "{{ lookup('env','OS_CACERT') }}"
+
+external_openstack_cloud_controller_image_tag: "latest"

roles/kubernetes-apps/external_cloud_controller/openstack/tasks/main.yml

@@ -0,0 +1,58 @@
+---
+- include_tasks: openstack-credential-check.yml
+  tags: external-openstack
+
+- name: External OpenStack Cloud Controller | Write cacert file
+  copy:
+    src: "{{ external_openstack_cacert }}"
+    dest: "{{ kube_config_dir }}/external-openstack-cacert.pem"
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  when:
+    - inventory_hostname in groups['k8s-cluster']
+    - external_openstack_cacert is defined
+    - external_openstack_cacert | length > 0
+  tags: external-openstack
+
+- name: External OpenStack Cloud Controller | Write External OpenStack cloud-config
+  template:
+    src: "external-openstack-cloud-config.j2"
+    dest: "{{ kube_config_dir }}/external_openstack_cloud_config"
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: external-openstack
+
+- name: External OpenStack Cloud Controller | Get base64 cloud-config
+  slurp:
+    src: "{{ kube_config_dir }}/external_openstack_cloud_config"
+  register: external_openstack_cloud_config_secret
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: external-openstack
+
+- name: External OpenStack Cloud Controller | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - {name: external-openstack-cloud-config-secret, file: external-openstack-cloud-config-secret.yml}
+    - {name: external-openstack-cloud-controller-manager-roles, file: external-openstack-cloud-controller-manager-roles.yml}
+    - {name: external-openstack-cloud-controller-manager-role-bindings, file: external-openstack-cloud-controller-manager-role-bindings.yml}
+    - {name: external-openstack-cloud-controller-manager-ds, file: external-openstack-cloud-controller-manager-ds.yml}
+  register: external_openstack_manifests
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: external-openstack
+
+- name: External OpenStack Cloud Controller | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ external_openstack_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+  tags: external-openstack

roles/kubernetes-apps/external_cloud_controller/openstack/tasks/openstack-credential-check.yml

@@ -0,0 +1,34 @@
+---
+- name: External OpenStack Cloud Controller | check external_openstack_auth_url value
+  fail:
+    msg: "external_openstack_auth_url is missing"
+  when: external_openstack_auth_url is not defined or not external_openstack_auth_url
+
+- name: External OpenStack Cloud Controller | check external_openstack_username value
+  fail:
+    msg: "external_openstack_username is missing"
+  when: external_openstack_username is not defined or not external_openstack_username
+
+- name: External OpenStack Cloud Controller | check external_openstack_password value
+  fail:
+    msg: "external_openstack_password is missing"
+  when: external_openstack_password is not defined or not external_openstack_password
+
+- name: External OpenStack Cloud Controller | check external_openstack_region value
+  fail:
+    msg: "external_openstack_region is missing"
+  when: external_openstack_region is not defined or not external_openstack_region
+
+- name: External OpenStack Cloud Controller | check external_openstack_tenant_id value
+  fail:
+    msg: "one of external_openstack_tenant_id or external_openstack_tenant_name must be specified"
+  when:
+    - external_openstack_tenant_id is not defined or not external_openstack_tenant_id
+    - external_openstack_tenant_name is not defined
+
+- name: External OpenStack Cloud Controller | check external_openstack_tenant_name value
+  fail:
+    msg: "one of external_openstack_tenant_id or external_openstack_tenant_name must be specified"
+  when:
+    - external_openstack_tenant_name is not defined or not external_openstack_tenant_name
+    - external_openstack_tenant_id is not defined

roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config-secret.yml.j2

@@ -0,0 +1,10 @@
+# This YAML file contains secret objects,
+# which are necessary to run external openstack cloud controller.
+
+kind: Secret
+apiVersion: v1
+metadata:
+  name: external-openstack-cloud-config
+  namespace: kube-system
+data:
+  cloud.conf: {{ external_openstack_cloud_config_secret.content }}

roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config.j2

@@ -0,0 +1,41 @@
+[Global]
+auth-url="{{ external_openstack_auth_url }}"
+username="{{ external_openstack_username }}"
+password="{{ external_openstack_password }}"
+region="{{ external_openstack_region }}"
+{% if external_openstack_tenant_id is defined and external_openstack_tenant_id != "" %}
+tenant-id="{{ external_openstack_tenant_id }}"
+{% endif %}
+{% if external_openstack_tenant_name is defined and external_openstack_tenant_name != "" %}
+tenant-name="{{ external_openstack_tenant_name }}"
+{% endif %}
+{% if external_openstack_domain_name is defined and external_openstack_domain_name != "" %}
+domain-name="{{ external_openstack_domain_name }}"
+{% elif external_openstack_domain_id is defined and external_openstack_domain_id != "" %}
+domain-id ="{{ external_openstack_domain_id }}"
+{% endif %}
+{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
+ca-file="{{ kube_config_dir }}/external-openstack-cacert.pem"
+{% endif %}
+
+[LoadBalancer]
+use-octavia={{ external_openstack_lbaas_use_octavia }}
+create-monitor={{ openstack_lbaas_create_monitor }}
+monitor-delay={{ openstack_lbaas_monitor_delay }}
+monitor-timeout={{ openstack_lbaas_monitor_timeout }}
+monitor-max-retries={{ openstack_lbaas_monitor_max_retries }}
+{% if external_openstack_lbaas_method is defined %}
+lb-method={{ external_openstack_lbaas_method }}
+{% endif %}
+{% if external_openstack_lbaas_network_id is defined %}
+network-id={{ external_openstack_lbaas_network_id }}
+{% endif %}
+{% if external_openstack_lbaas_subnet_id is defined %}
+subnet-id={{ external_openstack_lbaas_subnet_id }}
+{% endif %}
+{% if external_openstack_lbaas_floating_network_id is defined %}
+floating-network-id={{ external_openstack_lbaas_floating_network_id }}
+{% endif %}
+{% if external_openstack_lbaas_flaoting_subnet_id is defined %}
+floating-subnet-id={{ external_openstack_lbaas_floating_subnet_id }}
+{% endif %}

roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2

@@ -0,0 +1,92 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: openstack-cloud-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: openstack-cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      k8s-app: openstack-cloud-controller-manager
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: openstack-cloud-controller-manager
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      securityContext:
+        runAsUser: 1001
+      tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      serviceAccountName: cloud-controller-manager
+      containers:
+        - name: openstack-cloud-controller-manager
+          image: {{ docker_image_repo }}/k8scloudprovider/openstack-cloud-controller-manager:{{ external_openstack_cloud_controller_image_tag }}
+          args:
+            - /bin/openstack-cloud-controller-manager
+            - --v=1
+            - --cloud-config=$(CLOUD_CONFIG)
+            - --cloud-provider=openstack
+            - --use-service-account-credentials=true
+            - --address=127.0.0.1
+          volumeMounts:
+            - mountPath: /etc/kubernetes/pki
+              name: k8s-certs
+              readOnly: true
+            - mountPath: /etc/ssl/certs
+              name: ca-certs
+              readOnly: true
+            - mountPath: /etc/config
+              name: cloud-config-volume
+              readOnly: true
+{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
+            - mountPath: {{ kube_config_dir }}/external-openstack-cacert.pem
+              name: openstack-cacert
+              readOnly: true
+{% endif %}
+            - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+              name: flexvolume-dir
+          resources:
+            requests:
+              cpu: 200m
+          env:
+            - name: CLOUD_CONFIG
+              value: /etc/config/cloud.conf
+      hostNetwork: true
+      volumes:
+      - hostPath:
+          path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+          type: DirectoryOrCreate
+        name: flexvolume-dir
+      - hostPath:
+          path: /etc/kubernetes/pki
+          type: DirectoryOrCreate
+        name: k8s-certs
+      - hostPath:
+          path: /etc/ssl/certs
+          type: DirectoryOrCreate
+        name: ca-certs
+      - name: cloud-config-volume
+        secret:
+          secretName: external-openstack-cloud-config
+{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
+      - hostPath:
+          path: {{ kube_config_dir }}/external-openstack-cacert.pem
+          type: FileOrCreate
+        name: openstack-cacert
+{% endif %}

roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-role-bindings.yml.j2

@@ -0,0 +1,40 @@
+apiVersion: v1
+items:
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: system:cloud-node-controller
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: system:cloud-node-controller
+  subjects:
+  - kind: ServiceAccount
+    name: cloud-node-controller
+    namespace: kube-system
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: system:pvl-controller
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: system:pvl-controller
+  subjects:
+  - kind: ServiceAccount
+    name: pvl-controller
+    namespace: kube-system
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: system:cloud-controller-manager
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: system:cloud-controller-manager
+  subjects:
+  - kind: ServiceAccount
+    name: cloud-controller-manager
+    namespace: kube-system
+kind: List
+metadata: {}