AWS EBS CSI implementation (kubernetes-sigs#5549)

* AWS EBS CSI implementation

* Fixing image repos

* Add OWNERS file

* Fix expressions

* Add csi-driver tag

* Add AWS EBS prefix to variables

* Add AWS EBS CSI Driver documentation

docs/aws-ebs-csi.md

@@ -0,0 +1,87 @@
+# AWS EBS CSI Driver
+
+AWS EBS CSI driver allows you to provision EBS volumes for pods in EC2 instances. The old in-tree AWS cloud provider is deprecated and will be removed in future versions of Kubernetes. So transitioning to the CSI driver is advised.
+
+To enable AWS EBS CSI driver, uncomment the `aws_ebs_csi_enabled` option in `group_vars/all/aws.yml` and set it to `true`.
+
+To set the number of replicas for the AWS CSI controller, you can change `aws_ebs_csi_controller_replicas` option in `group_vars/all/aws.yml`.
+
+Make sure to add a role, for your EC2 instances hosting Kubernetes, that allows it to do the actions necessary to request a volume and attach it: [AWS CSI Policy](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/example-iam-policy.json)
+
+If you want to deploy the AWS EBS storage class used with the CSI Driver, you should set `persistent_volumes_enabled` in `group_vars/k8s-cluster/k8s-cluster.yml` to `true`.
+
+You can now run the kubespray playbook (cluster.yml) to deploy Kubernetes over AWS EC2 with EBS CSI Driver enabled.
+
+## Usage example
+
+To check if AWS EBS CSI Driver is deployed properly, check that the ebs-csi pods are running:
+
+```ShellSession
+$ kubectl -n kube-system get pods | grep ebs
+ebs-csi-controller-85d86bccc5-8gtq5                                  4/4     Running   4          40s
+ebs-csi-node-n4b99                                                   3/3     Running   3          40s
+```
+
+Check the associated storage class (if you enabled persistent_volumes):
+
+```ShellSession
+$ kubectl get storageclass
+NAME         PROVISIONER                AGE
+ebs-sc   ebs.csi.aws.com   45s
+```
+
+You can run a PVC and an example Pod using this file `ebs-pod.yml`:
+
+```yml
+--
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ebs-claim
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: ebs-sc
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: app
+spec:
+  containers:
+  - name: app
+    image: centos
+    command: ["/bin/sh"]
+    args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"]
+    volumeMounts:
+    - name: persistent-storage
+      mountPath: /data
+  volumes:
+  - name: persistent-storage
+    persistentVolumeClaim:
+      claimName: ebs-claim
+```
+
+Apply this conf to your cluster: ```kubectl apply -f ebs-pod.yml```
+
+You should see the PVC provisioned and bound:
+
+```ShellSession
+$ kubectl get pvc
+NAME                   STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
+ebs-claim   Bound    pvc-0034cb9e-1ddd-4b3f-bb9e-0b5edbf5194c   1Gi        RWO            ebs-sc         50s
+```
+
+And the volume mounted to the example Pod (wait until the Pod is Running):
+
+```ShellSession
+$ kubectl exec -it app -- df -h | grep data
+/dev/nvme1n1   1014M   34M  981M   4% /data
+```
+
+## More info
+
+For further information about the AWS EBS CSI Driver, you can refer to this page: [AWS EBS Driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/).

inventory/sample/group_vars/all/aws.yml

@@ -0,0 +1,8 @@
+## To use AWS EBS CSI Driver to provision volumes, uncomment the first value
+## and configure the parameters below
+# aws_ebs_csi_enabled: true
+# aws_ebs_csi_enable_volume_scheduling: true
+# aws_ebs_csi_enable_volume_snapshot: false
+# aws_ebs_csi_enable_volume_resizing: false
+# aws_ebs_csi_controller_replicas: 1
+# aws_ebs_csi_plugin_image_tag: latest

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -255,7 +255,7 @@ podsecuritypolicy_enabled: false
 ## See https://github.com/kubernetes-sigs/kubespray/issues/2141
 ## Set this variable to true to get rid of this issue
 volume_cross_zone_attachment: false
-# Add Persistent Volumes Storage Class for corresponding cloud provider ( OpenStack is only supported now )
+# Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI, AWS EBS CSI)
 persistent_volumes_enabled: false
 
 ## Container Engine Acceleration

roles/download/defaults/main.yml

@@ -494,18 +494,24 @@ addon_resizer_version: "1.8.8"
 addon_resizer_image_repo: "{{ kube_image_repo }}/addon-resizer"
 addon_resizer_image_tag: "{{ addon_resizer_version }}"
 
-cinder_csi_attacher_image_repo: "{{ quay_image_repo }}/k8scsi/csi-attacher"
-cinder_csi_attacher_image_tag: "v1.2.1"
-cinder_csi_provisioner_image_repo: "{{ quay_image_repo }}/k8scsi/csi-provisioner"
-cinder_csi_provisioner_image_tag: "v1.3.0"
-cinder_csi_snapshotter_image_repo: "{{ quay_image_repo }}/k8scsi/csi-snapshotter"
-cinder_csi_snapshotter_image_tag: "v1.2.0"
-cinder_csi_resizer_image_repo: "{{ quay_image_repo }}/k8scsi/csi-resizer"
-cinder_csi_resizer_image_tag: "v0.2.0"
+csi_attacher_image_repo: "{{ quay_image_repo }}/k8scsi/csi-attacher"
+csi_attacher_image_tag: "v1.2.1"
+csi_provisioner_image_repo: "{{ quay_image_repo }}/k8scsi/csi-provisioner"
+csi_provisioner_image_tag: "v1.3.0"
+csi_snapshotter_image_repo: "{{ quay_image_repo }}/k8scsi/csi-snapshotter"
+csi_snapshotter_image_tag: "v1.2.0"
+csi_resizer_image_repo: "{{ quay_image_repo }}/k8scsi/csi-resizer"
+csi_resizer_image_tag: "v0.2.0"
+csi_node_driver_registrar_image_repo: "{{ quay_image_repo }}/k8scsi/csi-node-driver-registrar"
+csi_node_driver_registrar_image_tag: "v1.1.0"
+csi_livenessprobe_image_repo: "{{ quay_image_repo }}/k8scsi/livenessprobe"
+csi_livenessprobe_image_tag: "v1.1.0"
+
 cinder_csi_plugin_image_repo: "{{ docker_image_repo }}/k8scloudprovider/cinder-csi-plugin"
 cinder_csi_plugin_image_tag: "latest"
-cinder_csi_node_driver_registrar_image_repo: "{{ quay_image_repo }}/k8scsi/csi-node-driver-registrar"
-cinder_csi_node_driver_registrar_image_tag: "v1.1.0"
+
+aws_ebs_csi_plugin_image_repo: "{{ docker_image_repo }}/amazon/aws-ebs-csi-driver"
+aws_ebs_csi_plugin_image_tag: "latest"
 
 dashboard_image_repo: "{{ gcr_image_repo }}/google_containers/kubernetes-dashboard-{{ image_arch }}"
 dashboard_image_tag: "v1.10.1"
@@ -1011,39 +1017,48 @@ downloads:
     groups:
       - kube-node
 
-  cinder_csi_attacher:
-    enabled: "{{ cinder_csi_enabled }}"
+  csi_attacher:
+    enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}"
     container: true
-    repo: "{{ cinder_csi_attacher_image_repo }}"
-    tag: "{{ cinder_csi_attacher_image_tag }}"
-    sha256: "{{ cinder_csi_attacher_digest_checksum|default(None) }}"
+    repo: "{{ csi_attacher_image_repo }}"
+    tag: "{{ csi_attacher_image_tag }}"
+    sha256: "{{ csi_attacher_digest_checksum|default(None) }}"
     groups:
       - kube-node
 
-  cinder_csi_provisioner:
-    enabled: "{{ cinder_csi_enabled }}"
+  csi_provisioner:
+    enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}"
     container: true
-    repo: "{{ cinder_csi_provisioner_image_repo }}"
-    tag: "{{ cinder_csi_provisioner_image_tag }}"
-    sha256: "{{ cinder_csi_provisioner_digest_checksum|default(None) }}"
+    repo: "{{ csi_provisioner_image_repo }}"
+    tag: "{{ csi_provisioner_image_tag }}"
+    sha256: "{{ csi_provisioner_digest_checksum|default(None) }}"
     groups:
       - kube-node
 
-  cinder_csi_snapshotter:
-    enabled: "{{ cinder_csi_enabled }}"
+  csi_snapshotter:
+    enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}"
     container: true
-    repo: "{{ cinder_csi_snapshotter_image_repo }}"
-    tag: "{{ cinder_csi_snapshotter_image_tag }}"
-    sha256: "{{ cinder_csi_snapshotter_digest_checksum|default(None) }}"
+    repo: "{{ csi_snapshotter_image_repo }}"
+    tag: "{{ csi_snapshotter_image_tag }}"
+    sha256: "{{ csi_snapshotter_digest_checksum|default(None) }}"
     groups:
       - kube-node
 
-  cinder_csi_resizer:
-    enabled: "{{ cinder_csi_enabled }}"
+  csi_resizer:
+    enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}"
     container: true
-    repo: "{{ cinder_csi_resizer_image_repo }}"
-    tag: "{{ cinder_csi_resizer_image_tag }}"
-    sha256: "{{ cinder_csi_resizer_digest_checksum|default(None) }}"
+    repo: "{{ csi_resizer_image_repo }}"
+    tag: "{{ csi_resizer_image_tag }}"
+    sha256: "{{ csi_resizer_digest_checksum|default(None) }}"
+    groups:
+      - kube-node
+
+  csi_node_driver_registrar:
+    enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}"
+    container: true
+    repo: "{{ csi_node_driver_registrar_image_repo }}"
+    tag: "{{ csi_node_driver_registrar_image_tag }}"
+    sha256: "{{ csi_node_driver_registrar_digest_checksum|default(None) }}"
     groups:
       - kube-node
 
@@ -1056,12 +1071,12 @@ downloads:
     groups:
       - kube-node
 
-  cinder_csi_node_driver_registrar:
-    enabled: "{{ cinder_csi_enabled }}"
+  aws_ebs_csi_plugin:
+    enabled: "{{ aws_ebs_csi_enabled }}"
     container: true
-    repo: "{{ cinder_csi_node_driver_registrar_image_repo }}"
-    tag: "{{ cinder_csi_node_driver_registrar_image_tag }}"
-    sha256: "{{ cinder_csi_node_driver_registrar_digest_checksum|default(None) }}"
+    repo: "{{ aws_ebs_csi_plugin_image_repo }}"
+    tag: "{{ aws_ebs_csi_plugin_image_tag }}"
+    sha256: "{{ aws_ebs_csi_plugin_digest_checksum|default(None) }}"
     groups:
       - kube-node
 

roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+aws_ebs_csi_enable_volume_scheduling: true
+aws_ebs_csi_enable_volume_snapshot: false
+aws_ebs_csi_enable_volume_resizing: false
+aws_ebs_csi_controller_replicas: 1
+aws_ebs_csi_plugin_image_tag: latest

roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: AWS CSI Driver | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - {name: aws-ebs-csi-driver, file: aws-ebs-csi-driver.yml}
+    - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice-rbac.yml}
+    - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice.yml}
+    - {name: aws-ebs-csi-nodeservice, file: aws-ebs-csi-nodeservice.yml}
+  register: aws_csi_manifests
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: aws-ebs-csi-driver
+
+- name: AWS CSI Driver | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ aws_csi_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+  tags: aws-ebs-csi-driver