From 27bb240f984ef2eda7e1a6e92daf2535a3af3817 Mon Sep 17 00:00:00 2001 From: Nikita Massalitin Date: Tue, 26 Nov 2019 18:39:01 +0300 Subject: [PATCH 1/4] delete tiller part --- roles/kubernetes-apps/helm/defaults/main.yml | 33 ------ .../helm/tasks/gen_helm_tiller_certs.yml | 108 ------------------ roles/kubernetes-apps/helm/tasks/main.yml | 88 -------------- .../helm/templates/helm-make-ssl.sh.j2 | 76 ------------ .../tiller-clusterrolebinding.yml.j2 | 29 ----- .../helm/templates/tiller-namespace.yml.j2 | 4 - .../helm/templates/tiller-sa.yml.j2 | 8 -- 7 files changed, 346 deletions(-) delete mode 100644 roles/kubernetes-apps/helm/tasks/gen_helm_tiller_certs.yml delete mode 100644 roles/kubernetes-apps/helm/templates/helm-make-ssl.sh.j2 delete mode 100644 roles/kubernetes-apps/helm/templates/tiller-clusterrolebinding.yml.j2 delete mode 100644 roles/kubernetes-apps/helm/templates/tiller-namespace.yml.j2 delete mode 100644 roles/kubernetes-apps/helm/templates/tiller-sa.yml.j2 diff --git a/roles/kubernetes-apps/helm/defaults/main.yml b/roles/kubernetes-apps/helm/defaults/main.yml index 1042b985c12..ad9d457a437 100644 --- a/roles/kubernetes-apps/helm/defaults/main.yml +++ b/roles/kubernetes-apps/helm/defaults/main.yml @@ -7,41 +7,8 @@ helm_home_dir: "/root/.helm" # Deployment mode: host or docker helm_deployment_type: host -# Wait until Tiller is running and ready to receive requests -tiller_wait: false - -# Do not download the local repository cache on helm init -helm_skip_refresh: false - -# Secure Tiller installation with TLS -tiller_enable_tls: false helm_config_dir: "{{ kube_config_dir }}/helm" helm_script_dir: "{{ bin_dir }}/helm-scripts" -# Store tiller release information as Secret instead of a ConfigMap -tiller_secure_release_info: false - -# Where private root key will be secured for TLS -helm_tiller_cert_dir: "{{ helm_config_dir }}/ssl" -tiller_tls_cert: "{{ helm_tiller_cert_dir }}/tiller.pem" -tiller_tls_key: "{{ helm_tiller_cert_dir }}/tiller-key.pem" -tiller_tls_ca_cert: "{{ helm_tiller_cert_dir }}/ca.pem" - -# Permission owner and group for helm client cert. Will be dependent on the helm_home_dir -helm_cert_group: root -helm_cert_owner: root - # Set URL for stable repository # helm_stable_repo_url: "https://kubernetes-charts.storage.googleapis.com" - -# Namespace for the Tiller Deployment. -tiller_namespace: kube-system - -# Set node selector options for Tiller Deployment manifest. -# tiller_node_selectors: "key1=val1,key2=val2" - -# Override values for the Tiller Deployment manifest. -# tiller_override: "key1=val1,key2=val2" - -# Limit the maximum number of revisions saved per release. Use 0 for no limit. -# tiller_max_history: 0 diff --git a/roles/kubernetes-apps/helm/tasks/gen_helm_tiller_certs.yml b/roles/kubernetes-apps/helm/tasks/gen_helm_tiller_certs.yml deleted file mode 100644 index f6500f7c1cd..00000000000 --- a/roles/kubernetes-apps/helm/tasks/gen_helm_tiller_certs.yml +++ /dev/null @@ -1,108 +0,0 @@ ---- -- name: "Gen_helm_tiller_certs | Create helm config directory (on {{ groups['kube-master'][0] }})" - run_once: yes - delegate_to: "{{ groups['kube-master'][0] }}" - file: - path: "{{ helm_config_dir }}" - state: directory - owner: kube - -- name: "Gen_helm_tiller_certs | Create helm script directory (on {{ groups['kube-master'][0] }})" - run_once: yes - delegate_to: "{{ groups['kube-master'][0] }}" - file: - path: "{{ helm_script_dir }}" - state: directory - owner: kube - -- name: Gen_helm_tiller_certs | Copy certs generation script - run_once: yes - delegate_to: "{{ groups['kube-master'][0] }}" - template: - src: "helm-make-ssl.sh.j2" - dest: "{{ helm_script_dir }}/helm-make-ssl.sh" - mode: 0700 - -- name: "Check_helm_certs | check if helm client certs have already been generated on first master (on {{ groups['kube-master'][0] }})" - find: - paths: "{{ helm_home_dir }}" - patterns: "*.pem" - get_checksum: true - delegate_to: "{{ groups['kube-master'][0] }}" - register: helmcert_master - run_once: true - -- name: Gen_helm_tiller_certs | run cert generation script - run_once: yes - delegate_to: "{{ groups['kube-master'][0] }}" - command: "{{ helm_script_dir }}/helm-make-ssl.sh -e {{ helm_home_dir }} -d {{ helm_tiller_cert_dir }}" - -- name: Check_helm_client_certs | Set helm_client_certs - set_fact: - helm_client_certs: ['ca.pem', 'cert.pem', 'key.pem'] - -- name: "Check_helm_client_certs | check if a cert already exists on master node" - find: - paths: "{{ helm_home_dir }}" - patterns: "*.pem" - get_checksum: true - register: helmcert_node - when: inventory_hostname != groups['kube-master'][0] - -- name: "Check_helm_client_certs | Set 'sync_helm_certs' to true on masters" - set_fact: - sync_helm_certs: (not item in helmcert_node.files | map(attribute='path') | map("basename") | list or helmcert_node.files | selectattr("path", "equalto", "{{ helm_home_dir }}/{{ item }}") | map(attribute="checksum")|first|default('') != helmcert_master.files | selectattr("path", "equalto", "{{ helm_home_dir }}/{{ item }}") | map(attribute="checksum")|first|default('')) - when: - - inventory_hostname != groups['kube-master'][0] - with_items: - - "{{ helm_client_certs }}" - -- name: Gen_helm_tiller_certs | Gather helm client certs - # noqa 303 - tar is called intentionally here, but maybe this should be done with the slurp module - shell: "tar cfz - -C {{ helm_home_dir }} -T /dev/stdin <<< {{ helm_client_certs|join(' ') }} | base64 --wrap=0" - args: - executable: /bin/bash - no_log: true - register: helm_client_cert_data - check_mode: no - delegate_to: "{{ groups['kube-master'][0] }}" - when: sync_helm_certs|default(false) and inventory_hostname != groups['kube-master'][0] - -- name: Gen_helm_tiller_certs | Use tempfile for unpacking certs on masters - tempfile: - state: file - path: /tmp - prefix: helmcertsXXXXX - suffix: tar.gz - register: helm_cert_tempfile - when: sync_helm_certs|default(false) and inventory_hostname != groups['kube-master'][0] - -- name: Gen_helm_tiller_certs | Write helm client certs to tempfile - copy: - content: "{{ helm_client_cert_data.stdout }}" - dest: "{{ helm_cert_tempfile.path }}" - owner: root - mode: "0600" - when: sync_helm_certs|default(false) and inventory_hostname != groups['kube-master'][0] - -- name: Gen_helm_tiller_certs | Unpack helm certs on masters - shell: "base64 -d < {{ helm_cert_tempfile.path }} | tar xz -C {{ helm_home_dir }}" - no_log: true - changed_when: false - check_mode: no - when: sync_helm_certs|default(false) and inventory_hostname != groups['kube-master'][0] - -- name: Gen_helm_tiller_certs | Cleanup tempfile on masters - file: - path: "{{ helm_cert_tempfile.path }}" - state: absent - when: sync_helm_certs|default(false) and inventory_hostname != groups['kube-master'][0] - -- name: Gen_certs | check certificate permissions - file: - path: "{{ helm_home_dir }}" - group: "{{ helm_cert_group }}" - state: directory - owner: "{{ helm_cert_owner }}" - mode: "u=rwX,g-rwx,o-rwx" - recurse: yes diff --git a/roles/kubernetes-apps/helm/tasks/main.yml b/roles/kubernetes-apps/helm/tasks/main.yml index f8a41e0d009..7d865b16c64 100644 --- a/roles/kubernetes-apps/helm/tasks/main.yml +++ b/roles/kubernetes-apps/helm/tasks/main.yml @@ -5,94 +5,6 @@ - name: Helm | Set up helm launcher include_tasks: "install_{{ helm_deployment_type }}.yml" -- name: Helm | Lay Down Helm Manifests (RBAC) - template: - src: "{{ item.file }}.j2" - dest: "{{ kube_config_dir }}/{{ item.file }}" - with_items: - - {name: tiller, file: tiller-namespace.yml, type: namespace} - - {name: tiller, file: tiller-sa.yml, type: sa} - - {name: tiller, file: tiller-clusterrolebinding.yml, type: clusterrolebinding} - register: manifests - when: - - dns_mode != 'none' - - inventory_hostname == groups['kube-master'][0] - -- name: Helm | Apply Helm Manifests (RBAC) - kube: - name: "{{ item.item.name }}" - namespace: "{{ tiller_namespace }}" - kubectl: "{{ bin_dir }}/kubectl" - resource: "{{ item.item.type }}" - filename: "{{ kube_config_dir }}/{{ item.item.file }}" - state: "latest" - with_items: "{{ manifests.results }}" - when: - - dns_mode != 'none' - - inventory_hostname == groups['kube-master'][0] - -# Generate necessary certs for securing Helm and Tiller connection with TLS -- name: Helm | Set up TLS - include_tasks: "gen_helm_tiller_certs.yml" - when: tiller_enable_tls - -- name: Helm | Install client on all masters - command: > - {{ bin_dir }}/helm init --tiller-namespace={{ tiller_namespace }} - {% if helm_skip_refresh %} --skip-refresh{% endif %} - {% if helm_stable_repo_url is defined %} --stable-repo-url {{ helm_stable_repo_url }}{% endif %} - --client-only - environment: "{{ proxy_env }}" - changed_when: false - -# FIXME: https://github.com/helm/helm/issues/6374 -- name: Helm | Install/upgrade helm - shell: > - {{ bin_dir }}/helm init --tiller-namespace={{ tiller_namespace }} - {% if helm_skip_refresh %} --skip-refresh{% endif %} - {% if helm_stable_repo_url is defined %} --stable-repo-url {{ helm_stable_repo_url }}{% endif %} - --upgrade --tiller-image={{ tiller_image_repo }}:{{ tiller_image_tag }} - {% if rbac_enabled %} --service-account=tiller{% endif %} - {% if tiller_node_selectors is defined %} --node-selectors {{ tiller_node_selectors }}{% endif %} - --override spec.template.spec.priorityClassName={% if tiller_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %} - {% if tiller_override is defined and tiller_override %} --override {{ tiller_override }}{% endif %} - {% if tiller_max_history is defined %} --history-max={{ tiller_max_history }}{% endif %} - {% if tiller_enable_tls %} --tiller-tls --tiller-tls-verify --tiller-tls-cert={{ tiller_tls_cert }} --tiller-tls-key={{ tiller_tls_key }} --tls-ca-cert={{ tiller_tls_ca_cert }} {% endif %} - {% if tiller_secure_release_info %} --override 'spec.template.spec.containers[0].command'='{/tiller,--storage=secret}' {% endif %} - --override spec.selector.matchLabels.'name'='tiller',spec.selector.matchLabels.'app'='helm' - {% if tiller_wait %} --wait{% endif %} - --output yaml - | sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' - | {{ bin_dir }}/kubectl apply -f - - register: install_helm - when: - - inventory_hostname == groups['kube-master'][0] - changed_when: false - environment: "{{ proxy_env }}" - -# FIXME: https://github.com/helm/helm/issues/4063 -- name: Helm | Force apply tiller overrides if necessary - shell: > - {{ bin_dir }}/helm init --upgrade --tiller-image={{ tiller_image_repo }}:{{ tiller_image_tag }} --tiller-namespace={{ tiller_namespace }} - {% if helm_skip_refresh %} --skip-refresh{% endif %} - {% if helm_stable_repo_url is defined %} --stable-repo-url {{ helm_stable_repo_url }}{% endif %} - {% if rbac_enabled %} --service-account=tiller{% endif %} - {% if tiller_node_selectors is defined %} --node-selectors {{ tiller_node_selectors }}{% endif %} - --override spec.template.spec.priorityClassName={% if tiller_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %} - {% if tiller_override is defined and tiller_override %} --override {{ tiller_override }}{% endif %} - {% if tiller_max_history is defined %} --history-max={{ tiller_max_history }}{% endif %} - {% if tiller_enable_tls %} --tiller-tls --tiller-tls-verify --tiller-tls-cert={{ tiller_tls_cert }} --tiller-tls-key={{ tiller_tls_key }} --tls-ca-cert={{ tiller_tls_ca_cert }} {% endif %} - {% if tiller_secure_release_info %} --override 'spec.template.spec.containers[0].command'='{/tiller,--storage=secret}' {% endif %} - --override spec.selector.matchLabels.'name'='tiller',spec.selector.matchLabels.'app'='helm' - {% if tiller_wait %} --wait{% endif %} - --output yaml - | sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' - | {{ bin_dir }}/kubectl apply -f - - changed_when: false - when: - - inventory_hostname == groups['kube-master'][0] - environment: "{{ proxy_env }}" - - name: Make sure bash_completion.d folder exists file: name: "/etc/bash_completion.d/" diff --git a/roles/kubernetes-apps/helm/templates/helm-make-ssl.sh.j2 b/roles/kubernetes-apps/helm/templates/helm-make-ssl.sh.j2 deleted file mode 100644 index f82d51c9c27..00000000000 --- a/roles/kubernetes-apps/helm/templates/helm-make-ssl.sh.j2 +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail - -usage() -{ - cat << EOF -Create self signed certificates - -Usage : $(basename $0) -f [-d ] - -h | --help : Show this message - -e | --helm-home : Helm home directory - -d | --ssldir : Directory where the certificates will be installed -EOF -} - -# Options parsing -while (($#)); do - case "$1" in - -h | --help) usage; exit 0;; - -e | --helm-home) HELM_HOME="${2}"; shift 2;; - -d | --ssldir) SSLDIR="${2}"; shift 2;; - *) - usage - echo "ERROR : Unknown option" - exit 3 - ;; - esac -done - -if [ -z ${SSLDIR} ]; then - SSLDIR="/etc/kubernetes/helm/ssl" -fi - -tmpdir=$(mktemp -d /tmp/helm_cacert.XXXXXX) -trap 'rm -rf "${tmpdir}"' EXIT -cd "${tmpdir}" - -mkdir -p "${SSLDIR}" - -# Root CA -if [ -e "$SSLDIR/ca-key.pem" ]; then - # Reuse existing CA - cp $SSLDIR/{ca.pem,ca-key.pem} . -else - openssl genrsa -out ca-key.pem 4096 > /dev/null 2>&1 - openssl req -x509 -new -nodes -key ca-key.pem -days {{certificates_duration}} -out ca.pem -subj "/CN=tiller-ca" > /dev/null 2>&1 -fi - -gen_key_and_cert() { - local name=$1 - local subject=$2 - openssl genrsa -out ${name}-key.pem 4096 > /dev/null 2>&1 - openssl req -new -key ${name}-key.pem -sha256 -out ${name}.csr -subj "${subject}" > /dev/null 2>&1 - openssl x509 -req -in ${name}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${name}.pem -days {{certificates_duration}} > /dev/null 2>&1 -} - -#Generate cert and key for Tiller if they don't exist -if ! [ -e "$SSLDIR/tiller.pem" ]; then - gen_key_and_cert "tiller" "/CN=tiller-server" -fi - -#Generate cert and key for Helm client if they don't exist -if ! [ -e "$SSLDIR/helm.pem" ]; then - gen_key_and_cert "helm" "/CN=helm-client" -fi - -# Secure certs to first master -mv *.pem ${SSLDIR}/ - -# Install Helm client certs to first master -# Copy using Helm default names for convenience -cp ${SSLDIR}/ca.pem ${HELM_HOME}/ca.pem -cp ${SSLDIR}/helm.pem ${HELM_HOME}/cert.pem -cp ${SSLDIR}/helm-key.pem ${HELM_HOME}/key.pem diff --git a/roles/kubernetes-apps/helm/templates/tiller-clusterrolebinding.yml.j2 b/roles/kubernetes-apps/helm/templates/tiller-clusterrolebinding.yml.j2 deleted file mode 100644 index 9bdfdde034c..00000000000 --- a/roles/kubernetes-apps/helm/templates/tiller-clusterrolebinding.yml.j2 +++ /dev/null @@ -1,29 +0,0 @@ ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: tiller - namespace: {{ tiller_namespace }} -subjects: - - kind: ServiceAccount - name: tiller - namespace: {{ tiller_namespace }} -roleRef: - kind: ClusterRole - name: cluster-admin - apiGroup: rbac.authorization.k8s.io -{% if podsecuritypolicy_enabled %} ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: psp:tiller -subjects: - - kind: ServiceAccount - name: tiller - namespace: {{ tiller_namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:privileged -{% endif %} diff --git a/roles/kubernetes-apps/helm/templates/tiller-namespace.yml.j2 b/roles/kubernetes-apps/helm/templates/tiller-namespace.yml.j2 deleted file mode 100644 index 455742185c2..00000000000 --- a/roles/kubernetes-apps/helm/templates/tiller-namespace.yml.j2 +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: "{{ tiller_namespace}}" diff --git a/roles/kubernetes-apps/helm/templates/tiller-sa.yml.j2 b/roles/kubernetes-apps/helm/templates/tiller-sa.yml.j2 deleted file mode 100644 index 09b8157250a..00000000000 --- a/roles/kubernetes-apps/helm/templates/tiller-sa.yml.j2 +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: tiller - namespace: {{ tiller_namespace }} - labels: - kubernetes.io/cluster-service: "true" From 4541c1aa1107b63493ce62d65a964a86c1e13eda Mon Sep 17 00:00:00 2001 From: Nikita Massalitin Date: Tue, 26 Nov 2019 18:53:55 +0300 Subject: [PATCH 2/4] ver bump --- roles/download/defaults/main.yml | 13 +------------ roles/kubernetes-apps/rotate_tokens/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index ebe5fcb5daf..7f83ee58023 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -299,11 +299,9 @@ test_image_repo: "{{ docker_image_repo }}/library/busybox" test_image_tag: latest busybox_image_repo: "{{ docker_image_repo }}/library/busybox" busybox_image_tag: 1.29.2 -helm_version: "v2.16.0" +helm_version: "v3.0.0" helm_image_repo: "{{ docker_image_repo }}/lachlanevenson/k8s-helm" helm_image_tag: "{{ helm_version }}" -tiller_image_repo: "{{ gcr_image_repo }}/kubernetes-helm/tiller" -tiller_image_tag: "{{ helm_version }}" registry_image_repo: "{{ docker_image_repo }}/library/registry" registry_image_tag: "2.6" @@ -714,15 +712,6 @@ downloads: groups: - kube-node - tiller: - enabled: "{{ helm_enabled }}" - container: true - repo: "{{ tiller_image_repo }}" - tag: "{{ tiller_image_tag }}" - sha256: "{{ tiller_digest_checksum|default(None) }}" - groups: - - kube-node - registry: enabled: "{{ registry_enabled }}" container: true diff --git a/roles/kubernetes-apps/rotate_tokens/tasks/main.yml b/roles/kubernetes-apps/rotate_tokens/tasks/main.yml index 347d1b4c2d2..f85ec26b86e 100644 --- a/roles/kubernetes-apps/rotate_tokens/tasks/main.yml +++ b/roles/kubernetes-apps/rotate_tokens/tasks/main.yml @@ -34,7 +34,7 @@ {{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf get secrets --all-namespaces -o 'jsonpath={range .items[*]}{"\n"}{.metadata.namespace}{" "}{.metadata.name}{" "}{.type}{end}' | grep kubernetes.io/service-account-token - | egrep 'default-token|kube-proxy|coredns|netchecker|weave|calico|canal|flannel|dashboard|cluster-proportional-autoscaler|tiller|local-volume-provisioner' + | egrep 'default-token|kube-proxy|coredns|netchecker|weave|calico|canal|flannel|dashboard|cluster-proportional-autoscaler|local-volume-provisioner' register: tokens_to_delete when: needs_rotation From 9f951059f2cb3bb4dbf2519d004ea57b3e4b98ec Mon Sep 17 00:00:00 2001 From: Nikita Massalitin Date: Thu, 28 Nov 2019 11:23:12 +0300 Subject: [PATCH 3/4] remove obsolete vars --- roles/kubernetes-apps/helm/defaults/main.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/roles/kubernetes-apps/helm/defaults/main.yml b/roles/kubernetes-apps/helm/defaults/main.yml index ad9d457a437..88061f723f9 100644 --- a/roles/kubernetes-apps/helm/defaults/main.yml +++ b/roles/kubernetes-apps/helm/defaults/main.yml @@ -6,9 +6,3 @@ helm_home_dir: "/root/.helm" # Deployment mode: host or docker helm_deployment_type: host - -helm_config_dir: "{{ kube_config_dir }}/helm" -helm_script_dir: "{{ bin_dir }}/helm-scripts" - -# Set URL for stable repository -# helm_stable_repo_url: "https://kubernetes-charts.storage.googleapis.com" From 47b3f1b6114011dc124e9ccae5e0b85c7ecf33bb Mon Sep 17 00:00:00 2001 From: Nikita Massalitin Date: Mon, 2 Dec 2019 13:15:20 +0300 Subject: [PATCH 4/4] add repo step --- roles/kubernetes-apps/helm/defaults/main.yml | 3 +++ roles/kubernetes-apps/helm/tasks/main.yml | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/roles/kubernetes-apps/helm/defaults/main.yml b/roles/kubernetes-apps/helm/defaults/main.yml index 88061f723f9..781ffaad2f7 100644 --- a/roles/kubernetes-apps/helm/defaults/main.yml +++ b/roles/kubernetes-apps/helm/defaults/main.yml @@ -6,3 +6,6 @@ helm_home_dir: "/root/.helm" # Deployment mode: host or docker helm_deployment_type: host + +# Set URL for stable repository +# helm_stable_repo_url: "https://kubernetes-charts.storage.googleapis.com" diff --git a/roles/kubernetes-apps/helm/tasks/main.yml b/roles/kubernetes-apps/helm/tasks/main.yml index 7d865b16c64..5620a532908 100644 --- a/roles/kubernetes-apps/helm/tasks/main.yml +++ b/roles/kubernetes-apps/helm/tasks/main.yml @@ -5,6 +5,12 @@ - name: Helm | Set up helm launcher include_tasks: "install_{{ helm_deployment_type }}.yml" +- name: Helm | Add helm stable repo + command: > + {{ bin_dir }}/helm repo add stable {{ helm_stable_repo_url }} + when: helm_stable_repo_url is defined + changed_when: false + - name: Make sure bash_completion.d folder exists file: name: "/etc/bash_completion.d/"