Fedora CoreOS support (kubernetes-sigs#5657)

* fedora coreos support - bootstrap and new fact for * fedora coreos support - fix bootstrap condition * fedora coreos support - allow customize packages for fedora coreos bootstrap * fedora coreos support - prevent install ptyhon3 and epel via dnf for fedora coreos * fedora coreos support - handle all ostree like os in same way * fedora coreos support - handle all ostree like os in same way for crio * fedora coreos support - add fcos documentations
libresh · Mar 17, 2020 · 876d4de · 876d4de
1 parent 974902a
commit 876d4de
Show file tree

Hide file tree

Showing 17 changed files with 222 additions and 48 deletions.
diff --git a/README.md b/README.md
@@ -83,6 +83,7 @@ vagrant up
 - [Network plugins](#network-plugins)
 - [Vagrant install](docs/vagrant.md)
 - [CoreOS bootstrap](docs/coreos.md)
+- [Fedora CoreOS bootstrap](docs/fcos.md)
 - [Debian Jessie setup](docs/debian.md)
 - [openSUSE setup](docs/opensuse.md)
 - [Downloaded artifacts](docs/downloads.md)
@@ -105,6 +106,7 @@ vagrant up
 - **CentOS/RHEL** 7
 - **Fedora** 28
 - **Fedora/CentOS** Atomic
+- **Fedora CoreOS** (experimental: see [fcos Note](docs/fcos.md)
 - **openSUSE** Leap 42.3/Tumbleweed
 - **Oracle Linux** 7
 

diff --git a/docs/fcos.md b/docs/fcos.md
@@ -0,0 +1,76 @@
+# Fedora CoreOS
+
+Tested with stable version 31.20200223.3.0
+Because package installation with `rpm-ostree` requires a reboot, playbook may fail while bootstrap.
+Restart playbook again.
+
+## Containers
+
+Tested with
+
+- docker
+- crio
+
+### docker
+
+OS base packages contains docker.
+
+### cri-o
+
+To use `cri-o` disable docker service with ignition:
+
+```yaml
+#workaround, see https://github.com/coreos/fedora-coreos-tracker/issues/229
+systemd:
+  units:
+    - name: docker.service
+      enabled: false
+      contents: |
+        [Unit]
+        Description=disable docker
+
+        [Service]
+
+        [Install]
+        WantedBy=multi-user.target
+```
+
+## libvirt setup
+
+### Prepare
+
+Prepare ignition and serve via http (a.e. python -m SimpleHTTPServer )
+
+```json
+{
+  "ignition": {
+     "version": "3.0.0"
+  },
+
+  "passwd": {
+    "users": [
+      {
+        "name": "adi",
+        "passwordHash": "$1$.RGu8J4x$U7uxcOg/eotTEIRxhk62I0",
+        "sshAuthorizedKeys": [
+          "ssh-rsa ..fillyouruser"
+        ],
+        "groups": [ "wheel" ]
+      }
+    ]
+  }
+}
+```
+
+### create guest
+
+```shell script
+fcos_version=31.20200223.3.0
+kernel=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-kernel-x86_64
+initrd=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-initramfs.x86_64.img
+ignition_url=http://mywebserver/fcos.ign
+kernel_args="ip=dhcp rd.neednet=1 console=tty0 coreos.liveiso=/ console=ttyS0 coreos.inst.install_dev=/dev/sda coreos.inst.stream=stable coreos.inst.ignition_url=${ignition_url}"
+sudo virt-install --name ${machine_name} --ram 4048 --graphics=none --vcpus 2 --disk size=20 \
+                --network bridge=virbr0 \
+                --install kernel=${kernel},initrd=${initrd},kernel_args_overwrite=yes,kernel_args="${kernel_args}"
+```
diff --git a/roles/bootstrap-os/defaults/main.yml b/roles/bootstrap-os/defaults/main.yml
@@ -13,6 +13,13 @@ coreos_locksmithd_disable: false
 # Install public repo on Oracle Linux
 use_oracle_public_repo: true
 
+fedora_coreos_packages:
+  - python
+  - libselinux-python3
+  - dbus-tools              # because of networkManager reload bug (https://bugzilla.redhat.com/show_bug.cgi?id=1745659)
+  - ethtool                 # required in kubeadm preflight phase for verifying the environment
+  - ipset                   # required in kubeadm preflight phase for verifying the environment
+
 ## General
 # Set the hostname to inventory_hostname
 override_system_hostname: true
diff --git a/roles/bootstrap-os/tasks/bootstrap-fedora-coreos.yml b/roles/bootstrap-os/tasks/bootstrap-fedora-coreos.yml
@@ -0,0 +1,35 @@
+---
+
+- name: Check if bootstrap is needed
+  raw: which python
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Install required packages on fedora coreos
+  raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree install {{ fedora_coreos_packages|join(' ') }}"
+  become: true
+  when: need_bootstrap.rc != 0
+
+# playbook fails because connection lost
+- name: Reboot immediately for updated ostree, please run playbook again if failed first time.
+  raw: "nohup bash -c 'sleep 5s && shutdown -r now'"
+  become: true
+  ignore_errors: yes
+  when: need_bootstrap.rc != 0
+
+- name: Wait for the reboot to complete
+  wait_for_connection:
+    timeout: 240
+    connect_timeout: 20
+    delay: 5
+    sleep: 5
+  when: need_bootstrap.rc != 0
+
+- name: Store the fact if this is an fedora core os host
+  set_fact:
+    is_fedora_coreos: True
+  tags:
+    - facts
diff --git a/roles/bootstrap-os/tasks/main.yml b/roles/bootstrap-os/tasks/main.yml
@@ -13,14 +13,21 @@
 - include_tasks: bootstrap-clearlinux.yml
   when: '"Clear Linux OS" in os_release.stdout'
 
+- include_tasks: bootstrap-fedora-coreos.yml
+  when: '"ID=fedora" in os_release.stdout and "VARIANT_ID=coreos" in os_release.stdout'
+
 - include_tasks: bootstrap-coreos.yml
-  when: '"CoreOS" in os_release.stdout or "Flatcar" in os_release.stdout'
+  when:
+    - '"CoreOS" in os_release.stdout or "Flatcar" in os_release.stdout'
+    - '"ID=fedora" not in os_release.stdout'
 
 - include_tasks: bootstrap-debian.yml
   when: '"Debian" in os_release.stdout or "Ubuntu" in os_release.stdout'
 
 - include_tasks: bootstrap-fedora.yml
-  when: '"Fedora" in os_release.stdout'
+  when:
+    - '"Fedora" in os_release.stdout'
+    - '"VARIANT_ID=coreos" not in os_release.stdout'
 
 - include_tasks: bootstrap-opensuse.yml
   when: '"openSUSE" in os_release.stdout'
@@ -43,7 +50,7 @@
     name: "{{ inventory_hostname }}"
   when:
     - override_system_hostname
-    - ansible_os_family not in ['Suse', 'Container Linux by CoreOS', 'Flatcar Container Linux by Kinvolk', 'ClearLinux']
+    - ansible_os_family not in ['Suse', 'Container Linux by CoreOS', 'Flatcar Container Linux by Kinvolk', 'ClearLinux'] and not is_fedora_coreos
 
 # (2/3)
 - name: Assign inventory name to unconfigured hostnames (CoreOS, non-Flatcar, Suse and ClearLinux only)
@@ -52,7 +59,7 @@
   changed_when: false
   when:
     - override_system_hostname
-    - ansible_os_family in ['Suse', 'Container Linux by CoreOS', 'Flatcar Container Linux by Kinvolk', 'ClearLinux']
+    - ansible_os_family in ['Suse', 'Container Linux by CoreOS', 'Flatcar Container Linux by Kinvolk', 'ClearLinux'] or is_fedora_coreos
 
 # (3/3)
 - name: Update hostname fact (CoreOS, Flatcar, Suse and ClearLinux only)
@@ -61,7 +68,7 @@
     filter: ansible_hostname
   when:
     - override_system_hostname
-    - ansible_os_family in ['Suse', 'Flatcar Container Linux by Kinvolk', 'Container Linux by CoreOS', 'ClearLinux']
+    - ansible_os_family in ['Suse', 'Flatcar Container Linux by Kinvolk', 'Container Linux by CoreOS', 'ClearLinux'] or is_fedora_coreos
 
 - name: "Install ceph-commmon package"
   package:

diff --git a/roles/container-engine/cri-o/tasks/main.yaml b/roles/container-engine/cri-o/tasks/main.yaml
@@ -1,4 +1,15 @@
 ---
+
+- name: check if atomic host or fedora coreos
+  stat:
+    path: /run/ostree-booted
+  register: ostree
+
+- name: set is_ostree
+  set_fact:
+    is_ostree: "{{ ostree.stat.exists }}"
+
+
 - name: gather os specific variables
   include_vars: "{{ item }}"
   with_first_found:
@@ -22,7 +33,7 @@
     description: OpenShift Origin Repo
     baseurl: "{{ crio_rhel_repo_base_url }}"
     gpgcheck: no
-  when: ansible_distribution in ["CentOS","RedHat","OracleLinux"] and not is_atomic
+  when: ansible_distribution in ["CentOS","RedHat","OracleLinux"] and not is_ostree
 
 - name: Add CRI-O PPA
   apt_repository:
@@ -51,8 +62,25 @@
   package:
     name: "{{ item }}"
     state: present
+  when: not is_ostree
   with_items: "{{ crio_packages }}"
 
+- name: Check if already installed
+  stat:
+    path: "/bin/crio"
+  register: need_bootstrap_crio
+  when: is_ostree
+
+- name: Install cri-o packages with osttree
+  raw: "export http_proxy={{ http_proxy | default('') }} && rpm-ostree install {{ crio_packages|join(' ') }}"
+  when: is_ostree and not need_bootstrap_crio.stat.exists
+  become: true
+
+- name: Reboot immediately for updated ostree
+  reboot:
+  become: true
+  when: is_ostree and not need_bootstrap_crio.stat.exists
+
 - name: Install cri-o config
   template:
     src: crio.conf.j2

diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2
@@ -104,7 +104,7 @@ selinux = {{ (preinstall_selinux_state == 'enforcing')|lower }}
 # for the runtime.
 {% if ansible_os_family == "ClearLinux" %}
 seccomp_profile = "/usr/share/defaults/crio/seccomp.json"
-{% elif ansible_distribution == "Ubuntu" %}
+{% elif ansible_distribution == "Ubuntu" or is_fedora_coreos %}
 seccomp_profile = ""
 {% else %}
 seccomp_profile = "/etc/crio/seccomp.json"
@@ -121,17 +121,17 @@ cgroup_manager = "cgroupfs"
 # only the capabilities defined in the containers json file by the user/kube
 # will be added.
 default_capabilities = [
-	"CHOWN", 
-	"DAC_OVERRIDE", 
-	"FSETID", 
-	"FOWNER", 
-	"NET_RAW", 
-	"SETGID", 
-	"SETUID", 
-	"SETPCAP", 
-	"NET_BIND_SERVICE", 
-	"SYS_CHROOT", 
-	"KILL", 
+	"CHOWN",
+	"DAC_OVERRIDE",
+	"FSETID",
+	"FOWNER",
+	"NET_RAW",
+	"SETGID",
+	"SETUID",
+	"SETPCAP",
+	"NET_BIND_SERVICE",
+	"SYS_CHROOT",
+	"KILL",
 ]
 
 # List of default sysctls. If it is empty or commented out, only the sysctls
@@ -154,7 +154,7 @@ hooks_dir = [
 # be removed in future versions in favor of default_mounts_file.
 default_mounts = [
 {% if ansible_os_family == "RedHat" %}
-	"/usr/share/rhel/secrets:/run/secrets", 
+	"/usr/share/rhel/secrets:/run/secrets",
 {% endif %}
 ]
 
@@ -216,7 +216,7 @@ ctr_stop_timeout = 0
   # The runtime to use is picked based on the runtime_handler provided by the CRI.
   # If no runtime_handler is provided, the runtime will be picked based on the level
   # of trust of the workload.
-  
+
   [crio.runtime.runtimes.runc]
 {% if ansible_os_family == "ClearLinux" or ansible_os_family == "RedHat" %}
   runtime_path = "/usr/bin/runc"
@@ -226,7 +226,7 @@ ctr_stop_timeout = 0
   runtime_path = "/usr/sbin/runc"
 {% endif %}
   runtime_type = "oci"
-  
+
 
 
 # The crio.image table contains settings pertaining to the management of OCI images.
@@ -242,7 +242,7 @@ ctr_stop_timeout = 0
 default_transport = "docker://"
 
 # The image used to instantiate infra containers.
-pause_image = "docker://k8s.gcr.io/pause:3.1"
+pause_image = "docker://{{kube_image_repo}}/pause:3.1"
 
 # If not empty, the path to a docker/config.json-like file containing credentials
 # necessary for pulling the image specified by pause_image above.

diff --git a/roles/container-engine/docker/handlers/main.yml b/roles/container-engine/docker/handlers/main.yml
@@ -15,7 +15,7 @@
   service:
     name: docker.socket
     state: restarted
-  when: ansible_os_family in ['Coreos', 'CoreOS', 'Container Linux by CoreOS', 'Flatcar', 'Flatcar Container Linux by Kinvolk']
+  when: ansible_os_family in ['Coreos', 'CoreOS', 'Container Linux by CoreOS', 'Flatcar', 'Flatcar Container Linux by Kinvolk'] or is_fedora_coreos
 
 - name: Docker | reload docker
   service: