feat: migrate to python for data validation (#1214)

* feat: migrate to python for data validation Signed-off-by: Devin Buhl <[email protected]> * fix: address PR comments Signed-off-by: Devin Buhl <[email protected]> * fix: add unused kwargs to validate functions Signed-off-by: Devin Buhl <[email protected]> * chore: update renovate pip and ansible regex Signed-off-by: Devin Buhl <[email protected]> * feat: add bootstrap_nodes test Signed-off-by: Devin Buhl <[email protected]> * fix: update taskfiles Signed-off-by: Devin Buhl <[email protected]> --------- Signed-off-by: Devin Buhl <[email protected]>
tuxpeople · Jan 21, 2024 · b89b7b3 · b89b7b3
1 parent 7456164
commit b89b7b3
Show file tree

Hide file tree

Showing 33 changed files with 340 additions and 537 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -9,10 +9,6 @@ charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
 
-[Makefile]
-indent_style = space
-indent_size = 4
-
-[*.{bash,sh}]
+[*.{bash,py,sh}]
 indent_style = space
 indent_size = 4
diff --git a/.github/renovate.json5 b/.github/renovate.json5
@@ -36,6 +36,16 @@
       "(^|/)kustomization\\.ya?ml(\\.j2)?(\\.j2)?$"
     ]
   },
+  "pip_requirements": {
+    "fileMatch": [
+      "(^|/)[\\w-]*requirements(-\\w+)?\\.(txt|pip)(\\.j2)?(\\.j2)?$"
+    ]
+  },
+  "ansible-galaxy": {
+    "fileMatch": [
+      "(^|/)(galaxy|requirements)(\\.ansible)?\\.ya?ml(\\.j2)?(\\.j2)?$"
+    ]
+  },
   // commit message topics
   "commitMessageTopic": "{{depName}}",
   "commitMessageExtra": "to {{newVersion}}",

diff --git a/.github/tests/config-k0s.yaml b/.github/tests/config-k0s.yaml
@@ -1,5 +1,5 @@
 ---
-ci_test: true
+skip_tests: true
 
 bootstrap_distribution: k0s
 bootstrap_github_username: onedr0p

diff --git a/.github/tests/config-k3s-ipv4.yaml b/.github/tests/config-k3s-ipv4.yaml
@@ -1,5 +1,5 @@
 ---
-ci_test: true
+skip_tests: true
 
 bootstrap_distribution: k3s
 bootstrap_github_username: onedr0p

diff --git a/.github/tests/config-k3s-ipv6.yaml b/.github/tests/config-k3s-ipv6.yaml
@@ -1,5 +1,5 @@
 ---
-ci_test: true
+skip_tests: true
 
 bootstrap_distribution: k3s
 bootstrap_github_username: onedr0p

diff --git a/.github/tests/config-k3s-no-kube-vip.yaml b/.github/tests/config-k3s-no-kube-vip.yaml
@@ -1,5 +1,5 @@
 ---
-ci_test: true
+skip_tests: true
 
 bootstrap_distribution: k3s
 bootstrap_github_username: onedr0p

diff --git a/.github/tests/config-talos.yaml b/.github/tests/config-talos.yaml
@@ -1,5 +1,5 @@
 ---
-ci_test: true
+skip_tests: true
 
 bootstrap_distribution: talos
 bootstrap_github_username: onedr0p

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -77,7 +77,7 @@ jobs:
         shell: bash
         run: brew install go-task
 
-      - name: Install Brew dependencies
+      - name: Run Workstation tasks
         if: ${{ github.event_name == 'pull_request' && steps.cache-homebrew-packages.outputs.cache-hit != 'true' }}
         shell: bash
         run: task workstation:brew
@@ -86,15 +86,15 @@ jobs:
         shell: bash
         run: direnv allow .
 
-      - name: Initialize Sops Age key
+      - name: Run Sops Age key task
         shell: bash
         run: task sops:age-keygen
 
-      - name: Install Ansible dependencies
+      - name: Run setup-virtual-env task
         shell: bash
-        run: task ansible:deps force=false
+        run: task setup-virtual-env
 
-      - name: Generate bootstrap config file
+      - name: Run init tasks
         shell: bash
         run: |
           task init
@@ -109,28 +109,26 @@ jobs:
         run: |
           echo "distribution=$(yq '.bootstrap_distribution' ./bootstrap/vars/config.yaml)" >> $GITHUB_OUTPUT
 
-      - name: Run configure
+      - name: Run configure task
         shell: bash
         run: task configure --yes
 
-      - name: Run Talos configure
+      - name: Run Talos tasks
         if: ${{ steps.config-env.outputs.distribution == 'talos' }}
         shell: bash
         run: |
           task talos:gensecret --yes
           task talos:genconfig
 
-      - name: Run Ansible lint
+      - name: Run Ansible tasks
         if: ${{ steps.config-env.outputs.distribution == 'k3s' || steps.config-env.outputs.distribution == 'k0s' }}
         shell: bash
-        run: task ansible:lint
-
-      - name: List Hosts with Ansible
-        if: ${{ steps.config-env.outputs.distribution == 'k3s' || steps.config-env.outputs.distribution == 'k0s' }}
-        shell: bash
-        run: task ansible:list
+        run: |
+          task ansible:deps force=false
+          task ansible:lint
+          task ansible:list
 
-      - name: Run repo clean and reset
+      - name: Run repo clean and reset tasks
         shell: bash
         run: |
           task repository:clean

diff --git a/.taskfiles/Ansible/Taskfile.yaml b/.taskfiles/Ansible/Taskfile.yaml
@@ -14,17 +14,28 @@ env:
 vars:
   ANSIBLE_LINT_FILE: "{{.ANSIBLE_DIR}}/.ansible-lint"
   ANSIBLE_INVENTORY_FILE: "{{.ANSIBLE_DIR}}/inventory/hosts.yaml"
-  ANSIBLE_REQUIREMENTS_FILE: "{{.ROOT_DIR}}/requirements.yaml"
-  PIP_REQUIREMENTS_FILE: "{{.ROOT_DIR}}/requirements.txt"
+  ANSIBLE_REQUIREMENTS_FILE: "{{.ANSIBLE_DIR}}/requirements.yaml"
+  ANSIBLE_PIP_REQUIREMENTS_FILE: "{{.ANSIBLE_DIR}}/requirements.txt"
 
 tasks:
 
   deps:
-    desc: Set up Ansible dependencies for the environment
+    desc: Set up Ansible dependencies
     cmds:
-      - task: .setup-virtual-env
-        vars:
-          force: '{{.force | default "true"}}'
+      - task: :setup-virtual-env
+      - .venv/bin/python3 -m pip install --upgrade --requirement "{{.ANSIBLE_PIP_REQUIREMENTS_FILE}}"
+      - .venv/bin/ansible-galaxy install --role-file "{{.ANSIBLE_REQUIREMENTS_FILE}}" {{if eq .force "true"}}--force{{end}}
+    preconditions:
+      - { msg: "Missing Ansible requirements file", sh: "test -f {{.ANSIBLE_REQUIREMENTS_FILE}}" }
+      - { msg: "Missing Pip requirements file", sh: "test -f {{.ANSIBLE_PIP_REQUIREMENTS_FILE}}" }
+    sources:
+      - "{{.ANSIBLE_REQUIREMENTS_FILE}}"
+      - "{{.ANSIBLE_PIP_REQUIREMENTS_FILE}}"
+    generates:
+      - "{{.VIRTUAL_ENV}}/bin/ansible"
+      - "{{.VIRTUAL_ENV}}/bin/ansible-galaxy"
+    vars:
+      force: '{{.force | default "true"}}'
 
   run:
     desc: Run an Ansible playbook for configuring a cluster
@@ -74,24 +85,6 @@ tasks:
     preconditions:
       - { msg: "Missing Ansible lint file", sh: "test -f {{.ANSIBLE_LINT_FILE}}" }
 
-  .setup-virtual-env:
-    internal: true
-    cmds:
-      - "{{.PYTHON_BIN}} -m venv {{.ROOT_DIR}}/.venv"
-      - .venv/bin/python3 -m pip install --upgrade pip setuptools wheel
-      - .venv/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}"
-      - .venv/bin/ansible-galaxy install --role-file "{{.ANSIBLE_REQUIREMENTS_FILE}}" {{if eq .force "true"}}--force{{end}}
-    sources:
-      - "{{.PIP_REQUIREMENTS_FILE}}"
-      - "{{.ANSIBLE_REQUIREMENTS_FILE}}"
-    generates:
-      - "{{.ROOT_DIR}}/.venv/pyvenv.cfg"
-    preconditions:
-      - { msg: "Missing Ansible requirements file", sh: "test -f {{.ANSIBLE_REQUIREMENTS_FILE}}" }
-      - { msg: "Missing Pip requirements file", sh: "test -f {{.PIP_REQUIREMENTS_FILE}}" }
-    vars:
-      force: '{{.force | default "true"}}'
-
   .reset:
     internal: true
     cmd: rm -rf {{.ANSIBLE_DIR}}
diff --git a/README.md b/README.md
@@ -46,7 +46,7 @@ If you are marching forward, now is a good time to choose whether you will deplo
 ### System requirements
 
 > [!IMPORTANT]
-> 1. The included behaviour of Talos, k0s or k3s is that all nodes are able to run workloads, **including** control nodes. Worker nodes are therefore optional.
+> 1. The included behaviour of Talos, k3s or k0s is that all nodes are able to run workloads, **including** control nodes. Worker nodes are therefore optional.
 > 2. Do you have 3 or more nodes? It is strongly recommended to make 3 of them control nodes for a highly available control plane.
 > 3. Running the cluster on Proxmox VE? My thoughts and recommendations about that are documented [here](https://onedr0p.github.io/home-ops/notes/proxmox-considerations.html).
 
@@ -193,12 +193,12 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g
     go-task workstation:yay
     ```
 
-4. Setup a Python virual env and install Ansible by running the following task command.
+4. Setup a Python virual env by running the following task command.
 
     📍 _This commands requires Python 3.11+ to be installed._
 
     ```sh
-    task ansible:deps
+    task setup-virtual-env
     ```
 
 5. Continue on to 🔧 [**Stage 3**](#-stage-3-do-bootstrap-configuration)
@@ -278,14 +278,7 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g
 
 7. Once done run the following command which will verify and generate all the files needed to continue.
 
-    > [!NOTE]
-    > The following configure task will create a `./ansible` directory for k3s or k0s and the following directories under `./kubernetes`.
-    > ```sh
-    > 📁 kubernetes      # Kubernetes cluster defined as code
-    > ├─📁 bootstrap     # Flux installation (not tracked by Flux)
-    > ├─📁 flux          # Main Flux configuration of repository
-    > └─📁 apps          # Apps deployed into the cluster grouped by namespace
-    > ```
+    📍 _The following configure task will create a `./ansible` directory for k3s or k0s and the following directories under `./kubernetes` for all distributions_
 
     ```sh
     task configure
@@ -305,25 +298,31 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g
 
 1. Ensure you are able to SSH into your nodes from your workstation using a private SSH key **without a passphrase** (for example using a SSH agent). This lets Ansible interact with your nodes.
 
-2. Verify Ansible can view your config
+3. Install the Ansible dependencies
+
+    ```sh
+    task ansible:deps
+    ```
+
+4. Verify Ansible can view your config
 
     ```sh
     task ansible:list
     ```
 
-3. Verify Ansible can ping your nodes
+5. Verify Ansible can ping your nodes
 
     ```sh
     task ansible:ping
     ```
 
-4. Run the Ansible prepare playbook (nodes wil reboot when done)
+6. Run the Ansible prepare playbook (nodes wil reboot when done)
 
     ```sh
     task ansible:run playbook=cluster-prepare
     ```
 
-5. Continue on to ⛵ [**Stage 5**](#-stage-5-install-kubernetes)
+7. Continue on to ⛵ [**Stage 5**](#-stage-5-install-kubernetes)
 
 ### ⛵ Stage 5: Install Kubernetes
 

diff --git a/Taskfile.yaml b/Taskfile.yaml
@@ -13,7 +13,7 @@ vars:
   BOOTSTRAP_CONFIG_FILE: "{{.BOOTSTRAP_DIR}}/vars/config.yaml"
   KUBECONFIG_FILE: "{{.ROOT_DIR}}/kubeconfig"
   MAKEJINJA_CONFIG_FILE: "{{.ROOT_DIR}}/makejinja.toml"
-  PYTHON_VERSION_CHECK_FILE: "{{.BOOTSTRAP_DIR}}/scripts/py_version_check.py"
+  PIP_REQUIREMENTS_FILE: "{{.ROOT_DIR}}/requirements.txt"
   SOPS_AGE_FILE: "{{.ROOT_DIR}}/age.key"
   # Binaries
   PYTHON_BIN: python3
@@ -37,8 +37,9 @@ tasks:
   default: task -l
 
   init:
-    desc: Initialize files and directories
+    desc: Initialize virtual env and configuration files
     cmds:
+      - task: setup-virtual-env
       - mkdir -p {{.PRIVATE_DIR}}
       - cp -n {{.BOOTSTRAP_ADDONS_FILE | replace ".yaml" ".sample.yaml"}} {{.BOOTSTRAP_ADDONS_FILE}}
       - cp -n {{.BOOTSTRAP_CONFIG_FILE | replace ".yaml" ".sample.yaml"}} {{.BOOTSTRAP_CONFIG_FILE}}
@@ -50,41 +51,40 @@ tasks:
         silent: true
       - cmd: echo {{.BOOTSTRAP_ADDONS_FILE}}
         silent: true
-    status:
-      - test -f {{.BOOTSTRAP_ADDONS_FILE}}
-      - test -f {{.BOOTSTRAP_CONFIG_FILE}}
 
   configure:
     desc: Configure repository from Ansible vars
     prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue?
     cmds:
-      - task: .pre-validate
       - task: .template
       - task: .post-validate
-    preconditions:
-      - { msg: "Missing Python version check script", sh: "test -f {{.PYTHON_VERSION_CHECK_FILE}}" }
-      - { msg: "Python version must be 3.11 or greater", sh: "{{.PYTHON_BIN}} {{.PYTHON_VERSION_CHECK_FILE}}" }
-
-  .pre-validate:
-    internal: true
-    cmd: ./.venv/bin/ansible-playbook {{.BOOTSTRAP_DIR}}/validate.yaml
-    env:
-      ANSIBLE_DISPLAY_SKIPPED_HOSTS: "false"
-    preconditions:
-      - { msg: "Missing virtual environment", sh: "test -d {{.ROOT_DIR}}/.venv" }
-      - { msg: "Missing bootstrap addons file", sh: "test -f {{.BOOTSTRAP_ADDONS_FILE}}" }
-      - { msg: "Missing bootstrap config file", sh: "test -f {{.BOOTSTRAP_CONFIG_FILE}}" }
 
   .template:
     internal: true
     cmds:
       - ./.venv/bin/makejinja
       - task: sops:encrypt
     preconditions:
+      - { msg: "Missing virtual environment", sh: "test -d {{.ROOT_DIR}}/.venv" }
       - { msg: "Missing Makejinja config file", sh: "test -f {{.MAKEJINJA_CONFIG_FILE}}" }
       - { msg: "Missing Makejinja loader file", sh: "test -f {{.BOOTSTRAP_DIR}}/scripts/loader.py" }
+      - { msg: "Missing bootstrap addons file", sh: "test -f {{.BOOTSTRAP_ADDONS_FILE}}" }
+      - { msg: "Missing bootstrap config file", sh: "test -f {{.BOOTSTRAP_CONFIG_FILE}}" }
 
   .post-validate:
     internal: true
     cmds:
       - task: kubernetes:kubeconform
+
+  setup-virtual-env:
+    desc: Set up virtual environment
+    cmds:
+      - "{{.PYTHON_BIN}} -m venv {{.ROOT_DIR}}/.venv"
+      - .venv/bin/python3 -m pip install --upgrade pip setuptools wheel
+      - .venv/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}"
+    sources:
+      - "{{.PIP_REQUIREMENTS_FILE}}"
+    generates:
+      - "{{.ROOT_DIR}}/.venv/pyvenv.cfg"
+    preconditions:
+      - { msg: "Missing Pip requirements file", sh: "test -f {{.PIP_REQUIREMENTS_FILE}}" }
diff --git a/bootstrap/scripts/loader.py b/bootstrap/scripts/loader.py
@@ -1,21 +1,40 @@
-import netaddr, bcrypt
+import bcrypt
+import netaddr
+
+import validation
 
 def nthhost(value: str, query: int) -> str:
     value = netaddr.IPNetwork(value)
-
     try:
         nth = int(query)
         if value.size > nth:
             return str(value[nth])
-
     except ValueError:
         return False
-
     return value
 
 def encrypt(value: str) -> str:
     return bcrypt.hashpw(value.encode(), bcrypt.gensalt(rounds=10)).decode("ascii")
 
 class Loader:
+    def __init__(self, data):
+        if data.get("skip_tests", False):
+            return
+        validation.validate_python_version()
+        validation.validate_cli_tools(data)
+        validation.validate_distribution(data)
+        validation.validate_github(data)
+        validation.validate_age(data)
+        validation.validate_timezone(data)
+        validation.validate_acme_email(data)
+        validation.validate_flux_github_webhook_token(data)
+        validation.validate_cloudflare(data)
+        validation.validate_host_network(data)
+        validation.validate_bootstrap_dns_server(data)
+        validation.validate_cilium_loadbalancer_mode(data)
+        validation.validate_local_storage_path(data)
+        validation.validate_cluster_cidrs(data)
+        validation.validate_nodes(data)
+
     def filters(self):
         return [nthhost, encrypt]
diff --git a/bootstrap/scripts/py_version_check.py b/bootstrap/scripts/py_version_check.py