diff --git a/bootstrap/scripts/plugin.py b/bootstrap/scripts/plugin.py
index 8944f38de..d474921cf 100644
--- a/bootstrap/scripts/plugin.py
+++ b/bootstrap/scripts/plugin.py
@@ -11,6 +11,20 @@
import validation
+# Return the filename of a path without the j2 extension
+def basename(value: str) -> str:
+ return Path(value).stem
+
+
+# Return a list of files in the talos patches directory
+def talos_patches(value: str) -> list[str]:
+ path = Path(f'bootstrap/templates/kubernetes/bootstrap/talos/patches/{value}')
+ if not path.is_dir():
+ return []
+ return [str(f) for f in path.glob('*.yaml.j2') if f.is_file()]
+
+
+# Return the nth host in a CIDR range
def nthhost(value: str, query: int) -> str:
value = IPNetwork(value)
try:
@@ -50,7 +64,11 @@ def __init__(self, data: dict[str, Any], config: makejinja.config.Config):
def filters(self) -> makejinja.plugin.Filters:
- return [nthhost]
+ return [basename, nthhost]
+
+
+ def functions(self) -> makejinja.plugin.Functions:
+ return [talos_patches]
def path_filters(self):
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/README.md.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/README.md.j2
new file mode 100644
index 000000000..b96818887
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/README.md.j2
@@ -0,0 +1,15 @@
+# Talos Patching
+
+This directory contains Kustomization patches that are added to the talhelper configuration file.
+
+
+
+## Patch Directories
+
+Under this `patches` directory, there are several sub-directories that can contain patches that are added to the talhelper configuration file.
+Each directory is optional and therefore might not created by default.
+
+- `global/`: patches that are applied to both the controller and worker configurations
+- `controller/`: patches that are applied to the controller configurations
+- `worker/`: patches that are applied to the worker configurations
+- `${node-hostname}/`: patches that are applied to the node with the specified name
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/api-access.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/api-access.yaml.j2
new file mode 100644
index 000000000..772328442
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/api-access.yaml.j2
@@ -0,0 +1,8 @@
+machine:
+ features:
+ kubernetesTalosAPIAccess:
+ enabled: true
+ allowedRoles:
+ - os:admin
+ allowedKubernetesNamespaces:
+ - system-upgrade
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/cluster.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/cluster.yaml.j2
new file mode 100644
index 000000000..aa3a9f226
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/cluster.yaml.j2
@@ -0,0 +1,12 @@
+cluster:
+ allowSchedulingOnControlPlanes: true
+ controllerManager:
+ extraArgs:
+ bind-address: 0.0.0.0
+ coreDNS:
+ disabled: true
+ proxy:
+ disabled: true
+ scheduler:
+ extraArgs:
+ bind-address: 0.0.0.0
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/disable-admission-controller.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/disable-admission-controller.yaml.j2
new file mode 100644
index 000000000..e311789f4
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/disable-admission-controller.yaml.j2
@@ -0,0 +1,2 @@
+- op: remove
+ path: /cluster/apiServer/admissionControl
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/etcd.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/etcd.yaml.j2
new file mode 100644
index 000000000..df35aa5d0
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/etcd.yaml.j2
@@ -0,0 +1,6 @@
+cluster:
+ etcd:
+ extraArgs:
+ listen-metrics-urls: http://0.0.0.0:2381
+ advertisedSubnets:
+ - #{ bootstrap_node_network }#
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/cluster-discovery.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/cluster-discovery.yaml.j2
new file mode 100644
index 000000000..586a07abb
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/cluster-discovery.yaml.j2
@@ -0,0 +1,7 @@
+cluster:
+ discovery:
+ registries:
+ kubernetes:
+ disabled: false
+ service:
+ disabled: false
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/containerd.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/containerd.yaml.j2
new file mode 100644
index 000000000..2952d6b41
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/containerd.yaml.j2
@@ -0,0 +1,12 @@
+machine:
+ files:
+ - op: create
+ path: /etc/cri/conf.d/20-customization.part
+ content: |-
+ [plugins."io.containerd.grpc.v1.cri"]
+ enable_unprivileged_ports = true
+ enable_unprivileged_icmp = true
+ [plugins."io.containerd.grpc.v1.cri".containerd]
+ discard_unpacked_layers = false
+ [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+ discard_unpacked_layers = false
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/disable-search-domain.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/disable-search-domain.yaml.j2
new file mode 100644
index 000000000..8ba647c49
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/disable-search-domain.yaml.j2
@@ -0,0 +1,3 @@
+machine:
+ network:
+ disableSearchDomain: true
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/hostdns.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/hostdns.yaml.j2
new file mode 100644
index 000000000..36c5e94f7
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/hostdns.yaml.j2
@@ -0,0 +1,6 @@
+machine:
+ features:
+ hostDNS:
+ enabled: true
+ resolveMemberNames: true
+ forwardKubeDNSToHost: false
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/kubelet.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/kubelet.yaml.j2
new file mode 100644
index 000000000..ee71c2809
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/kubelet.yaml.j2
@@ -0,0 +1,7 @@
+machine:
+ kubelet:
+ extraArgs:
+ rotate-server-certificates: true
+ nodeIP:
+ validSubnets:
+ - #{ bootstrap_node_network }#
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/openebs-local.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/openebs-local.yaml.j2
new file mode 100644
index 000000000..e4095d171
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/openebs-local.yaml.j2
@@ -0,0 +1,10 @@
+machine:
+ kubelet:
+ extraMounts:
+ - destination: /var/openebs/local
+ type: bind
+ source: /var/openebs/local
+ options:
+ - bind
+ - rshared
+ - rw
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/sysctl.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/sysctl.yaml.j2
new file mode 100644
index 000000000..90361d7bb
--- /dev/null
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/sysctl.yaml.j2
@@ -0,0 +1,7 @@
+machine:
+ sysctls:
+ fs.inotify.max_queued_events: "65536"
+ fs.inotify.max_user_watches: "524288"
+ fs.inotify.max_user_instances: "8192"
+ net.core.rmem_max: "2500000"
+ net.core.wmem_max: "2500000"
diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2
index 35b4171d6..17ddf0349 100644
--- a/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2
+++ b/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2
@@ -82,10 +82,6 @@ nodes:
ip: "#{ bootstrap_controller_vip }#"
#% endif %#
#% endif %#
- #% if bootstrap_user_patches %#
- patches:
- - "@./patches/node_#{ item.name }#.yaml"
- #% endif %#
#% if item.manifests %#
extraManifests:
#% for manifest in item.manifests %#
@@ -110,62 +106,19 @@ nodes:
#% endif %#
#% endfor %#
#% endif %#
+ #% for file in talos_patches('%s' % (item.name)) %#
+ #% if loop.index == 1 %#
+ patches:
+ #% endif %#
+ - "@./patches/#{ item.name }#/#{ file | basename }#"
+ #% endfor %#
#% endfor %#
+# Global patches
patches:
- # Configure containerd
- - |-
- machine:
- files:
- - op: create
- path: /etc/cri/conf.d/20-customization.part
- content: |-
- [plugins."io.containerd.grpc.v1.cri"]
- enable_unprivileged_ports = true
- enable_unprivileged_icmp = true
- [plugins."io.containerd.grpc.v1.cri".containerd]
- discard_unpacked_layers = false
- [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
- discard_unpacked_layers = false
-
- # Disable search domain everywhere
- - |-
- machine:
- network:
- disableSearchDomain: true
-
- # Enable cluster discovery
- - |-
- cluster:
- discovery:
- registries:
- kubernetes:
- disabled: false
- service:
- disabled: false
-
- # Configure kubelet
- - |-
- machine:
- kubelet:
- extraArgs:
- rotate-server-certificates: true
- nodeIP:
- validSubnets:
- - #{ bootstrap_node_network }#
-
- # Enable host dns
- - |-
- machine:
- features:
- hostDNS:
- enabled: true
- resolveMemberNames: true
- forwardKubeDNSToHost: false
-
#% if bootstrap_dns_servers | length %#
- # Force nameserver
- - |-
+ - # Force nameserver
+ |-
machine:
network:
nameservers:
@@ -173,10 +126,9 @@ patches:
- #{ item }#
#% endfor %#
#% endif %#
-
#% if bootstrap_ntp_servers | length %#
- # Configure NTP
- - |-
+ - # Configure NTP
+ |-
machine:
time:
disabled: false
@@ -185,33 +137,9 @@ patches:
- #{ item }#
#% endfor %#
#% endif %#
-
- # Custom sysctl settings
- - |-
- machine:
- sysctls:
- fs.inotify.max_queued_events: "65536"
- fs.inotify.max_user_watches: "524288"
- fs.inotify.max_user_instances: "8192"
- net.core.rmem_max: "2500000"
- net.core.wmem_max: "2500000"
-
- # Mount openebs-hostpath in kubelet
- - |-
- machine:
- kubelet:
- extraMounts:
- - destination: /var/openebs/local
- type: bind
- source: /var/openebs/local
- options:
- - bind
- - rshared
- - rw
-
#% if bootstrap_secureboot.enabled and bootstrap_secureboot.encrypt_disk_with_tpm %#
- # Encrypt system disk with TPM
- - |-
+ - # Encrypt system disk with TPM
+ |-
machine:
systemDiskEncryption:
ephemeral:
@@ -225,60 +153,26 @@ patches:
- slot: 0
tpm: {}
#% endif %#
+ #% for file in talos_patches('global') %#
+ - "@./patches/global/#{ file | basename }#"
+ #% endfor %#
- #% if bootstrap_user_patches %#
- # User specified global patches
- - "@./patches/global.yaml"
- #% endif %#
-
+#% for file in talos_patches('controller') %#
+#% if loop.index == 1 %#
+# Controller patches
controlPlane:
patches:
- # Cluster configuration
- - |-
- cluster:
- allowSchedulingOnControlPlanes: true
- controllerManager:
- extraArgs:
- bind-address: 0.0.0.0
- coreDNS:
- disabled: true
- proxy:
- disabled: true
- scheduler:
- extraArgs:
- bind-address: 0.0.0.0
-
- # ETCD configuration
- - |-
- cluster:
- etcd:
- extraArgs:
- listen-metrics-urls: http://0.0.0.0:2381
- advertisedSubnets:
- - #{ bootstrap_node_network }#
-
- # Disable default API server admission plugins.
- - |-
- - op: remove
- path: /cluster/apiServer/admissionControl
+#% endif %#
+ - "@./patches/controller/#{ file | basename }#"
+#% endfor %#
- # Enable K8s Talos API Access
- - |-
- machine:
- features:
- kubernetesTalosAPIAccess:
- enabled: true
- allowedRoles:
- - os:admin
- allowedKubernetesNamespaces:
- - system-upgrade
- #% if bootstrap_user_patches %#
- # User specified controlPlane patches
- - "@./patches/controlPlane.yaml"
- #% endif %#
-#% if ((bootstrap_user_patches) and (bootstrap_node_inventory | selectattr('controller', 'equalto', False) | list | length)) %#
+#% if (bootstrap_node_inventory | selectattr('controller', 'equalto', False) | list | length) and (talos_patches('worker') | length) %#
+#% for file in talos_patches('worker') %#
+#% if loop.index == 1 %#
+# Worker patches
worker:
patches:
- # User specified worker patches
- - "@./patches/worker.yaml"
+#% endif %#
+ - "@./patches/worker/#{ file | basename }#"
+#% endfor %#
#% endif %#
diff --git a/config.sample.yaml b/config.sample.yaml
index 9444cb095..94171738c 100644
--- a/config.sample.yaml
+++ b/config.sample.yaml
@@ -111,16 +111,6 @@ bootstrap_secureboot:
# See: https://www.talos.dev/v1.6/talos-guides/install/bare-metal-platforms/secureboot/#disk-encryption-with-tpm
encrypt_disk_with_tpm: false
-# (Optional) Add includes for user provided patches to generated talconfig.yaml.
-# See: https://github.com/budimanjojo/talhelper/blob/179ba9ed42f70069c7842109bea24f769f7af6eb/example/extraKernelArgs-patch.yaml
-# Patches are applied in this order. (global overrides cp/worker which overrides node-specific).
-# Create these files to allow talos:bootstrap-genconfig to complete (empty files are ok).
-# kubernetes/bootstrap/talos/patches/node_.yaml # Patches for individual nodes
-# kubernetes/bootstrap/talos/patches/controlPlane.yaml # Patches for controlplane nodes
-# kubernetes/bootstrap/talos/patches/worker.yaml # Patches for worker nodes
-# kubernetes/bootstrap/talos/patches/global.yaml # Patches for ALL nodes
-bootstrap_user_patches: false
-
#
# 2. (Required) Flux details - Flux is used to manage the cluster configuration.
#
@@ -219,6 +209,4 @@ bootstrap_cloudflare:
# # IMPORTANT: I am looking for people to help maintain IPv6 support since I cannot test it.
# # Ref: https://github.com/onedr0p/cluster-template/issues/1148
# # Keep in mind that Cilium does not currently support IPv6 L2 announcements.
-# # Make sure you set cluster.pod_cidr and cluster.service_cidr
-# # to a valid dual stack CIDRs, e.g. "10.42.0.0/16,fd00:10:244::/64"
# dual_stack_ipv4_first: false