diff --git a/bootstrap/scripts/plugin.py b/bootstrap/scripts/plugin.py index 8944f38de..d474921cf 100644 --- a/bootstrap/scripts/plugin.py +++ b/bootstrap/scripts/plugin.py @@ -11,6 +11,20 @@ import validation +# Return the filename of a path without the j2 extension +def basename(value: str) -> str: + return Path(value).stem + + +# Return a list of files in the talos patches directory +def talos_patches(value: str) -> list[str]: + path = Path(f'bootstrap/templates/kubernetes/bootstrap/talos/patches/{value}') + if not path.is_dir(): + return [] + return [str(f) for f in path.glob('*.yaml.j2') if f.is_file()] + + +# Return the nth host in a CIDR range def nthhost(value: str, query: int) -> str: value = IPNetwork(value) try: @@ -50,7 +64,11 @@ def __init__(self, data: dict[str, Any], config: makejinja.config.Config): def filters(self) -> makejinja.plugin.Filters: - return [nthhost] + return [basename, nthhost] + + + def functions(self) -> makejinja.plugin.Functions: + return [talos_patches] def path_filters(self): diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/README.md.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/README.md.j2 new file mode 100644 index 000000000..b96818887 --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/README.md.j2 @@ -0,0 +1,15 @@ +# Talos Patching + +This directory contains Kustomization patches that are added to the talhelper configuration file. + + + +## Patch Directories + +Under this `patches` directory, there are several sub-directories that can contain patches that are added to the talhelper configuration file. +Each directory is optional and therefore might not created by default. + +- `global/`: patches that are applied to both the controller and worker configurations +- `controller/`: patches that are applied to the controller configurations +- `worker/`: patches that are applied to the worker configurations +- `${node-hostname}/`: patches that are applied to the node with the specified name diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/api-access.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/api-access.yaml.j2 new file mode 100644 index 000000000..772328442 --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/api-access.yaml.j2 @@ -0,0 +1,8 @@ +machine: + features: + kubernetesTalosAPIAccess: + enabled: true + allowedRoles: + - os:admin + allowedKubernetesNamespaces: + - system-upgrade diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/cluster.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/cluster.yaml.j2 new file mode 100644 index 000000000..aa3a9f226 --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/cluster.yaml.j2 @@ -0,0 +1,12 @@ +cluster: + allowSchedulingOnControlPlanes: true + controllerManager: + extraArgs: + bind-address: 0.0.0.0 + coreDNS: + disabled: true + proxy: + disabled: true + scheduler: + extraArgs: + bind-address: 0.0.0.0 diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/disable-admission-controller.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/disable-admission-controller.yaml.j2 new file mode 100644 index 000000000..e311789f4 --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/disable-admission-controller.yaml.j2 @@ -0,0 +1,2 @@ +- op: remove + path: /cluster/apiServer/admissionControl diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/etcd.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/etcd.yaml.j2 new file mode 100644 index 000000000..df35aa5d0 --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/etcd.yaml.j2 @@ -0,0 +1,6 @@ +cluster: + etcd: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2381 + advertisedSubnets: + - #{ bootstrap_node_network }# diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/cluster-discovery.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/cluster-discovery.yaml.j2 new file mode 100644 index 000000000..586a07abb --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/cluster-discovery.yaml.j2 @@ -0,0 +1,7 @@ +cluster: + discovery: + registries: + kubernetes: + disabled: false + service: + disabled: false diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/containerd.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/containerd.yaml.j2 new file mode 100644 index 000000000..2952d6b41 --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/containerd.yaml.j2 @@ -0,0 +1,12 @@ +machine: + files: + - op: create + path: /etc/cri/conf.d/20-customization.part + content: |- + [plugins."io.containerd.grpc.v1.cri"] + enable_unprivileged_ports = true + enable_unprivileged_icmp = true + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + discard_unpacked_layers = false diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/disable-search-domain.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/disable-search-domain.yaml.j2 new file mode 100644 index 000000000..8ba647c49 --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/disable-search-domain.yaml.j2 @@ -0,0 +1,3 @@ +machine: + network: + disableSearchDomain: true diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/hostdns.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/hostdns.yaml.j2 new file mode 100644 index 000000000..36c5e94f7 --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/hostdns.yaml.j2 @@ -0,0 +1,6 @@ +machine: + features: + hostDNS: + enabled: true + resolveMemberNames: true + forwardKubeDNSToHost: false diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/kubelet.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/kubelet.yaml.j2 new file mode 100644 index 000000000..ee71c2809 --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/kubelet.yaml.j2 @@ -0,0 +1,7 @@ +machine: + kubelet: + extraArgs: + rotate-server-certificates: true + nodeIP: + validSubnets: + - #{ bootstrap_node_network }# diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/openebs-local.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/openebs-local.yaml.j2 new file mode 100644 index 000000000..e4095d171 --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/openebs-local.yaml.j2 @@ -0,0 +1,10 @@ +machine: + kubelet: + extraMounts: + - destination: /var/openebs/local + type: bind + source: /var/openebs/local + options: + - bind + - rshared + - rw diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/sysctl.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/sysctl.yaml.j2 new file mode 100644 index 000000000..90361d7bb --- /dev/null +++ b/bootstrap/templates/kubernetes/bootstrap/talos/patches/global/sysctl.yaml.j2 @@ -0,0 +1,7 @@ +machine: + sysctls: + fs.inotify.max_queued_events: "65536" + fs.inotify.max_user_watches: "524288" + fs.inotify.max_user_instances: "8192" + net.core.rmem_max: "2500000" + net.core.wmem_max: "2500000" diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2 index 35b4171d6..17ddf0349 100644 --- a/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2 +++ b/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2 @@ -82,10 +82,6 @@ nodes: ip: "#{ bootstrap_controller_vip }#" #% endif %# #% endif %# - #% if bootstrap_user_patches %# - patches: - - "@./patches/node_#{ item.name }#.yaml" - #% endif %# #% if item.manifests %# extraManifests: #% for manifest in item.manifests %# @@ -110,62 +106,19 @@ nodes: #% endif %# #% endfor %# #% endif %# + #% for file in talos_patches('%s' % (item.name)) %# + #% if loop.index == 1 %# + patches: + #% endif %# + - "@./patches/#{ item.name }#/#{ file | basename }#" + #% endfor %# #% endfor %# +# Global patches patches: - # Configure containerd - - |- - machine: - files: - - op: create - path: /etc/cri/conf.d/20-customization.part - content: |- - [plugins."io.containerd.grpc.v1.cri"] - enable_unprivileged_ports = true - enable_unprivileged_icmp = true - [plugins."io.containerd.grpc.v1.cri".containerd] - discard_unpacked_layers = false - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - discard_unpacked_layers = false - - # Disable search domain everywhere - - |- - machine: - network: - disableSearchDomain: true - - # Enable cluster discovery - - |- - cluster: - discovery: - registries: - kubernetes: - disabled: false - service: - disabled: false - - # Configure kubelet - - |- - machine: - kubelet: - extraArgs: - rotate-server-certificates: true - nodeIP: - validSubnets: - - #{ bootstrap_node_network }# - - # Enable host dns - - |- - machine: - features: - hostDNS: - enabled: true - resolveMemberNames: true - forwardKubeDNSToHost: false - #% if bootstrap_dns_servers | length %# - # Force nameserver - - |- + - # Force nameserver + |- machine: network: nameservers: @@ -173,10 +126,9 @@ patches: - #{ item }# #% endfor %# #% endif %# - #% if bootstrap_ntp_servers | length %# - # Configure NTP - - |- + - # Configure NTP + |- machine: time: disabled: false @@ -185,33 +137,9 @@ patches: - #{ item }# #% endfor %# #% endif %# - - # Custom sysctl settings - - |- - machine: - sysctls: - fs.inotify.max_queued_events: "65536" - fs.inotify.max_user_watches: "524288" - fs.inotify.max_user_instances: "8192" - net.core.rmem_max: "2500000" - net.core.wmem_max: "2500000" - - # Mount openebs-hostpath in kubelet - - |- - machine: - kubelet: - extraMounts: - - destination: /var/openebs/local - type: bind - source: /var/openebs/local - options: - - bind - - rshared - - rw - #% if bootstrap_secureboot.enabled and bootstrap_secureboot.encrypt_disk_with_tpm %# - # Encrypt system disk with TPM - - |- + - # Encrypt system disk with TPM + |- machine: systemDiskEncryption: ephemeral: @@ -225,60 +153,26 @@ patches: - slot: 0 tpm: {} #% endif %# + #% for file in talos_patches('global') %# + - "@./patches/global/#{ file | basename }#" + #% endfor %# - #% if bootstrap_user_patches %# - # User specified global patches - - "@./patches/global.yaml" - #% endif %# - +#% for file in talos_patches('controller') %# +#% if loop.index == 1 %# +# Controller patches controlPlane: patches: - # Cluster configuration - - |- - cluster: - allowSchedulingOnControlPlanes: true - controllerManager: - extraArgs: - bind-address: 0.0.0.0 - coreDNS: - disabled: true - proxy: - disabled: true - scheduler: - extraArgs: - bind-address: 0.0.0.0 - - # ETCD configuration - - |- - cluster: - etcd: - extraArgs: - listen-metrics-urls: http://0.0.0.0:2381 - advertisedSubnets: - - #{ bootstrap_node_network }# - - # Disable default API server admission plugins. - - |- - - op: remove - path: /cluster/apiServer/admissionControl +#% endif %# + - "@./patches/controller/#{ file | basename }#" +#% endfor %# - # Enable K8s Talos API Access - - |- - machine: - features: - kubernetesTalosAPIAccess: - enabled: true - allowedRoles: - - os:admin - allowedKubernetesNamespaces: - - system-upgrade - #% if bootstrap_user_patches %# - # User specified controlPlane patches - - "@./patches/controlPlane.yaml" - #% endif %# -#% if ((bootstrap_user_patches) and (bootstrap_node_inventory | selectattr('controller', 'equalto', False) | list | length)) %# +#% if (bootstrap_node_inventory | selectattr('controller', 'equalto', False) | list | length) and (talos_patches('worker') | length) %# +#% for file in talos_patches('worker') %# +#% if loop.index == 1 %# +# Worker patches worker: patches: - # User specified worker patches - - "@./patches/worker.yaml" +#% endif %# + - "@./patches/worker/#{ file | basename }#" +#% endfor %# #% endif %# diff --git a/config.sample.yaml b/config.sample.yaml index 9444cb095..94171738c 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -111,16 +111,6 @@ bootstrap_secureboot: # See: https://www.talos.dev/v1.6/talos-guides/install/bare-metal-platforms/secureboot/#disk-encryption-with-tpm encrypt_disk_with_tpm: false -# (Optional) Add includes for user provided patches to generated talconfig.yaml. -# See: https://github.com/budimanjojo/talhelper/blob/179ba9ed42f70069c7842109bea24f769f7af6eb/example/extraKernelArgs-patch.yaml -# Patches are applied in this order. (global overrides cp/worker which overrides node-specific). -# Create these files to allow talos:bootstrap-genconfig to complete (empty files are ok). -# kubernetes/bootstrap/talos/patches/node_.yaml # Patches for individual nodes -# kubernetes/bootstrap/talos/patches/controlPlane.yaml # Patches for controlplane nodes -# kubernetes/bootstrap/talos/patches/worker.yaml # Patches for worker nodes -# kubernetes/bootstrap/talos/patches/global.yaml # Patches for ALL nodes -bootstrap_user_patches: false - # # 2. (Required) Flux details - Flux is used to manage the cluster configuration. # @@ -219,6 +209,4 @@ bootstrap_cloudflare: # # IMPORTANT: I am looking for people to help maintain IPv6 support since I cannot test it. # # Ref: https://github.com/onedr0p/cluster-template/issues/1148 # # Keep in mind that Cilium does not currently support IPv6 L2 announcements. -# # Make sure you set cluster.pod_cidr and cluster.service_cidr -# # to a valid dual stack CIDRs, e.g. "10.42.0.0/16,fd00:10:244::/64" # dual_stack_ipv4_first: false