diff --git a/Makefile b/Makefile index 7f56e4e63d6..f19b4053a75 100644 --- a/Makefile +++ b/Makefile @@ -387,6 +387,7 @@ create-cluster: ## Create a workload development Kubernetes cluster on Azure in EXP_MACHINE_POOL=true \ EXP_EDGEZONE=true \ EXP_ASO_API=true \ + EXP_APISERVER_ILB=false \ $(MAKE) create-management-cluster \ create-workload-cluster @@ -746,7 +747,7 @@ kind-create: $(KUBECTL) ## Create capz kind cluster if needed. aks-create: $(KUBECTL) ## Create aks cluster as mgmt cluster. ./scripts/aks-as-mgmt.sh -.PHONY: tilt-up +.PHONY: tilt-up ## TODO: EXP_APISERVER_ILB should be set to "true" either at the template level or via the env as being done here. tilt-up: install-tools ## Start tilt and build kind cluster if needed. @if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \ export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \ diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index dddc46d6a47..cbc1152fbbc 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -24,7 +24,7 @@ spec: - --leader-elect - "--diagnostics-address=${CAPZ_DIAGNOSTICS_ADDRESS:=:8443}" - "--insecure-diagnostics=${CAPZ_INSECURE_DIAGNOSTICS:=false}" - - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true}" + - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true},APIServerILB=${EXP_APISERVER_ILB:=false}" - "--v=0" image: controller:latest imagePullPolicy: Always diff --git a/docs/book/src/developers/development.md b/docs/book/src/developers/development.md index a99b7be31da..9065b9499db 100644 --- a/docs/book/src/developers/development.md +++ b/docs/book/src/developers/development.md @@ -21,6 +21,8 @@ - [Tilt for dev in CAPZ](#tilt-for-dev-in-capz) - [Tilt for dev in both CAPZ and CAPI](#tilt-for-dev-in-both-capz-and-capi) - [Deploying a workload cluster](#deploying-a-workload-cluster) + - [Tilt for development (Microsoft Tenant)](#tilt-for-development-microsoft-tenant) + - [Flavors for development(Microsoft Tenant)](#flavors-for-development-microsoft-tenant) - [Viewing Telemetry](#viewing-telemetry) - [Debugging](#debugging) - [Manual Testing](#manual-testing) @@ -151,6 +153,8 @@ development will span both CAPZ and CAPI, then follow the [CAPI and CAPZ instruc #### Tilt for dev in CAPZ +If you are a MS Tenant and want to develop in CAPZ, navigate to [Tilt for development (Microsoft Tenant)](#tilt-for-development-microsoft-tenant). + If you want to develop in CAPZ and get a local development cluster working quickly, this is the path for you. Create a file named `tilt-settings.yaml` in the root of the CAPZ repository with the following contents: @@ -247,6 +251,34 @@ make delete-workload-cluster > Check out the [self-managed](../self-managed/troubleshooting.md) and [managed](../managed/troubleshooting.md) troubleshooting guides for common errors you might run into. +#### Tilt for development (Microsoft Tenant) + +The major difference between developing in CAPZ for MS Tenant and developing in CAPZ for non-MS Tenant is the enabling of the `APIServerILB` feature flag. +Microsoft tenants are required to use the `APIServerILB` feature flag to facilitate intern-node communication in the workload cluster. +We also encourage you to use AKS cluster as your management cluster. + +Outline of the steps: +- `make clean` +- `make generate` +- Set `REGISTRY` in your env. Preferably an Azure Container Registry. +- Run `docker-build-all` to build all the images. +- Run `make acr-login` to login to your ACR. +- Run `docker-push-all` to push all the images to your ACR. +- Run `make aks-create` to create an AKS cluster. _Notice the changes that get applied to the `tilt-settings.yaml` file._ +- Run `make tilt-up` to start Tilt. + +``` +TODO: +1. Come up with a shorter make target to do all the above steps for MS Tenants. +2. VNet peering should be exported out as a shell script for users to run. +``` + +##### Flavors for development (Microsoft Tenant) + +There are two flavors available for development in CAPZ for MSFT Tenant: +- [apiserver-ilb](../../../../templates/cluster-template-apiserver-ilb.yaml): VM based default flavor that brings up native K8s clusters with Linux nodes. +- [apiserver-ilb-windows](../../../../templates/cluster-template-windows-apiserver-ilb.yaml): VM based flavor that brings up native K8s clusters with Linux and Windows nodes. + #### Viewing Telemetry The CAPZ controller emits tracing and metrics data. When run in Tilt, the KinD management cluster is diff --git a/feature/feature.go b/feature/feature.go index b4af07d23fd..81eb0faa572 100644 --- a/feature/feature.go +++ b/feature/feature.go @@ -52,6 +52,12 @@ const ( // owner: @nojnhuh // alpha: v1.15 ASOAPI featuregate.Feature = "ASOAPI" + + // APIServerILB is a CAPZ feature gate to create an internal LB for the API Server of the workload cluster. + // Defaults to false. + // owner: @nawazkh + // alpha: v1.18 + APIServerILB featuregate.Feature = "APIServerILB" ) func init() { @@ -66,4 +72,5 @@ var defaultCAPZFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ AKSResourceHealth: {Default: false, PreRelease: featuregate.Alpha}, EdgeZone: {Default: false, PreRelease: featuregate.Alpha}, ASOAPI: {Default: true, PreRelease: featuregate.Alpha}, + APIServerILB: {Default: false, PreRelease: featuregate.Alpha}, } diff --git a/hack/observability/opentelemetry/controller-manager-patch.yaml b/hack/observability/opentelemetry/controller-manager-patch.yaml index 2ba355b841d..a952b4cbd05 100644 --- a/hack/observability/opentelemetry/controller-manager-patch.yaml +++ b/hack/observability/opentelemetry/controller-manager-patch.yaml @@ -12,5 +12,5 @@ spec: - "--diagnostics-address=:8080" - "--insecure-diagnostics" - "--leader-elect" - - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true}" + - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true},APIServerILB=${EXP_APISERVER_ILB:=false}" - "--enable-tracing"