From 9a708e0356d0235ff22b082a7ffe217bcf4983ce Mon Sep 17 00:00:00 2001 From: Justin Bertrand Date: Wed, 15 Mar 2023 15:06:24 -0400 Subject: [PATCH 1/3] feat(istio/kubeflow-gateway): update all configs to reference CNP kubeflow gateway. --- .../centraldashboard/base/kustomization.yaml | 9 +++ .../jupyter-web-app/base/kustomization.yaml | 9 +++ kustomize/apps/jupyter-web-app/base/vs.yaml | 4 +- kustomize/apps/katib/base/kustomization.yaml | 9 +++ .../base/kustomization.yaml | 7 +++ .../apps/profiles/base/kustomization.yaml | 10 ++- .../common/knative/base/config-istio.yaml | 62 ------------------- .../common/knative/base/kustomization.yaml | 14 ++++- .../kserve/base/kfserve-config-configmap.yaml | 7 +++ .../contrib/kserve/base/kustomization.yaml | 1 + .../contrib/seldon/base/kustomization.yaml | 16 +++++ 11 files changed, 82 insertions(+), 66 deletions(-) delete mode 100644 kustomize/common/knative/base/config-istio.yaml create mode 100644 kustomize/contrib/kserve/base/kfserve-config-configmap.yaml diff --git a/kustomize/apps/centraldashboard/base/kustomization.yaml b/kustomize/apps/centraldashboard/base/kustomization.yaml index 3c04dca..5a4158a 100644 --- a/kustomize/apps/centraldashboard/base/kustomization.yaml +++ b/kustomize/apps/centraldashboard/base/kustomization.yaml @@ -8,3 +8,12 @@ patchesStrategicMerge: - deployment.yaml - centraldashboard-config.yaml - authorizationpolicy.yaml + +patches: + - patch: |- + - op: replace + path: /spec/gateways + value: ["istio-system/kubeflow-istio-ingress-gateway-https"] + target: + group: networking.istio.io + kind: VirtualService diff --git a/kustomize/apps/jupyter-web-app/base/kustomization.yaml b/kustomize/apps/jupyter-web-app/base/kustomization.yaml index 7fc2c27..7f0f4ca 100644 --- a/kustomize/apps/jupyter-web-app/base/kustomization.yaml +++ b/kustomize/apps/jupyter-web-app/base/kustomization.yaml @@ -20,3 +20,12 @@ configMapGenerator: generatorOptions: disableNameSuffixHash: true + +patches: + - patch: |- + - op: replace + path: /spec/gateways + value: ["istio-system/kubeflow-istio-ingress-gateway-https"] + target: + group: networking.istio.io + kind: VirtualService diff --git a/kustomize/apps/jupyter-web-app/base/vs.yaml b/kustomize/apps/jupyter-web-app/base/vs.yaml index 9c1ec2f..90283af 100644 --- a/kustomize/apps/jupyter-web-app/base/vs.yaml +++ b/kustomize/apps/jupyter-web-app/base/vs.yaml @@ -4,7 +4,7 @@ metadata: name: jupyter-web-app-jupyter-web-app spec: gateways: - - kubeflow-gateway + - istio-system/kubeflow-istio-ingress-gateway-https hosts: - '*' http: @@ -23,4 +23,4 @@ spec: - destination: host: jupyter-web-app-service.$(JWA_NAMESPACE).svc.$(JWA_CLUSTER_DOMAIN) port: - number: 80 \ No newline at end of file + number: 80 diff --git a/kustomize/apps/katib/base/kustomization.yaml b/kustomize/apps/katib/base/kustomization.yaml index ad0f7ad..bbbf3b6 100644 --- a/kustomize/apps/katib/base/kustomization.yaml +++ b/kustomize/apps/katib/base/kustomization.yaml @@ -20,3 +20,12 @@ patchesJson6902: - op: replace path: "/apiVersion" value: cert-manager.io/v1 + +patches: + - patch: |- + - op: replace + path: /spec/gateways + value: ["istio-system/kubeflow-istio-ingress-gateway-https"] + target: + group: networking.istio.io + kind: VirtualService diff --git a/kustomize/apps/notebook-controller/base/kustomization.yaml b/kustomize/apps/notebook-controller/base/kustomization.yaml index ee02819..1ba6c35 100644 --- a/kustomize/apps/notebook-controller/base/kustomization.yaml +++ b/kustomize/apps/notebook-controller/base/kustomization.yaml @@ -6,3 +6,10 @@ resources: patchesStrategicMerge: - deployment.yaml + +# Update the link to the Gateway to use. +configMapGenerator: +- name: config + behavior: merge + literals: + - ISTIO_GATEWAY=istio-system/kubeflow-istio-ingress-gateway-https diff --git a/kustomize/apps/profiles/base/kustomization.yaml b/kustomize/apps/profiles/base/kustomization.yaml index 12c910e..de31e1a 100644 --- a/kustomize/apps/profiles/base/kustomization.yaml +++ b/kustomize/apps/profiles/base/kustomization.yaml @@ -12,6 +12,14 @@ configMapGenerator: files: - namespace-labels.yaml - patchesStrategicMerge: - deployment.yaml + +patches: + - patch: |- + - op: replace + path: /spec/gateways + value: ["istio-system/kubeflow-istio-ingress-gateway-https"] + target: + group: networking.istio.io + kind: VirtualService diff --git a/kustomize/common/knative/base/config-istio.yaml b/kustomize/common/knative/base/config-istio.yaml deleted file mode 100644 index 511916a..0000000 --- a/kustomize/common/knative/base/config-istio.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/component: knative-serving - app.kubernetes.io/name: knative-serving - kustomize.component: knative - networking.knative.dev/ingress-provider: istio - serving.knative.dev/release: v0.22.1 - name: config-istio - namespace: knative-serving -data: - _example: | - ################################ - # # - # EXAMPLE CONFIGURATION # - # # - ################################ - - # This block is not actually functional configuration, - # but serves to illustrate the available configuration - # options and document them in a way that is accessible - # to users that `kubectl edit` this config map. - # - # These sample configuration options may be copied out of - # this example block and unindented to be in the data block - # to actually change the configuration. - - # Default Knative Gateway after v0.3. It points to the Istio - # standard istio-ingressgateway, instead of a custom one that we - # used pre-0.3. The configuration format should be `gateway. - # {{gateway_namespace}}.{{gateway_name}}: "{{ingress_name}}. - # {{ingress_namespace}}.svc.cluster.local"`. The {{gateway_namespace}} - # is optional; when it is omitted, the system will search for - # the gateway in the serving system namespace `knative-serving` - gateway.knative-serving.knative-ingress-gateway: "istio-ingressgateway.istio-system.svc.cluster.local" - - # A cluster local gateway to allow pods outside of the mesh to access - # Services and Routes not exposing through an ingress. If the users - # do have a service mesh setup, this isn't required and can be removed. - # - # An example use case is when users want to use Istio without any - # sidecar injection (like Knative's istio-ci-no-mesh.yaml). Since every pod - # is outside of the service mesh in that case, a cluster-local service - # will need to be exposed to a cluster-local gateway to be accessible. - # The configuration format should be `local-gateway.{{local_gateway_namespace}}. - # {{local_gateway_name}}: "{{cluster_local_gateway_name}}. - # {{cluster_local_gateway_namespace}}.svc.cluster.local"`. The - # {{local_gateway_namespace}} is optional; when it is omitted, the system - # will search for the local gateway in the serving system namespace - # `knative-serving` - local-gateway.knative-serving.cluster-local-gateway: "cluster-local-gateway.istio-system.svc.cluster.local" - - # To use only Istio service mesh and no cluster-local-gateway, replace - # all local-gateway.* entries by the following entry. - local-gateway.mesh: "mesh" - - # This is commented out as we don't want it exposed on the gateway since the authenticaion only works for kubeflow - gateway.kubeflow.kubeflow-gateway: istio-ingressgateway.istio-system.svc.cluster.local - gateway.ingress-general-system.general-istio-ingress-gateway-https: general.ingress-general-system.svc.cluster.local - local-gateway.istio-system.cluster-local-gateway: cluster-local-gateway.istio-system.svc.cluster.local - local-gateway.mesh: mesh diff --git a/kustomize/common/knative/base/kustomization.yaml b/kustomize/common/knative/base/kustomization.yaml index fe0fea9..c6883ec 100644 --- a/kustomize/common/knative/base/kustomization.yaml +++ b/kustomize/common/knative/base/kustomization.yaml @@ -7,7 +7,7 @@ resources: patchesStrategicMerge: - config-domain.yaml -- config-istio.yaml +# - config-istio.yaml patches: - path: istio-webhook-deployment-patch.json @@ -60,3 +60,15 @@ patchesJson6902: - op: replace path: "/spec/minReplicas" value: 3 + +configMapGenerator: +- name: config-istio + namespace: knative-serving + behavior: replace + options: + disableNameSuffixHash: true + literals: + - gateway.ingress-general-system.general-istio-ingress-gateway-https=general.ingress-general-system.svc.cluster.local + - gateway.istio-system.kubeflow-istio-ingress-gateway-https=kubeflow.istio-system.svc.cluster.local + - local-gateway.istio-system.cluster-local-gateway=cluster-local-gateway.istio-system.svc.cluster.local + - local-gateway.mesh=mesh diff --git a/kustomize/contrib/kserve/base/kfserve-config-configmap.yaml b/kustomize/contrib/kserve/base/kfserve-config-configmap.yaml new file mode 100644 index 0000000..f324657 --- /dev/null +++ b/kustomize/contrib/kserve/base/kfserve-config-configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +data: + ingressGateway: istio-system/kubeflow-istio-ingress-gateway-https +kind: ConfigMap +metadata: + name: kserve-config + namespace: kubeflow diff --git a/kustomize/contrib/kserve/base/kustomization.yaml b/kustomize/contrib/kserve/base/kustomization.yaml index 8bb1a20..f7bb3a8 100644 --- a/kustomize/contrib/kserve/base/kustomization.yaml +++ b/kustomize/contrib/kserve/base/kustomization.yaml @@ -8,6 +8,7 @@ resources: patchesStrategicMerge: - inferenceservice-configmap.yaml +- kfserve-config-configmap.yaml ## Cert Manager patchesJson6902: diff --git a/kustomize/contrib/seldon/base/kustomization.yaml b/kustomize/contrib/seldon/base/kustomization.yaml index 7df4e5d..2401238 100644 --- a/kustomize/contrib/seldon/base/kustomization.yaml +++ b/kustomize/contrib/seldon/base/kustomization.yaml @@ -15,3 +15,19 @@ patchesJson6902: - op: replace path: "/apiVersion" value: cert-manager.io/v1 + +patches: + - patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: seldon-controller-manager + namespace: kubeflow + spec: + template: + spec: + containers: + - name: manager + env: + - name: ISTIO_GATEWAY + value: istio-system/kubeflow-istio-ingress-gateway-https From 37784b91451e46795183d220b767136026db4f2d Mon Sep 17 00:00:00 2001 From: Justin Bertrand Date: Thu, 16 Mar 2023 07:26:41 -0400 Subject: [PATCH 2/3] chore(knative): remove unneeded comment. --- kustomize/common/knative/base/kustomization.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/kustomize/common/knative/base/kustomization.yaml b/kustomize/common/knative/base/kustomization.yaml index c6883ec..098c44c 100644 --- a/kustomize/common/knative/base/kustomization.yaml +++ b/kustomize/common/knative/base/kustomization.yaml @@ -7,7 +7,6 @@ resources: patchesStrategicMerge: - config-domain.yaml -# - config-istio.yaml patches: - path: istio-webhook-deployment-patch.json From 45f197a8196d5e06b61604914a09520c68af844b Mon Sep 17 00:00:00 2001 From: Justin Bertrand Date: Thu, 16 Mar 2023 07:44:50 -0400 Subject: [PATCH 3/3] chore: add some comments to help identify kustimizations. --- kustomize/apps/centraldashboard/base/kustomization.yaml | 1 + kustomize/apps/jupyter-web-app/base/kustomization.yaml | 1 + kustomize/apps/katib/base/kustomization.yaml | 1 + kustomize/apps/notebook-controller/base/kustomization.yaml | 3 ++- kustomize/apps/profiles/base/kustomization.yaml | 1 + kustomize/common/knative/base/kustomization.yaml | 3 +++ 6 files changed, 9 insertions(+), 1 deletion(-) diff --git a/kustomize/apps/centraldashboard/base/kustomization.yaml b/kustomize/apps/centraldashboard/base/kustomization.yaml index 5a4158a..28e96ed 100644 --- a/kustomize/apps/centraldashboard/base/kustomization.yaml +++ b/kustomize/apps/centraldashboard/base/kustomization.yaml @@ -10,6 +10,7 @@ patchesStrategicMerge: - authorizationpolicy.yaml patches: + # Patch all VirtualServices to use centralized Kubeflow ingress gateway - patch: |- - op: replace path: /spec/gateways diff --git a/kustomize/apps/jupyter-web-app/base/kustomization.yaml b/kustomize/apps/jupyter-web-app/base/kustomization.yaml index 7f0f4ca..3af1145 100644 --- a/kustomize/apps/jupyter-web-app/base/kustomization.yaml +++ b/kustomize/apps/jupyter-web-app/base/kustomization.yaml @@ -22,6 +22,7 @@ generatorOptions: disableNameSuffixHash: true patches: + # Patch all VirtualServices to use centralized Kubeflow ingress gateway - patch: |- - op: replace path: /spec/gateways diff --git a/kustomize/apps/katib/base/kustomization.yaml b/kustomize/apps/katib/base/kustomization.yaml index bbbf3b6..327d348 100644 --- a/kustomize/apps/katib/base/kustomization.yaml +++ b/kustomize/apps/katib/base/kustomization.yaml @@ -22,6 +22,7 @@ patchesJson6902: value: cert-manager.io/v1 patches: + # Patch all VirtualServices to use centralized Kubeflow ingress gateway - patch: |- - op: replace path: /spec/gateways diff --git a/kustomize/apps/notebook-controller/base/kustomization.yaml b/kustomize/apps/notebook-controller/base/kustomization.yaml index 1ba6c35..5ad9d60 100644 --- a/kustomize/apps/notebook-controller/base/kustomization.yaml +++ b/kustomize/apps/notebook-controller/base/kustomization.yaml @@ -7,8 +7,9 @@ resources: patchesStrategicMerge: - deployment.yaml -# Update the link to the Gateway to use. configMapGenerator: +# Update the link to the Gateway to use. +# https://github.com/kubeflow/manifests/blob/c2795524afd97dc0776b804521bb7d937646e98f/apps/jupyter/notebook-controller/upstream/manager/kustomization.yaml#L5 - name: config behavior: merge literals: diff --git a/kustomize/apps/profiles/base/kustomization.yaml b/kustomize/apps/profiles/base/kustomization.yaml index de31e1a..04b17f4 100644 --- a/kustomize/apps/profiles/base/kustomization.yaml +++ b/kustomize/apps/profiles/base/kustomization.yaml @@ -16,6 +16,7 @@ patchesStrategicMerge: - deployment.yaml patches: + # Patch all VirtualServices to use centralized Kubeflow ingress gateway - patch: |- - op: replace path: /spec/gateways diff --git a/kustomize/common/knative/base/kustomization.yaml b/kustomize/common/knative/base/kustomization.yaml index 098c44c..a71373a 100644 --- a/kustomize/common/knative/base/kustomization.yaml +++ b/kustomize/common/knative/base/kustomization.yaml @@ -61,6 +61,9 @@ patchesJson6902: value: 3 configMapGenerator: +# Replace the example ConfigMap and the patch coming in: +# https://github.com/kubeflow/manifests/blob/c2795524afd97dc0776b804521bb7d937646e98f/common/knative/knative-serving/base/patches/configmap-patch.yaml#L8 +# https://github.com/kubeflow/manifests/blob/c2795524afd97dc0776b804521bb7d937646e98f/common/knative/knative-serving/base/upstream/net-istio.yaml#L63 - name: config-istio namespace: knative-serving behavior: replace