diff --git a/docs/test-version.md b/docs/test-version.md index 8b136419a..ebd2cfcfa 100644 --- a/docs/test-version.md +++ b/docs/test-version.md @@ -15,5 +15,5 @@ The following Gateway API version and Ingress were tested as part of the release | Ingress | Tested version | Unavailable features | | ------- | ----------------------- | ------------------------------ | -| Istio | v1.21.1 | retry,httpoption,update | -| Contour | v1.28.3 | httpoption,update | +| Istio | v1.21.1 | retry,httpoption | +| Contour | v1.28.3 | httpoption | diff --git a/hack/test-env.sh b/hack/test-env.sh index 1d3b98375..c2ec82f7f 100755 --- a/hack/test-env.sh +++ b/hack/test-env.sh @@ -16,6 +16,6 @@ export GATEWAY_API_VERSION="v1.0.0" export ISTIO_VERSION="1.21.1" -export ISTIO_UNSUPPORTED_E2E_TESTS="retry,httpoption,update" +export ISTIO_UNSUPPORTED_E2E_TESTS="retry,httpoption" export CONTOUR_VERSION="v1.28.3" -export CONTOUR_UNSUPPORTED_E2E_TESTS="httpoption,update" +export CONTOUR_UNSUPPORTED_E2E_TESTS="httpoption" diff --git a/pkg/reconciler/ingress/controller.go b/pkg/reconciler/ingress/controller.go index dc0c231ea..3e8bf79a8 100644 --- a/pkg/reconciler/ingress/controller.go +++ b/pkg/reconciler/ingress/controller.go @@ -23,11 +23,9 @@ import ( "k8s.io/client-go/tools/cache" "knative.dev/networking/pkg/apis/networking" - "knative.dev/networking/pkg/apis/networking/v1alpha1" ingressinformer "knative.dev/networking/pkg/client/injection/informers/networking/v1alpha1/ingress" ingressreconciler "knative.dev/networking/pkg/client/injection/reconciler/networking/v1alpha1/ingress" networkcfg "knative.dev/networking/pkg/config" - "knative.dev/networking/pkg/status" endpointsinformer "knative.dev/pkg/client/injection/kube/informers/core/v1/endpoints" "knative.dev/pkg/configmap" "knative.dev/pkg/controller" @@ -39,6 +37,7 @@ import ( httprouteinformer "knative.dev/net-gateway-api/pkg/client/injection/informers/apis/v1beta1/httproute" referencegrantinformer "knative.dev/net-gateway-api/pkg/client/injection/informers/apis/v1beta1/referencegrant" "knative.dev/net-gateway-api/pkg/reconciler/ingress/config" + "knative.dev/net-gateway-api/pkg/status" ) const ( @@ -105,13 +104,11 @@ func NewController( statusProber := status.NewProber( logger.Named("status-manager"), NewProbeTargetLister(logger, endpointsInformer.Lister()), - func(ing *v1alpha1.Ingress) { - logger.Debugf("Ready callback triggered for ingress: %s/%s", ing.Namespace, ing.Name) - impl.EnqueueKey(types.NamespacedName{Namespace: ing.Namespace, Name: ing.Name}) + func(ing types.NamespacedName) { + logger.Debugf("Ready callback triggered for ingress: %v", ing) + impl.EnqueueKey(ing) }) c.statusManager = statusProber - // TODO: Bring up gateway-api community to discuss about probing. - // related to https://github.com/knative-sandbox/net-gateway-api/issues/18 statusProber.Start(ctx.Done()) // Make sure trackers are deleted once the observers are removed. diff --git a/pkg/reconciler/ingress/fixtures_test.go b/pkg/reconciler/ingress/fixtures_test.go new file mode 100644 index 000000000..1a36b5bfd --- /dev/null +++ b/pkg/reconciler/ingress/fixtures_test.go @@ -0,0 +1,232 @@ +/* +Copyright 2021 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ingress + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "knative.dev/networking/pkg/apis/networking" + "knative.dev/networking/pkg/http/header" + gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayapi "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +type RuleBuilder interface { + Build() gatewayapi.HTTPRouteRule +} + +type HTTPRoute struct { + Namespace string + Name string + Hostnames []string + Hostname string + Rules []RuleBuilder + StatusConditions []metav1.Condition +} + +func (r HTTPRoute) Build() *gatewayapi.HTTPRoute { + hostnames := r.Hostnames + + if len(hostnames) == 0 && r.Hostname == "" { + hostnames = []string{"example.com"} + } + + if r.Hostname != "" { + hostnames = append(hostnames, r.Hostname) + } + + route := gatewayapi.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.Name, + Namespace: r.Namespace, + Annotations: map[string]string{ + networking.IngressClassAnnotationKey: gatewayAPIIngressClassName, + }, + Labels: map[string]string{ + networking.VisibilityLabelKey: "", + }, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "networking.internal.knative.dev/v1alpha1", + Kind: "Ingress", + Name: "name", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }}, + }, + Spec: gatewayapi.HTTPRouteSpec{ + CommonRouteSpec: gatewayapi.CommonRouteSpec{ + ParentRefs: []gatewayapi.ParentReference{{ + Group: ptr.To[gatewayapi.Group]("gateway.networking.k8s.io"), + Kind: ptr.To[gatewayapi.Kind]("Gateway"), + Namespace: ptr.To[gatewayapi.Namespace]("istio-system"), + Name: "istio-gateway", + }}, + }, + }, + } + + for _, hostname := range hostnames { + route.Spec.Hostnames = append( + route.Spec.Hostnames, + gatewayapi.Hostname(hostname), + ) + } + + if route.Status.Parents == nil { + route.Status.Parents = []gatewayapi.RouteParentStatus{{}} + } + + route.Status.RouteStatus.Parents[0].Conditions = append( + route.Status.RouteStatus.Parents[0].Conditions, + r.StatusConditions..., + ) + + for _, rule := range r.Rules { + route.Spec.Rules = append(route.Spec.Rules, rule.Build()) + } + + return &route +} + +type EndpointProbeRule struct { + Namespace string + Name string + Hash string + Path string + Port int + Headers []string +} + +func (p EndpointProbeRule) Build() gatewayapi.HTTPRouteRule { + path := p.Path + if path == "" { + path = "/" + } + rule := gatewayapi.HTTPRouteRule{ + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To(path), + }, + Headers: []gatewayapi.HTTPHeaderMatch{{ + Type: ptr.To(gatewayapiv1.HeaderMatchExact), + Name: header.HashKey, + Value: header.HashValueOverride, + }}, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: header.HashKey, + Value: p.Hash, + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + Filters: []gatewayapiv1.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "K-Serving-Namespace", + Value: p.Namespace, + }, { + Name: "K-Serving-Revision", + Value: p.Name, + }}, + }, + }}, + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](100), + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: gatewayapi.ObjectName(p.Name), + Port: ptr.To[gatewayapi.PortNumber](gatewayapi.PortNumber(p.Port)), + }, + }, + }}, + } + + for i := 0; i < len(p.Headers); i += 2 { + k, v := p.Headers[i], p.Headers[i+1] + rule.BackendRefs[0].Filters[0].RequestHeaderModifier.Set = append( + rule.BackendRefs[0].Filters[0].RequestHeaderModifier.Set, + gatewayapi.HTTPHeader{Name: gatewayapiv1.HTTPHeaderName(k), Value: v}, + ) + } + + return rule +} + +type NormalRule struct { + Namespace string + Name string + Path string + Port int + Headers []string + Weight int +} + +func (p NormalRule) Build() gatewayapi.HTTPRouteRule { + path := p.Path + if path == "" { + path = "/" + } + rule := gatewayapi.HTTPRouteRule{ + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To(path), + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + Filters: []gatewayapiv1.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "K-Serving-Namespace", + Value: p.Namespace, + }, { + Name: "K-Serving-Revision", + Value: p.Name, + }}, + }, + }}, + BackendRef: gatewayapi.BackendRef{ + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: gatewayapi.ObjectName(p.Name), + Port: ptr.To[gatewayapi.PortNumber](gatewayapi.PortNumber(p.Port)), + }, + Weight: ptr.To[int32](int32(p.Weight)), + }, + }, + }, + } + + for i := 0; i < len(p.Headers); i += 2 { + k, v := p.Headers[i], p.Headers[i+1] + rule.BackendRefs[0].Filters[0].RequestHeaderModifier.Set = append( + rule.BackendRefs[0].Filters[0].RequestHeaderModifier.Set, + gatewayapi.HTTPHeader{Name: gatewayapiv1.HTTPHeaderName(k), Value: v}, + ) + } + + return rule +} diff --git a/pkg/reconciler/ingress/ingress.go b/pkg/reconciler/ingress/ingress.go index 3062fd1d7..71141efe5 100644 --- a/pkg/reconciler/ingress/ingress.go +++ b/pkg/reconciler/ingress/ingress.go @@ -19,14 +19,17 @@ package ingress import ( "context" "fmt" + "net/url" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "knative.dev/net-gateway-api/pkg/reconciler/ingress/config" + "knative.dev/net-gateway-api/pkg/status" "knative.dev/networking/pkg/apis/networking/v1alpha1" ingressreconciler "knative.dev/networking/pkg/client/injection/reconciler/networking/v1alpha1/ingress" + "knative.dev/networking/pkg/http/header" "knative.dev/networking/pkg/ingress" - "knative.dev/networking/pkg/status" "knative.dev/pkg/network" pkgreconciler "knative.dev/pkg/reconciler" @@ -86,29 +89,44 @@ func (c *Reconciler) reconcileIngress(ctx context.Context, ing *v1alpha1.Ingress // in this getting written back to the API Server, but lets downstream logic make // assumptions about defaulting. ing.SetDefaults(ctx) - before := ing.DeepCopy() - ing.Status.InitializeConditions() - if _, err := ingress.InsertProbe(ing); err != nil { + var ( + hash string + err error + ) + + if hash, err = ingress.InsertProbe(ing); err != nil { return fmt.Errorf("failed to add knative probe header: %w", err) } + backends := status.Backends{ + Version: hash, + Key: types.NamespacedName{ + Name: ing.Name, + Namespace: ing.Namespace, + }, + } + for _, rule := range ing.Spec.Rules { rule := rule - httproutes, err := c.reconcileHTTPRoute(ctx, ing, &rule) + httproute, err := c.reconcileHTTPRoute(ctx, &hash, ing, &rule) if err != nil { return err } - if isHTTPRouteReady(httproutes) { + if isHTTPRouteReady(httproute) { ing.Status.MarkNetworkConfigured() + gatherProbes(&backends, httproute, rule.Visibility) } else { ing.Status.MarkIngressNotReady("HTTPRouteNotReady", "Waiting for HTTPRoute becomes Ready.") } } + // Hash might have changed depending on HTTPRoute reconciliation + backends.Version = hash + externalIngressTLS := ing.GetIngressTLSForVisibility(v1alpha1.IngressVisibilityExternalIP) listeners := make([]*gatewayapi.Listener, 0, len(externalIngressTLS)) for _, tls := range externalIngressTLS { @@ -132,13 +150,12 @@ func (c *Reconciler) reconcileIngress(ctx context.Context, ing *v1alpha1.Ingress } // TODO: check Gateway readiness before reporting Ingress ready - - ready, err := c.statusManager.IsReady(ctx, before) + state, err := c.statusManager.DoProbes(ctx, backends) if err != nil { return fmt.Errorf("failed to probe Ingress: %w", err) } - if ready { + if state.Ready { namespacedNameService := gatewayConfig.Gateways[v1alpha1.IngressVisibilityExternalIP].Service publicLbs := []v1alpha1.LoadBalancerIngressStatus{ {DomainInternal: network.GetServiceHostname(namespacedNameService.Name, namespacedNameService.Namespace)}, @@ -157,6 +174,28 @@ func (c *Reconciler) reconcileIngress(ctx context.Context, ing *v1alpha1.Ingress return nil } +func gatherProbes(b *status.Backends, r *gatewayapi.HTTPRoute, visibility v1alpha1.IngressVisibility) { + if visibility == "" { + visibility = v1alpha1.IngressVisibilityExternalIP + } + + for _, rule := range r.Spec.Rules { + for _, match := range rule.Matches { + for _, headers := range match.Headers { + // Skip non-probe matches + if headers.Name != header.HashKey { + continue + } + + for _, hostname := range r.Spec.Hostnames { + url := url.URL{Host: string(hostname), Path: *match.Path.Value} + b.AddURL(visibility, url) + } + } + } + } +} + // isHTTPRouteReady will check the status conditions of the ingress and return true if // all gateways have been admitted. func isHTTPRouteReady(r *gatewayapi.HTTPRoute) bool { diff --git a/pkg/reconciler/ingress/ingress_test.go b/pkg/reconciler/ingress/ingress_test.go index 430ec6be6..7ec1f7514 100644 --- a/pkg/reconciler/ingress/ingress_test.go +++ b/pkg/reconciler/ingress/ingress_test.go @@ -28,22 +28,25 @@ import ( "k8s.io/apimachinery/pkg/types" clientgotesting "k8s.io/client-go/testing" "k8s.io/utils/pointer" + "k8s.io/utils/ptr" fakegwapiclientset "knative.dev/net-gateway-api/pkg/client/injection/client/fake" "knative.dev/net-gateway-api/pkg/reconciler/ingress/config" "knative.dev/net-gateway-api/pkg/reconciler/ingress/resources" + "knative.dev/net-gateway-api/pkg/status" "knative.dev/networking/pkg/apis/networking" "knative.dev/networking/pkg/apis/networking/v1alpha1" fakeingressclient "knative.dev/networking/pkg/client/injection/client/fake" ingressreconciler "knative.dev/networking/pkg/client/injection/reconciler/networking/v1alpha1/ingress" networkcfg "knative.dev/networking/pkg/config" + "knative.dev/networking/pkg/http/header" "knative.dev/networking/pkg/ingress" - "knative.dev/networking/pkg/status" "knative.dev/pkg/configmap" "knative.dev/pkg/controller" "knative.dev/pkg/logging" "knative.dev/pkg/network" + gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1" gatewayapi "sigs.k8s.io/gateway-api/apis/v1beta1" . "knative.dev/net-gateway-api/pkg/reconciler/testing" @@ -202,8 +205,11 @@ func TestReconcile(t *testing.T) { httprouteLister: listers.GetHTTPRouteLister(), gatewayLister: listers.GetGatewayLister(), statusManager: &fakeStatusManager{ - FakeIsReady: func(context.Context, *v1alpha1.Ingress) (bool, error) { - return true, nil + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: true}, nil + }, + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + return status.ProbeState{Ready: true}, true }, }, } @@ -350,9 +356,14 @@ func TestReconcileTLS(t *testing.T) { httprouteLister: listers.GetHTTPRouteLister(), referenceGrantLister: listers.GetReferenceGrantLister(), gatewayLister: listers.GetGatewayLister(), - statusManager: &fakeStatusManager{FakeIsReady: func(context.Context, *v1alpha1.Ingress) (bool, error) { - return true, nil - }}, + statusManager: &fakeStatusManager{ + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: true}, nil + }, + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + return status.ProbeState{Ready: true}, true + }, + }, } // The fake tracker's `Add` method incorrectly pluralizes "gatewaies" using UnsafeGuessKindToResource, // so create this via explicit call (per note in client-go/testing/fixture.go in tracker.Add) @@ -383,8 +394,8 @@ func TestReconcileProbing(t *testing.T) { Name: "first reconciler probe returns false", Key: "ns/name", Ctx: withStatusManager(&fakeStatusManager{ - FakeIsReady: func(context.Context, *v1alpha1.Ingress) (bool, error) { - return false, nil + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: false}, nil }, }), Objects: append([]runtime.Object{ @@ -412,8 +423,8 @@ func TestReconcileProbing(t *testing.T) { Name: "first reconcile probe returns an error", Key: "ns/name", Ctx: withStatusManager(&fakeStatusManager{ - FakeIsReady: func(context.Context, *v1alpha1.Ingress) (bool, error) { - return false, errors.New("this is the error") + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: false}, errors.New("this is the error") }, }), WantErr: true, @@ -443,8 +454,11 @@ func TestReconcileProbing(t *testing.T) { Name: "prober callback all endpoints ready", Key: "ns/name", Ctx: withStatusManager(&fakeStatusManager{ - FakeIsReady: func(context.Context, *v1alpha1.Ingress) (bool, error) { - return true, nil + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: true}, nil + }, + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + return status.ProbeState{Ready: true}, true }, }), Objects: append([]runtime.Object{ @@ -454,6 +468,776 @@ func TestReconcileProbing(t *testing.T) { WantStatusUpdates: []clientgotesting.UpdateActionImpl{ {Object: ing(withBasicSpec, withGatewayAPIclass, withFinalizer, makeItReady)}, }, + }, { + Name: "updated ingress - new backends used for endpoint probing", + Key: "ns/name", + Objects: append([]runtime.Object{ + ing(withSecondRevisionSpec, withGatewayAPIclass, withFinalizer, makeItReady), + httpRoute(t, ing(withBasicSpec, withGatewayAPIclass), httpRouteReady), + }, servicesAndEndpoints...), + Ctx: withStatusManager(&fakeStatusManager{ + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + return status.ProbeState{Ready: true, Version: "previous"}, true + }, + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: false}, nil + }, + }), + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: ing(withSecondRevisionSpec, + withGatewayAPIclass, + withFinalizer, + makeItReady, + makeLoadBalancerNotReady, + ), + }}, + WantUpdates: []clientgotesting.UpdateActionImpl{{ + Object: HTTPRoute{ + Name: "example.com", + Namespace: "ns", + Hostname: "example.com", + Rules: []RuleBuilder{ + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + NormalRule{ + Namespace: "ns", + Name: "goo", + Port: 123, + Weight: 100, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "second-revision", + Path: "/.well-known/knative/revision/ns/second-revision", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Path: "/.well-known/knative/revision/ns/goo", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + }, + StatusConditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }.Build(), + }}, + }, { + Name: "steady state ingress - endpoint probing still not ready", + Key: "ns/name", + Ctx: withStatusManager(&fakeStatusManager{ + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + return status.ProbeState{ + Ready: false, + Version: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + }, true + }, + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: false}, nil + }, + }), + Objects: append([]runtime.Object{ + ing(withSecondRevisionSpec, + withGatewayAPIclass, + withFinalizer, + makeItReady, + makeLoadBalancerNotReady, + ), + HTTPRoute{ + Name: "example.com", + Namespace: "ns", + Hostname: "example.com", + Rules: []RuleBuilder{ + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + NormalRule{ + Namespace: "ns", + Name: "goo", + Port: 123, + Weight: 100, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "second-revision", + Path: "/.well-known/knative/revision/ns/second-revision", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Path: "/.well-known/knative/revision/ns/goo", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + }, + StatusConditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }.Build(), + }, servicesAndEndpoints...), + }, { + Name: "endpoints are ready - transition to new backends", + Key: "ns/name", + Ctx: withStatusManager(&fakeStatusManager{ + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + state := status.ProbeState{Ready: true, Version: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2"} + return state, true + }, + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: false}, nil + }, + }), + Objects: append([]runtime.Object{ + ing(withSecondRevisionSpec, + withGatewayAPIclass, + withFinalizer, + makeItReady, + makeLoadBalancerNotReady, + ), + HTTPRoute{ + Name: "example.com", + Namespace: "ns", + Hostname: "example.com", + Rules: []RuleBuilder{ + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + NormalRule{ + Namespace: "ns", + Name: "goo", + Port: 123, + Weight: 100, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "second-revision", + Path: "/.well-known/knative/revision/ns/second-revision", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Path: "/.well-known/knative/revision/ns/goo", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + }, + StatusConditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }.Build(), + }, servicesAndEndpoints...), + WantUpdates: []clientgotesting.UpdateActionImpl{{ + Object: HTTPRoute{ + Name: "example.com", + Namespace: "ns", + Hostname: "example.com", + Rules: []RuleBuilder{ + EndpointProbeRule{ + Namespace: "ns", + Name: "second-revision", + Hash: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + NormalRule{ + Namespace: "ns", + Name: "second-revision", + Port: 123, + Weight: 100, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "second-revision", + Path: "/.well-known/knative/revision/ns/second-revision", + Hash: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Path: "/.well-known/knative/revision/ns/goo", + Hash: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + }, + StatusConditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }.Build(), + }}, + }, { + Name: "steady state - transition probing still not ready", + Key: "ns/name", + Ctx: withStatusManager(&fakeStatusManager{ + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + state := status.ProbeState{Ready: false, Version: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2"} + return state, true + }, + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: false}, nil + }, + }), + Objects: append([]runtime.Object{ + ing(withSecondRevisionSpec, + withGatewayAPIclass, + withFinalizer, + makeItReady, + makeLoadBalancerNotReady, + ), + &gatewayapi.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example.com", + Namespace: "ns", + Annotations: map[string]string{ + networking.IngressClassAnnotationKey: gatewayAPIIngressClassName, + }, + Labels: map[string]string{ + networking.VisibilityLabelKey: "", + }, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "networking.internal.knative.dev/v1alpha1", + Kind: "Ingress", + Name: "name", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }}, + }, + Spec: gatewayapi.HTTPRouteSpec{ + CommonRouteSpec: gatewayapi.CommonRouteSpec{ + ParentRefs: []gatewayapi.ParentReference{{ + Group: ptr.To[gatewayapi.Group]("gateway.networking.k8s.io"), + Kind: ptr.To[gatewayapi.Kind]("Gateway"), + Namespace: ptr.To[gatewayapi.Namespace]("istio-system"), + Name: "istio-gateway", + }}, + }, + Hostnames: []gatewayapi.Hostname{"example.com"}, + Rules: []gatewayapi.HTTPRouteRule{{ + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/"), + }, + Headers: []gatewayapi.HTTPHeaderMatch{{ + Type: ptr.To(gatewayapiv1.HeaderMatchExact), + Name: header.HashKey, + Value: header.HashValueOverride, + }}, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: header.HashKey, + Value: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + Filters: []gatewayapiv1.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "K-Serving-Revision", + Value: "second-revision", + }, { + Name: "K-Serving-Namespace", + Value: "ns", + }}, + }, + }}, + BackendRef: gatewayapi.BackendRef{ + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: "second-revision", + Port: ptr.To[gatewayapi.PortNumber](123), + }, + Weight: ptr.To[int32](100), + }, + }}, + }, { + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/"), + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + Filters: []gatewayapiv1.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "K-Serving-Revision", + Value: "second-revision", + }, { + Name: "K-Serving-Namespace", + Value: "ns", + }}, + }, + }}, + BackendRef: gatewayapi.BackendRef{ + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: "second-revision", + Port: ptr.To[gatewayapi.PortNumber](123), + }, + Weight: ptr.To[int32](100), + }, + }}, + }, { + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/.well-known/knative/revision/ns/second-revision"), + }, + Headers: []gatewayapi.HTTPHeaderMatch{{ + Type: ptr.To(gatewayapiv1.HeaderMatchExact), + Name: header.HashKey, + Value: header.HashValueOverride, + }}, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: header.HashKey, + Value: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + Filters: []gatewayapiv1.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "K-Serving-Namespace", + Value: "ns", + }, { + Name: "K-Serving-Revision", + Value: "second-revision", + }}, + }, + }}, + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](100), + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: "second-revision", + Port: ptr.To[gatewayapi.PortNumber](123), + }, + }, + }}, + }, { + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/.well-known/knative/revision/ns/goo"), + }, + Headers: []gatewayapi.HTTPHeaderMatch{{ + Type: ptr.To(gatewayapiv1.HeaderMatchExact), + Name: header.HashKey, + Value: header.HashValueOverride, + }}, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: header.HashKey, + Value: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + Filters: []gatewayapiv1.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "K-Serving-Namespace", + Value: "ns", + }, { + Name: "K-Serving-Revision", + Value: "goo", + }}, + }, + }}, + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](100), + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: "goo", + Port: ptr.To[gatewayapi.PortNumber](123), + }, + }, + }}, + }}, + }, + Status: gatewayapi.HTTPRouteStatus{ + RouteStatus: gatewayapi.RouteStatus{ + Parents: []gatewayapi.RouteParentStatus{{ + Conditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }}, + }, + }, + }, + }, + servicesAndEndpoints...), + }, { + Name: "transition probe complete - drop endpoint probes", + Key: "ns/name", + Ctx: withStatusManager(&fakeStatusManager{ + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + state := status.ProbeState{Ready: true, Version: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2"} + return state, true + }, + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: false}, nil + }, + }), + Objects: append([]runtime.Object{ + ing(withSecondRevisionSpec, + withGatewayAPIclass, + withFinalizer, + makeItReady, + makeLoadBalancerNotReady, + ), + HTTPRoute{ + Name: "example.com", + Namespace: "ns", + Hostname: "example.com", + Rules: []RuleBuilder{ + EndpointProbeRule{ + Namespace: "ns", + Name: "second-revision", + Hash: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + NormalRule{ + Namespace: "ns", + Name: "second-revision", + Port: 123, + Weight: 100, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "second-revision", + Path: "/.well-known/knative/revision/ns/second-revision", + Hash: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + }, + StatusConditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }.Build(), + }, + servicesAndEndpoints...), + WantUpdates: []clientgotesting.UpdateActionImpl{{ + Object: HTTPRoute{ + Name: "example.com", + Namespace: "ns", + Hostname: "example.com", + Rules: []RuleBuilder{ + EndpointProbeRule{ + Namespace: "ns", + Name: "second-revision", + Hash: "9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + NormalRule{ + Namespace: "ns", + Name: "second-revision", + Port: 123, + Weight: 100, + }, + }, + StatusConditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }.Build(), + }}, + }, { + Name: "dropping endpoint probes complete - mark ingress ready", + Key: "ns/name", + Ctx: withStatusManager(&fakeStatusManager{ + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + state := status.ProbeState{Ready: true, Version: "9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2"} + return state, true + }, + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: true}, nil + }, + }), + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: ing(withSecondRevisionSpec, + withGatewayAPIclass, + withFinalizer, + makeItReady, + ), + }}, + Objects: append([]runtime.Object{ + ing(withSecondRevisionSpec, + withGatewayAPIclass, + withFinalizer, + makeItReady, + makeLoadBalancerNotReady, + ), + HTTPRoute{ + Name: "example.com", + Namespace: "ns", + Hostname: "example.com", + Rules: []RuleBuilder{ + EndpointProbeRule{ + Namespace: "ns", + Name: "second-revision", + Hash: "9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + NormalRule{ + Namespace: "ns", + Name: "second-revision", + Port: 123, + Weight: 100, + }, + }, + StatusConditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }.Build(), + }, + servicesAndEndpoints...), + }, { + Name: "endpoints are ready - wrong hash", + // When the endpoints are ready but the hash is incorrect we do + // not transition the backend + Key: "ns/name", + Ctx: withStatusManager(&fakeStatusManager{ + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + state := status.ProbeState{Ready: true, Version: "bad-hash"} + return state, true + }, + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: false}, nil + }, + }), + Objects: append([]runtime.Object{ + ing(withSecondRevisionSpec, + withGatewayAPIclass, + withFinalizer, + makeItReady, + makeLoadBalancerNotReady, + ), + HTTPRoute{ + Name: "example.com", + Namespace: "ns", + Hostname: "example.com", + Rules: []RuleBuilder{ + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Port: 123, + }, + NormalRule{ + Namespace: "ns", + Name: "goo", + Port: 123, + Weight: 100, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "second-revision", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Path: "/.well-known/knative/revision/ns/second-revision", + Port: 123, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Path: "/.well-known/knative/revision/ns/goo", + Port: 123, + }, + }, + StatusConditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }.Build(), + }, servicesAndEndpoints...), + }, { + Name: "updated ingress - while endpoint probing in progress", + // Here we want the existing probe to stop and then new backends added + // to the endpoint probes + Key: "ns/name", + Ctx: withStatusManager(&fakeStatusManager{ + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + return status.ProbeState{Ready: false, Version: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2"}, true + }, + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: false}, nil + }, + }), + Objects: append([]runtime.Object{ + ing(withThirdRevisionSpec, withGatewayAPIclass, withFinalizer, makeItReady, makeLoadBalancerNotReady), + HTTPRoute{ + Name: "example.com", + Namespace: "ns", + Hostname: "example.com", + Rules: []RuleBuilder{ + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + }, + NormalRule{ + Namespace: "ns", + Name: "goo", + Port: 123, + Weight: 100, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "second-revision", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Path: "/.well-known/knative/revision/ns/second-revision", + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Hash: "ep-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2", + Path: "/.well-known/knative/revision/ns/goo", + }, + }, + StatusConditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }.Build(), + }, + servicesAndEndpoints...), + WantUpdates: []clientgotesting.UpdateActionImpl{{ + Object: HTTPRoute{ + Name: "example.com", + Namespace: "ns", + Hostname: "example.com", + Rules: []RuleBuilder{ + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Hash: "ep-40e40e812e47b79d9bae1f1d0ecec5bcb481030dad90a1aa6200f3389c31d374", + }, + NormalRule{ + Namespace: "ns", + Name: "goo", + Port: 123, + Weight: 100, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "third-revision", + Hash: "ep-40e40e812e47b79d9bae1f1d0ecec5bcb481030dad90a1aa6200f3389c31d374", + Path: "/.well-known/knative/revision/ns/third-revision", + Port: 123, + }, + EndpointProbeRule{ + Namespace: "ns", + Name: "goo", + Hash: "ep-40e40e812e47b79d9bae1f1d0ecec5bcb481030dad90a1aa6200f3389c31d374", + Path: "/.well-known/knative/revision/ns/goo", + Port: 123, + }, + }, + StatusConditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }.Build(), + }}, + }, { + Name: "updated ingress - backend headers change", + Key: "ns/name", + Objects: append([]runtime.Object{ + ing(withBasicSpec, + withGatewayAPIclass, + withFinalizer, + makeItReady, + withBackendAppendHeaders("key", "value")), + httpRoute(t, ing(withBasicSpec, withGatewayAPIclass), httpRouteReady), + }, servicesAndEndpoints...), + Ctx: withStatusManager(&fakeStatusManager{ + FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) { + return status.ProbeState{Ready: true, Version: "previous"}, true + }, + FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) { + return status.ProbeState{Ready: false}, nil + }, + }), + WantStatusUpdates: []clientgotesting.UpdateActionImpl{{ + Object: ing(withBasicSpec, + withGatewayAPIclass, + withFinalizer, + makeItReady, + makeLoadBalancerNotReady, + withBackendAppendHeaders("key", "value"), + ), + }}, + WantUpdates: []clientgotesting.UpdateActionImpl{{ + Object: HTTPRoute{ + Name: "example.com", + Namespace: "ns", + Rules: []RuleBuilder{ + EndpointProbeRule{ + Name: "goo", + Namespace: "ns", + Hash: "3531718c72349578ea2293f8ec1cd980d551f70295c1b0b4c10abfc0b2a248f8", + Headers: []string{"key", "value"}, + Port: 123, + }, + NormalRule{ + Name: "goo", + Namespace: "ns", + Headers: []string{"key", "value"}, + Port: 123, + Weight: 100, + }, + }, + StatusConditions: []metav1.Condition{{ + Type: string(gatewayapi.RouteConditionAccepted), + Status: metav1.ConditionTrue, + }}, + }.Build(), + }}, }} table.Test(t, MakeFactory(func(ctx context.Context, listers *Listers, cmw configmap.Watcher) controller.Reconciler { @@ -474,6 +1258,10 @@ func TestReconcileProbing(t *testing.T) { })) } +func makeLoadBalancerNotReady(i *v1alpha1.Ingress) { + i.Status.MarkLoadBalancerNotReady() +} + func makeItReady(i *v1alpha1.Ingress) { i.Status.InitializeConditions() i.Status.MarkNetworkConfigured() @@ -519,11 +1307,16 @@ func withStatusManager(f *fakeStatusManager) context.Context { } type fakeStatusManager struct { - FakeIsReady func(context.Context, *v1alpha1.Ingress) (bool, error) + FakeDoProbes func(context.Context, status.Backends) (status.ProbeState, error) + FakeIsProbeActive func(types.NamespacedName) (status.ProbeState, bool) +} + +func (m *fakeStatusManager) DoProbes(ctx context.Context, backends status.Backends) (status.ProbeState, error) { + return m.FakeDoProbes(ctx, backends) } -func (m *fakeStatusManager) IsReady(ctx context.Context, ing *v1alpha1.Ingress) (bool, error) { - return m.FakeIsReady(ctx, ing) +func (m *fakeStatusManager) IsProbeActive(ing types.NamespacedName) (status.ProbeState, bool) { + return m.FakeIsProbeActive(ing) } type testConfigStore struct { diff --git a/pkg/reconciler/ingress/lister.go b/pkg/reconciler/ingress/lister.go index 994f88902..a4a6a292a 100644 --- a/pkg/reconciler/ingress/lister.go +++ b/pkg/reconciler/ingress/lister.go @@ -19,16 +19,15 @@ package ingress import ( "context" "fmt" - "net/url" "strconv" "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/sets" corev1listers "k8s.io/client-go/listers/core/v1" "knative.dev/networking/pkg/apis/networking/v1alpha1" - "knative.dev/networking/pkg/status" "knative.dev/net-gateway-api/pkg/reconciler/ingress/config" + "knative.dev/net-gateway-api/pkg/status" ) func NewProbeTargetLister(logger *zap.SugaredLogger, endpointsLister corev1listers.EndpointsLister) status.ProbeTargetLister { @@ -43,76 +42,60 @@ type gatewayPodTargetLister struct { endpointsLister corev1listers.EndpointsLister } -func (l *gatewayPodTargetLister) ListProbeTargets(ctx context.Context, ing *v1alpha1.Ingress) ([]status.ProbeTarget, error) { - result := make([]status.ProbeTarget, 0, len(ing.Spec.Rules)) - for _, rule := range ing.Spec.Rules { - eps, err := l.getRuleProbes(ctx, rule, ing.Spec.HTTPOption) - if err != nil { - return nil, err - } - result = append(result, eps...) - } - return result, nil -} - -func (l *gatewayPodTargetLister) getRuleProbes(ctx context.Context, rule v1alpha1.IngressRule, sslOpt v1alpha1.HTTPOption) ([]status.ProbeTarget, error) { +func (l *gatewayPodTargetLister) BackendsToProbeTargets(ctx context.Context, backends status.Backends) ([]status.ProbeTarget, error) { gatewayConfig := config.FromContext(ctx).Gateway - service := gatewayConfig.Gateways[rule.Visibility].Service - eps, err := l.endpointsLister.Endpoints(service.Namespace).Get(service.Name) - if err != nil { - return nil, fmt.Errorf("failed to get endpoints: %w", err) - } - - targets := make([]status.ProbeTarget, 0, len(eps.Subsets)) foundTargets := 0 - for _, sub := range eps.Subsets { - scheme := "http" - // Istio uses "http2" for the http port - // Contour uses "http-80" for the http port - matchSchemes := sets.New("http", "http2", "http-80") - if rule.Visibility == v1alpha1.IngressVisibilityExternalIP && sslOpt == v1alpha1.HTTPOptionRedirected { - scheme = "https" - matchSchemes = sets.New("https", "https-443") + targets := make([]status.ProbeTarget, 0, len(backends.URLs)) + + for visibility, urls := range backends.URLs { + service := gatewayConfig.Gateways[visibility].Service + eps, err := l.endpointsLister.Endpoints(service.Namespace).Get(service.Name) + if err != nil { + return nil, fmt.Errorf("failed to get endpoints: %w", err) } - pt := status.ProbeTarget{PodIPs: sets.New[string]()} - - portNumber := sub.Ports[0].Port - for _, port := range sub.Ports { - if matchSchemes.Has(port.Name) { - // Prefer to match the name exactly - portNumber = port.Port - break + for _, sub := range eps.Subsets { + scheme := "http" + // Istio uses "http2" for the http port + // Contour uses "http-80" for the http port + matchSchemes := sets.New("http", "http2", "http-80") + if visibility == v1alpha1.IngressVisibilityExternalIP && backends.HTTPOption == v1alpha1.HTTPOptionRedirected { + scheme = "https" + matchSchemes = sets.New("https", "https-443") } - if port.AppProtocol != nil && matchSchemes.Has(*port.AppProtocol) { - portNumber = port.Port + pt := status.ProbeTarget{PodIPs: sets.New[string]()} + + portNumber := sub.Ports[0].Port + for _, port := range sub.Ports { + if matchSchemes.Has(port.Name) { + // Prefer to match the name exactly + portNumber = port.Port + break + } + if port.AppProtocol != nil && matchSchemes.Has(*port.AppProtocol) { + portNumber = port.Port + } } - } - pt.PodPort = strconv.Itoa(int(portNumber)) + pt.PodPort = strconv.Itoa(int(portNumber)) - for _, address := range sub.Addresses { - pt.PodIPs.Insert(address.IP) - } - foundTargets += len(pt.PodIPs) + for _, address := range sub.Addresses { + pt.PodIPs.Insert(address.IP) + } - pt.URLs = domainsToURL(rule.Hosts, scheme) - targets = append(targets, pt) + for url := range urls { + url := url + url.Scheme = scheme + pt.URLs = append(pt.URLs, &url) + } + + if len(pt.URLs) > 0 { + foundTargets += len(pt.PodIPs) + targets = append(targets, pt) + } + } } if foundTargets == 0 { return nil, fmt.Errorf("no gateway pods available") } return targets, nil } - -func domainsToURL(domains []string, scheme string) []*url.URL { - urls := make([]*url.URL, 0, len(domains)) - for _, domain := range domains { - url := &url.URL{ - Scheme: scheme, - Host: domain, - Path: "/", - } - urls = append(urls, url) - } - return urls -} diff --git a/pkg/reconciler/ingress/lister_test.go b/pkg/reconciler/ingress/lister_test.go index 7b8aca244..1f3b5cec6 100644 --- a/pkg/reconciler/ingress/lister_test.go +++ b/pkg/reconciler/ingress/lister_test.go @@ -20,6 +20,8 @@ import ( "context" "fmt" "net/url" + "slices" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -28,9 +30,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" + + "knative.dev/net-gateway-api/pkg/status" "knative.dev/networking/pkg/apis/networking" "knative.dev/networking/pkg/apis/networking/v1alpha1" - "knative.dev/networking/pkg/status" "knative.dev/pkg/kmeta" . "knative.dev/net-gateway-api/pkg/reconciler/testing" @@ -42,20 +45,26 @@ var ( privateName = "knative-local-gateway" ) -func TestListProbeTargets(t *testing.T) { - tests := []struct { - name string - ing *v1alpha1.Ingress - objects []runtime.Object - want []status.ProbeTarget - wantErr error +func TestBackendsToProbeTargets(t *testing.T) { + cases := []struct { + name string + backends status.Backends + objects []runtime.Object + want []status.ProbeTarget + wantErr error }{{ name: "single address to probe", objects: []runtime.Object{ privateEndpointsOneAddr, publicEndpointsOneAddr, }, - ing: ing(withBasicSpec, withGatewayAPIClass), + backends: status.Backends{ + URLs: map[v1alpha1.IngressVisibility]status.URLSet{ + v1alpha1.IngressVisibilityExternalIP: sets.New( + url.URL{Host: "example.com", Path: "/"}, + ), + }, + }, want: []status.ProbeTarget{ { PodIPs: sets.New("1.2.3.4"), @@ -72,30 +81,54 @@ func TestListProbeTargets(t *testing.T) { objects: []runtime.Object{ publicEndpointsOneAddr, }, - ing: ing(withBasicSpec, withInternalSpec, withGatewayAPIClass), + backends: status.Backends{ + URLs: map[v1alpha1.IngressVisibility]status.URLSet{ + v1alpha1.IngressVisibilityClusterLocal: sets.New( + url.URL{Host: "example.com", Path: "/"}, + ), + }, + }, wantErr: fmt.Errorf("failed to get endpoints: endpoints %q not found", privateName), }, { name: "no external endpoint to probe", objects: []runtime.Object{ - privateEndpointsOneAddr, + privateEndpointsNoAddr, + }, + backends: status.Backends{ + URLs: map[v1alpha1.IngressVisibility]status.URLSet{ + v1alpha1.IngressVisibilityExternalIP: sets.New( + url.URL{Host: "example.com", Path: "/"}, + ), + }, }, - ing: ing(withBasicSpec, withGatewayAPIClass), - wantErr: fmt.Errorf("failed to get endpoints: endpoints %q not found", "istio-gateway"), + wantErr: fmt.Errorf("failed to get endpoints: endpoints %q not found", publicName), }, { name: "local endpoint without address to probe", objects: []runtime.Object{ privateEndpointsNoAddr, publicEndpointsOneAddr, }, - ing: ing(withBasicSpec, withInternalSpec, withGatewayAPIClass), + backends: status.Backends{ + URLs: map[v1alpha1.IngressVisibility]status.URLSet{ + v1alpha1.IngressVisibilityClusterLocal: sets.New( + url.URL{Host: "example.com", Path: "/"}, + ), + }, + }, wantErr: fmt.Errorf("no gateway pods available"), }, { - name: "external endpoint without address to probe", + name: "local endpoint without address to probe", objects: []runtime.Object{ privateEndpointsOneAddr, publicEndpointsNoAddr, }, - ing: ing(withBasicSpec, withGatewayAPIClass), + backends: status.Backends{ + URLs: map[v1alpha1.IngressVisibility]status.URLSet{ + v1alpha1.IngressVisibilityExternalIP: sets.New( + url.URL{Host: "example.com", Path: "/"}, + ), + }, + }, wantErr: fmt.Errorf("no gateway pods available"), }, { name: "endpoint with single address to probe (https redirected)", @@ -103,7 +136,14 @@ func TestListProbeTargets(t *testing.T) { privateEndpointsOneAddr, publicSslEndpointsOneAddr, }, - ing: ing(withBasicSpec, withGatewayAPIClass, withHTTPOption(v1alpha1.HTTPOptionRedirected)), + backends: status.Backends{ + HTTPOption: v1alpha1.HTTPOptionRedirected, + URLs: map[v1alpha1.IngressVisibility]status.URLSet{ + v1alpha1.IngressVisibilityExternalIP: sets.New( + url.URL{Host: "example.com", Path: "/"}, + ), + }, + }, want: []status.ProbeTarget{{ PodIPs: sets.New("1.2.3.4"), PodPort: "8443", @@ -119,7 +159,14 @@ func TestListProbeTargets(t *testing.T) { privateEndpointsMultiAddrMultiSubset, publicEndpointsMultiAddrMultiSubset, }, - ing: ing(withBasicSpec, withGatewayAPIClass), + backends: status.Backends{ + HTTPOption: v1alpha1.HTTPOptionRedirected, + URLs: map[v1alpha1.IngressVisibility]status.URLSet{ + v1alpha1.IngressVisibilityClusterLocal: sets.New( + url.URL{Host: "example.com", Path: "/"}, + ), + }, + }, want: []status.ProbeTarget{ { PodIPs: sets.New("2.3.4.5"), @@ -138,9 +185,76 @@ func TestListProbeTargets(t *testing.T) { Path: "/", }}, }}, + }, { + name: "complex case", + objects: []runtime.Object{ + privateEndpointsMultiAddrMultiSubset, + publicEndpointsMultiAddrMultiSubset, + }, + backends: status.Backends{ + URLs: map[v1alpha1.IngressVisibility]status.URLSet{ + v1alpha1.IngressVisibilityExternalIP: sets.New( + url.URL{Host: "example.com", Path: "/"}, + url.URL{Host: "example.com", Path: "/.well-known/knative"}, + ), + v1alpha1.IngressVisibilityClusterLocal: sets.New( + url.URL{Host: "rev.default.svc.cluster.local", Path: "/"}, + url.URL{Host: "rev.default.svc.cluster.local", Path: "/.well-known/knative"}, + ), + }, + }, + want: []status.ProbeTarget{{ + PodIPs: sets.New("2.3.4.6"), + PodPort: "1230", + URLs: []*url.URL{{ + Scheme: "http", + Host: "example.com", + Path: "/", + }, { + Scheme: "http", + Host: "example.com", + Path: "/.well-known/knative", + }}, + }, { + PodIPs: sets.New("3.4.5.7", "4.3.2.0"), + PodPort: "4320", + URLs: []*url.URL{{ + Scheme: "http", + Host: "example.com", + Path: "/", + }, { + Scheme: "http", + Host: "example.com", + Path: "/.well-known/knative", + }}, + }, { + PodIPs: sets.New("2.3.4.5"), + PodPort: "1234", + URLs: []*url.URL{{ + Scheme: "http", + Host: "rev.default.svc.cluster.local", + Path: "/", + }, { + Scheme: "http", + Host: "rev.default.svc.cluster.local", + Path: "/.well-known/knative", + }}, + }, { + PodIPs: sets.New("3.4.5.6", "4.3.2.1"), + PodPort: "4321", + URLs: []*url.URL{{ + Scheme: "http", + Host: "rev.default.svc.cluster.local", + Path: "/", + }, { + Scheme: "http", + Host: "rev.default.svc.cluster.local", + Path: "/.well-known/knative", + }}, + }}, }} - for _, test := range tests { + for _, test := range cases { t.Run(test.name, func(t *testing.T) { tl := NewListers(test.objects) @@ -151,15 +265,37 @@ func TestListProbeTargets(t *testing.T) { cfg := defaultConfig.DeepCopy() ctx := (&testConfigStore{config: cfg}).ToContext(context.Background()) - got, gotErr := l.ListProbeTargets(ctx, test.ing) + got, gotErr := l.BackendsToProbeTargets(ctx, test.backends) if (gotErr != nil) != (test.wantErr != nil) { - t.Fatalf("ListProbeTargets() = %v, wanted %v", gotErr, test.wantErr) + t.Fatalf("BackendsToProbeTargets() = %v, wanted %v", gotErr, test.wantErr) } else if gotErr != nil && test.wantErr != nil && gotErr.Error() != test.wantErr.Error() { - t.Fatalf("ListProbeTargets() = %v, wanted %v", gotErr, test.wantErr) + t.Fatalf("BackendsToProbeTargets() = %v, wanted %v", gotErr, test.wantErr) + } + + // Ensure stable comparison + urlSortFunc := func(a, b *url.URL) int { + return strings.Compare(a.String(), b.String()) + } + for _, target := range test.want { + slices.SortFunc(target.URLs, urlSortFunc) + } + for _, target := range got { + slices.SortFunc(target.URLs, urlSortFunc) } - if !cmp.Equal(test.want, got) { - t.Error("ListProbeTargets (-want, +got) =", cmp.Diff(test.want, got)) + sortFunc := func(a, b status.ProbeTarget) int { + cmp := slices.Compare(sets.List(a.PodIPs), sets.List(b.PodIPs)) + if cmp == 0 { + return strings.Compare(a.PodPort, b.PodPort) + } + return cmp + } + + slices.SortFunc(test.want, sortFunc) + slices.SortFunc(got, sortFunc) + + if diff := cmp.Diff(test.want, got); diff != "" { + t.Error("BackendsToProbeTargets(-want, +got) =", diff) } }) } @@ -250,20 +386,20 @@ var ( Subsets: []corev1.EndpointSubset{{ Ports: []corev1.EndpointPort{{ Name: "asdf", - Port: 1234, + Port: 1230, }}, Addresses: []corev1.EndpointAddress{{ - IP: "2.3.4.5", + IP: "2.3.4.6", }}, }, { Ports: []corev1.EndpointPort{{ Name: "asdf", - Port: 4321, + Port: 4320, }}, Addresses: []corev1.EndpointAddress{{ - IP: "3.4.5.6", + IP: "3.4.5.7", }, { - IP: "4.3.2.1", + IP: "4.3.2.0", }}, }}, } @@ -301,6 +437,10 @@ func withBasicSpec(i *v1alpha1.Ingress) { HTTP: &v1alpha1.HTTPIngressRuleValue{ Paths: []v1alpha1.HTTPIngressPath{{ Splits: []v1alpha1.IngressBackendSplit{{ + AppendHeaders: map[string]string{ + "K-Serving-Revision": "goo", + "K-Serving-Namespace": "ns", + }, IngressBackend: v1alpha1.IngressBackend{ ServiceName: "goo", ServiceNamespace: i.Namespace, @@ -313,6 +453,24 @@ func withBasicSpec(i *v1alpha1.Ingress) { }) } +func withSecondRevisionSpec(i *v1alpha1.Ingress) { + withBasicSpec(i) + i.Spec.Rules[0].HTTP.Paths[0].Splits[0].ServiceName = "second-revision" + i.Spec.Rules[0].HTTP.Paths[0].Splits[0].AppendHeaders["K-Serving-Revision"] = "second-revision" +} + +func withThirdRevisionSpec(i *v1alpha1.Ingress) { + withBasicSpec(i) + i.Spec.Rules[0].HTTP.Paths[0].Splits[0].ServiceName = "third-revision" + i.Spec.Rules[0].HTTP.Paths[0].Splits[0].AppendHeaders["K-Serving-Revision"] = "third-revision" +} + +func withBackendAppendHeaders(key, val string) IngressOption { + return func(i *v1alpha1.Ingress) { + i.Spec.Rules[0].HTTP.Paths[0].Splits[0].AppendHeaders[key] = val + } +} + func withInternalSpec(i *v1alpha1.Ingress) { i.Spec.Rules = append(i.Spec.Rules, v1alpha1.IngressRule{ Hosts: []string{"foo.svc", "foo.svc.cluster.local"}, @@ -320,6 +478,10 @@ func withInternalSpec(i *v1alpha1.Ingress) { HTTP: &v1alpha1.HTTPIngressRuleValue{ Paths: []v1alpha1.HTTPIngressPath{{ Splits: []v1alpha1.IngressBackendSplit{{ + AppendHeaders: map[string]string{ + "K-Serving-Revision": "goo", + "K-Serving-Namespace": "ns", + }, IngressBackend: v1alpha1.IngressBackend{ ServiceName: "goo", ServiceNamespace: i.Namespace, diff --git a/pkg/reconciler/ingress/reconcile_resources.go b/pkg/reconciler/ingress/reconcile_resources.go index 2d28f4a25..485849878 100644 --- a/pkg/reconciler/ingress/reconcile_resources.go +++ b/pkg/reconciler/ingress/reconcile_resources.go @@ -19,12 +19,15 @@ package ingress import ( "context" "fmt" + "slices" + "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1" gatewayapi "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -32,6 +35,7 @@ import ( "knative.dev/net-gateway-api/pkg/reconciler/ingress/config" "knative.dev/net-gateway-api/pkg/reconciler/ingress/resources" netv1alpha1 "knative.dev/networking/pkg/apis/networking/v1alpha1" + "knative.dev/networking/pkg/http/header" "knative.dev/pkg/controller" ) @@ -39,7 +43,9 @@ const listenerPrefix = "kni-" // reconcileHTTPRoute reconciles HTTPRoute. func (c *Reconciler) reconcileHTTPRoute( - ctx context.Context, ing *netv1alpha1.Ingress, + ctx context.Context, + hash *string, + ing *netv1alpha1.Ingress, rule *netv1alpha1.IngressRule, ) (*gatewayapi.HTTPRoute, error) { recorder := controller.GetEventRecorder(ctx) @@ -60,33 +66,105 @@ func (c *Reconciler) reconcileHTTPRoute( return httproute, nil } else if err != nil { return nil, err + } + + return c.reconcileHTTPRouteUpdate(ctx, hash, ing, rule, httproute.DeepCopy()) +} + +func (c *Reconciler) reconcileHTTPRouteUpdate( + ctx context.Context, + hash *string, + ing *netv1alpha1.Ingress, + rule *netv1alpha1.IngressRule, + httproute *gatewayapi.HTTPRoute, +) (*gatewayapi.HTTPRoute, error) { + + const ( + endpointPrefix = "ep-" + transitionPrefix = "tr-" + ) + + var ( + probeKey = types.NamespacedName{ + Name: ing.Name, + Namespace: ing.Namespace, + } + original = httproute.DeepCopy() + recorder = controller.GetEventRecorder(ctx) + desired *gatewayapi.HTTPRoute + err error + probe, _ = c.statusManager.IsProbeActive(probeKey) + + wasEndpointProbe = strings.HasPrefix(probe.Version, endpointPrefix) + wasTransitionProbe = strings.HasPrefix(probe.Version, transitionPrefix) + ) + + probeHash := strings.TrimPrefix(probe.Version, endpointPrefix) + probeHash = strings.TrimPrefix(probeHash, transitionPrefix) + + newBackends, oldBackends := computeBackends(httproute, rule) + + if wasTransitionProbe && probeHash == *hash && probe.Ready { + desired, err = resources.MakeHTTPRoute(ctx, ing, rule) + } else if wasEndpointProbe && probeHash == *hash && probe.Ready { + *hash = transitionPrefix + *hash + + desired, err = resources.MakeHTTPRoute(ctx, ing, rule) + resources.UpdateProbeHash(desired, *hash) + + resources.RemoveEndpointProbes(httproute) + for _, backend := range newBackends { + resources.AddEndpointProbe(desired, *hash, backend) + } + for _, backend := range oldBackends { + resources.AddOldBackend(desired, *hash, backend) + } + } else if len(newBackends) > 0 { + *hash = endpointPrefix + *hash + desired = httproute.DeepCopy() + resources.UpdateProbeHash(desired, *hash) + resources.RemoveEndpointProbes(desired) + for _, backend := range newBackends { + resources.AddEndpointProbe(desired, *hash, backend) + } + for _, backend := range oldBackends { + resources.AddOldBackend(desired, *hash, backend) + } + } else if probeHash != *hash { + desired, err = resources.MakeHTTPRoute(ctx, ing, rule) } else { - desired, err := resources.MakeHTTPRoute(ctx, ing, rule) - if err != nil { - return nil, err + // Noop + if probe.Version != "" { + *hash = probe.Version } + // desired, err = resources.MakeHTTPRoute(ctx, ing, rule) + return httproute, nil + } - if !equality.Semantic.DeepEqual(httproute.Spec, desired.Spec) || - !equality.Semantic.DeepEqual(httproute.Annotations, desired.Annotations) || - !equality.Semantic.DeepEqual(httproute.Labels, desired.Labels) { - - // Don't modify the informers copy. - origin := httproute.DeepCopy() - origin.Spec = desired.Spec - origin.Annotations = desired.Annotations - origin.Labels = desired.Labels - - updated, err := c.gwapiclient.GatewayV1beta1().HTTPRoutes(origin.Namespace).Update( - ctx, origin, metav1.UpdateOptions{}) - if err != nil { - recorder.Eventf(ing, corev1.EventTypeWarning, "UpdateFailed", "Failed to update HTTPRoute: %v", err) - return nil, fmt.Errorf("failed to update HTTPRoute: %w", err) - } - return updated, nil + if err != nil { + return nil, err + } + + if !equality.Semantic.DeepEqual(original.Spec, desired.Spec) || + !equality.Semantic.DeepEqual(original.Annotations, desired.Annotations) || + !equality.Semantic.DeepEqual(original.Labels, desired.Labels) { + + // Don't modify the informers copy. + original.Spec = desired.Spec + original.Annotations = desired.Annotations + original.Labels = desired.Labels + + updated, err := c.gwapiclient.GatewayV1beta1().HTTPRoutes(original.Namespace). + Update(ctx, original, metav1.UpdateOptions{}) + + if err != nil { + recorder.Eventf(ing, corev1.EventTypeWarning, "UpdateFailed", "Failed to update HTTPRoute: %v", err) + return nil, fmt.Errorf("failed to update HTTPRoute: %w", err) } + return updated, nil } - return httproute, err + return httproute, nil } func (c *Reconciler) reconcileTLS( @@ -280,3 +358,66 @@ func (c *Reconciler) clearGatewayListeners(ctx context.Context, ing *netv1alpha1 return nil } + +func computeBackends( + route *gatewayapi.HTTPRoute, + rule *netv1alpha1.IngressRule, +) ([]netv1alpha1.IngressBackendSplit, []gatewayapi.HTTPBackendRef) { + newBackends := []netv1alpha1.IngressBackendSplit{} + oldBackends := []gatewayapi.HTTPBackendRef{} + oldNames := sets.Set[types.NamespacedName]{} + +oldbackends: + for _, rule := range route.Spec.Rules { + // We want to skip probes + for _, match := range rule.Matches { + for _, headers := range match.Headers { + if headers.Name == header.HashKey { + continue oldbackends + } + } + } + + for _, backend := range rule.BackendRefs { + nn := types.NamespacedName{ + Name: string(backend.Name), + } + if backend.Namespace != nil { + nn.Namespace = string(*backend.Namespace) + } else { + nn.Namespace = route.Namespace + + } + oldNames.Insert(nn) + oldBackends = append(oldBackends, backend) + } + } + +newbackends: + for _, path := range rule.HTTP.Paths { + // We want to skip probes + for k := range path.Headers { + if k == header.HashKey { + continue newbackends + } + } + + for _, split := range path.Splits { + service := types.NamespacedName{ + Name: split.ServiceName, + Namespace: split.ServiceNamespace, + } + + if oldNames.Has(service) { + continue + } + + newBackends = append(newBackends, split) + } + } + + slices.SortFunc(newBackends, func(a, b netv1alpha1.IngressBackendSplit) int { + return strings.Compare(a.ServiceName, b.ServiceName) + }) + return newBackends, oldBackends +} diff --git a/pkg/reconciler/ingress/resources/httproute.go b/pkg/reconciler/ingress/resources/httproute.go index 6c7be8cd6..1a7ba2de7 100644 --- a/pkg/reconciler/ingress/resources/httproute.go +++ b/pkg/reconciler/ingress/resources/httproute.go @@ -18,7 +18,10 @@ package resources import ( "context" + "fmt" + "slices" "sort" + "strings" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,9 +32,154 @@ import ( "knative.dev/net-gateway-api/pkg/reconciler/ingress/config" "knative.dev/networking/pkg/apis/networking" netv1alpha1 "knative.dev/networking/pkg/apis/networking/v1alpha1" + "knative.dev/networking/pkg/http/header" "knative.dev/pkg/kmeta" ) +func UpdateProbeHash(r *gatewayapi.HTTPRoute, hash string) { + // Note: we use indices and references to avoid mutating copies + for rIdx := range r.Spec.Rules { + rule := &r.Spec.Rules[rIdx] + + for fIdx := range rule.Filters { + filter := &rule.Filters[fIdx] + + if filter.Type != gatewayapiv1.HTTPRouteFilterRequestHeaderModifier { + continue + } + + if filter.RequestHeaderModifier == nil { + continue + } + + for hIdx := range filter.RequestHeaderModifier.Set { + h := &filter.RequestHeaderModifier.Set[hIdx] + if h.Name == header.HashKey { + h.Value = hash + } + } + } + } +} + +func RemoveEndpointProbes(r *gatewayapi.HTTPRoute) { + rules := r.Spec.Rules + r.Spec.Rules = make([]gatewayapi.HTTPRouteRule, 0, len(rules)) + + // Remove old endpoint probes +outer: + for _, rule := range rules { + for _, match := range rule.Matches { + if match.Path != nil && match.Path.Value != nil && + strings.HasPrefix(*match.Path.Value, "/.well-known/knative") { + continue outer + } + r.Spec.Rules = append(r.Spec.Rules, rule) + } + } +} + +func AddEndpointProbe(r *gatewayapi.HTTPRoute, hash string, backend netv1alpha1.IngressBackendSplit) { + rule := gatewayapi.HTTPRouteRule{ + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To(fmt.Sprintf("/.well-known/knative/revision/%s/%s", backend.ServiceNamespace, backend.ServiceName)), + }, + Headers: []gatewayapi.HTTPHeaderMatch{{ + Type: ptr.To(gatewayapiv1.HeaderMatchExact), + Name: header.HashKey, + Value: header.HashValueOverride, + }}, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: header.HashKey, + Value: hash, + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](100), + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: gatewayapi.ObjectName(backend.ServiceName), + Port: ptr.To[gatewayapi.PortNumber](gatewayapi.PortNumber(backend.ServicePort.IntValue())), + }, + }, + }}, + } + + if len(backend.AppendHeaders) > 0 { + headers := make([]gatewayapi.HTTPHeader, 0, len(backend.AppendHeaders)) + + for k, v := range backend.AppendHeaders { + headers = append(headers, gatewayapi.HTTPHeader{ + Name: gatewayapiv1.HTTPHeaderName(k), + Value: v, + }) + } + + slices.SortFunc(headers, compareHTTPHeader) + + rule.BackendRefs[0].Filters = append(rule.BackendRefs[0].Filters, + gatewayapiv1.HTTPRouteFilter{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: headers, + }, + }, + ) + } + + r.Spec.Rules = append(r.Spec.Rules, rule) +} + +func AddOldBackend(r *gatewayapi.HTTPRoute, hash string, old gatewayapi.HTTPBackendRef) { + backend := *old.DeepCopy() + backend.Weight = ptr.To[int32](100) + + // KIngress only supports AppendHeaders so there's only this filter + for _, filters := range backend.Filters { + if filters.RequestHeaderModifier != nil { + + slices.SortFunc(filters.RequestHeaderModifier.Set, func(a, b gatewayapi.HTTPHeader) int { + return strings.Compare(string(a.Name), string(b.Name)) + }) + } + } + + rule := gatewayapi.HTTPRouteRule{ + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To(fmt.Sprintf("/.well-known/knative/revision/%s/%s", r.Namespace, backend.Name)), + }, + Headers: []gatewayapi.HTTPHeaderMatch{{ + Type: ptr.To(gatewayapiv1.HeaderMatchExact), + Name: header.HashKey, + Value: header.HashValueOverride, + }}, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: header.HashKey, + Value: hash, + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{backend}, + } + + r.Spec.Rules = append(r.Spec.Rules, rule) +} + // MakeHTTPRoute creates HTTPRoute to set up routing rules. func MakeHTTPRoute( ctx context.Context, @@ -109,7 +257,7 @@ func makeHTTPRouteRule(rule *netv1alpha1.IngressRule) []gatewayapi.HTTPRouteRule } // Sort HTTPHeader as the order is random. - sort.Sort(HTTPHeaderList(headers)) + slices.SortFunc(headers, compareHTTPHeader) preFilters = []gatewayapi.HTTPRouteFilter{{ Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, @@ -138,7 +286,7 @@ func makeHTTPRouteRule(rule *netv1alpha1.IngressRule) []gatewayapi.HTTPRouteRule } // Sort HTTPHeader as the order is random. - sort.Sort(HTTPHeaderList(headers)) + slices.SortFunc(headers, compareHTTPHeader) name := split.ServiceName backendRef := gatewayapi.HTTPBackendRef{ @@ -221,3 +369,7 @@ func (h HTTPHeaderMatchList) Less(i, j int) bool { func (h HTTPHeaderMatchList) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +func compareHTTPHeader(a, b gatewayapi.HTTPHeader) int { + return strings.Compare(string(a.Name), string(b.Name)) +} diff --git a/pkg/reconciler/ingress/resources/httproute_test.go b/pkg/reconciler/ingress/resources/httproute_test.go index 35efc315b..a21533ebd 100644 --- a/pkg/reconciler/ingress/resources/httproute_test.go +++ b/pkg/reconciler/ingress/resources/httproute_test.go @@ -28,6 +28,7 @@ import ( "knative.dev/net-gateway-api/pkg/reconciler/ingress/config" "knative.dev/networking/pkg/apis/networking" "knative.dev/networking/pkg/apis/networking/v1alpha1" + "knative.dev/networking/pkg/http/header" "knative.dev/pkg/kmeta" "knative.dev/pkg/reconciler" gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -53,6 +54,51 @@ var ( } testHosts = []string{"hello-example.default.example.com"} + + testIngress = &v1alpha1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: testIngressName, + Namespace: testNamespace, + Labels: map[string]string{ + networking.IngressLabelKey: testIngressName, + }, + }, + Spec: v1alpha1.IngressSpec{ + Rules: []v1alpha1.IngressRule{{ + Hosts: testHosts, + Visibility: v1alpha1.IngressVisibilityExternalIP, + HTTP: &v1alpha1.HTTPIngressRuleValue{ + Paths: []v1alpha1.HTTPIngressPath{{ + AppendHeaders: map[string]string{ + "Foo": "bar", + }, + Splits: []v1alpha1.IngressBackendSplit{{ + IngressBackend: v1alpha1.IngressBackend{ + ServiceName: "goo", + ServiceNamespace: testNamespace, + ServicePort: intstr.FromInt(123), + }, + Percent: 12, + AppendHeaders: map[string]string{ + "Baz": "blah", + "Bleep": "bloop", + }, + }, { + IngressBackend: v1alpha1.IngressBackend{ + ServiceName: "doo", + ServiceNamespace: testNamespace, + ServicePort: intstr.FromInt(124), + }, + Percent: 88, + AppendHeaders: map[string]string{ + "Baz": "blurg", + }, + }}, + }}, + }, + }}, + }, + } ) func TestMakeHTTPRoute(t *testing.T) { @@ -164,14 +210,14 @@ func TestMakeHTTPRoute(t *testing.T) { Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ Set: []gatewayapi.HTTPHeader{ - { - Name: "Bleep", - Value: "bloop", - }, { Name: "Baz", Value: "blah", }, + { + Name: "Bleep", + Value: "bloop", + }, }}}}, }, { BackendRef: gatewayapi.BackendRef{ @@ -249,14 +295,14 @@ func TestMakeHTTPRoute(t *testing.T) { Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ Set: []gatewayapi.HTTPHeader{ - { - Name: "Bleep", - Value: "bloop", - }, { Name: "Baz", Value: "blah", }, + { + Name: "Bleep", + Value: "bloop", + }, }}}}, }, { BackendRef: gatewayapi.BackendRef{ @@ -510,6 +556,565 @@ func TestMakeHTTPRoute(t *testing.T) { } } +func TestAddEndpointProbes(t *testing.T) { + tcs := &testConfigStore{config: testConfig} + ctx := tcs.ToContext(context.Background()) + + ing := testIngress.DeepCopy() + rule := &ing.Spec.Rules[0] + route, err := MakeHTTPRoute(ctx, ing, rule) + if err != nil { + t.Fatal("MakeHTTPRoute failed:", err) + } + + AddEndpointProbe(route, "hash", rule.HTTP.Paths[0].Splits[0]) + AddEndpointProbe(route, "hash", rule.HTTP.Paths[0].Splits[1]) + + expected := &gatewayapi.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: LongestHost(testHosts), + Namespace: testNamespace, + Labels: map[string]string{ + networking.IngressLabelKey: testIngressName, + "networking.knative.dev/visibility": "", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(ing)}, + }, + Spec: gatewayapi.HTTPRouteSpec{ + CommonRouteSpec: gatewayapi.CommonRouteSpec{ + ParentRefs: []gatewayapi.ParentReference{{ + Group: (*gatewayapi.Group)(ptr.To("gateway.networking.k8s.io")), + Kind: (*gatewayapi.Kind)(ptr.To("Gateway")), + Namespace: ptr.To[gatewayapi.Namespace]("test-ns"), + Name: gatewayapi.ObjectName("foo"), + }}, + }, + Hostnames: []gatewayapi.Hostname{externalHost}, + Rules: []gatewayapi.HTTPRouteRule{{ + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/"), + }, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapiv1.HTTPHeader{{ + Name: "Foo", + Value: "bar", + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](12), + BackendObjectReference: gatewayapi.BackendObjectReference{ + Group: (*gatewayapi.Group)(ptr.To("")), + Kind: (*gatewayapi.Kind)(ptr.To("Service")), + Port: ptr.To(gatewayapiv1.PortNumber(123)), + Name: "goo", + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Baz", + Value: "blah", + }, { + Name: "Bleep", + Value: "bloop", + }}, + }}, + }, + }, { + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](88), + BackendObjectReference: gatewayapi.BackendObjectReference{ + Group: (*gatewayapi.Group)(ptr.To("")), + Kind: (*gatewayapi.Kind)(ptr.To("Service")), + Port: ptr.To(gatewayapiv1.PortNumber(124)), + Name: "doo", + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Baz", + Value: "blurg", + }}, + }}, + }}, + }, + }, { + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/.well-known/knative/revision/test-ns/goo"), + }, + Headers: []gatewayapi.HTTPHeaderMatch{{ + Type: ptr.To(gatewayapiv1.HeaderMatchExact), + Name: header.HashKey, + Value: header.HashValueOverride, + }}, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: header.HashKey, + Value: "hash", + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](100), + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: "goo", + Port: ptr.To[gatewayapi.PortNumber](123), + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Baz", + Value: "blah", + }, { + Name: "Bleep", + Value: "bloop", + }}, + }, + }}, + }}, + }, { + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/.well-known/knative/revision/test-ns/doo"), + }, + Headers: []gatewayapi.HTTPHeaderMatch{{ + Type: ptr.To(gatewayapiv1.HeaderMatchExact), + Name: header.HashKey, + Value: header.HashValueOverride, + }}, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: header.HashKey, + Value: "hash", + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](100), + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: "doo", + Port: ptr.To[gatewayapi.PortNumber](124), + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Baz", + Value: "blurg", + }}, + }, + }}, + }}, + }}, + }, + } + + if diff := cmp.Diff(expected, route); diff != "" { + t.Fatal("Unexpected (-want, +got): ", diff) + } +} + +func TestRemoveEndpointProbes(t *testing.T) { + tcs := &testConfigStore{config: testConfig} + ctx := tcs.ToContext(context.Background()) + + ing := testIngress.DeepCopy() + rule := &ing.Spec.Rules[0] + route, err := MakeHTTPRoute(ctx, ing, rule) + if err != nil { + t.Fatal("MakeHTTPRoute failed:", err) + } + + expected := route.DeepCopy() + + AddEndpointProbe(route, "hash", rule.HTTP.Paths[0].Splits[0]) + AddEndpointProbe(route, "hash", rule.HTTP.Paths[0].Splits[1]) + RemoveEndpointProbes(route) + + if diff := cmp.Diff(expected, route); diff != "" { + t.Fatal("Unexpected (-want, +got): ", diff) + } +} + +func TestUpdateProbeHash(t *testing.T) { + tcs := &testConfigStore{config: testConfig} + ctx := tcs.ToContext(context.Background()) + ing := testIngress.DeepCopy() + rule := &ing.Spec.Rules[0] + route, err := MakeHTTPRoute(ctx, ing, rule) + if err != nil { + t.Fatal("MakeHTTPRoute failed:", err) + } + + AddEndpointProbe(route, "hash", rule.HTTP.Paths[0].Splits[0]) + AddEndpointProbe(route, "hash", rule.HTTP.Paths[0].Splits[1]) + UpdateProbeHash(route, "second-hash") + + expected := &gatewayapi.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: LongestHost(testHosts), + Namespace: testNamespace, + Labels: map[string]string{ + networking.IngressLabelKey: testIngressName, + "networking.knative.dev/visibility": "", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(ing)}, + }, + Spec: gatewayapi.HTTPRouteSpec{ + CommonRouteSpec: gatewayapi.CommonRouteSpec{ + ParentRefs: []gatewayapi.ParentReference{{ + Group: (*gatewayapi.Group)(ptr.To("gateway.networking.k8s.io")), + Kind: (*gatewayapi.Kind)(ptr.To("Gateway")), + Namespace: ptr.To[gatewayapi.Namespace]("test-ns"), + Name: gatewayapi.ObjectName("foo"), + }}, + }, + Hostnames: []gatewayapi.Hostname{externalHost}, + Rules: []gatewayapi.HTTPRouteRule{{ + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/"), + }, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapiv1.HTTPHeader{{ + Name: "Foo", + Value: "bar", + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](12), + BackendObjectReference: gatewayapi.BackendObjectReference{ + Group: (*gatewayapi.Group)(ptr.To("")), + Kind: (*gatewayapi.Kind)(ptr.To("Service")), + Port: ptr.To(gatewayapiv1.PortNumber(123)), + Name: "goo", + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Baz", + Value: "blah", + }, { + Name: "Bleep", + Value: "bloop", + }}, + }}, + }, + }, { + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](88), + BackendObjectReference: gatewayapi.BackendObjectReference{ + Group: (*gatewayapi.Group)(ptr.To("")), + Kind: (*gatewayapi.Kind)(ptr.To("Service")), + Port: ptr.To(gatewayapiv1.PortNumber(124)), + Name: "doo", + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Baz", + Value: "blurg", + }}, + }}, + }}, + }, + }, { + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/.well-known/knative/revision/test-ns/goo"), + }, + Headers: []gatewayapi.HTTPHeaderMatch{{ + Type: ptr.To(gatewayapiv1.HeaderMatchExact), + Name: header.HashKey, + Value: header.HashValueOverride, + }}, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: header.HashKey, + Value: "second-hash", + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](100), + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: "goo", + Port: ptr.To[gatewayapi.PortNumber](123), + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Baz", + Value: "blah", + }, { + Name: "Bleep", + Value: "bloop", + }}, + }, + }}, + }}, + }, { + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/.well-known/knative/revision/test-ns/doo"), + }, + Headers: []gatewayapi.HTTPHeaderMatch{{ + Type: ptr.To(gatewayapiv1.HeaderMatchExact), + Name: header.HashKey, + Value: header.HashValueOverride, + }}, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: header.HashKey, + Value: "second-hash", + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](100), + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: "doo", + Port: ptr.To[gatewayapi.PortNumber](124), + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Baz", + Value: "blurg", + }}, + }, + }}, + }}, + }}, + }, + } + + if diff := cmp.Diff(expected, route); diff != "" { + t.Fatal("Unexpected (-want, +got): ", diff) + } +} + +func TestAddOldBackend(t *testing.T) { + tcs := &testConfigStore{config: testConfig} + ctx := tcs.ToContext(context.Background()) + ing := testIngress.DeepCopy() + + rule := &ing.Spec.Rules[0] + route, err := MakeHTTPRoute(ctx, ing, rule) + if err != nil { + t.Fatal("MakeHTTPRoute failed:", err) + } + + AddOldBackend(route, "hash", gatewayapi.HTTPBackendRef{ + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](100), + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: "blah", + Namespace: ptr.To[gatewayapi.Namespace]("test-ns"), + Port: ptr.To[gatewayapi.PortNumber](127), + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Foo", + Value: "bar", + }}, + }, + }}, + }) + + expected := &gatewayapi.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: LongestHost(testHosts), + Namespace: testNamespace, + Labels: map[string]string{ + networking.IngressLabelKey: testIngressName, + "networking.knative.dev/visibility": "", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(ing)}, + }, + Spec: gatewayapi.HTTPRouteSpec{ + CommonRouteSpec: gatewayapi.CommonRouteSpec{ + ParentRefs: []gatewayapi.ParentReference{{ + Group: (*gatewayapi.Group)(ptr.To("gateway.networking.k8s.io")), + Kind: (*gatewayapi.Kind)(ptr.To("Gateway")), + Namespace: ptr.To[gatewayapi.Namespace]("test-ns"), + Name: gatewayapi.ObjectName("foo"), + }}, + }, + Hostnames: []gatewayapi.Hostname{externalHost}, + Rules: []gatewayapi.HTTPRouteRule{{ + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/"), + }, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapiv1.HTTPHeader{{ + Name: "Foo", + Value: "bar", + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](12), + BackendObjectReference: gatewayapi.BackendObjectReference{ + Group: (*gatewayapi.Group)(ptr.To("")), + Kind: (*gatewayapi.Kind)(ptr.To("Service")), + Port: ptr.To(gatewayapiv1.PortNumber(123)), + Name: "goo", + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Baz", + Value: "blah", + }, { + Name: "Bleep", + Value: "bloop", + }}, + }}, + }, + }, { + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](88), + BackendObjectReference: gatewayapi.BackendObjectReference{ + Group: (*gatewayapi.Group)(ptr.To("")), + Kind: (*gatewayapi.Kind)(ptr.To("Service")), + Port: ptr.To(gatewayapiv1.PortNumber(124)), + Name: "doo", + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Baz", + Value: "blurg", + }}, + }}, + }}, + }, + }, { + Matches: []gatewayapi.HTTPRouteMatch{{ + Path: &gatewayapi.HTTPPathMatch{ + Type: ptr.To(gatewayapiv1.PathMatchPathPrefix), + Value: ptr.To("/.well-known/knative/revision/test-ns/blah"), + }, + Headers: []gatewayapi.HTTPHeaderMatch{{ + Type: ptr.To(gatewayapiv1.HeaderMatchExact), + Name: header.HashKey, + Value: header.HashValueOverride, + }}, + }}, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: header.HashKey, + Value: "hash", + }}, + }, + }}, + BackendRefs: []gatewayapi.HTTPBackendRef{{ + BackendRef: gatewayapi.BackendRef{ + Weight: ptr.To[int32](100), + BackendObjectReference: gatewayapiv1.BackendObjectReference{ + Group: ptr.To[gatewayapi.Group](""), + Kind: ptr.To[gatewayapi.Kind]("Service"), + Name: "blah", + Namespace: ptr.To[gatewayapi.Namespace]("test-ns"), + Port: ptr.To[gatewayapi.PortNumber](127), + }, + }, + Filters: []gatewayapi.HTTPRouteFilter{{ + Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, + RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{ + Set: []gatewayapi.HTTPHeader{{ + Name: "Foo", + Value: "bar", + }}, + }, + }}, + }}, + }}, + }, + } + + if diff := cmp.Diff(expected, route); diff != "" { + t.Fatal("Unexpected (-want, +got): ", diff) + } +} + type testConfigStore struct { config *config.Config } diff --git a/vendor/knative.dev/networking/pkg/status/status.go b/vendor/knative.dev/networking/pkg/status/status.go deleted file mode 100644 index 4ea161001..000000000 --- a/vendor/knative.dev/networking/pkg/status/status.go +++ /dev/null @@ -1,486 +0,0 @@ -/* -Copyright 2019 The Knative Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package status - -import ( - "context" - "crypto/tls" - "fmt" - "net" - "net/http" - "net/url" - "path" - "reflect" - "sync" - "time" - - "go.uber.org/atomic" - "go.uber.org/zap" - "golang.org/x/time/rate" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/util/workqueue" - - "knative.dev/networking/pkg/apis/networking/v1alpha1" - nethttp "knative.dev/networking/pkg/http" - "knative.dev/networking/pkg/http/header" - "knative.dev/networking/pkg/ingress" - "knative.dev/networking/pkg/prober" - "knative.dev/pkg/kmeta" - "knative.dev/pkg/logging" -) - -const ( - // probeConcurrency defines how many probing calls can be issued simultaneously - probeConcurrency = 15 - // probeTimeout defines the maximum amount of time a request will wait - probeTimeout = 1 * time.Second - // initialDelay defines the delay before enqueuing a probing request the first time. - // It gives times for the change to propagate and prevents unnecessary retries. - initialDelay = 200 * time.Millisecond -) - -var dialContext = (&net.Dialer{Timeout: probeTimeout}).DialContext - -// ingressState represents the probing state of an Ingress -type ingressState struct { - hash string - ing *v1alpha1.Ingress - - // pendingCount is the number of pods that haven't been successfully probed yet - pendingCount atomic.Int32 - lastAccessed time.Time - - cancel func() -} - -// podState represents the probing state of a Pod (for a specific Ingress) -type podState struct { - // pendingCount is the number of probes for the Pod - pendingCount atomic.Int32 - - cancel func() -} - -// cancelContext is a pair of a Context and its cancel function -type cancelContext struct { - context context.Context - cancel func() -} - -type workItem struct { - ingressState *ingressState - podState *podState - context context.Context - url *url.URL - podIP string - podPort string - logger *zap.SugaredLogger -} - -// ProbeTarget contains the URLs to probes for a set of Pod IPs serving out of the same port. -type ProbeTarget struct { - PodIPs sets.Set[string] - PodPort string - Port string - URLs []*url.URL -} - -// ProbeTargetLister lists all the targets that requires probing. -type ProbeTargetLister interface { - // ListProbeTargets returns a list of targets to be probed. - ListProbeTargets(ctx context.Context, ingress *v1alpha1.Ingress) ([]ProbeTarget, error) -} - -// Manager provides a way to check if an Ingress is ready -type Manager interface { - IsReady(ctx context.Context, ing *v1alpha1.Ingress) (bool, error) -} - -// Prober provides a way to check if a VirtualService is ready by probing the Envoy pods -// handling that VirtualService. -type Prober struct { - logger *zap.SugaredLogger - - // mu guards ingressStates and podContexts - mu sync.Mutex - ingressStates map[types.NamespacedName]*ingressState - podContexts map[string]cancelContext - - workQueue workqueue.RateLimitingInterface - - targetLister ProbeTargetLister - - readyCallback func(*v1alpha1.Ingress) - - probeConcurrency int -} - -// NewProber creates a new instance of Prober -func NewProber( - logger *zap.SugaredLogger, - targetLister ProbeTargetLister, - readyCallback func(*v1alpha1.Ingress)) *Prober { - return &Prober{ - logger: logger, - ingressStates: make(map[types.NamespacedName]*ingressState), - podContexts: make(map[string]cancelContext), - workQueue: workqueue.NewNamedRateLimitingQueue( - workqueue.NewMaxOfRateLimiter( - // Per item exponential backoff - workqueue.NewItemExponentialFailureRateLimiter(50*time.Millisecond, 30*time.Second), - // Global rate limiter - &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(50), 100)}, - ), - "ProbingQueue"), - targetLister: targetLister, - readyCallback: readyCallback, - probeConcurrency: probeConcurrency, - } -} - -// IsReady checks if the provided Ingress is ready, i.e. the Envoy pods serving the Ingress -// have all been updated. This function is designed to be used by the Ingress controller, i.e. it -// will be called in the order of reconciliation. This means that if IsReady is called on an Ingress, -// this Ingress is the latest known version and therefore anything related to older versions can be ignored. -// Also, it means that IsReady is not called concurrently. -func (m *Prober) IsReady(ctx context.Context, ing *v1alpha1.Ingress) (bool, error) { - ingressKey := types.NamespacedName{Namespace: ing.Namespace, Name: ing.Name} - logger := logging.FromContext(ctx) - - bytes, err := ingress.ComputeHash(ing) - if err != nil { - return false, fmt.Errorf("failed to compute the hash of the Ingress: %w", err) - } - hash := fmt.Sprintf("%x", bytes) - - if ready, ok := func() (bool, bool) { - m.mu.Lock() - defer m.mu.Unlock() - if state, ok := m.ingressStates[ingressKey]; ok { - if state.hash == hash { - state.lastAccessed = time.Now() - return state.pendingCount.Load() == 0, true - } - - // Cancel the polling for the outdated version - state.cancel() - delete(m.ingressStates, ingressKey) - } - return false, false - }(); ok { - return ready, nil - } - - ingCtx, cancel := context.WithCancel(context.Background()) - ingressState := &ingressState{ - hash: hash, - ing: ing, - lastAccessed: time.Now(), - cancel: cancel, - } - - // Get the probe targets and group them by IP - targets, err := m.targetLister.ListProbeTargets(ctx, ing) - if err != nil { - return false, err - } - workItems := make(map[string][]*workItem) - for _, target := range targets { - for ip := range target.PodIPs { - for _, url := range target.URLs { - workItems[ip] = append(workItems[ip], &workItem{ - ingressState: ingressState, - url: url, - podIP: ip, - podPort: target.PodPort, - logger: logger, - }) - } - } - } - - ingressState.pendingCount.Store(int32(len(workItems))) - - for ip, ipWorkItems := range workItems { - // Get or create the context for that IP - ipCtx := func() context.Context { - m.mu.Lock() - defer m.mu.Unlock() - cancelCtx, ok := m.podContexts[ip] - if !ok { - ctx, cancel := context.WithCancel(context.Background()) - cancelCtx = cancelContext{ - context: ctx, - cancel: cancel, - } - m.podContexts[ip] = cancelCtx - } - return cancelCtx.context - }() - - podCtx, cancel := context.WithCancel(ingCtx) - podState := &podState{ - pendingCount: *atomic.NewInt32(int32(len(ipWorkItems))), - cancel: cancel, - } - - // Quick and dirty way to join two contexts (i.e. podCtx is cancelled when either ingCtx or ipCtx are cancelled) - go func() { - select { - case <-podCtx.Done(): - // This is the actual context, there is nothing to do except - // break to avoid leaking this goroutine. - break - case <-ipCtx.Done(): - // Cancel podCtx - cancel() - } - }() - - // Update the states when probing is cancelled - go func() { - <-podCtx.Done() - m.onProbingCancellation(ingressState, podState) - }() - - for _, wi := range ipWorkItems { - wi.podState = podState - wi.context = podCtx - m.workQueue.AddAfter(wi, initialDelay) - logger.Infof("Queuing probe for %s, IP: %s:%s (depth: %d)", - wi.url, wi.podIP, wi.podPort, m.workQueue.Len()) - } - } - - func() { - m.mu.Lock() - defer m.mu.Unlock() - m.ingressStates[ingressKey] = ingressState - }() - return len(workItems) == 0, nil -} - -// Start starts the Manager background operations -func (m *Prober) Start(done <-chan struct{}) chan struct{} { - var wg sync.WaitGroup - - // Start the worker goroutines - for i := 0; i < m.probeConcurrency; i++ { - wg.Add(1) - go func() { - defer wg.Done() - //nolint:all - for m.processWorkItem() { - } - }() - } - - // Stop processing the queue when cancelled - go func() { - <-done - m.workQueue.ShutDown() - }() - - // Return a channel closed when all work is done - ch := make(chan struct{}) - go func() { - wg.Wait() - close(ch) - }() - return ch -} - -// CancelIngressProbing cancels probing of the provided Ingress -func (m *Prober) CancelIngressProbing(obj interface{}) { - acc, err := kmeta.DeletionHandlingAccessor(obj) - if err != nil { - return - } - - key := types.NamespacedName{Namespace: acc.GetNamespace(), Name: acc.GetName()} - m.CancelIngressProbingByKey(key) -} - -// CancelIngressProbingByKey cancels probing of the Ingress identified by the provided key. -func (m *Prober) CancelIngressProbingByKey(key types.NamespacedName) { - m.mu.Lock() - defer m.mu.Unlock() - if state, ok := m.ingressStates[key]; ok { - state.cancel() - delete(m.ingressStates, key) - } -} - -// CancelPodProbing cancels probing of the provided Pod IP. -// -// TODO(#6269): make this cancellation based on Pod x port instead of just Pod. -func (m *Prober) CancelPodProbing(obj interface{}) { - if pod, ok := obj.(*corev1.Pod); ok { - m.mu.Lock() - defer m.mu.Unlock() - - if ctx, ok := m.podContexts[pod.Status.PodIP]; ok { - ctx.cancel() - delete(m.podContexts, pod.Status.PodIP) - } - } -} - -// processWorkItem processes a single work item from workQueue. -// It returns false when there is no more items to process, true otherwise. -func (m *Prober) processWorkItem() bool { - obj, shutdown := m.workQueue.Get() - if shutdown { - return false - } - - defer m.workQueue.Done(obj) - - // Crash if the item is not of the expected type - item, ok := obj.(*workItem) - if !ok { - m.logger.Fatalf("Unexpected work item type: want: %s, got: %s\n", - reflect.TypeOf(&workItem{}).Name(), reflect.TypeOf(obj).Name()) - } - item.logger.Infof("Processing probe for %s, IP: %s:%s (depth: %d)", - item.url, item.podIP, item.podPort, m.workQueue.Len()) - - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = &tls.Config{ - //nolint:gosec - // We only want to know that the Gateway is configured, not that the configuration is valid. - // Therefore, we can safely ignore any TLS certificate validation. - InsecureSkipVerify: true, - } - transport.DialContext = func(ctx context.Context, network, addr string) (conn net.Conn, e error) { - // Requests with the IP as hostname and the Host header set do no pass client-side validation - // because the HTTP client validates that the hostname (not the Host header) matches the server - // TLS certificate Common Name or Alternative Names. Therefore, http.Request.URL is set to the - // hostname and it is substituted it here with the target IP. - return dialContext(ctx, network, net.JoinHostPort(item.podIP, item.podPort)) - } - - probeURL := deepCopy(item.url) - probeURL.Path = path.Join(probeURL.Path, nethttp.HealthCheckPath) - - ctx, cancel := context.WithTimeout(item.context, probeTimeout) - defer cancel() - ok, err := prober.Do( - ctx, - transport, - probeURL.String(), - prober.WithHeader(header.UserAgentKey, header.IngressReadinessUserAgent), - prober.WithHeader(header.ProbeKey, header.ProbeValue), - prober.WithHeader(header.HashKey, header.HashValueOverride), - m.probeVerifier(item)) - - // In case of cancellation, drop the work item - select { - case <-item.context.Done(): - m.workQueue.Forget(obj) - return true - default: - } - - if err != nil || !ok { - // In case of error, enqueue for retry - m.workQueue.AddRateLimited(obj) - item.logger.Errorf("Probing of %s failed, IP: %s:%s, ready: %t, error: %v (depth: %d)", - item.url, item.podIP, item.podPort, ok, err, m.workQueue.Len()) - } else { - m.onProbingSuccess(item.ingressState, item.podState) - } - return true -} - -func (m *Prober) onProbingSuccess(ingressState *ingressState, podState *podState) { - // The last probe call for the Pod succeeded, the Pod is ready - if podState.pendingCount.Dec() == 0 { - // Unlock the goroutine blocked on <-podCtx.Done() - podState.cancel() - - // This is the last pod being successfully probed, the Ingress is ready - if ingressState.pendingCount.Dec() == 0 { - m.readyCallback(ingressState.ing) - } - } -} - -func (m *Prober) onProbingCancellation(ingressState *ingressState, podState *podState) { - for { - pendingCount := podState.pendingCount.Load() - if pendingCount <= 0 { - // Probing succeeded, nothing to do - return - } - - // Attempt to set pendingCount to 0. - if podState.pendingCount.CAS(pendingCount, 0) { - // This is the last pod being successfully probed, the Ingress is ready - if ingressState.pendingCount.Dec() == 0 { - m.readyCallback(ingressState.ing) - } - return - } - } -} - -func (m *Prober) probeVerifier(item *workItem) prober.Verifier { - return func(r *http.Response, _ []byte) (bool, error) { - // In the happy path, the probe request is forwarded to Activator or Queue-Proxy and the response (HTTP 200) - // contains the "K-Network-Hash" header that can be compared with the expected hash. If the hashes match, - // probing is successful, if they don't match, a new probe will be sent later. - // An HTTP 404/503 is expected in the case of the creation of a new Knative service because the rules will - // not be present in the Envoy config until the new VirtualService is applied. - // No information can be extracted from any other scenario (e.g. HTTP 302), therefore in that case, - // probing is assumed to be successful because it is better to say that an Ingress is Ready before it - // actually is Ready than never marking it as Ready. It is best effort. - switch r.StatusCode { - case http.StatusOK: - hash := r.Header.Get(header.HashKey) - switch hash { - case "": - item.logger.Errorf("Probing of %s abandoned, IP: %s:%s: the response doesn't contain the %q header", - item.url, item.podIP, item.podPort, header.HashKey) - return true, nil - case item.ingressState.hash: - return true, nil - default: - return false, fmt.Errorf("unexpected hash: want %q, got %q", item.ingressState.hash, hash) - } - - case http.StatusNotFound, http.StatusServiceUnavailable: - return false, fmt.Errorf("unexpected status code: want %v, got %v", http.StatusOK, r.StatusCode) - - default: - item.logger.Errorf("Probing of %s abandoned, IP: %s:%s: the response status is %v, expected one of: %v", - item.url, item.podIP, item.podPort, r.StatusCode, - []int{http.StatusOK, http.StatusNotFound, http.StatusServiceUnavailable}) - return true, nil - } - } -} - -// deepCopy copies a URL into a new one -func deepCopy(in *url.URL) *url.URL { - // Safe to ignore the error since this is a deep copy - newURL, _ := url.Parse(in.String()) - return newURL -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 43b88b8d3..90a887cd2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -922,7 +922,6 @@ knative.dev/networking/pkg/http/stats knative.dev/networking/pkg/ingress knative.dev/networking/pkg/k8s knative.dev/networking/pkg/prober -knative.dev/networking/pkg/status knative.dev/networking/test knative.dev/networking/test/conformance/ingress knative.dev/networking/test/defaultsystem