Skip to content

Commit

Permalink
Add a third phase to the probing
Browse files Browse the repository at this point in the history 
We can't just drop the probing rules and move the backends into
the main rules. This still results in 503s with Contour.

Instead we now move the newer backends into the main rules and then
once that 'succeeds' we drop the probes.
  • Loading branch information
@dprotaso
dprotaso committed Apr 15, 2024
1 parent 6fcd834 commit f621578
Show file tree
Hide file tree
Showing 2 changed files with 207 additions and 9 deletions.
183 changes: 181 additions & 2 deletions pkg/reconciler/ingress/ingress_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1055,7 +1055,7 @@ func TestReconcileProbing(t *testing.T) {
RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{
Set: []gatewayapi.HTTPHeader{{
Name: header.HashKey,
Value: "9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2",
Value: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2",
}},
},
}},
Expand Down Expand Up @@ -1112,6 +1112,49 @@ func TestReconcileProbing(t *testing.T) {
Weight: ptr.To[int32](100),
},
}},
}, {
Matches: []gatewayapi.HTTPRouteMatch{{
Path: &gatewayapi.HTTPPathMatch{
Type: ptr.To(gatewayapiv1.PathMatchPathPrefix),
Value: ptr.To("/.well-known/knative/revision/ns/second-revision"),
},
Headers: []gatewayapi.HTTPHeaderMatch{{
Type: ptr.To(gatewayapiv1.HeaderMatchExact),
Name: header.HashKey,
Value: header.HashValueOverride,
}},
}},
Filters: []gatewayapi.HTTPRouteFilter{{
Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier,
RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{
Set: []gatewayapi.HTTPHeader{{
Name: header.HashKey,
Value: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2",
}},
},
}},
BackendRefs: []gatewayapi.HTTPBackendRef{{
Filters: []gatewayapiv1.HTTPRouteFilter{{
Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier,
RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{
Set: []gatewayapi.HTTPHeader{{
Name: "K-Serving-Namespace",
Value: "ns",
}, {
Name: "K-Serving-Revision",
Value: "second-revision",
}},
},
}},
BackendRef: gatewayapi.BackendRef{
BackendObjectReference: gatewayapiv1.BackendObjectReference{
Group: ptr.To[gatewayapi.Group](""),
Kind: ptr.To[gatewayapi.Kind]("Service"),
Name: "second-revision",
Port: ptr.To[gatewayapi.PortNumber](123),
},
},
}},
}},
},
Status: gatewayapi.HTTPRouteStatus{
Expand All @@ -1127,7 +1170,143 @@ func TestReconcileProbing(t *testing.T) {
},
}},
}, {
Name: "backends ready - mark ingress ready",
Name: "transition probe complete - drop endpoint probes",
Key: "ns/name",
Ctx: withStatusManager(&fakeStatusManager{
FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) {
state := status.ProbeState{Ready: true, Version: "tr-9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2"}
return state, true
},
FakeDoProbes: func(context.Context, status.Backends) (status.ProbeState, error) {
return status.ProbeState{Ready: false}, nil
},
}),
Objects: append([]runtime.Object{
ing(withSecondRevisionSpec,
withGatewayAPIclass,
withFinalizer,
makeItReady,
makeLoadBalancerNotReady,
),
&gatewayapi.HTTPRoute{
ObjectMeta: metav1.ObjectMeta{
Name: "example.com",
Namespace: "ns",
Annotations: map[string]string{
networking.IngressClassAnnotationKey: gatewayAPIIngressClassName,
},
Labels: map[string]string{
networking.VisibilityLabelKey: "",
},
OwnerReferences: []metav1.OwnerReference{{
APIVersion: "networking.internal.knative.dev/v1alpha1",
Kind: "Ingress",
Name: "name",
Controller: ptr.To(true),
BlockOwnerDeletion: ptr.To(true),
}},
},
Spec: gatewayapi.HTTPRouteSpec{
CommonRouteSpec: gatewayapi.CommonRouteSpec{
ParentRefs: []gatewayapi.ParentReference{{
Group: ptr.To[gatewayapi.Group]("gateway.networking.k8s.io"),
Kind: ptr.To[gatewayapi.Kind]("Gateway"),
Namespace: ptr.To[gatewayapi.Namespace]("istio-system"),
Name: "istio-gateway",
}},
},
Hostnames: []gatewayapi.Hostname{"example.com"},
Rules: []gatewayapi.HTTPRouteRule{{
Matches: []gatewayapi.HTTPRouteMatch{{
Path: &gatewayapi.HTTPPathMatch{
Type: ptr.To(gatewayapiv1.PathMatchPathPrefix),
Value: ptr.To("/"),
},
Headers: []gatewayapi.HTTPHeaderMatch{{
Type: ptr.To(gatewayapiv1.HeaderMatchExact),
Name: header.HashKey,
Value: header.HashValueOverride,
}},
}},
Filters: []gatewayapi.HTTPRouteFilter{{
Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier,
RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{
Set: []gatewayapi.HTTPHeader{{
Name: header.HashKey,
Value: "9333a9a68409bb44f2a5f538d2d7c617e5338b6b6c1ebc5e00a19612a5c962c2",
}},
},
}},
BackendRefs: []gatewayapi.HTTPBackendRef{{
Filters: []gatewayapiv1.HTTPRouteFilter{{
Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier,
RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{
Set: []gatewayapi.HTTPHeader{{
Name: "K-Serving-Revision",
Value: "second-revision",
}, {
Name: "K-Serving-Namespace",
Value: "ns",
}},
},
}},
BackendRef: gatewayapi.BackendRef{
BackendObjectReference: gatewayapiv1.BackendObjectReference{
Group: ptr.To[gatewayapi.Group](""),
Kind: ptr.To[gatewayapi.Kind]("Service"),
Name: "second-revision",
Port: ptr.To[gatewayapi.PortNumber](123),
},
Weight: ptr.To[int32](100),
},
}},
}, {
Matches: []gatewayapi.HTTPRouteMatch{{
Path: &gatewayapi.HTTPPathMatch{
Type: ptr.To(gatewayapiv1.PathMatchPathPrefix),
Value: ptr.To("/"),
},
}},
BackendRefs: []gatewayapi.HTTPBackendRef{{
Filters: []gatewayapiv1.HTTPRouteFilter{{
Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier,
RequestHeaderModifier: &gatewayapi.HTTPHeaderFilter{
Set: []gatewayapi.HTTPHeader{{
Name: "K-Serving-Revision",
Value: "second-revision",
}, {
Name: "K-Serving-Namespace",
Value: "ns",
}},
},
}},
BackendRef: gatewayapi.BackendRef{
BackendObjectReference: gatewayapiv1.BackendObjectReference{
Group: ptr.To[gatewayapi.Group](""),
Kind: ptr.To[gatewayapi.Kind]("Service"),
Name: "second-revision",
Port: ptr.To[gatewayapi.PortNumber](123),
},
Weight: ptr.To[int32](100),
},
}},
}},
},
Status: gatewayapi.HTTPRouteStatus{
RouteStatus: gatewayapi.RouteStatus{
Parents: []gatewayapi.RouteParentStatus{{
Conditions: []metav1.Condition{{
Type: string(gatewayapi.RouteConditionAccepted),
Status: metav1.ConditionTrue,
}},
}},
},
},
},
},
servicesAndEndpoints...),
}, {
Name: "dropping endpoint probes complete - mark ingress ready",
Key: "ns/name",
Ctx: withStatusManager(&fakeStatusManager{
FakeIsProbeActive: func(types.NamespacedName) (status.ProbeState, bool) {
Expand Down
33 changes: 26 additions & 7 deletions pkg/reconciler/ingress/reconcile_resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,22 +79,41 @@ func (c *Reconciler) reconcileHTTPRouteUpdate(
httproute *gatewayapi.HTTPRoute,
) (*gatewayapi.HTTPRoute, error) {

const (
endpointPrefix = "ep-"
transitionPrefix = "tr-"
)

var (
probeKey = types.NamespacedName{
Name: ing.Name,
Namespace: ing.Namespace,
}
recorder = controller.GetEventRecorder(ctx)
desired *gatewayapi.HTTPRoute
err error
probe, _ = c.statusManager.IsProbeActive(probeKey)
probeHash = strings.TrimPrefix(probe.Version, "ep-")
recorder = controller.GetEventRecorder(ctx)
desired *gatewayapi.HTTPRoute
err error
probe, _ = c.statusManager.IsProbeActive(probeKey)

wasEndpointProbe = strings.HasPrefix(probe.Version, endpointPrefix)
wasTransitionProbe = strings.HasPrefix(probe.Version, transitionPrefix)
)

if probeHash == *hash && probe.Ready {
probeHash := strings.TrimPrefix(probe.Version, endpointPrefix)
probeHash = strings.TrimPrefix(probeHash, transitionPrefix)

if wasTransitionProbe && probeHash == *hash && probe.Ready {
desired, err = resources.MakeHTTPRoute(ctx, ing, rule)
} else if wasEndpointProbe && probeHash == *hash && probe.Ready {
*hash = transitionPrefix + *hash

desired, err = resources.MakeHTTPRoute(ctx, ing, rule)
resources.UpdateProbeHash(desired, *hash)

for _, backend := range newBackends(httproute, rule) {
resources.UpdateEndpointProbes(desired, *hash, backend)
}
} else if newBackends := newBackends(httproute, rule); len(newBackends) > 0 {
*hash = "ep-" + *hash
*hash = endpointPrefix + *hash
desired = httproute.DeepCopy()
resources.UpdateProbeHash(desired, *hash)

Expand Down

0 comments on commit f621578

Please sign in to comment.