This is an automated email from the ASF dual-hosted git repository.
AlinsRan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git
The following commit(s) were added to refs/heads/master by this push:
new d779fa2c fix: preserve shared GatewayProxy resources when deleting
Gateways (#2905)
d779fa2c is described below
commit d779fa2c14dc75c852ec479e42926055cad1acea
Author: AlinsRan <[email protected]>
AuthorDate: Tue Oct 27 08:51:37 2026 +0800
fix: preserve shared GatewayProxy resources when deleting Gateways (#2905)
---
internal/controller/grpcroute_controller.go | 12 +-
internal/controller/httproute_controller.go | 13 +-
internal/controller/httproute_controller_test.go | 10 +-
internal/controller/route_parent_resolver.go | 101 ++++++++++++
internal/controller/route_parent_resolver_test.go | 192 ++++++++++++++++++++++
internal/controller/tcproute_controller.go | 12 +-
internal/controller/tlsroute_controller.go | 12 +-
internal/controller/udproute_controller.go | 12 +-
internal/controller/utils.go | 32 ++--
internal/controller/utils_parentref_test.go | 60 +++----
internal/provider/apisix/gateway_delete_test.go | 85 ++++++++++
internal/provider/apisix/provider.go | 71 ++++----
internal/provider/apisix/provider_test.go | 107 ++++++++++++
internal/provider/common/configmanager.go | 13 ++
test/conformance/README.md | 20 +++
test/conformance/conformance_test.go | 16 +-
test/e2e/crds/v2/tls.go | 16 +-
test/e2e/ingress/annotations.go | 31 ++--
test/e2e/scaffold/apisix_deployer.go | 6 +
19 files changed, 660 insertions(+), 161 deletions(-)
diff --git a/internal/controller/grpcroute_controller.go
b/internal/controller/grpcroute_controller.go
index 1218afcb..8bc90458 100644
--- a/internal/controller/grpcroute_controller.go
+++ b/internal/controller/grpcroute_controller.go
@@ -57,6 +57,8 @@ type GRPCRouteReconciler struct { //nolint:revive
Provider provider.Provider
+ parentResolver routeParentResolver
+
genericEvent chan event.GenericEvent
Updater status.Updater
@@ -94,7 +96,7 @@ func (r *GRPCRouteReconciler) SetupWithManager(mgr
ctrl.Manager) error {
return false
},
DeleteFunc: func(e event.DeleteEvent)
bool {
- return false
+ return true
},
CreateFunc: func(e event.CreateEvent)
bool {
return true
@@ -164,6 +166,7 @@ func (r *GRPCRouteReconciler) Reconcile(ctx
context.Context, req ctrl.Request) (
gr := new(gatewayv1.GRPCRoute)
if err := r.Get(ctx, req.NamespacedName, gr); err != nil {
if client.IgnoreNotFound(err) == nil {
+ r.parentResolver.Forget(req.NamespacedName)
gr.Namespace = req.Namespace
gr.Name = req.Name
@@ -192,17 +195,12 @@ func (r *GRPCRouteReconciler) Reconcile(ctx
context.Context, req ctrl.Request) (
msg: "Route is accepted",
}
- gateways, unresolvedParents, err := ParseRouteParentRefs(ctx, r.Client,
r.Log, gr, gr.Spec.ParentRefs)
+ gateways, err := r.parentResolver.Resolve(ctx, r.Client, r.Log, gr,
gr.Spec.ParentRefs)
if err != nil {
return ctrl.Result{}, err
}
if len(gateways) == 0 {
- if unresolvedParents {
- // See the HTTPRoute reconciler: an unresolvable
parentRef leaves ownership
- // unknown rather than disproven, so the data plane
must be left alone.
- return ctrl.Result{}, nil
- }
// See the HTTPRoute reconciler: a route that no longer
references a
// Gateway managed by this controller must have its previously
pushed
// configuration removed, or the data plane keeps serving it.
diff --git a/internal/controller/httproute_controller.go
b/internal/controller/httproute_controller.go
index 5050cea9..d3b8a670 100644
--- a/internal/controller/httproute_controller.go
+++ b/internal/controller/httproute_controller.go
@@ -60,6 +60,8 @@ type HTTPRouteReconciler struct { //nolint:revive
Provider provider.Provider
+ parentResolver routeParentResolver
+
genericEvent chan event.GenericEvent
Updater status.Updater
@@ -97,7 +99,7 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr
ctrl.Manager) error {
return false
},
DeleteFunc: func(e event.DeleteEvent)
bool {
- return false
+ return true
},
CreateFunc: func(e event.CreateEvent)
bool {
return true
@@ -143,6 +145,7 @@ func (r *HTTPRouteReconciler) Reconcile(ctx
context.Context, req ctrl.Request) (
hr := new(gatewayv1.HTTPRoute)
if err := r.Get(ctx, req.NamespacedName, hr); err != nil {
if client.IgnoreNotFound(err) == nil {
+ r.parentResolver.Forget(req.NamespacedName)
if err := r.updateHTTPRoutePolicyStatusOnDeleting(ctx,
req.NamespacedName); err != nil {
return ctrl.Result{}, err
}
@@ -174,18 +177,12 @@ func (r *HTTPRouteReconciler) Reconcile(ctx
context.Context, req ctrl.Request) (
msg: "Route is accepted",
}
- gateways, unresolvedParents, err := ParseRouteParentRefs(ctx, r.Client,
r.Log, hr, hr.Spec.ParentRefs)
+ gateways, err := r.parentResolver.Resolve(ctx, r.Client, r.Log, hr,
hr.Spec.ParentRefs)
if err != nil {
return ctrl.Result{}, err
}
if len(gateways) == 0 {
- if unresolvedParents {
- // A missing Gateway or GatewayClass leaves ownership
unknown rather than
- // disproven. GatewayClass is cluster-scoped, so while
one is absent every
- // route under it resolves empty and deleting would
drain the data plane.
- return ctrl.Result{}, nil
- }
// The route does not reference any Gateway managed by this
controller.
// It may have referenced one before, e.g. when its parentRefs
are
// repointed at a Gateway belonging to another GatewayClass, so
the
diff --git a/internal/controller/httproute_controller_test.go
b/internal/controller/httproute_controller_test.go
index 00876c02..38b77234 100644
--- a/internal/controller/httproute_controller_test.go
+++ b/internal/controller/httproute_controller_test.go
@@ -40,11 +40,7 @@ type noopReadier struct {
func (noopReadier) Done(client.Object, k8stypes.NamespacedName) {}
-// TestHTTPRouteReconcile_EmptyGateways covers the two ways a route resolves
to no
-// Gateway of this controller. Only a resolved parent naming another controller
-// proves the route is not ours; an unresolvable parent must leave the data
plane
-// alone, since a missing cluster-scoped GatewayClass empties the list for
every
-// route under it at once.
+// Routes with no known managed parents retract any stale configuration.
func TestHTTPRouteReconcile_EmptyGateways(t *testing.T) {
scheme := parentRefTestScheme(t)
@@ -82,9 +78,9 @@ func TestHTTPRouteReconcile_EmptyGateways(t *testing.T) {
wantDelete: true,
},
{
- name: "gatewayclass missing",
+ name: "gatewayclass never resolved",
objects: []client.Object{route, gw("gone")},
- wantDelete: false,
+ wantDelete: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
diff --git a/internal/controller/route_parent_resolver.go
b/internal/controller/route_parent_resolver.go
new file mode 100644
index 00000000..89cab1c8
--- /dev/null
+++ b/internal/controller/route_parent_resolver.go
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controller
+
+import (
+ "context"
+ "sync"
+
+ "github.com/go-logr/logr"
+ "k8s.io/apimachinery/pkg/types"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+
+ "github.com/apache/apisix-ingress-controller/internal/controller/config"
+)
+
+type resolvedParentClass struct {
+ gatewayUID types.UID
+ class *gatewayv1.GatewayClass
+}
+
+type routeParentOwnership struct {
+ uid types.UID
+ parents map[client.ObjectKey]resolvedParentClass
+}
+
+// routeParentResolver remembers ownership only for parents previously resolved
+// for this route. A never-resolved sibling must not prevent deletion cleanup.
+// Like the provider's desired configuration, this state is rebuilt on startup.
+type routeParentResolver struct {
+ mu sync.Mutex
+ parents map[client.ObjectKey]routeParentOwnership
+}
+
+func (r *routeParentResolver) Forget(key client.ObjectKey) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ delete(r.parents, key)
+}
+
+func (r *routeParentResolver) Resolve(ctx context.Context, cli client.Client,
log logr.Logger, route client.Object, refs []gatewayv1.ParentReference)
([]RouteParentRefContext, error) {
+ // controller-runtime never reconciles one route concurrently, so the
lock only
+ // guards the map, not the parent lookups between the read and the
write.
+ routeKey := client.ObjectKeyFromObject(route)
+ r.mu.Lock()
+ previous := r.parents[routeKey].parents
+ if r.parents[routeKey].uid != route.GetUID() {
+ previous = nil
+ }
+ r.mu.Unlock()
+ next := make(map[client.ObjectKey]resolvedParentClass)
+ gateways, err := parseRouteParentRefs(ctx, cli, log, route, refs,
func(gw *gatewayv1.Gateway) (*gatewayv1.GatewayClass, error) {
+ key := client.ObjectKeyFromObject(gw)
+ gc := &gatewayv1.GatewayClass{}
+ if err := cli.Get(ctx, client.ObjectKey{Name:
string(gw.Spec.GatewayClassName)}, gc); err != nil {
+ if client.IgnoreNotFound(err) != nil {
+ return nil, err
+ }
+ known, ok := previous[key]
+ if !ok || known.gatewayUID != gw.UID ||
known.class.Name != string(gw.Spec.GatewayClassName) {
+ return nil, nil
+ }
+ // Keep resolving the surviving Gateway's current
listeners without
+ // retaining configuration belonging to a different,
deleted parent.
+ gc = known.class
+ }
+ if string(gc.Spec.ControllerName) ==
config.ControllerConfig.ControllerName {
+ next[key] = resolvedParentClass{gatewayUID: gw.UID,
class: gc}
+ }
+ return gc, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if len(next) == 0 {
+ delete(r.parents, routeKey)
+ } else {
+ if r.parents == nil {
+ r.parents =
make(map[client.ObjectKey]routeParentOwnership)
+ }
+ r.parents[routeKey] = routeParentOwnership{uid: route.GetUID(),
parents: next}
+ }
+ return gateways, nil
+}
diff --git a/internal/controller/route_parent_resolver_test.go
b/internal/controller/route_parent_resolver_test.go
new file mode 100644
index 00000000..4ab3f505
--- /dev/null
+++ b/internal/controller/route_parent_resolver_test.go
@@ -0,0 +1,192 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package controller
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/go-logr/logr"
+ "github.com/stretchr/testify/require"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/utils/ptr"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+ gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+)
+
+func TestRouteParentResolverGatewayDeletion(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ route client.Object
+ protocol gatewayv1.ProtocolType
+ }{
+ {"HTTPRoute", &gatewayv1.HTTPRoute{},
gatewayv1.HTTPProtocolType},
+ {"GRPCRoute", &gatewayv1.GRPCRoute{},
gatewayv1.HTTPProtocolType},
+ {"TCPRoute", &gatewayv1.TCPRoute{}, gatewayv1.TCPProtocolType},
+ {"UDPRoute", &gatewayv1.UDPRoute{}, gatewayv1.UDPProtocolType},
+ {"TLSRoute", &gatewayv1.TLSRoute{}, gatewayv1.TLSProtocolType},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ for _, knownSibling := range []bool{false, true} {
+ name := "never-resolved sibling"
+ if knownSibling {
+ name = "previously-resolved sibling"
+ }
+ t.Run(name, func(t *testing.T) {
+ ctx := context.Background()
+ route :=
tc.route.DeepCopyObject().(client.Object)
+ route.SetNamespace("default")
+ route.SetName("route")
+ route.SetUID("route-uid")
+ classA := newParentRefGatewayClass()
+ classB := classA.DeepCopy()
+ classB.Name = "sibling-class"
+ listener := gatewayv1.Listener{Name:
"listener", Port: 9000, Protocol: tc.protocol}
+ if tc.protocol ==
gatewayv1.TLSProtocolType {
+ listener.TLS =
&gatewayv1.ListenerTLSConfig{Mode: ptr.To(gatewayv1.TLSModePassthrough)}
+ }
+ a := &gatewayv1.Gateway{
+ ObjectMeta:
metav1.ObjectMeta{Namespace: "default", Name: "a", UID: "a-uid"},
+ Spec:
gatewayv1.GatewaySpec{GatewayClassName: gatewayv1.ObjectName(classA.Name),
Listeners: []gatewayv1.Listener{listener}},
+ }
+ b := a.DeepCopy()
+ b.Name, b.UID = "b", "b-uid"
+ b.Spec.GatewayClassName =
gatewayv1.ObjectName(classB.Name)
+ objects := []client.Object{classA, a, b}
+ if knownSibling {
+ objects = append(objects,
classB)
+ }
+ cli :=
fake.NewClientBuilder().WithScheme(parentRefTestScheme(t)).WithObjects(objects...).Build()
+ refs :=
[]gatewayv1.ParentReference{{Name: "a"}, {Name: "b"}}
+ resolver := &routeParentResolver{}
+ parents, err := resolver.Resolve(ctx,
cli, logr.Discard(), route, refs)
+ require.NoError(t, err)
+ expected := 1
+ if knownSibling {
+ expected = 2
+ }
+ require.Len(t, parents, expected)
+ require.True(t,
isRouteAccepted(parents))
+ if knownSibling {
+ require.NoError(t,
cli.Delete(ctx, classB))
+ }
+ require.NoError(t, cli.Delete(ctx, a))
+ parents, err = resolver.Resolve(ctx,
cli, logr.Discard(), route, refs)
+ require.NoError(t, err)
+ if knownSibling {
+ require.Len(t, parents, 1)
+ require.Equal(t, "b",
parents[0].Gateway.Name)
+ require.True(t,
isRouteAccepted(parents))
+ // Listener changes still apply
while its class is absent.
+
b.Spec.Listeners[0].AllowedRoutes = &gatewayv1.AllowedRoutes{
+ Kinds:
[]gatewayv1.RouteGroupKind{{Kind: "UnsupportedRoute"}},
+ }
+ require.NoError(t,
cli.Update(ctx, b))
+ parents, err =
resolver.Resolve(ctx, cli, logr.Discard(), route, refs)
+ require.NoError(t, err)
+ require.False(t,
isRouteAccepted(parents))
+ require.NoError(t,
cli.Delete(ctx, b))
+ parents, err =
resolver.Resolve(ctx, cli, logr.Discard(), route, refs)
+ require.NoError(t, err)
+ }
+ require.Empty(t, parents)
+ require.Empty(t, resolver.parents)
+ })
+ }
+ })
+ }
+}
+
+func TestRouteParentResolverDoesNotReuseOwnership(t *testing.T) {
+ for _, change := range []string{"gateway recreated", "route recreated",
"class changed", "route deleted", "parent removed", "class reassigned"} {
+ t.Run(change, func(t *testing.T) {
+ ctx := context.Background()
+ gc := newParentRefGatewayClass()
+ gw := &gatewayv1.Gateway{
+ ObjectMeta: metav1.ObjectMeta{Namespace:
"default", Name: "gw", UID: "gateway-uid"},
+ Spec:
gatewayv1.GatewaySpec{GatewayClassName: gatewayv1.ObjectName(gc.Name),
Listeners: []gatewayv1.Listener{{Name: "http", Port: 80, Protocol:
gatewayv1.HTTPProtocolType}}},
+ }
+ route := &gatewayv1.HTTPRoute{ObjectMeta:
metav1.ObjectMeta{Namespace: "default", Name: "route", UID: "route-uid"}}
+ cli :=
fake.NewClientBuilder().WithScheme(parentRefTestScheme(t)).WithObjects(gc,
gw).Build()
+ resolver := &routeParentResolver{}
+ refs := []gatewayv1.ParentReference{{Name: "gw"}}
+ parents, err := resolver.Resolve(ctx, cli,
logr.Discard(), route, refs)
+ require.NoError(t, err)
+ require.Len(t, parents, 1)
+ require.NoError(t, cli.Delete(ctx, gc))
+ switch change {
+ case "gateway recreated":
+ require.NoError(t, cli.Delete(ctx, gw))
+ gw.ResourceVersion = ""
+ gw.UID = "new-gateway-uid"
+ require.NoError(t, cli.Create(ctx, gw))
+ case "route recreated":
+ route.UID = "new-route-uid"
+ case "class changed":
+ gw.Spec.GatewayClassName = "different-class"
+ require.NoError(t, cli.Update(ctx, gw))
+ case "route deleted":
+
resolver.Forget(client.ObjectKeyFromObject(route))
+ case "parent removed":
+ _, err = resolver.Resolve(ctx, cli,
logr.Discard(), route, nil)
+ require.NoError(t, err)
+ case "class reassigned":
+ gc.ResourceVersion = ""
+ gc.Spec.ControllerName =
"other.example/controller"
+ require.NoError(t, cli.Create(ctx, gc))
+ }
+ parents, err = resolver.Resolve(ctx, cli,
logr.Discard(), route, refs)
+ require.NoError(t, err)
+ require.Empty(t, parents)
+ })
+ }
+}
+
+func TestRouteParentResolverPreservesOwnershipOnReadError(t *testing.T) {
+ ctx := context.Background()
+ gc := newParentRefGatewayClass()
+ gw := &gatewayv1.Gateway{
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "gw",
UID: "gateway-uid"},
+ Spec: gatewayv1.GatewaySpec{GatewayClassName:
gatewayv1.ObjectName(gc.Name), Listeners: []gatewayv1.Listener{{Name: "http",
Port: 80, Protocol: gatewayv1.HTTPProtocolType}}},
+ }
+ route := &gatewayv1.HTTPRoute{ObjectMeta: metav1.ObjectMeta{Namespace:
"default", Name: "route"}}
+ fail := false
+ readErr := errors.New("temporary API failure")
+ cli :=
fake.NewClientBuilder().WithScheme(parentRefTestScheme(t)).WithObjects(gc, gw).
+ WithInterceptorFuncs(interceptor.Funcs{Get: func(ctx
context.Context, cli client.WithWatch, key client.ObjectKey, obj client.Object,
opts ...client.GetOption) error {
+ if _, ok := obj.(*gatewayv1.GatewayClass); ok && fail {
+ return readErr
+ }
+ return cli.Get(ctx, key, obj, opts...)
+ }}).Build()
+ resolver := &routeParentResolver{}
+ refs := []gatewayv1.ParentReference{{Name: "gw"}}
+ _, err := resolver.Resolve(ctx, cli, logr.Discard(), route, refs)
+ require.NoError(t, err)
+ fail = true
+ _, err = resolver.Resolve(ctx, cli, logr.Discard(), route, refs)
+ require.ErrorIs(t, err, readErr)
+ fail = false
+ require.NoError(t, cli.Delete(ctx, gc))
+ parents, err := resolver.Resolve(ctx, cli, logr.Discard(), route, refs)
+ require.NoError(t, err)
+ require.Len(t, parents, 1)
+}
diff --git a/internal/controller/tcproute_controller.go
b/internal/controller/tcproute_controller.go
index 8428a55e..8116f482 100644
--- a/internal/controller/tcproute_controller.go
+++ b/internal/controller/tcproute_controller.go
@@ -55,6 +55,8 @@ type TCPRouteReconciler struct { //nolint:revive
Provider provider.Provider
+ parentResolver routeParentResolver
+
Updater status.Updater
Readier readiness.ReadinessManager
@@ -86,7 +88,7 @@ func (r *TCPRouteReconciler) SetupWithManager(mgr
ctrl.Manager) error {
return false
},
DeleteFunc: func(e event.DeleteEvent)
bool {
- return false
+ return true
},
CreateFunc: func(e event.CreateEvent)
bool {
return true
@@ -253,6 +255,7 @@ func (r *TCPRouteReconciler) Reconcile(ctx context.Context,
req ctrl.Request) (c
tr := new(gatewayv1.TCPRoute)
if err := r.Get(ctx, req.NamespacedName, tr); err != nil {
if client.IgnoreNotFound(err) == nil {
+ r.parentResolver.Forget(req.NamespacedName)
tr.Namespace = req.Namespace
tr.Name = req.Name
@@ -283,17 +286,12 @@ func (r *TCPRouteReconciler) Reconcile(ctx
context.Context, req ctrl.Request) (c
msg: "Route is accepted",
}
- gateways, unresolvedParents, err := ParseRouteParentRefs(ctx, r.Client,
r.Log, tr, tr.Spec.ParentRefs)
+ gateways, err := r.parentResolver.Resolve(ctx, r.Client, r.Log, tr,
tr.Spec.ParentRefs)
if err != nil {
return ctrl.Result{}, err
}
if len(gateways) == 0 {
- if unresolvedParents {
- // See the HTTPRoute reconciler: an unresolvable
parentRef leaves ownership
- // unknown rather than disproven, so the data plane
must be left alone.
- return ctrl.Result{}, nil
- }
// See the HTTPRoute reconciler: a route that no longer
references a
// Gateway managed by this controller must have its previously
pushed
// configuration removed, or the data plane keeps serving it.
diff --git a/internal/controller/tlsroute_controller.go
b/internal/controller/tlsroute_controller.go
index 1346c3ef..1ba9f4f5 100644
--- a/internal/controller/tlsroute_controller.go
+++ b/internal/controller/tlsroute_controller.go
@@ -55,6 +55,8 @@ type TLSRouteReconciler struct { //nolint:revive
Provider provider.Provider
+ parentResolver routeParentResolver
+
Updater status.Updater
Readier readiness.ReadinessManager
@@ -86,7 +88,7 @@ func (r *TLSRouteReconciler) SetupWithManager(mgr
ctrl.Manager) error {
return false
},
DeleteFunc: func(e event.DeleteEvent)
bool {
- return false
+ return true
},
CreateFunc: func(e event.CreateEvent)
bool {
return true
@@ -253,6 +255,7 @@ func (r *TLSRouteReconciler) Reconcile(ctx context.Context,
req ctrl.Request) (c
tr := new(gatewayv1.TLSRoute)
if err := r.Get(ctx, req.NamespacedName, tr); err != nil {
if client.IgnoreNotFound(err) == nil {
+ r.parentResolver.Forget(req.NamespacedName)
tr.Namespace = req.Namespace
tr.Name = req.Name
@@ -283,17 +286,12 @@ func (r *TLSRouteReconciler) Reconcile(ctx
context.Context, req ctrl.Request) (c
msg: "Route is accepted",
}
- gateways, unresolvedParents, err := ParseRouteParentRefs(ctx, r.Client,
r.Log, tr, tr.Spec.ParentRefs)
+ gateways, err := r.parentResolver.Resolve(ctx, r.Client, r.Log, tr,
tr.Spec.ParentRefs)
if err != nil {
return ctrl.Result{}, err
}
if len(gateways) == 0 {
- if unresolvedParents {
- // See the HTTPRoute reconciler: an unresolvable
parentRef leaves ownership
- // unknown rather than disproven, so the data plane
must be left alone.
- return ctrl.Result{}, nil
- }
// See the HTTPRoute reconciler: a route that no longer
references a
// Gateway managed by this controller must have its previously
pushed
// configuration removed, or the data plane keeps serving it.
diff --git a/internal/controller/udproute_controller.go
b/internal/controller/udproute_controller.go
index a9475941..3f3255d4 100644
--- a/internal/controller/udproute_controller.go
+++ b/internal/controller/udproute_controller.go
@@ -55,6 +55,8 @@ type UDPRouteReconciler struct { //nolint:revive
Provider provider.Provider
+ parentResolver routeParentResolver
+
Updater status.Updater
Readier readiness.ReadinessManager
@@ -86,7 +88,7 @@ func (r *UDPRouteReconciler) SetupWithManager(mgr
ctrl.Manager) error {
return false
},
DeleteFunc: func(e event.DeleteEvent)
bool {
- return false
+ return true
},
CreateFunc: func(e event.CreateEvent)
bool {
return true
@@ -253,6 +255,7 @@ func (r *UDPRouteReconciler) Reconcile(ctx context.Context,
req ctrl.Request) (c
tr := new(gatewayv1.UDPRoute)
if err := r.Get(ctx, req.NamespacedName, tr); err != nil {
if client.IgnoreNotFound(err) == nil {
+ r.parentResolver.Forget(req.NamespacedName)
tr.Namespace = req.Namespace
tr.Name = req.Name
@@ -283,17 +286,12 @@ func (r *UDPRouteReconciler) Reconcile(ctx
context.Context, req ctrl.Request) (c
msg: "Route is accepted",
}
- gateways, unresolvedParents, err := ParseRouteParentRefs(ctx, r.Client,
r.Log, tr, tr.Spec.ParentRefs)
+ gateways, err := r.parentResolver.Resolve(ctx, r.Client, r.Log, tr,
tr.Spec.ParentRefs)
if err != nil {
return ctrl.Result{}, err
}
if len(gateways) == 0 {
- if unresolvedParents {
- // See the HTTPRoute reconciler: an unresolvable
parentRef leaves ownership
- // unknown rather than disproven, so the data plane
must be left alone.
- return ctrl.Result{}, nil
- }
// See the HTTPRoute reconciler: a route that no longer
references a
// Gateway managed by this controller must have its previously
pushed
// configuration removed, or the data plane keeps serving it.
diff --git a/internal/controller/utils.go b/internal/controller/utils.go
index bc328f4c..e5814c9b 100644
--- a/internal/controller/utils.go
+++ b/internal/controller/utils.go
@@ -338,21 +338,15 @@ func parentRefTargetsListenerExplicitly(parentRef
gatewayv1.ParentReference) boo
return parentRef.Port != nil
}
-// ParseRouteParentRefs resolves the parentRefs of a route to the Gateways this
-// controller manages. The second return value reports that at least one
-// parentRef could not be resolved, because its Gateway or that Gateway's
-// GatewayClass does not exist. An empty gateway list then means "ownership
-// unknown", not "owned by another controller", and callers must not act on the
-// route's data plane configuration.
-func ParseRouteParentRefs(
+func parseRouteParentRefs(
ctx context.Context,
mgrc client.Client,
log logr.Logger,
route client.Object,
parentRefs []gatewayv1.ParentReference,
-) ([]RouteParentRefContext, bool, error) {
+ resolveClass func(*gatewayv1.Gateway) (*gatewayv1.GatewayClass, error),
+) ([]RouteParentRefContext, error) {
gateways := make([]RouteParentRefContext, 0)
- unresolved := false
for _, parentRef := range parentRefs {
namespace := route.GetNamespace()
if parentRef.Namespace != nil {
@@ -370,24 +364,18 @@ func ParseRouteParentRefs(
Name: name,
}, &gateway); err != nil {
if client.IgnoreNotFound(err) == nil {
- unresolved = true
+ // A deleted Gateway no longer authorizes its
routes to serve traffic.
continue
}
- return nil, false, fmt.Errorf("failed to retrieve
gateway for route: %w", err)
+ return nil, fmt.Errorf("failed to retrieve gateway for
route: %w", err)
}
- gatewayClass := gatewayv1.GatewayClass{}
- if err := mgrc.Get(ctx, client.ObjectKey{
- Name: string(gateway.Spec.GatewayClassName),
- }, &gatewayClass); err != nil {
- if client.IgnoreNotFound(err) == nil {
- unresolved = true
- continue
- }
- return nil, false, fmt.Errorf("failed to retrieve
gatewayclass for gateway: %w", err)
+ gatewayClass, err := resolveClass(&gateway)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve gatewayclass
for gateway: %w", err)
}
- if string(gatewayClass.Spec.ControllerName) !=
config.ControllerConfig.ControllerName {
+ if gatewayClass == nil ||
string(gatewayClass.Spec.ControllerName) !=
config.ControllerConfig.ControllerName {
continue
}
@@ -522,7 +510,7 @@ func ParseRouteParentRefs(
}
}
- return gateways, unresolved, nil
+ return gateways, nil
}
// reuseUnchangedListenerStatus keeps the previously published status when
diff --git a/internal/controller/utils_parentref_test.go
b/internal/controller/utils_parentref_test.go
index 4a6bf3b3..3b60f028 100644
--- a/internal/controller/utils_parentref_test.go
+++ b/internal/controller/utils_parentref_test.go
@@ -95,7 +95,7 @@ func TestParseRouteParentRefs_ReasonOrderIndependent(t
*testing.T) {
cli := fake.NewClientBuilder().WithScheme(scheme).
WithObjects(newParentRefGatewayClass(),
gw).Build()
- got, _, err :=
ParseRouteParentRefs(context.Background(), cli, logr.Discard(), route,
+ got, err :=
(&routeParentResolver{}).Resolve(context.Background(), cli, logr.Discard(),
route,
[]gatewayv1.ParentReference{{Name: "gw"}})
require.NoError(t, err)
require.Len(t, got, 1)
@@ -146,7 +146,7 @@ func TestParseRouteParentRefs_ExplicitListenerMatch(t
*testing.T) {
port8080 := gatewayv1.PortNumber(8080)
t.Run("invalid explicit ref is not satisfied by another gateway's
listener", func(t *testing.T) {
- got, _, err := ParseRouteParentRefs(context.Background(), cli,
logr.Discard(), route,
+ got, err :=
(&routeParentResolver{}).Resolve(context.Background(), cli, logr.Discard(),
route,
[]gatewayv1.ParentReference{
// Explicit sectionName "web" on gw-a, which
has no such listener.
{Name: "gw-a", SectionName: &webSection},
@@ -172,7 +172,7 @@ func TestParseRouteParentRefs_ExplicitListenerMatch(t
*testing.T) {
})
t.Run("explicit sectionName on the owning gateway is an explicit
match", func(t *testing.T) {
- got, _, err := ParseRouteParentRefs(context.Background(), cli,
logr.Discard(), route,
+ got, err :=
(&routeParentResolver{}).Resolve(context.Background(), cli, logr.Discard(),
route,
[]gatewayv1.ParentReference{{Name: "gw-b", SectionName:
&webSection}})
require.NoError(t, err)
require.Len(t, got, 1)
@@ -181,7 +181,7 @@ func TestParseRouteParentRefs_ExplicitListenerMatch(t
*testing.T) {
})
t.Run("explicit port on the owning gateway is an explicit match",
func(t *testing.T) {
- got, _, err := ParseRouteParentRefs(context.Background(), cli,
logr.Discard(), route,
+ got, err :=
(&routeParentResolver{}).Resolve(context.Background(), cli, logr.Discard(),
route,
[]gatewayv1.ParentReference{{Name: "gw-b", Port:
&port8080}})
require.NoError(t, err)
require.Len(t, got, 1)
@@ -190,7 +190,7 @@ func TestParseRouteParentRefs_ExplicitListenerMatch(t
*testing.T) {
})
t.Run("implicit ref is not an explicit match", func(t *testing.T) {
- got, _, err := ParseRouteParentRefs(context.Background(), cli,
logr.Discard(), route,
+ got, err :=
(&routeParentResolver{}).Resolve(context.Background(), cli, logr.Discard(),
route,
[]gatewayv1.ParentReference{{Name: "gw-b"}})
require.NoError(t, err)
require.Len(t, got, 1)
@@ -223,7 +223,7 @@ func TestParseRouteParentRefs_ConflictingTLSModePort(t
*testing.T) {
cli := fake.NewClientBuilder().WithScheme(scheme).
WithObjects(newParentRefGatewayClass(), gw).Build()
- got, _, err := ParseRouteParentRefs(context.Background(), cli,
logr.Discard(), route,
+ got, err := (&routeParentResolver{}).Resolve(context.Background(), cli,
logr.Discard(), route,
[]gatewayv1.ParentReference{{Name: "gw"}})
require.NoError(t, err)
require.Len(t, got, 1)
@@ -233,11 +233,7 @@ func TestParseRouteParentRefs_ConflictingTLSModePort(t
*testing.T) {
assert.Equal(t, string(gatewayv1.RouteReasonNotAllowedByListeners),
cond.Reason)
}
-// TestParseRouteParentRefs_UnresolvedParents verifies the second return value
-// distinguishes "no parentRef named this controller" from "a parentRef could
not
-// be resolved". Callers delete data plane configuration on an empty gateway
list,
-// which is only correct in the first case: a GatewayClass is cluster-scoped,
so
-// while one is missing every route under it resolves to an empty list at once.
+// Unknown parents must not prevent cleanup of previously managed
configuration.
func TestParseRouteParentRefs_UnresolvedParents(t *testing.T) {
scheme := parentRefTestScheme(t)
@@ -260,46 +256,40 @@ func TestParseRouteParentRefs_UnresolvedParents(t
*testing.T) {
}
for _, tc := range []struct {
- name string
- objects []client.Object
- parentRef string
- wantGateways int
- wantUnresolved bool
+ name string
+ objects []client.Object
+ parentRef string
+ wantGateways int
}{
{
- name: "gateway missing",
- objects:
[]client.Object{newParentRefGatewayClass()},
- parentRef: "gw",
- wantUnresolved: true,
+ name: "gateway missing retracts stale routes",
+ objects: []client.Object{newParentRefGatewayClass()},
+ parentRef: "gw",
},
{
- name: "gatewayclass missing",
- objects: []client.Object{gw("gw", "apisix")},
- parentRef: "gw",
- wantUnresolved: true,
+ name: "gatewayclass missing",
+ objects: []client.Object{gw("gw", "apisix")},
+ parentRef: "gw",
},
{
- name: "gateway of another controller",
- objects: []client.Object{foreignClass, gw("gw",
"other")},
- parentRef: "gw",
- wantUnresolved: false,
+ name: "gateway of another controller",
+ objects: []client.Object{foreignClass, gw("gw",
"other")},
+ parentRef: "gw",
},
{
- name: "gateway of this controller",
- objects:
[]client.Object{newParentRefGatewayClass(), gw("gw", "apisix")},
- parentRef: "gw",
- wantGateways: 1,
- wantUnresolved: false,
+ name: "gateway of this controller",
+ objects:
[]client.Object{newParentRefGatewayClass(), gw("gw", "apisix")},
+ parentRef: "gw",
+ wantGateways: 1,
},
} {
t.Run(tc.name, func(t *testing.T) {
cli :=
fake.NewClientBuilder().WithScheme(scheme).WithObjects(tc.objects...).Build()
- got, unresolved, err :=
ParseRouteParentRefs(context.Background(), cli, logr.Discard(), route,
+ got, err :=
(&routeParentResolver{}).Resolve(context.Background(), cli, logr.Discard(),
route,
[]gatewayv1.ParentReference{{Name:
gatewayv1.ObjectName(tc.parentRef)}})
require.NoError(t, err)
assert.Len(t, got, tc.wantGateways)
- assert.Equal(t, tc.wantUnresolved, unresolved)
})
}
}
diff --git a/internal/provider/apisix/gateway_delete_test.go
b/internal/provider/apisix/gateway_delete_test.go
new file mode 100644
index 00000000..3ba0cdc8
--- /dev/null
+++ b/internal/provider/apisix/gateway_delete_test.go
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package apisix
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "testing"
+
+ "github.com/go-logr/logr"
+ "github.com/stretchr/testify/require"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ k8stypes "k8s.io/apimachinery/pkg/types"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+ gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+
+ adctypes "github.com/apache/apisix-ingress-controller/api/adc"
+ "github.com/apache/apisix-ingress-controller/internal/controller"
+ "github.com/apache/apisix-ingress-controller/internal/manager/readiness"
+ "github.com/apache/apisix-ingress-controller/internal/types"
+ "github.com/apache/apisix-ingress-controller/internal/utils"
+)
+
+type gatewayDeleteReadier struct{ readiness.ReadinessManager }
+
+func (gatewayDeleteReadier) Done(client.Object, k8stypes.NamespacedName) {}
+
+func TestGatewayDeletionRetractsRouteWithUnknownSibling(t *testing.T) {
+ withMockADCServer(t, func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewEncoder(w).Encode(adctypes.SyncResult{Status:
adctypes.StatusSuccess})
+ })
+ d := newTestProvider(t)
+ gateway := &gatewayv1.Gateway{
+ TypeMeta: metav1.TypeMeta{Kind: "Gateway", APIVersion:
gatewayv1.GroupVersion.String()},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name:
"deleted"},
+ }
+ route := &gatewayv1.HTTPRoute{
+ TypeMeta: metav1.TypeMeta{Kind: "HTTPRoute", APIVersion:
gatewayv1.GroupVersion.String()},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name:
"route"},
+ Spec: gatewayv1.HTTPRouteSpec{CommonRouteSpec:
gatewayv1.CommonRouteSpec{
+ ParentRefs: []gatewayv1.ParentReference{{Name:
"deleted"}, {Name: "unresolved"}},
+ }},
+ }
+ sibling := &gatewayv1.Gateway{
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name:
"unresolved"},
+ Spec: gatewayv1.GatewaySpec{GatewayClassName:
"missing-class"},
+ }
+ proxy := types.NamespacedNameKind{Kind: "GatewayProxy", Namespace:
"default", Name: "proxy"}
+ configs := map[types.NamespacedNameKind]adctypes.Config{proxy: {Name:
proxy.String(), BackendType: "apisix", ServerAddrs:
[]string{"http://apisix:9180"}}}
+ owner := utils.NamespacedNameKind(gateway)
+ routeOwner := utils.NamespacedNameKind(route)
+ require.NoError(t, d.applyResourceState(owner, configs,
[]string{adctypes.TypeSSL}, &adctypes.Resources{}, labelsOf(owner),
pluginsNone))
+ require.NoError(t, d.applyResourceState(routeOwner, configs,
[]string{adctypes.TypeService}, &adctypes.Resources{
+ Services: []*adctypes.Service{{Metadata: adctypes.Metadata{ID:
"stale-route", Labels: labelsOf(routeOwner)}}},
+ }, labelsOf(routeOwner), pluginsNone))
+ require.NoError(t, d.Delete(context.Background(), gateway))
+ scheme := runtime.NewScheme()
+ require.NoError(t, gatewayv1.Install(scheme))
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(route,
sibling).Build()
+ r := &controller.HTTPRouteReconciler{Client: cli, Scheme: scheme, Log:
logr.Discard(), Provider: d, Readier: gatewayDeleteReadier{}}
+ _, err := r.Reconcile(context.Background(),
ctrl.Request{NamespacedName: client.ObjectKeyFromObject(route)})
+ require.NoError(t, err)
+ resources, err := d.store.GetResources(proxy.String())
+ require.NoError(t, err)
+ require.Empty(t, resources.Services, "deleted Gateway's route survives
reconciliation with an unresolved sibling")
+}
diff --git a/internal/provider/apisix/provider.go
b/internal/provider/apisix/provider.go
index 403d1528..29f67d2e 100644
--- a/internal/provider/apisix/provider.go
+++ b/internal/provider/apisix/provider.go
@@ -236,16 +236,22 @@ func (d *apisixProvider) Delete(ctx context.Context, obj
client.Object) error {
resourceTypes []string
labels map[string]string
plugins pluginSource
- // wholeConfig is set for a Gateway or IngressClass, whose
deletion wipes every
- // config it referenced.
- wholeConfig bool
+ // IngressClass deletion clears its config; Gateways can share
a config.
+ wholeConfig bool
+ immediateSync bool
)
switch obj.(type) {
case *gatewayv1.HTTPRoute, *apiv2.ApisixRoute, *gatewayv1.GRPCRoute,
*gatewayv1.TCPRoute, *gatewayv1.UDPRoute, *gatewayv1.TLSRoute:
resourceTypes = append(resourceTypes, adctypes.TypeService)
labels = label.GenLabel(obj)
- case *gatewayv1.Gateway, *networkingv1.IngressClass:
+ case *gatewayv1.Gateway:
+ resourceTypes = append(resourceTypes, adctypes.TypeSSL)
+ labels = label.GenLabel(obj)
+ plugins = pluginsFromGatewayProxy
+ immediateSync = true
+ case *networkingv1.IngressClass:
wholeConfig = true
+ immediateSync = true
case *networkingv1.Ingress:
resourceTypes = append(resourceTypes, adctypes.TypeService,
adctypes.TypeSSL)
labels = label.GenLabel(obj)
@@ -268,11 +274,10 @@ func (d *apisixProvider) Delete(ctx context.Context, obj
client.Object) error {
if err != nil {
return err
}
- // Full synchronization is performed on a gateway by gateway basis
- // and it is not possible to perform scheduled synchronization
- // on deleted gateway level resources
- if wholeConfig {
- d.syncEvictedConfigsNow(ctx, removed, resourceTypes, labels)
+ // Push deletions immediately even when the deleted resource was the
last
+ // source of a config and no scheduled sync will pick it up.
+ if immediateSync {
+ d.syncEvictedConfigsNow(ctx, removed)
return nil
}
// Syncing pushes the whole store to every data plane. Objects this
controller never
@@ -357,7 +362,8 @@ func (d *apisixProvider) removeResourceState(
// evictFromStore deletes rk's contribution from each of the given configs'
cached
// snapshots. global_rules and plugin_metadata sourced from a GatewayProxy
belong to that
-// GatewayProxy's own config and are left in place. Callers must already hold
d.Lock.
+// GatewayProxy's own config and are left in place until no Gateway or
IngressClass
+// references it any more. Callers must already hold d.Lock.
func (d *apisixProvider) evictFromStore(
rk types.NamespacedNameKind,
configs map[types.NamespacedNameKind]adctypes.Config,
@@ -365,19 +371,31 @@ func (d *apisixProvider) evictFromStore(
labels map[string]string,
plugins pluginSource,
) error {
- for _, cfg := range configs {
+ for gatewayProxy, cfg := range configs {
if err := d.store.Delete(cfg.Name, resourceTypes, labels); err
!= nil {
return fmt.Errorf("store delete failed for config %s:
%w", cfg.Name, err)
}
- if plugins == pluginsFromResource {
- if err := d.store.SetGlobalRules(cfg.Name, rk, nil);
err != nil {
- return fmt.Errorf("store global rules failed
for config %s: %w", cfg.Name, err)
+ owner := rk
+ switch {
+ case plugins == pluginsFromGatewayProxy &&
!d.configManager.HasResource(gatewayProxy, isGatewayProxyReferrer):
+ owner = gatewayProxy
+ if err := d.store.SetPluginMetadata(cfg.Name, nil); err
!= nil {
+ return fmt.Errorf("store plugin metadata failed
for config %s: %w", cfg.Name, err)
}
+ case plugins != pluginsFromResource:
+ continue
+ }
+ if err := d.store.SetGlobalRules(cfg.Name, owner, nil); err !=
nil {
+ return fmt.Errorf("store global rules failed for config
%s: %w", cfg.Name, err)
}
}
return nil
}
+func isGatewayProxyReferrer(rk types.NamespacedNameKind) bool {
+ return rk.Kind == types.KindGateway || rk.Kind == types.KindIngressClass
+}
+
// syncConfigNow reads name's current data (via build, called only once this
cacheKey's
// lock is actually held) and pushes it, one atomic read-then-push step per
cacheKey, so
// whichever caller is granted the lock decides what to push only once it
holds it: nothing
@@ -465,28 +483,23 @@ func toADCExecutionError(name string, err error)
types.ADCExecutionError {
return types.ADCExecutionError{Name: name, FailedErrors:
[]types.ADCExecutionServerAddrError{{Err: err.Error()}}}
}
-// syncEvictedConfigsNow pushes an empty resource set for each of the given
configs
-// immediately, instead of waiting for the next scheduled sync round, through
the same
-// per-cacheKey lock the periodic sync uses, so it can never race a periodic
round for the
-// same GatewayProxy. Used only when the deleted resource is a Gateway or
IngressClass,
-// where resourceTypes is empty, so the preceding removeResourceState call
already
-// reset each config's whole cached snapshot via Store.Delete, and that reset
should reach
-// the data plane promptly. Failures are logged, not surfaced as a status
update, matching
-// the deferred path, which only reports through the next scheduled sync round.
+// syncEvictedConfigsNow pushes the remaining snapshot immediately after
deleting
+// a Gateway or IngressClass. Read it under the sync lock so a concurrent
update
+// cannot be overwritten by a stale snapshot or an unconditional empty push.
func (d *apisixProvider) syncEvictedConfigsNow(
ctx context.Context,
configs map[types.NamespacedNameKind]adctypes.Config,
- resourceTypes []string,
- labels map[string]string,
) {
for _, cfg := range configs {
_, err := d.syncConfigNow(ctx, cfg.Name, func()
(adcclient.SyncInput, error) {
+ resources, err := d.store.GetResources(cfg.Name)
+ if err != nil {
+ return adcclient.SyncInput{}, err
+ }
return adcclient.SyncInput{
- Name: cfg.Name,
- Config: cfg,
- Resources: &adctypes.Resources{},
- ResourceTypes: resourceTypes,
- Labels: labels,
+ Name: cfg.Name,
+ Config: cfg,
+ Resources: resources,
}, nil
})
if err != nil {
diff --git a/internal/provider/apisix/provider_test.go
b/internal/provider/apisix/provider_test.go
index 1bd8b6b9..55eba787 100644
--- a/internal/provider/apisix/provider_test.go
+++ b/internal/provider/apisix/provider_test.go
@@ -347,3 +347,110 @@ func
TestRemoveResourceStateRemovesAnApisixGlobalRulesPlugins(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, adctypes.GlobalRule{"cors": map[string]any{}},
resources.GlobalRules, "only the deleted resource's own plugins go away")
}
+
+func TestDeleteGatewayPreservesSharedProxyResources(t *testing.T) {
+ var mu sync.Mutex
+ var received adcclient.ADCServerRequest
+ withMockADCServer(t, func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ defer mu.Unlock()
+ require.NoError(t, json.NewDecoder(r.Body).Decode(&received))
+ w.WriteHeader(http.StatusOK)
+ _ = json.NewEncoder(w).Encode(adctypes.SyncResult{Status:
adctypes.StatusSuccess})
+ })
+ d := newTestProvider(t)
+ gateway := &gatewayv1.Gateway{
+ TypeMeta: metav1.TypeMeta{Kind: "Gateway", APIVersion:
gatewayv1.GroupVersion.String()},
+ ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name:
"deleted"},
+ }
+ owner := utils.NamespacedNameKind(gateway)
+ other := types.NamespacedNameKind{Kind: "Gateway", Namespace:
"default", Name: "remaining"}
+ route := types.NamespacedNameKind{Kind: "HTTPRoute", Namespace:
"default", Name: "route"}
+ proxy := types.NamespacedNameKind{Kind: "GatewayProxy", Namespace:
"default", Name: "proxy"}
+ configs := map[types.NamespacedNameKind]adctypes.Config{proxy: {
+ Name: proxy.String(), BackendType: "apisix", ServerAddrs:
[]string{"http://apisix:9180"},
+ }}
+ for _, key := range []types.NamespacedNameKind{owner, other} {
+ resources := &adctypes.Resources{
+ SSLs: []*adctypes.SSL{{Metadata:
adctypes.Metadata{ID: key.Name, Labels: labelsOf(key)}}},
+ GlobalRules: adctypes.GlobalRule{"cors":
map[string]any{}},
+ PluginMetadata: adctypes.PluginMetadata{"http-logger":
map[string]any{}},
+ }
+ require.NoError(t, d.applyResourceState(key, configs,
[]string{adctypes.TypeSSL}, resources, labelsOf(key), pluginsFromGatewayProxy))
+ }
+ require.NoError(t, d.applyResourceState(route, configs,
[]string{adctypes.TypeService}, &adctypes.Resources{
+ Services: []*adctypes.Service{{Metadata: adctypes.Metadata{ID:
"route", Labels: labelsOf(route)}}},
+ }, labelsOf(route), pluginsNone))
+
+ require.NoError(t, d.Delete(context.Background(), gateway))
+ resources, err := d.store.GetResources(proxy.String())
+ require.NoError(t, err)
+ require.Len(t, resources.SSLs, 1)
+ assert.Equal(t, "remaining", resources.SSLs[0].ID)
+ require.Len(t, resources.Services, 1)
+ assert.Equal(t, "route", resources.Services[0].ID)
+ assert.Contains(t, resources.GlobalRules, "cors")
+ assert.Contains(t, resources.PluginMetadata, "http-logger")
+ mu.Lock()
+ defer mu.Unlock()
+ assert.Equal(t, resources.Services, received.Task.Config.Services)
+ assert.Equal(t, resources.SSLs, received.Task.Config.SSLs)
+ assert.Equal(t, resources.GlobalRules, received.Task.Config.GlobalRules)
+ assert.Equal(t, resources.PluginMetadata,
received.Task.Config.PluginMetadata)
+ assert.Empty(t, d.configManager.Get(owner))
+ assert.NotEmpty(t, d.configManager.Get(other))
+ assert.NotEmpty(t, d.configManager.Get(route))
+}
+
+func TestDeleteLastGatewayClearsProxyPlugins(t *testing.T) {
+ var mu sync.Mutex
+ var received adcclient.ADCServerRequest
+ withMockADCServer(t, func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ defer mu.Unlock()
+ received = adcclient.ADCServerRequest{}
+ assert.NoError(t, json.NewDecoder(r.Body).Decode(&received))
+ _ = json.NewEncoder(w).Encode(adctypes.SyncResult{Status:
adctypes.StatusSuccess})
+ })
+ proxy := types.NamespacedNameKind{Kind: "GatewayProxy", Namespace:
"default", Name: "proxy"}
+ configs := map[types.NamespacedNameKind]adctypes.Config{proxy: {
+ Name: proxy.String(), BackendType: "apisix", ServerAddrs:
[]string{"http://apisix:9180"},
+ }}
+ plugins := &adctypes.Resources{
+ GlobalRules: adctypes.GlobalRule{"cors": map[string]any{}},
+ PluginMetadata: adctypes.PluginMetadata{"http-logger":
map[string]any{}},
+ }
+ for name, referrers := range map[string][]types.NamespacedNameKind{
+ "last referrer": nil,
+ "IngressClass remains": {{Kind: "IngressClass", Name:
"apisix"}},
+ "another Gateway remains": {{Kind: "Gateway", Namespace:
"default", Name: "remaining"}},
+ } {
+ t.Run(name, func(t *testing.T) {
+ d := newTestProvider(t)
+ gateway := &gatewayv1.Gateway{
+ TypeMeta: metav1.TypeMeta{Kind: "Gateway",
APIVersion: gatewayv1.GroupVersion.String()},
+ ObjectMeta: metav1.ObjectMeta{Namespace:
"default", Name: "deleted"},
+ }
+ route := types.NamespacedNameKind{Kind: "HTTPRoute",
Namespace: "default", Name: "route"}
+ for _, key := range append(referrers,
utils.NamespacedNameKind(gateway)) {
+ require.NoError(t, d.applyResourceState(key,
configs, []string{adctypes.TypeSSL}, plugins, labelsOf(key),
pluginsFromGatewayProxy))
+ }
+ require.NoError(t, d.applyResourceState(route, configs,
[]string{adctypes.TypeService}, &adctypes.Resources{}, labelsOf(route),
pluginsNone))
+
+ require.NoError(t, d.Delete(context.Background(),
gateway))
+ resources, err := d.store.GetResources(proxy.String())
+ require.NoError(t, err)
+ if len(referrers) == 0 {
+ assert.Empty(t, resources.GlobalRules, "an
attached route must not keep the GatewayProxy's plugins alive")
+ assert.Empty(t, resources.PluginMetadata)
+ } else {
+ assert.Equal(t, plugins.GlobalRules,
resources.GlobalRules)
+ assert.Equal(t, plugins.PluginMetadata,
resources.PluginMetadata)
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ assert.Equal(t, resources.GlobalRules,
received.Task.Config.GlobalRules, "the deletion push must carry the remaining
global rules")
+ assert.Equal(t, resources.PluginMetadata,
received.Task.Config.PluginMetadata)
+ })
+ }
+}
diff --git a/internal/provider/common/configmanager.go
b/internal/provider/common/configmanager.go
index 387537b0..e8735839 100644
--- a/internal/provider/common/configmanager.go
+++ b/internal/provider/common/configmanager.go
@@ -18,6 +18,7 @@
package common
import (
+ "slices"
"sync"
)
@@ -148,6 +149,18 @@ func (s *ConfigManager[K, T]) Update(
return discard
}
+// HasResource reports whether a resource accepted by match still references
config.
+func (s *ConfigManager[K, T]) HasResource(config K, match func(K) bool) bool {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for resource, configKeys := range s.resourceConfigKeys {
+ if match(resource) && slices.Contains(configKeys, config) {
+ return true
+ }
+ }
+ return false
+}
+
func (s *ConfigManager[K, T]) Set(key K, cfg T) {
s.mu.Lock()
defer s.mu.Unlock()
diff --git a/test/conformance/README.md b/test/conformance/README.md
index c4ffbdcf..c97643a5 100644
--- a/test/conformance/README.md
+++ b/test/conformance/README.md
@@ -114,3 +114,23 @@ that capability.
runs in parallel and those tests report after the report is written, so a
report can read `Failed: 0` while `make conformance-test` exited non-zero;
the exit status is what to trust.
+
+## Coverage and remaining skips
+
+The default profiles are `GATEWAY-HTTP`, `GATEWAY-GRPC`, and `GATEWAY-TLS`.
+Both Admin API and standalone providers run the same tests. Tests requiring
+undeclared Extended features or other profiles are also skipped by the upstream
+suite; those skips are separate from the explicit exclusions below.
+
+| Test | Level | Remaining constraint |
+| --- | --- | --- |
+| `HTTPRouteHTTPSListener` | Core | The suite generates a certificate with a
bare `*` SAN. The translator excludes that SAN, and port 443 in this deployment
is assigned to stream TLS passthrough rather than HTTP TLS termination. |
+| `HTTPRouteMultipleGateways` | Core | Both Gateways share an address and
listener, but the test requires their dedicated `/` routes to reach different
backends independently. Separate data plane listeners or addresses are needed. |
+| `TLSRouteHostnameIntersection` | Core | Gateways share the same stream
listener and SNI routing table. A wildcard accepted by one Gateway can also
answer traffic sent to another Gateway's identical address. |
+| `HTTPRouteRedirectPortAndScheme` | Extended | Uses HTTP TLS termination,
with the same certificate and port constraints as `HTTPRouteHTTPSListener`. |
+| `TLSRouteTerminateSimpleSameNamespace` | Extended, provisional | Requires a
terminating stream listener on the selected port, while this deployment exposes
passthrough on port 443. |
+
+The invalid-backend Core tests for unknown kinds, missing Services, and denied
+cross-namespace references are enabled. They check both `ResolvedRefs=False`
+and HTTP 500 responses; these responses must remain synchronizable through both
+providers even when the translated upstream has no nodes.
diff --git a/test/conformance/conformance_test.go
b/test/conformance/conformance_test.go
index c583d253..55188ac7 100644
--- a/test/conformance/conformance_test.go
+++ b/test/conformance/conformance_test.go
@@ -30,17 +30,9 @@ var skippedTestsForSSL = []string{
tests.HTTPRouteRedirectPortAndScheme.ShortName,
}
-// Known gaps tracked for follow-up. These are genuine feature gaps rather than
-// architectural limits, so they are expected to shrink over time.
-var skippedTestsForKnownGaps = []string{
- // An unresolvable or unknown-kind backendRef must respond 500. The
- // translator already injects fault-injection for these, but they still
trip
- // against the APISIX admin API: UnknownKind fails consistently (passes
in
- // standalone), while the cross-namespace and nonexistent cases are
flaky
- // across runs. Kept skipped until the empty-upstream sync is sorted
out.
- tests.HTTPRouteInvalidBackendRefUnknownKind.ShortName,
- tests.HTTPRouteInvalidCrossNamespaceBackendRef.ShortName,
- tests.HTTPRouteInvalidNonExistentBackendRef.ShortName,
+// These tests require listener or data plane isolation that the shared
+// conformance deployment does not provide.
+var skippedTestsForSharedDataplane = []string{
// Terminate mode itself is covered by
TLSRouteListenerTerminateSupportedKinds
// and by the e2e TLSRoute suite. This provisional test additionally
requires a
// standalone Gateway with no GatewayProxy attached to reach
Accepted=True, and
@@ -71,7 +63,7 @@ func TestGatewayAPIConformance(t *testing.T) {
opts.CleanupBaseResources = true
opts.GatewayClassName = gatewayClassName
opts.SkipTests = append(opts.SkipTests, skippedTestsForSSL...)
- opts.SkipTests = append(opts.SkipTests, skippedTestsForKnownGaps...)
+ opts.SkipTests = append(opts.SkipTests,
skippedTestsForSharedDataplane...)
// Implementation is left to the flags DefaultOptions already applied.
// Assigning it here would override them and pin the report to a stale
version.
diff --git a/test/e2e/crds/v2/tls.go b/test/e2e/crds/v2/tls.go
index 8a6bc32a..d45af78f 100644
--- a/test/e2e/crds/v2/tls.go
+++ b/test/e2e/crds/v2/tls.go
@@ -162,13 +162,15 @@ spec:
assert.ElementsMatch(GinkgoT(), []string{host},
tls[0].Snis)
By("test HTTPS request to dataplane")
- Eventually(func() int {
- return s.NewAPISIXHttpsClient("api6.com").
- GET("/get").
- WithHost("api6.com").
- Expect().
- Raw().StatusCode
- }).WithTimeout(30 * time.Second).ProbeEvery(1 *
time.Second).Should(Equal(http.StatusOK))
+ s.RequestAssert(&scaffold.RequestAssert{
+ Client: s.NewAPISIXHttpsClient("api6.com"),
+ Method: "GET",
+ Path: "/get",
+ Host: "api6.com",
+ Check:
scaffold.WithExpectedStatus(http.StatusOK),
+ Timeout: 30 * time.Second,
+ Interval: time.Second,
+ })
s.NewAPISIXHttpsClient("api6.com").
GET("/get").
diff --git a/test/e2e/ingress/annotations.go b/test/e2e/ingress/annotations.go
index 622d0cb8..2f813c88 100644
--- a/test/e2e/ingress/annotations.go
+++ b/test/e2e/ingress/annotations.go
@@ -623,19 +623,24 @@ spec:
Check:
scaffold.WithExpectedStatus(http.StatusPermanentRedirect),
})
- _ = s.NewAPISIXClient().
- GET("/get").
- WithHost("httpbin.example").
- Expect().
- Status(http.StatusMovedPermanently).
-
Header("Location").IsEqual("https://httpbin.example:9443/get";)
-
- _ = s.NewAPISIXClient().
- GET("/ip").
- WithHost("httpbin.example").
- Expect().
- Status(http.StatusPermanentRedirect).
- Header("Location").IsEqual("/anything/ip")
+ s.RequestAssert(&scaffold.RequestAssert{
+ Method: "GET",
+ Path: "/get",
+ Host: "httpbin.example",
+ Checks: []scaffold.ResponseCheckFunc{
+
scaffold.WithExpectedStatus(http.StatusMovedPermanently),
+ scaffold.WithExpectedHeader("Location",
"https://httpbin.example:9443/get";),
+ },
+ })
+ s.RequestAssert(&scaffold.RequestAssert{
+ Method: "GET",
+ Path: "/ip",
+ Host: "httpbin.example",
+ Checks: []scaffold.ResponseCheckFunc{
+
scaffold.WithExpectedStatus(http.StatusPermanentRedirect),
+ scaffold.WithExpectedHeader("Location",
"/anything/ip"),
+ },
+ })
})
It("csrf", func() {
diff --git a/test/e2e/scaffold/apisix_deployer.go
b/test/e2e/scaffold/apisix_deployer.go
index 3d5b3b5c..6e722d4b 100644
--- a/test/e2e/scaffold/apisix_deployer.go
+++ b/test/e2e/scaffold/apisix_deployer.go
@@ -285,6 +285,12 @@ func (s *APISIXDeployer) deployDataplane(opts
*APISIXDeployOptions) *corev1.Serv
LabelSelector: "app.kubernetes.io/name=apisix",
})
Expect(err).ToNot(HaveOccurred(), "waiting for gateway pod
ready")
+ } else {
+ // A terminating pod can still be Ready and selected by a new
port-forward.
+ err = k8s.WaitUntilNumPodsCreatedE(s.GinkgoT, kubectlOpts,
metav1.ListOptions{
+ LabelSelector: "app.kubernetes.io/name=apisix",
+ }, 0, 60, time.Second)
+ Expect(err).ToNot(HaveOccurred(), "waiting for gateway pods to
terminate")
}
Eventually(func() bool {
