This is an automated email from the ASF dual-hosted git repository. tokers pushed a commit to branch fix/connect-to-apisix in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git
commit 0b3fd0413df76bc6840c7accbb5f7b68ee6039a2 Author: Chao Zhang <[email protected]> AuthorDate: Fri Nov 26 16:08:03 2021 +0800 fix: delete the cluster object when give up the leadership In the previous implementation, the cluster object won't be removed if the controller gives up the leadership, which causes some error state is cached even if the next leader takes office, this hurts the running. In this pull request, the cluster is deleted once the leadership is given up, and the retry-logic will be aborted (for the cluster added in last term) if the context is canceld. Signed-off-by: Chao Zhang <[email protected]> --- pkg/apisix/apisix.go | 11 +++++++++++ pkg/apisix/cluster.go | 12 +++++++++--- pkg/apisix/plugin_test.go | 4 ++-- pkg/apisix/schema_test.go | 3 ++- pkg/ingress/controller.go | 9 +++++++++ pkg/kube/apisix/apis/config/v1/zz_generated.deepcopy.go | 1 + .../apisix/apis/config/v2alpha1/zz_generated.deepcopy.go | 1 + pkg/kube/apisix/apis/config/v2beta1/zz_generated.deepcopy.go | 1 + pkg/kube/apisix/apis/config/v2beta2/zz_generated.deepcopy.go | 1 + pkg/types/apisix/v1/zz_generated.deepcopy.go | 1 + tools.go | 1 + 11 files changed, 39 insertions(+), 6 deletions(-) diff --git a/pkg/apisix/apisix.go b/pkg/apisix/apisix.go index fe89c86..a9cb3f8 100644 --- a/pkg/apisix/apisix.go +++ b/pkg/apisix/apisix.go @@ -32,6 +32,8 @@ type APISIX interface { UpdateCluster(context.Context, *ClusterOptions) error // ListClusters lists all APISIX clusters. ListClusters() []Cluster + // DeleteCluster deletes the target APISIX cluster by its name. + DeleteCluster(string) } // Cluster defines specific operations that can be applied in an APISIX @@ -203,3 +205,12 @@ func (c *apisix) UpdateCluster(ctx context.Context, co *ClusterOptions) error { c.clusters[co.Name] = cluster return nil } + +func (c *apisix) DeleteCluster(name string) { + c.mu.Lock() + defer c.mu.Unlock() + + // Don't have to close or free some resources in that cluster, so + // just delete its index. + delete(c.clusters, name) +} diff --git a/pkg/apisix/cluster.go b/pkg/apisix/cluster.go index 42a7f7a..4d947e1 100644 --- a/pkg/apisix/cluster.go +++ b/pkg/apisix/cluster.go @@ -177,10 +177,16 @@ func (c *cluster) syncCache(ctx context.Context) { Steps: 5, } var lastSyncErr error - err := wait.ExponentialBackoff(backoff, func() (done bool, _ error) { + err := wait.ExponentialBackoff(backoff, func() (done bool, err error) { // impossibly return: false, nil // so can safe used done, lastSyncErr = c.syncCacheOnce(ctx) + select { + case <-ctx.Done(): + err = context.Canceled + default: + break + } return }) if err != nil { @@ -197,7 +203,7 @@ func (c *cluster) syncCache(ctx context.Context) { func (c *cluster) syncCacheOnce(ctx context.Context) (bool, error) { routes, err := c.route.List(ctx) if err != nil { - log.Errorf("failed to list route in APISIX: %s", err) + log.Errorf("failed to list routes in APISIX: %s", err) return false, err } upstreams, err := c.upstream.List(ctx) @@ -327,7 +333,7 @@ func (c *cluster) syncSchema(ctx context.Context, interval time.Duration) { for { if err := c.syncSchemaOnce(ctx); err != nil { - log.Warnf("failed to sync schema: %s", err) + log.Errorf("failed to sync schema: %s", err) c.metricsCollector.IncrSyncOperation("schema", "failure") } diff --git a/pkg/apisix/plugin_test.go b/pkg/apisix/plugin_test.go index 159e887..fa36537 100644 --- a/pkg/apisix/plugin_test.go +++ b/pkg/apisix/plugin_test.go @@ -22,10 +22,10 @@ import ( "strings" "testing" - "github.com/apache/apisix-ingress-controller/pkg/metrics" "github.com/stretchr/testify/assert" - "golang.org/x/net/nettest" + + "github.com/apache/apisix-ingress-controller/pkg/metrics" ) type fakeAPISIXPluginSrv struct { diff --git a/pkg/apisix/schema_test.go b/pkg/apisix/schema_test.go index a9cb4a1..2d6c08e 100644 --- a/pkg/apisix/schema_test.go +++ b/pkg/apisix/schema_test.go @@ -21,9 +21,10 @@ import ( "strings" "testing" - "github.com/apache/apisix-ingress-controller/pkg/metrics" "github.com/stretchr/testify/assert" "golang.org/x/net/nettest" + + "github.com/apache/apisix-ingress-controller/pkg/metrics" ) type fakeAPISIXSchemaSrv struct { diff --git a/pkg/ingress/controller.go b/pkg/ingress/controller.go index ec3deb6..06ec18d 100644 --- a/pkg/ingress/controller.go +++ b/pkg/ingress/controller.go @@ -347,6 +347,11 @@ func (c *Controller) Run(stop chan struct{}) error { zap.String("namespace", c.namespace), zap.String("pod", c.name), ) + c.MetricsCollector.ResetLeader(false) + // delete the old APISIX cluster, so that the cached state + // like synchronization won't be used next time the candidate + // becomes the leader again. + c.apisix.DeleteCluster(c.cfg.APISIX.DefaultClusterName) } }, OnStoppedLeading: func() { @@ -355,6 +360,10 @@ func (c *Controller) Run(stop chan struct{}) error { zap.String("pod", c.name), ) c.MetricsCollector.ResetLeader(false) + // delete the old APISIX cluster, so that the cached state + // like synchronization won't be used next time the candidate + // becomes the leader again. + c.apisix.DeleteCluster(c.cfg.APISIX.DefaultClusterName) }, }, // Set it to false as current leaderelection implementation will report diff --git a/pkg/kube/apisix/apis/config/v1/zz_generated.deepcopy.go b/pkg/kube/apisix/apis/config/v1/zz_generated.deepcopy.go index 0d234e0..d6ae7cb 100644 --- a/pkg/kube/apisix/apis/config/v1/zz_generated.deepcopy.go +++ b/pkg/kube/apisix/apis/config/v1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated // Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/pkg/kube/apisix/apis/config/v2alpha1/zz_generated.deepcopy.go b/pkg/kube/apisix/apis/config/v2alpha1/zz_generated.deepcopy.go index b6d05ce..5cd4721 100644 --- a/pkg/kube/apisix/apis/config/v2alpha1/zz_generated.deepcopy.go +++ b/pkg/kube/apisix/apis/config/v2alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated // Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/pkg/kube/apisix/apis/config/v2beta1/zz_generated.deepcopy.go b/pkg/kube/apisix/apis/config/v2beta1/zz_generated.deepcopy.go index e835ddf..1cfb91c 100644 --- a/pkg/kube/apisix/apis/config/v2beta1/zz_generated.deepcopy.go +++ b/pkg/kube/apisix/apis/config/v2beta1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated // Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/pkg/kube/apisix/apis/config/v2beta2/zz_generated.deepcopy.go b/pkg/kube/apisix/apis/config/v2beta2/zz_generated.deepcopy.go index 7172824..cf41588 100644 --- a/pkg/kube/apisix/apis/config/v2beta2/zz_generated.deepcopy.go +++ b/pkg/kube/apisix/apis/config/v2beta2/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated // Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/pkg/types/apisix/v1/zz_generated.deepcopy.go b/pkg/types/apisix/v1/zz_generated.deepcopy.go index a30ef8a..a705821 100644 --- a/pkg/types/apisix/v1/zz_generated.deepcopy.go +++ b/pkg/types/apisix/v1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated // Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/tools.go b/tools.go index f9e2de6..73aca5b 100644 --- a/tools.go +++ b/tools.go @@ -1,3 +1,4 @@ +//go:build tools // +build tools // Licensed to the Apache Software Foundation (ASF) under one or more
