yangkaa opened a new issue, #2298:
URL: https://github.com/apache/apisix-ingress-controller/issues/2298
### Current Behavior
When the ingress-apisix-controller has been running for too long, it will
restart
### Expected Behavior
Stable operation without triggering a restart.
### Error Logs
```
3564 2024-09-18T08:04:01+08:00 ^[[33mwarn^[[0m
apisix/apisix_route.go:486 sync ApisixRoute failed, will retry
{"object": {"Type":4,"Object":{"Key":"test001/53gref0d61-8080-daxf2mr5-app.fs-0
1-0001.test.xxx.com","OldObject":null,"GroupVersion":"apisix.apache.org/v2"},"OldObject":null,"Tombstone":null},
"error": "service.spec.ports: port not defined"}
3565 2024-09-18T08:04:01+08:00 ^[[31merror^[[0m
translation/apisix_route.go:629 ApisixRoute refers to non-existent Service port
{"namespace": "test001", "port": "8080"}
3566 fatal error: concurrent map iteration and map write
3567
3568 goroutine 269 [running]:
3569 reflect.mapiternext(0x4aabef?)
3570 /usr/local/go/src/runtime/map.go:1380 +0x19
3571 reflect.(*MapIter).Next(0xc003c222c0?)
3572 /usr/local/go/src/reflect/value.go:1924 +0x7e
3573 encoding/json.mapEncoder.encode({0x21aa76b?}, 0xc0037fcb00,
{0x1f1c3c0?, 0xc000a18018?, 0x7fbee7274878?}, {0x9?, 0x0?})
3574 /usr/local/go/src/encoding/json/encode.go:797 +0x33e
3575 encoding/json.structEncoder.encode({{{0xc000a7b200?, 0xc000c42050?,
0xede6b4b3b?}, 0xc000a60b10?}}, 0xc0037fcb00, {0x2016540?, 0xc000a18000?,
0x17?}, {0x0, 0x0})
3576 /usr/local/go/src/encoding/json/encode.go:759 +0x1f4
3577 encoding/json.arrayEncoder.encode({0x6000ef?}, 0xc0037fcb00,
{0x1d3a480?, 0xc000c42100?, 0xc003c22550?}, {0x45?, 0xc6?})
3578 /usr/local/go/src/encoding/json/encode.go:914 +0xd5
3579 encoding/json.sliceEncoder.encode({0xc003c22698?}, 0xc0037fcb00,
{0x1d3a480?, 0xc000c42100?, 0xc000c42020?}, {0xa?, 0x0?})
3580 /usr/local/go/src/encoding/json/encode.go:887 +0x32f
3581 encoding/json.structEncoder.encode({{{0xc002634000?, 0xc000a44650?,
0xede6b4b00?}, 0xc000a60e40?}}, 0xc0037fcb00, {0x21097a0?, 0xc000c42000?,
0x4?}, {0x0, 0x0})
3582 /usr/local/go/src/encoding/json/encode.go:759 +0x1f4
3583 encoding/json.arrayEncoder.encode({0x5fe6fa?}, 0xc0037fcb00,
{0x1d3a3c0?, 0xc000e107f8?, 0x1b?}, {0x50?, 0x1c?})
3584 /usr/local/go/src/encoding/json/encode.go:914 +0xd5
3585 encoding/json.sliceEncoder.encode({0x1da0380?}, 0xc0037fcb00,
{0x1d3a3c0?, 0xc000e107f8?, 0x6?}, {0x7?, 0x0?})
3586 /usr/local/go/src/encoding/json/encode.go:887 +0x32f
3587 encoding/json.structEncoder.encode({{{0xc000a7a6c0?, 0x3fc?,
0x1da0380?}, 0xc000a610e0?}}, 0xc0037fcb00, {0x1fb91c0?, 0xc000e107e8?, 0x7?},
{0x0, 0x0})
3588 /usr/local/go/src/encoding/json/encode.go:759 +0x1f4
3589 encoding/json.structEncoder.encode({{{0xc002606900?, 0x0?,
0xc0005aa400?}, 0xc000a61290?}}, 0xc0037fcb00, {0x2016300?, 0xc000e106e0?,
0x414016?}, {0x0, 0x0}qDqDqDqDqDqDqDqDqDqDqDqDqDqDqDqDqDqDqDqDqDq D)
3590 /usr/local/go/src/encoding/json/encode.go:759 +0x1f4
3591 encoding/json.ptrEncoder.encode({0xc003c22c88?}, 0xc0037fcb00,
{0x215c540?, 0xc000e106e0?, 0x215c540?}, {0x4c?, 0x0?})
3592 /usr/local/go/src/encoding/json/encode.go:943 +0x21c
3593 encoding/json.(*encodeState).reflectValue(0x20bfc20?, {0x215c540?,
0xc000e106e0?, 0x50?}, {0x0?, 0x0?})
3594 /usr/local/go/src/encoding/json/encode.go:358 +0x78
3595 encoding/json.interfaceEncoder(0xc0037fcb00, {0x20bfc20?,
0xc00360ff08?, 0x0?}, {0xc0?, 0x51?})
3596 /usr/local/go/src/encoding/json/encode.go:714 +0xc8
3597 encoding/json.structEncoder.encode({{{0xc002640240?, 0x0?,
0x21c2d1f?}, 0xc000a61350?}}, 0xc0037fcb00, {0x2159de0?, 0xc00360fef0?,
0xc003c22f88?}, {0x0, 0x0})
3598 /usr/local/go/src/encoding/json/encode.go:759 +0x1f4
3599 encoding/json.ptrEncoder.encode({0x0?}, 0xc0037fcb00, {0x214b500?,
0xc00360fef0?, 0x214b500?}, {0x30?, 0x30?})
3600 /usr/local/go/src/encoding/json/encode.go:943 +0x21c
3601 encoding/json.(*encodeState).reflectValue(0xc003c23028?,
{0x214b500?, 0xc00360fef0?, 0x4?}, {0x20?, 0xa7?})
3602 /usr/local/go/src/encoding/json/encode.go:358 +0x78
3603 encoding/json.(*encodeState).marshal(0x7fbebfaa01b8?, {0x214b500?,
0xc00360fef0?}, {0x0?, 0xa4?})
3604 /usr/local/go/src/encoding/json/encode.go:330 +0xfa
3605 encoding/json.(*Encoder).Encode(0xc0025d3090, {0x214b500,
0xc00360fef0})
3606 /usr/local/go/src/encoding/json/stream.go:209 +0xf3
3607 go.uber.org/zap/zapcore.(*jsonEncoder).encodeReflected(0xc0017ca480,
{0x214b500, 0xc00360fef0})
3608
/go/pkg/mod/go.uber.org/[email protected]/zapcore/json_encoder.go:172 +0x5c
3609 go.uber.org/zap/zapcore.(*jsonEncoder).AddReflected(0xc0017ca480,
{0x21b2c2c, 0x6}, {0x214b500?, 0xc00360fef0?})
3610
/go/pkg/mod/go.uber.org/[email protected]/zapcore/json_encoder.go:180 +0x45
3611 go.uber.org/zap/zapcore.Field.AddTo({{0x21b2c2c, 0x6}, 0x17, 0x0,
{0x0, 0x0}, {0x214b500, 0xc00360fef0}}, {0x2511120, 0xc0017ca480})
3612 /go/pkg/mod/go.uber.org/[email protected]/zapcore/field.go:170
+0x80e
3613 go.uber.org/zap/zapcore.addFields(...)
3614 /go/pkg/mod/go.uber.org/[email protected]/zapcore/field.go:210
3615 go.uber.org/zap/zapcore.consoleEncoder.writeContext({0x24d95a0?},
0xc0027768a0, {0xc0036c1a80, 0x2, 0x1?})
3616
/go/pkg/mod/go.uber.org/[email protected]/zapcore/console_encoder.go:141 +0x185
3617 go.uber.org/zap/zapcore.consoleEncoder.EncodeEntry({0x2a867f8?},
{0x2, {0xc1b2a41c57deedf6, 0x768ec763fe48, 0x3598de0}, {0x0, 0x0}, {0x21e5fa8,
0x22}, {0x1, ...}, ...}, ...)
3618
/go/pkg/mod/go.uber.org/[email protected]/zapcore/console_encoder.go:119 +0x6ba
3619 go.uber.org/zap/zapcore.(*ioCore).Write(0xc00054ea20, {0x2,
{0xc1b2a41c57deedf6, 0x768ec763fe48, 0x3598de0}, {0x0, 0x0}, {0x21e5fa8, 0x22},
{0x1, ...}, ...}, ...)
3620 /go/pkg/mod/go.uber.org/[email protected]/zapcore/core.go:95 +0x7c
3621
github.com/apache/apisix-ingress-controller/pkg/log.(*Logger).write(0xc00054e9f0,
0x2, {0x21e5fa8, 0x22}, {0xc0036c1a80, 0x2, 0x2})
3622 /build/pkg/log/logger.go:59 +0x1f8
3623
github.com/apache/apisix-ingress-controller/pkg/log.(*Logger).Errorw(...)
3624 /build/pkg/log/logger.go:168
3625 github.com/apache/apisix-ingress-controller/pkg/log.Errorw(...)
3626 /build/pkg/log/default_logger.go:98
3627
github.com/apache/apisix-ingress-controller/pkg/providers/apisix.(*apisixRouteController).sync(0xc00072e700,
{0x24f9640, 0xc000649090}, 0xc002c9c640)
3628 /build/pkg/providers/apisix/apisix_route.go:327 +0x954
3629
github.com/apache/apisix-ingress-controller/pkg/providers/apisix.(*apisixRouteController).runWorker(0xc00072e700,
{0x24f9640, 0xc000649090})
3630 /build/pkg/providers/apisix/apisix_route.go:126 +0x86
3631 created by
github.com/apache/apisix-ingress-controller/pkg/providers/apisix.(*apisixRouteController).run
3632 /build/pkg/providers/apisix/apisix_route.go:111 +0x265
3633
3634 goroutine 1 [chan receive, 2172 minutes]:
3635
github.com/apache/apisix-ingress-controller/pkg/providers.(*Controller).run(0xc000147000,
{0x24f9640, 0xc000648af0})
3636 /build/pkg/providers/controller.go:566 +0x138f
3637
github.com/apache/apisix-ingress-controller/pkg/providers.(*Controller).Run(0xc000147000,
{0x24f9640?, 0xc00057a0f0?})
3638 /build/pkg/providers/controller.go:186 +0x35e
3639
github.com/apache/apisix-ingress-controller/cmd/ingress.NewIngressCommand.func1(0xc000004c00?,
{0x21aa43b?, 0x15?, 0x16?})
3640 /build/cmd/ingress/ingress.go:148 +0x811
3641 github.com/spf13/cobra.(*Command).execute(0xc000004c00,
{0xc0003bd8c0, 0x15, 0x16})
3642 /go/pkg/mod/github.com/spf13/[email protected]/command.go:987
+0xa91
3643 github.com/spf13/cobra.(*Command).ExecuteC(0xc000004900)
3644 /go/pkg/mod/github.com/spf13/[email protected]/command.go:1115
+0x425
3645 github.com/spf13/cobra.(*Command).Execute(0x4069d9?)
3646 /go/pkg/mod/github.com/spf13/[email protected]/command.go:1039
+0x19
3647 main.main()
3648 /build/main.go:26 +0x1e
3649
3650 goroutine 9 [chan receive, 2172 minutes]:
3651
github.com/apache/apisix-ingress-controller/cmd/ingress.contextWithSignalCancel.func1()
3652 /build/cmd/ingress/ingress.go:51 +0x89
3653 created by
github.com/apache/apisix-ingress-controller/cmd/ingress.contextWithSignalCancel
3654 /build/cmd/ingress/ingress.go:47 +0xab
3655
3656 goroutine 23 [syscall, 2172 minutes]:
3657 os/signal.signal_recv()
3658 /usr/local/go/src/runtime/sigqueue.go:152 +0x2f
3659 os/signal.loop()
3660 /usr/local/go/src/os/signal/signal_unix.go:23 +0x19
3661 created by os/signal.Notify.func1.1
3662 /usr/local/go/src/os/signal/signal.go:151 +0x2a
3663
3664 goroutine 10 [chan receive]:
3665 k8s.io/apimachinery/pkg/watch.(*Broadcaster).loop(0xc000648460)
3666 /go/pkg/mod/k8s.io/[email protected]/pkg/watch/mux.go:268
+0x65
3667 created by k8s.io/apimachinery/pkg/watch.NewLongQueueBroadcaster
3668 /go/pkg/mod/k8s.io/[email protected]/pkg/watch/mux.go:93
+0x116
3669
3670 goroutine 11 [runnable]:
3671 sync.(*RWMutex).Lock(0xc00062b920?)
3672 /usr/local/go/src/sync/rwmutex.go:141 +0x85
3673
k8s.io/client-go/tools/record.(*eventLogger).eventObserve(0xc00062b920,
0xc00283b900, {0xc0027b5200, 0x102})
3674
/go/pkg/mod/k8s.io/[email protected]/tools/record/events_cache.go:335 +0xf6
3675
k8s.io/client-go/tools/record.(*EventCorrelator).EventCorrelate(0xc0002c6c00,
0xc0006e7ed0?)
3676
/go/pkg/mod/k8s.io/[email protected]/tools/record/events_cache.go:515 +0x54
3677
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).recordToSink(0xc0000b5980,
{0x24f7aa0, 0xc0006e7ed0}, 0xc0021ebb80, 0xc003b7bf01?)
3678
/go/pkg/mod/k8s.io/[email protected]/tools/record/event.go:214 +0xc5
3679
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartRecordingToSink.func1(0xc002d7e780?)
3680
/go/pkg/mod/k8s.io/[email protected]/tools/record/event.go:200 +0x2f
3681
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1()
3682
/go/pkg/mod/k8s.io/[email protected]/tools/record/event.go:327 +0xa2
3683 created by
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher
3684
/go/pkg/mod/k8s.io/[email protected]/tools/record/event.go:318 +0x13d
3685
3686 goroutine 36 [IO wait]:
3687 internal/poll.runtime_pollWait(0x7fbebfc91d28, 0x72)
3688 /usr/local/go/src/runtime/netpoll.go:306 +0x89
3689 internal/poll.(*pollDesc).wait(0xc0000b5d00?, 0xc00220f500?, 0x0)
3690 /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
3691 internal/poll.(*pollDesc).waitRead(...)
3692 /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
3693 internal/poll.(*FD).Read(0xc0000b5d00, {0xc00220f500, 0x5500,
0x5500})
3694 /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299
3695 net.(*netFD).Read(0xc0000b5d00, {0xc00220f500?, 0xc00220f505?,
0x35b?})
3696 /usr/local/go/src/net/fd_posix.go:55 +0x29
3697 net.(*conn).Read(0xc000532020, {0xc00220f500?, 0x7d3fa3?,
0xc0002f08b0?})
3698 /usr/local/go/src/net/net.go:183 +0x45
3699 crypto/tls.(*atLeastReader).Read(0xc0039490e0, {0xc00220f500?,
0xc0039490e0?, 0x0?})
3700 /usr/local/go/src/crypto/tls/conn.go:788 +0x3d
3701 bytes.(*Buffer).ReadFrom(0xc0002f0990, {0x24d69e0, 0xc0039490e0})
3702 /usr/local/go/src/bytes/buffer.go:202 +0x98
3703 crypto/tls.(*Conn).readFromUntil(0xc0002f0700, {0x24db540?,
0xc000532020}, 0x5500?)
3704 /usr/local/go/src/crypto/tls/conn.go:810 +0xe5
3705 crypto/tls.(*Conn).readRecordOrCCS(0xc0002f0700, 0x0)
3706 /usr/local/go/src/crypto/tls/conn.go:617 +0x116
3707 crypto/tls.(*Conn).readRecord(...)
3708 /usr/local/go/src/crypto/tls/conn.go:583
3709 crypto/tls.(*Conn).Read(0xc0002f0700, {0xc00074b000, 0x1000,
0xc00395e18a?})
3710 /usr/local/go/src/crypto/tls/conn.go:1316 +0x16f
3711 bufio.(*Reader).Read(0xc0001fff80, {0xc00072c200, 0x9,
0xc000b27d38?})
3712 /usr/local/go/src/bufio/bufio.go:237 +0x1bb
3713 io.ReadAtLeast({0x24d67e0, 0xc0001fff80}, {0xc00072c200, 0x9, 0x9},
0x9)
3714 /usr/local/go/src/io/io.go:332 +0x9a
3715 io.ReadFull(...)
3716 /usr/local/go/src/io/io.go:351
3717 golang.org/x/net/http2.readFrameHeader({0xc00072c200?, 0x9?,
0xc000000000?}, {0x24d67e0?, 0xc0001fff80?})
3718 /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
3719 golang.org/x/net/http2.(*Framer).ReadFrame(0xc00072c1c0)
3720 /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
3721 golang.org/x/net/http2.(*clientConnReadLoop).run(0xc000b27f98)
3722 /go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2275
+0x12e
3723 golang.org/x/net/http2.(*ClientConn).readLoop(0xc000732180)
3724 /go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2170
+0x6f
3725 created by golang.org/x/net/http2.(*Transport).newClientConn
3726 /go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:821
+0xc1f
```
### Steps to Reproduce
1. Install APISIX Ingress controller with the Helm chart.
2. Create a Route with ApisixRoute resource.
3. After running for a long time, I noticed it had restarted, so I checked
the logs.
### Environment
- APISIX Ingress controller version (run `apisix-ingress-controller version
--long`)
```
Version: 1.8.2
Git SHA: no-git-module
Go Version: go1.20.14
Building OS/Arch: linux/amd64
Running OS/Arch: linux/amd64
```
- Kubernetes cluster version (run `kubectl version`)
```
Client Version: version.Info{Major:"1", Minor:"25", GitVersion:"v1.25.10",
GitCommit:"e770bdbb87cccdc2daa790ecd69f40cf4df3cc9d", GitTreeState:"clean",
BuildDate:"2023-05-17T14:12:20Z", GoVersion:"go1.19.9", Compiler:"gc",
Platform:"linux/amd64"}
Kustomize Version: v4.5.7
Server Version: version.Info{Major:"1", Minor:"25", GitVersion:"v1.25.10",
GitCommit:"e770bdbb87cccdc2daa790ecd69f40cf4df3cc9d", GitTreeState:"clean",
BuildDate:"2023-05-17T14:06:35Z", GoVersion:"go1.19.9", Compiler:"gc",
Platform:"linux/amd64"}
```
- OS version if running APISIX Ingress controller in a bare-metal
environment (run `uname -a`)
```
Linux master01-172-21-228-13 5.4.188-1.el7.elrepo.x86_64 #1 SMP Mon Mar 28
09:10:07 EDT 2022 x86_64 x86_64 x86_64 GNU/Linux
```
```
CentOS 7
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
