[ https://issues.apache.org/jira/browse/FLINK-27821?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Hector Miuler Malpica Gallegos updated FLINK-27821: --------------------------------------------------- Description: My operator was installed with following command: {{git clone g...@github.com:apache/flink-kubernetes-operator.git}} {{git checkout 207b17b}} {{cd flink-kubernetes-operator}} {{helm -debug upgrade -i flink-kubernetes-operator helm/flink-kubernetes-operator --set image.repository=ghcr.io/apache/flink-kubernetes-operator -set image.tag=207b17b}} Then I create a flinkDeployment and flinkSessionJob, then I deleted the deployment that generated the flinkDeployment, then reinstall operator and finally I wanted to delete the flinkdeployment kubectl logs -f pod/flink-kubernetes-operator-5cf66cbbcb-bpl9p {{2022-05-27 13:40:22,027 o.a.f.k.o.c.FlinkDeploymentController [INFO ][flink-system/migration] Deleting FlinkDeployment}} {{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.c.AbstractChannel [WARN ] Force-closing a channel whose registration task was not accepted by an event loop: [id: 0xb2062900]}} {{java.util.concurrent.RejectedExecutionException: event executor terminated}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe.register(AbstractChannel.java:483)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:87)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:81)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.MultithreadEventLoopGroup.register(MultithreadEventLoopGroup.java:86)}} {{ at org.apache.flink.shaded.netty4.io.netty.bootstrap.AbstractBootstrap.initAndRegister(AbstractBootstrap.java:323)}} {{ at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.doResolveAndConnect(Bootstrap.java:155)}} {{ at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:139)}} {{ at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:123)}} {{ at org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:467)}} {{ at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}} {{ at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}} {{ at org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}} {{ at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown Source)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}} {{ at java.base/java.lang.Thread.run(Unknown Source)}} {{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.u.c.D.rejectedExecution [ERROR] Failed to submit a listener notification task. Event loop shut down?}} {{java.util.concurrent.RejectedExecutionException: event executor terminated}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:499)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:184)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:95)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:30)}} {{ at org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:471)}} {{ at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}} {{ at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}} {{ at org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}} {{ at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown Source)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}} {{ at java.base/java.lang.Thread.run(Unknown Source)}} {{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [ERROR][flink-system/migration-cosmosdb] REST service in session cluster is bad now}} {{java.util.concurrent.TimeoutException}} {{ at java.base/java.util.concurrent.CompletableFuture.timedGet(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture.get(Unknown Source)}} {{ at org.apache.flink.kubernetes.operator.service.FlinkService.listJobs(FlinkService.java:331)}} {{ at org.apache.flink.kubernetes.operator.observer.deployment.SessionObserver.observeFlinkCluster(SessionObserver.java:45)}} {{ at org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:92)}} {{ at org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:56)}} {{ at org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:101)}} {{ at org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:59)}} {{ at io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:68)}} {{ at io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:50)}} {{ at io.javaoperatorsdk.operator.api.monitoring.Metrics.timeControllerExecution(Metrics.java:34)}} {{ at io.javaoperatorsdk.operator.processing.Controller.cleanup(Controller.java:49)}} {{ at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleCleanup(ReconciliationDispatcher.java:252)}} {{ at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleDispatch(ReconciliationDispatcher.java:72)}} {{ at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleExecution(ReconciliationDispatcher.java:50)}} {{ at io.javaoperatorsdk.operator.processing.event.EventProcessor$ControllerExecution.run(EventProcessor.java:349)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}} {{ at java.base/java.lang.Thread.run(Unknown Source)}} {{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [INFO ][flink-system/migration-cosmosdb] Observing JobManager deployment. Previous status: READY}} {{2022-05-27 13:40:34,049 o.a.f.k.o.o.d.SessionObserver [ERROR][flink-system/migration-cosmosdb] Missing JobManager deployment}} was: My operator was installed with following command: {{git clone g...@github.com:apache/flink-kubernetes-operator.git}} {{git checkout 207b17b}} {{cd flink-kubernetes-operator}} {{helm -debug upgrade -i flink-kubernetes-operator helm/flink-kubernetes-operator --set image.repository=ghcr.io/apache/flink-kubernetes-operator -set image.tag=207b17b}} Then I create a flinkDeployment and flinkSessionJob, then I deleted the deployment that generated the flinkDeployment, then reinstall operator and finally I wanted to delete the flinkdeployment kubectl logs -f pod/flink-kubernetes-operator-5cf66cbbcb-bpl9p {{2022-05-27 13:40:22,027 o.a.f.k.o.c.FlinkDeploymentController [INFO ][flink-system/migration] Deleting FlinkDeployment}} {{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.c.AbstractChannel [WARN ] Force-closing a channel whose registration task was not accepted by an event loop: [id: 0xb2062900]}} {{java.util.concurrent.RejectedExecutionException: event executor terminated}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe.register(AbstractChannel.java:483)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:87)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:81)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.MultithreadEventLoopGroup.register(MultithreadEventLoopGroup.java:86)}} {{ at org.apache.flink.shaded.netty4.io.netty.bootstrap.AbstractBootstrap.initAndRegister(AbstractBootstrap.java:323)}} {{ at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.doResolveAndConnect(Bootstrap.java:155)}} {{ at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:139)}} {{ at org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:123)}} {{ at org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:467)}} {{ at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}} {{ at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}} {{ at org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}} {{ at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown Source)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}} {{ at java.base/java.lang.Thread.run(Unknown Source)}} {{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.u.c.D.rejectedExecution [ERROR] Failed to submit a listener notification task. Event loop shut down?}} {{java.util.concurrent.RejectedExecutionException: event executor terminated}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:499)}} {{ at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:184)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:95)}} {{ at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:30)}} {{ at org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:471)}} {{ at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}} {{ at org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}} {{ at org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}} {{ at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown Source)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}} {{ at java.base/java.lang.Thread.run(Unknown Source)}} {{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [ERROR][flink-system/migration-cosmosdb] REST service in session cluster is bad now}} {{java.util.concurrent.TimeoutException}} {{ at java.base/java.util.concurrent.CompletableFuture.timedGet(Unknown Source)}} {{ at java.base/java.util.concurrent.CompletableFuture.get(Unknown Source)}} {{ at org.apache.flink.kubernetes.operator.service.FlinkService.listJobs(FlinkService.java:331)}} {{ at org.apache.flink.kubernetes.operator.observer.deployment.SessionObserver.observeFlinkCluster(SessionObserver.java:45)}} {{ at org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:92)}} {{ at org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:56)}} {{ at org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:101)}} {{ at org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:59)}} {{ at io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:68)}} {{ at io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:50)}} {{ at io.javaoperatorsdk.operator.api.monitoring.Metrics.timeControllerExecution(Metrics.java:34)}} {{ at io.javaoperatorsdk.operator.processing.Controller.cleanup(Controller.java:49)}} {{ at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleCleanup(ReconciliationDispatcher.java:252)}} {{ at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleDispatch(ReconciliationDispatcher.java:72)}} {{ at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleExecution(ReconciliationDispatcher.java:50)}} {{ at io.javaoperatorsdk.operator.processing.event.EventProcessor$ControllerExecution.run(EventProcessor.java:349)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}} {{ at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}} {{ at java.base/java.lang.Thread.run(Unknown Source)}} {{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [INFO ][flink-system/migration-cosmosdb] Observing JobManager deployment. Previous status: READY}} {{2022-05-27 13:40:34,049 o.a.f.k.o.o.d.SessionObserver [ERROR][flink-system/migration-cosmosdb] Missing JobManager deployment}} > Cannot delete flinkdeployment when the pod and deployment deleted manually > -------------------------------------------------------------------------- > > Key: FLINK-27821 > URL: https://issues.apache.org/jira/browse/FLINK-27821 > Project: Flink > Issue Type: Bug > Components: Kubernetes Operator > Affects Versions: kubernetes-operator-1.1.0 > Reporter: Hector Miuler Malpica Gallegos > Priority: Major > > My operator was installed with following command: > > {{git clone g...@github.com:apache/flink-kubernetes-operator.git}} > {{git checkout 207b17b}} > {{cd flink-kubernetes-operator}} > {{helm -debug upgrade -i flink-kubernetes-operator > helm/flink-kubernetes-operator --set > image.repository=ghcr.io/apache/flink-kubernetes-operator -set > image.tag=207b17b}} > > Then I create a flinkDeployment and flinkSessionJob, then I deleted the > deployment that generated the flinkDeployment, then reinstall operator and > finally I wanted to delete the flinkdeployment > > kubectl logs -f pod/flink-kubernetes-operator-5cf66cbbcb-bpl9p > > {{2022-05-27 13:40:22,027 o.a.f.k.o.c.FlinkDeploymentController [INFO > ][flink-system/migration] Deleting FlinkDeployment}} > {{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.c.AbstractChannel [WARN ] > Force-closing a channel whose registration task was not accepted by an event > loop: [id: 0xb2062900]}} > {{java.util.concurrent.RejectedExecutionException: event executor terminated}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AbstractUnsafe.register(AbstractChannel.java:483)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:87)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.channel.SingleThreadEventLoop.register(SingleThreadEventLoop.java:81)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.channel.MultithreadEventLoopGroup.register(MultithreadEventLoopGroup.java:86)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.bootstrap.AbstractBootstrap.initAndRegister(AbstractBootstrap.java:323)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.doResolveAndConnect(Bootstrap.java:155)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:139)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap.connect(Bootstrap.java:123)}} > {{ at > org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:467)}} > {{ at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}} > {{ at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}} > {{ at > org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}} > {{ at > java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown > Source)}} > {{ at > java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown > Source)}} > {{ at > java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source)}} > {{ at > java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown > Source)}} > {{ at > java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown > Source)}} > {{ at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}} > {{ at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}} > {{ at java.base/java.lang.Thread.run(Unknown Source)}} > {{2022-05-27 13:40:34,047 o.a.f.s.n.i.n.u.c.D.rejectedExecution [ERROR] > Failed to submit a listener notification task. Event loop shut down?}} > {{java.util.concurrent.RejectedExecutionException: event executor terminated}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:841)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:499)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:184)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:95)}} > {{ at > org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:30)}} > {{ at > org.apache.flink.runtime.rest.RestClient.submitRequest(RestClient.java:471)}} > {{ at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:390)}} > {{ at > org.apache.flink.runtime.rest.RestClient.sendRequest(RestClient.java:304)}} > {{ at > org.apache.flink.client.program.rest.RestClusterClient.lambda$null$32(RestClusterClient.java:863)}} > {{ at > java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown > Source)}} > {{ at > java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown > Source)}} > {{ at > java.base/java.util.concurrent.CompletableFuture.postFire(Unknown Source)}} > {{ at > java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown > Source)}} > {{ at > java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown > Source)}} > {{ at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}} > {{ at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}} > {{ at java.base/java.lang.Thread.run(Unknown Source)}} > {{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver > [ERROR][flink-system/migration-cosmosdb] REST service in session cluster is > bad now}} > {{java.util.concurrent.TimeoutException}} > {{ at > java.base/java.util.concurrent.CompletableFuture.timedGet(Unknown Source)}} > {{ at java.base/java.util.concurrent.CompletableFuture.get(Unknown > Source)}} > {{ at > org.apache.flink.kubernetes.operator.service.FlinkService.listJobs(FlinkService.java:331)}} > {{ at > org.apache.flink.kubernetes.operator.observer.deployment.SessionObserver.observeFlinkCluster(SessionObserver.java:45)}} > {{ at > org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:92)}} > {{ at > org.apache.flink.kubernetes.operator.observer.deployment.AbstractDeploymentObserver.observe(AbstractDeploymentObserver.java:56)}} > {{ at > org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:101)}} > {{ at > org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:59)}} > {{ at > io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:68)}} > {{ at > io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:50)}} > {{ at > io.javaoperatorsdk.operator.api.monitoring.Metrics.timeControllerExecution(Metrics.java:34)}} > {{ at > io.javaoperatorsdk.operator.processing.Controller.cleanup(Controller.java:49)}} > {{ at > io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleCleanup(ReconciliationDispatcher.java:252)}} > {{ at > io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleDispatch(ReconciliationDispatcher.java:72)}} > {{ at > io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleExecution(ReconciliationDispatcher.java:50)}} > {{ at > io.javaoperatorsdk.operator.processing.event.EventProcessor$ControllerExecution.run(EventProcessor.java:349)}} > {{ at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)}} > {{ at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)}} > {{ at java.base/java.lang.Thread.run(Unknown Source)}} > {{2022-05-27 13:40:34,048 o.a.f.k.o.o.d.SessionObserver [INFO > ][flink-system/migration-cosmosdb] Observing JobManager deployment. Previous > status: READY}} > {{2022-05-27 13:40:34,049 o.a.f.k.o.o.d.SessionObserver > [ERROR][flink-system/migration-cosmosdb] Missing JobManager deployment}} > > -- This message was sent by Atlassian Jira (v8.20.7#820007)