[
https://issues.apache.org/jira/browse/IGNITE-20988?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Valery Shorin updated IGNITE-20988:
-----------------------------------
Labels: ise (was: )
> Ignite can't execute tasks - "Topology projection is empty"
> -----------------------------------------------------------
>
> Key: IGNITE-20988
> URL: https://issues.apache.org/jira/browse/IGNITE-20988
> Project: Ignite
> Issue Type: Bug
> Affects Versions: 2.16
> Reporter: Valery Shorin
> Priority: Critical
> Labels: ise
> Attachments: image-2023-11-28-16-52-32-145.png
>
>
> On our Ignite cluster logs are flooded by the following exceptions:
> {code:java}
> 2023-11-28 16:39:54.253 [INFO
> ][db-checkpoint-thread-#92][org.apache.ignite.internal.processors.cache.persistence.checkpoint.Checkpointer]
> Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=6ms,
> checkpointLockWait=0ms, checkpointListenersExecuteTime=5ms,
> checkpointLockHoldTime=6ms, reason='timeout']
> 2023-11-28 16:40:18.905 [WARN
> ][client-connector-#83][org.apache.ignite.internal.processors.odbc.ClientListenerNioListener]
> Failed to process client request
> [req=o.a.i.i.processors.platform.client.compute.ClientExecuteTaskRequest@33b4781e,
> msg=Topology projection is empty.]
> org.apache.ignite.internal.processors.platform.client.IgniteClientException:
> Topology projection is empty.
> at
> org.apache.ignite.internal.processors.platform.client.compute.ClientComputeTask.execute(ClientComputeTask.java:117)
> ~[ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.platform.client.compute.ClientExecuteTaskRequest.process(ClientExecuteTaskRequest.java:89)
> ~[ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.platform.client.ClientRequestHandler.handle(ClientRequestHandler.java:101)
> ~[ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:211)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:57)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.nio.GridNioFilterChain$TailFilter.onMessageReceived(GridNioFilterChain.java:279)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.nio.GridNioFilterAdapter.proceedMessageReceived(GridNioFilterAdapter.java:109)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.nio.GridNioAsyncNotifyFilter$3.body(GridNioAsyncNotifyFilter.java:97)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.worker.GridWorkerPool$1.run(GridWorkerPool.java:70)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> [?:?]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> [?:?]
> at java.lang.Thread.run(Thread.java:834) [?:?]
> 2023-11-28 16:40:48.895 [WARN
> ][client-connector-#82][org.apache.ignite.internal.processors.odbc.ClientListenerNioListener]
> Failed to process client request
> [req=o.a.i.i.processors.platform.client.compute.ClientExecuteTaskRequest@aa13d84,
> msg=Topology projection is empty.]
> org.apache.ignite.internal.processors.platform.client.IgniteClientException:
> Topology projection is empty.
> at
> org.apache.ignite.internal.processors.platform.client.compute.ClientComputeTask.execute(ClientComputeTask.java:117)
> ~[ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.platform.client.compute.ClientExecuteTaskRequest.process(ClientExecuteTaskRequest.java:89)
> ~[ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.platform.client.ClientRequestHandler.handle(ClientRequestHandler.java:101)
> ~[ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:211)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:57)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.nio.GridNioFilterChain$TailFilter.onMessageReceived(GridNioFilterChain.java:279)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.nio.GridNioFilterAdapter.proceedMessageReceived(GridNioFilterAdapter.java:109)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.nio.GridNioAsyncNotifyFilter$3.body(GridNioAsyncNotifyFilter.java:97)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.worker.GridWorkerPool$1.run(GridWorkerPool.java:70)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> [?:?]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> [?:?]
> at java.lang.Thread.run(Thread.java:834) [?:?]
> 2023-11-28 16:41:00.885 [INFO
> ][client-connector-#82][org.apache.ignite.internal.managers.deployment.GridDeploymentLocalStore]
> Task locally deployed: class
> com.sbt.gridcenter.plugin.cluster.ClusterInfoSQLTask
> 2023-11-28 16:41:00.955 [WARN
> ][client-connector-#83][org.apache.ignite.internal.processors.odbc.ClientListenerNioListener]
> Failed to process client request
> [req=o.a.i.i.processors.platform.client.compute.ClientExecuteTaskRequest@3084b0a4,
> msg=Topology projection is empty.]
> org.apache.ignite.internal.processors.platform.client.IgniteClientException:
> Topology projection is empty.
> at
> org.apache.ignite.internal.processors.platform.client.compute.ClientComputeTask.execute(ClientComputeTask.java:117)
> ~[ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.platform.client.compute.ClientExecuteTaskRequest.process(ClientExecuteTaskRequest.java:89)
> ~[ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.platform.client.ClientRequestHandler.handle(ClientRequestHandler.java:101)
> ~[ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:211)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:57)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.nio.GridNioFilterChain$TailFilter.onMessageReceived(GridNioFilterChain.java:279)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.nio.GridNioFilterAdapter.proceedMessageReceived(GridNioFilterAdapter.java:109)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.nio.GridNioAsyncNotifyFilter$3.body(GridNioAsyncNotifyFilter.java:97)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> org.apache.ignite.internal.util.worker.GridWorkerPool$1.run(GridWorkerPool.java:70)
> [ignite-core-16.0.0-SNAPSHOT.jar:16.0.0-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> [?:?]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> [?:?]
> at java.lang.Thread.run(Thread.java:834) [?:?]
> {code}
> From my understanding this issue might be caused by
> https://issues.apache.org/jira/browse/IGNITE-18545 fix where in
> {{GridTaskProcessor}} the following change was made:
> !image-2023-11-28-16-52-32-145.png!
> With this dirty fix "empty topology" exception is no longer reproducible:
> {code:java}
> Index:
> modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
> IDEA additional info:
> Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
> <+>UTF-8
> ===================================================================
> diff --git
> a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
>
> b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
> ---
> a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
> (revision d943c4cf1f8d00c0376af51945c2da453eeb745c)
> +++
> b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
> (date 1701174982188)
> @@ -651,7 +651,7 @@
> if (topPred == null) {
> final Collection<ClusterNode> nodes = opts.projection();
>
> - top = nodes != null ? F.nodeIds(nodes) : null;
> + top = nodes == null || nodes.isEmpty() ? null : F.nodeIds(nodes);
> }
>
> boolean internal = false;
> Index:
> modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
> IDEA additional info:
> Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
> <+>UTF-8
> ===================================================================
> diff --git
> a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
>
> b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
> ---
> a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
> (revision d943c4cf1f8d00c0376af51945c2da453eeb745c)
> +++
> b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
> (date 1701174982174)
> @@ -686,7 +686,7 @@
> private List<ClusterNode> getTaskTopology() throws
> IgniteCheckedException {
> Collection<UUID> top = ses.getTopology();
>
> - Collection<? extends ClusterNode> subgrid = top != null ?
> ctx.discovery().nodes(top) : ctx.discovery().allNodes();
> + Collection<? extends ClusterNode> subgrid = top == null ||
> top.isEmpty() ? ctx.discovery().allNodes() : ctx.discovery().nodes(top);
>
> int size = subgrid.size();
>
> Index:
> modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionImpl.java
> IDEA additional info:
> Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
> <+>UTF-8
> ===================================================================
> diff --git
> a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionImpl.java
>
> b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionImpl.java
> ---
> a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionImpl.java
> (revision d943c4cf1f8d00c0376af51945c2da453eeb745c)
> +++
> b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionImpl.java
> (date 1701174982180)
> @@ -828,7 +828,7 @@
> if (topPred != null)
> return F.viewReadOnly(ctx.discovery().allNodes(), F.node2id(),
> topPred);
>
> - return top != null ? top : F.nodeIds(ctx.discovery().allNodes());
> + return top == null || top.isEmpty() ?
> F.nodeIds(ctx.discovery().allNodes()) : top;
> }
>
> /**
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
