This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new c7400e8750c [To dev/1.3] Fix stale query cleanup for abandoned queries
and query statements in executeBatch
c7400e8750c is described below
commit c7400e8750ce8420daa2ded0cc312a6275f22d7a
Author: Jackie Tien <[email protected]>
AuthorDate: Fri Nov 20 14:41:40 2026 +0800
[To dev/1.3] Fix stale query cleanup for abandoned queries and query
statements in executeBatch
---
.../apache/iotdb/db/it/IoTDBExecuteBatchIT.java | 27 +++
.../org/apache/iotdb/db/it/IoTDBShowQueriesIT.java | 119 ++++++++++
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 36 ++-
.../iotdb/db/queryengine/plan/Coordinator.java | 10 +-
.../plan/execution/IQueryExecution.java | 2 +
.../queryengine/plan/execution/QueryExecution.java | 4 +-
.../plan/execution/config/ConfigExecution.java | 2 +-
.../queryengine/execution/ConfigExecutionTest.java | 24 ++
.../iotdb/db/queryengine/plan/CoordinatorTest.java | 258 +++++++++++++++++++++
.../execution/QueryExecutionLifecycleTest.java | 75 ++++++
10 files changed, 549 insertions(+), 8 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java
index f1822fd4bb4..4f266f03bfa 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBExecuteBatchIT.java
@@ -81,6 +81,33 @@ public class IoTDBExecuteBatchIT {
}
}
+ @Test
+ public void testJDBCExecuteBatchWithSelectInto() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.addBatch("insert into root.batch_into.src(timestamp,s1)
values(1,1.0)");
+ statement.addBatch("insert into root.batch_into.src(timestamp,s1)
values(2,2.0)");
+ statement.addBatch("select s1 into root.batch_into.dst(s1) from
root.batch_into.src");
+ statement.executeBatch();
+ statement.clearBatch();
+ // A batch has no result set to fetch, so the rows must be written when
it returns.
+ ResultSet resultSet = statement.executeQuery("select s1 from
root.batch_into.dst");
+ int count = 0;
+
+ long[] timestamps = {1, 2};
+ double[] values = {1.0, 2.0};
+
+ while (resultSet.next()) {
+ assertEquals(timestamps[count],
resultSet.getLong(ColumnHeaderConstant.TIME));
+ assertEquals(values[count],
resultSet.getDouble("root.batch_into.dst.s1"), 0.0);
+ count++;
+ }
+ assertEquals(timestamps.length, count);
+ } catch (SQLException e) {
+ fail(e.getMessage());
+ }
+ }
+
@Test
public void testJDBCExecuteBatchForCreateMultiTimeSeriesPlan() {
try (Connection connection = EnvFactory.getEnv().getConnection();
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBShowQueriesIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBShowQueriesIT.java
new file mode 100644
index 00000000000..1cf8cb05159
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBShowQueriesIT.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.isession.ISession;
+import org.apache.iotdb.isession.SessionDataSet;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static
org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.QUERY_ID;
+import static
org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.STATEMENT;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({LocalStandaloneIT.class, ClusterIT.class})
+public class IoTDBShowQueriesIT {
+ private static final String DATABASE = "root.show_queries";
+ private static final String DEVICE = DATABASE + ".d1";
+ private static final String TIMESERIES = DEVICE + ".s1";
+ private static final String BATCH_QUERY = "SELECT s1 FROM " + DEVICE;
+ private static final String SHOW_QUERIES = "SHOW QUERIES";
+ private static final String TEMPLATE = "show_queries_template";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvFactory.getEnv().initClusterEnvironment();
+ try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ session.executeNonQueryStatement("CREATE DATABASE " + DATABASE);
+ session.executeNonQueryStatement("CREATE TIMESERIES " + TIMESERIES + "
INT32");
+ session.executeNonQueryStatement("INSERT INTO " + DEVICE + "(time, s1)
VALUES (1, 1)");
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void queryInBatchReleaseTest() throws Exception {
+ try (ISession adminSession = EnvFactory.getEnv().getSessionConnection();
+ Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ Set<String> queryIdsBefore = getOtherQueryIds(adminSession);
+ statement.addBatch(BATCH_QUERY);
+ statement.executeBatch();
+ // A batch only returns the status of its statements, so nobody can
fetch or close the query.
+ assertNoQueryIsLeft(adminSession, queryIdsBefore);
+ }
+ }
+
+ @Test
+ public void schemaTemplateQueryReleaseTest() throws Exception {
+ try (ISession adminSession = EnvFactory.getEnv().getSessionConnection()) {
+ adminSession.executeNonQueryStatement("CREATE DEVICE TEMPLATE " +
TEMPLATE + " (s1 INT32)");
+ try {
+ Set<String> queryIdsBefore = getOtherQueryIds(adminSession);
+ // The measurements come back in the response, so nothing is left to
fetch or close.
+ Assert.assertEquals(
+ Collections.singletonList("s1"),
adminSession.showMeasurementsInTemplate(TEMPLATE));
+ assertNoQueryIsLeft(adminSession, queryIdsBefore);
+ } finally {
+ adminSession.executeNonQueryStatement("DROP DEVICE TEMPLATE " +
TEMPLATE);
+ }
+ }
+ }
+
+ /** Ids of the registered queries, except the SHOW QUERIES statement that
lists them. */
+ private static Set<String> getOtherQueryIds(ISession session) throws
Exception {
+ Set<String> queryIds = new HashSet<>();
+ try (SessionDataSet dataSet = session.executeQueryStatement(SHOW_QUERIES))
{
+ SessionDataSet.DataIterator iterator = dataSet.iterator();
+ while (iterator.next()) {
+ if (!SHOW_QUERIES.equals(iterator.getString(STATEMENT))) {
+ queryIds.add(iterator.getString(QUERY_ID));
+ }
+ }
+ }
+ return queryIds;
+ }
+
+ private static void assertNoQueryIsLeft(ISession session, Set<String>
queryIdsBefore)
+ throws Exception {
+ Set<String> leftQueryIds = getOtherQueryIds(session);
+ leftQueryIds.removeAll(queryIdsBefore);
+ Assert.assertEquals(Collections.emptySet(), leftQueryIds);
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
index 9607e51e7e7..9e352550a69 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
@@ -1616,6 +1616,19 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
return executeStatementInternal(req, OLD_SELECT_RESULT);
}
+ /** Runs a query to its end and discards its result. */
+ private static void consumeResult(IQueryExecution queryExecution) throws
IoTDBException {
+ if (queryExecution == null) {
+ // only queries are registered, any other statement has already been
executed
+ return;
+ }
+ while (queryExecution.hasNextResult()) {
+ if (!queryExecution.getBatchResult().isPresent()) {
+ break;
+ }
+ }
+ }
+
@Override
public TSStatus executeBatchStatement(TSExecuteBatchStatementReq req) {
long t1 = System.nanoTime();
@@ -1632,6 +1645,8 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
long t2 = System.nanoTime();
String type = null;
OperationQuota quota = null;
+ Long queryId = null;
+ Throwable t = null;
try {
Statement s = StatementGenerator.createStatement(statement,
clientSession.getZoneId());
if (s == null) {
@@ -1652,7 +1667,7 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
AuditLogger.log(statement, s);
}
- long queryId = SESSION_MANAGER.requestQueryId();
+ queryId = SESSION_MANAGER.requestQueryId();
type = s.getType() == null ? null : s.getType().name();
// create and cache dataset
@@ -1685,9 +1700,15 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
if (result.status.getCode() !=
TSStatusCode.SUCCESS_STATUS.getStatusCode()
&& result.status.getCode() !=
TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) {
isAllSuccessful = false;
+ } else if (queryId != null) {
+ // A batch returns no result set, so nobody fetches the result of
a statement that is a
+ // query. Without a consumer such a statement is not run to its
end, which matters for
+ // the ones that write, such as SELECT INTO.
+ consumeResult(COORDINATOR.getQueryExecution(queryId));
}
results.add(result.status);
} catch (Exception e) {
+ t = e;
LOGGER.warn("Error occurred when executing executeBatchStatement: ",
e);
TSStatus status =
onQueryException(
@@ -1695,6 +1716,10 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
isAllSuccessful = false;
results.add(status);
} finally {
+ if (queryId != null) {
+ // Nobody can fetch or close the query of this statement once the
batch has returned.
+ COORDINATOR.cleanupQueryExecution(queryId, req, t);
+ }
CommonUtils.addStatementExecutionLatency(
OperationType.EXECUTE_STATEMENT, type, System.nanoTime() - t2);
if (quota != null) {
@@ -2511,6 +2536,8 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
Statement statement, TSQueryTemplateReq req, TSQueryTemplateResp resp) {
long startTime = System.nanoTime();
OperationQuota quota = null;
+ Long queryId = null;
+ Throwable t = null;
try {
IClientSession clientSession =
SESSION_MANAGER.getCurrSessionAndUpdateIdleTime();
// permission check
@@ -2527,7 +2554,7 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
if (ENABLE_AUDIT_LOG) {
AuditLogger.log(String.format("execute Query: %s", statement),
statement);
}
- long queryId = SESSION_MANAGER.requestQueryId();
+ queryId = SESSION_MANAGER.requestQueryId();
// create and cache dataset
ExecutionResult executionResult =
COORDINATOR.executeForTreeModel(
@@ -2571,10 +2598,15 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
return resp;
}
} catch (Exception e) {
+ t = e;
resp.setStatus(
onQueryException(e, "\"" + statement + "\". " +
OperationType.EXECUTE_STATEMENT));
return null;
} finally {
+ if (queryId != null) {
+ // The whole result is part of the response, so nobody can fetch or
close the query later.
+ COORDINATOR.cleanupQueryExecution(queryId, req, t);
+ }
CommonUtils.addStatementExecutionLatency(
OperationType.EXECUTE_STATEMENT,
statement.getType().name(),
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
index 00fd3344119..0e6e4d6e96b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
@@ -331,14 +331,18 @@ public class Coordinator {
* We need to reclaim resources from queries that have exceeded their
timeout by more than one
* minute. This indicates that the associated clients have failed to perform
proper resource
* cleanup.
+ *
+ * <p>Whether an RPC of the query is in progress does not matter here. The
timeout bounds the
+ * whole lifetime of a query, so a query without an RPC in progress has been
abandoned by its
+ * client, and an RPC that is still in progress this long after the timeout
is stuck, e.g. in a
+ * dispatch that never returns, and is only unblocked by stopping the query.
Sparing the queries
+ * with an RPC in progress would also keep forever every query whose caller
never reports the end
+ * of its RPC through {@link #recordExecutionTime}.
*/
public void cleanUpStaleQueries() {
long currentTime = System.currentTimeMillis();
queryExecutionMap.forEach(
(queryId, queryExecution) -> {
- if (queryExecution.isActive()) {
- return;
- }
long timeout = queryExecution.getTimeout();
long queryStartTime = queryExecution.getStartExecutionTime();
long executeTime = currentTime - queryStartTime;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java
index d41db48afec..97db9772bef 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java
@@ -78,6 +78,8 @@ public interface IQueryExecution {
* finishes. On the other hand, if there is no active RPC, it means that the
client has finished
* fetching results or has not started fetching results yet, and the
QueryExecution can be safely
* cleaned up.
+ *
+ * @return true if an RPC of this query is in progress, false otherwise
*/
boolean isActive();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
index c99af2a8ffe..dd8a050ac8b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
@@ -432,7 +432,7 @@ public class QueryExecution implements IQueryExecution {
* ClientRPCServiceImpl is responsible for performing the cleanup.
*/
private synchronized void cleanUpCoordinatorContextMapIfNeeded(Throwable t) {
- if (isActive()) {
+ if (!isActive()) {
Coordinator.getInstance().cleanupQueryExecution(context.getLocalQueryId(),
null, t);
}
}
@@ -446,7 +446,7 @@ public class QueryExecution implements IQueryExecution {
* cleaned up.
*/
public synchronized boolean isActive() {
- return startTimeOfCurrentRpc == -1;
+ return startTimeOfCurrentRpc != -1;
}
/**
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java
index 33173a5e598..b4ca53f27e1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java
@@ -257,7 +257,7 @@ public class ConfigExecution implements IQueryExecution {
@Override
public boolean isActive() {
- return startTimeOfCurrentRpc == -1;
+ return startTimeOfCurrentRpc != -1;
}
@Override
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/ConfigExecutionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/ConfigExecutionTest.java
index 32dcd88f42d..1fc7d5c389e 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/ConfigExecutionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/ConfigExecutionTest.java
@@ -46,6 +46,7 @@ import java.util.concurrent.ExecutorService;
import static com.google.common.util.concurrent.Futures.immediateFuture;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class ConfigExecutionTest {
@@ -157,6 +158,29 @@ public class ConfigExecutionTest {
}
}
+ @Test
+ public void isActiveTellsWhetherAnRpcOfTheQueryIsInProgress() {
+ ExecutorService executor = getExecutor();
+ try {
+ IConfigTask task =
+ (clientManager) -> immediateFuture(new
ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
+ ConfigExecution execution = new ConfigExecution(genMPPQueryContext(),
executor, task);
+ // the execution is created by the RPC that executes the statement
+ assertTrue(execution.isActive());
+ // that RPC returns, and the client has not asked for the result yet
+ execution.recordExecutionTime(1);
+ assertFalse(execution.isActive());
+ // an RPC that fetches the result arrives
+ execution.updateCurrentRpcStartTime(System.nanoTime());
+ assertTrue(execution.isActive());
+ // and returns as well
+ execution.recordExecutionTime(1);
+ assertFalse(execution.isActive());
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
private MPPQueryContext genMPPQueryContext() {
MPPQueryContext context = new MPPQueryContext(new QueryId("query1"));
context.setQueryType(QueryType.WRITE);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/CoordinatorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/CoordinatorTest.java
new file mode 100644
index 00000000000..b2dd8c33f59
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/CoordinatorTest.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.common.SessionInfo;
+import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
+import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution;
+import org.apache.iotdb.db.queryengine.plan.execution.QueryExecution;
+import org.apache.iotdb.db.queryengine.plan.planner.IPlanner;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CoordinatorTest {
+
+ // A query id is never reused, as a terminated query releases itself
asynchronously by its id
+ private static final AtomicLong NEXT_QUERY_ID = new AtomicLong(10_001L);
+
+ private static final long QUERY_TIMEOUT_IN_MS = 2_000L;
+ // Coordinator.cleanUpStaleQueries spares a query until it exceeds its
timeout by this long
+ private static final long STALE_QUERY_GRACE_PERIOD_IN_MS = 60_000L;
+
+ private static int previousDataNodeId;
+ private static ExecutorService stateMachineExecutor;
+
+ private final List<Long> executedQueryIds = new ArrayList<>();
+
+ @BeforeClass
+ public static void setUp() {
+ // the coordinator builds its query id generator from the data node id of
this node
+ previousDataNodeId =
IoTDBDescriptor.getInstance().getConfig().getDataNodeId();
+ IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0);
+ stateMachineExecutor = Executors.newSingleThreadExecutor();
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ stateMachineExecutor.shutdownNow();
+
IoTDBDescriptor.getInstance().getConfig().setDataNodeId(previousDataNodeId);
+ }
+
+ @After
+ public void releaseExecutedQueries() {
+ // a query that had to be left alone is still registered
+ executedQueryIds.forEach(Coordinator.getInstance()::cleanupQueryExecution);
+ }
+
+ @Test
+ public void testStaleQueryWithoutRpcInProgressIsCleanedUp() throws Throwable
{
+ QueryExecution execution = executeQuery(QUERY_TIMEOUT_IN_MS);
+ // the RPC that executed the statement returns, and the client never
fetches the result
+ finishRpc(execution);
+ setElapsedTime(execution, QUERY_TIMEOUT_IN_MS +
STALE_QUERY_GRACE_PERIOD_IN_MS + 1_000L);
+
+ Coordinator.getInstance().cleanUpStaleQueries();
+
+
assertNull(Coordinator.getInstance().getQueryExecution(queryIdOf(execution)));
+ }
+
+ @Test
+ public void testStaleQueryWithRpcInProgressIsCleanedUp() throws Throwable {
+ // the RPC that executed the statement has not returned yet, or its caller
never reports that
+ // it returned
+ QueryExecution executing = executeQuery(QUERY_TIMEOUT_IN_MS);
+ // that RPC returned, and a later RPC is fetching the result
+ QueryExecution fetching = executeQuery(QUERY_TIMEOUT_IN_MS);
+ finishRpc(fetching);
+ fetching.updateCurrentRpcStartTime(System.nanoTime());
+ long elapsedTime = QUERY_TIMEOUT_IN_MS + STALE_QUERY_GRACE_PERIOD_IN_MS +
1_000L;
+ setElapsedTime(executing, elapsedTime);
+ setElapsedTime(fetching, elapsedTime);
+
+ Coordinator.getInstance().cleanUpStaleQueries();
+
+ // an RPC still in progress that long after the timeout is stuck, and
stopping the query is
+ // what unblocks it
+ for (QueryExecution execution : new QueryExecution[] {executing,
fetching}) {
+
assertNull(Coordinator.getInstance().getQueryExecution(queryIdOf(execution)));
+ }
+ }
+
+ @Test
+ public void testQueryWithRpcInProgressWithinGracePeriodIsLeftAlone() throws
Throwable {
+ QueryExecution execution = executeQuery(QUERY_TIMEOUT_IN_MS);
+ // timed out, but not yet by more than the grace period
+ setElapsedTime(execution, QUERY_TIMEOUT_IN_MS +
STALE_QUERY_GRACE_PERIOD_IN_MS / 2);
+
+ Coordinator.getInstance().cleanUpStaleQueries();
+
+ assertLeftAlone(execution);
+ }
+
+ @Test
+ public void testIdleQueryWithinGracePeriodIsLeftAlone() throws Throwable {
+ QueryExecution execution = executeQuery(QUERY_TIMEOUT_IN_MS);
+ finishRpc(execution);
+ // timed out, but not yet by more than the grace period
+ setElapsedTime(execution, QUERY_TIMEOUT_IN_MS +
STALE_QUERY_GRACE_PERIOD_IN_MS / 2);
+
+ Coordinator.getInstance().cleanUpStaleQueries();
+
+ assertLeftAlone(execution);
+ }
+
+ @Test
+ public void testIdleQueryWithoutPositiveTimeoutIsLeftAlone() throws
Throwable {
+ QueryExecution zeroTimeout = executeQuery(0L);
+ QueryExecution negativeTimeout = executeQuery(-1L);
+ finishRpc(zeroTimeout);
+ finishRpc(negativeTimeout);
+ setElapsedTime(zeroTimeout, 10 * STALE_QUERY_GRACE_PERIOD_IN_MS);
+ setElapsedTime(negativeTimeout, 10 * STALE_QUERY_GRACE_PERIOD_IN_MS);
+
+ Coordinator.getInstance().cleanUpStaleQueries();
+
+ assertLeftAlone(zeroTimeout);
+ assertLeftAlone(negativeTimeout);
+ }
+
+ @Test
+ public void testKilledQueryIsReleasedUnlessAnRpcIsInProgress() throws
Throwable {
+ // the RPC that executed the statement has not returned yet
+ QueryExecution executing = executeQuery(QUERY_TIMEOUT_IN_MS);
+ // that RPC returned, and the client has not fetched the result yet
+ QueryExecution idle = executeQuery(QUERY_TIMEOUT_IN_MS);
+ finishRpc(idle);
+
+ executing.cancel();
+ idle.cancel();
+
+ // the state machine thread answers getStatus after it has run the cleanup
of the killed query
+ assertEquals(TSStatusCode.QUERY_WAS_KILLED.getStatusCode(),
statusCodeOf(executing));
+ assertEquals(TSStatusCode.QUERY_WAS_KILLED.getStatusCode(),
statusCodeOf(idle));
+ // the RPC in progress releases its query when it returns, nothing would
release the idle one
+ assertSame(executing,
Coordinator.getInstance().getQueryExecution(queryIdOf(executing)));
+ assertNull(Coordinator.getInstance().getQueryExecution(queryIdOf(idle)));
+ }
+
+ /**
+ * Executes a query the way a client RPC does. When this method returns the
query is RUNNING and
+ * registered in the coordinator, and the RPC that created it is still in
progress. The result of
+ * the query is already in memory, so no fragment instance is dispatched.
+ */
+ private QueryExecution executeQuery(long timeout) throws Throwable {
+ long queryId = NEXT_QUERY_ID.getAndIncrement();
+ executedQueryIds.add(queryId);
+ IAnalysis analysis = mock(IAnalysis.class);
+ when(analysis.canSkipExecute(any(MPPQueryContext.class))).thenReturn(true);
+ when(analysis.constructResultForMemorySource(any(MPPQueryContext.class)))
+ .thenReturn(new TsBlock(0));
+ IPlanner planner = mock(IPlanner.class);
+ when(planner.analyze(any(MPPQueryContext.class))).thenReturn(analysis);
+ QueryExecution[] createdExecution = new QueryExecution[1];
+ execute(
+ queryId,
+ "SELECT s FROM t1",
+ (queryContext, startTime) -> {
+ queryContext.setTimeOut(timeout);
+ queryContext.setStartTime(startTime);
+ createdExecution[0] = new QueryExecution(planner, queryContext,
stateMachineExecutor);
+ return createdExecution[0];
+ });
+ return createdExecution[0];
+ }
+
+ /** What the RPC in progress does when it returns to the client. */
+ private static void finishRpc(QueryExecution execution) {
+ Coordinator.getInstance().recordExecutionTime(queryIdOf(execution), 1L);
+ }
+
+ /** Makes the query look as if it had been started {@code elapsedTimeInMs}
ago. */
+ private static void setElapsedTime(QueryExecution execution, long
elapsedTimeInMs) {
+ execution.getContext().setStartTime(System.currentTimeMillis() -
elapsedTimeInMs);
+ }
+
+ private static long queryIdOf(QueryExecution execution) {
+ return execution.getContext().getLocalQueryId();
+ }
+
+ private static int statusCodeOf(QueryExecution execution) {
+ return execution.getStatus().status.getCode();
+ }
+
+ private static void assertLeftAlone(QueryExecution execution) {
+ assertSame(execution,
Coordinator.getInstance().getQueryExecution(queryIdOf(execution)));
+ // still running, so its result can still be fetched
+ assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(),
statusCodeOf(execution));
+ }
+
+ private static void execute(
+ long queryId,
+ String sql,
+ BiFunction<MPPQueryContext, Long, IQueryExecution> queryExecutionFactory)
+ throws Throwable {
+ Method execution =
+ Coordinator.class.getDeclaredMethod(
+ "execution",
+ long.class,
+ SessionInfo.class,
+ String.class,
+ boolean.class,
+ boolean.class,
+ BiFunction.class);
+ execution.setAccessible(true);
+ try {
+ execution.invoke(
+ Coordinator.getInstance(),
+ queryId,
+ new SessionInfo(0, "test", ZoneId.systemDefault(), "test"),
+ sql,
+ false,
+ false,
+ queryExecutionFactory);
+ } catch (InvocationTargetException e) {
+ throw e.getCause();
+ }
+ }
+}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecutionLifecycleTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecutionLifecycleTest.java
new file mode 100644
index 00000000000..3226505a490
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecutionLifecycleTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.execution;
+
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.common.QueryId;
+import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
+import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
+import org.apache.iotdb.db.queryengine.plan.planner.IPlanner;
+
+import org.junit.Test;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class QueryExecutionLifecycleTest {
+
+ @Test
+ public void isActiveTellsWhetherAnRpcOfTheQueryIsInProgress() throws
Exception {
+ ExecutorService notifications = Executors.newSingleThreadExecutor();
+ try {
+ QueryExecution execution = new QueryExecution(planner(), context(),
notifications);
+ // the execution is created by the RPC that executes the statement
+ assertTrue(execution.isActive());
+ // that RPC returns, and the client has not asked for the result yet
+ execution.recordExecutionTime(1);
+ assertFalse(execution.isActive());
+ // an RPC that fetches the result arrives
+ execution.updateCurrentRpcStartTime(System.nanoTime());
+ assertTrue(execution.isActive());
+ // and returns as well
+ execution.recordExecutionTime(1);
+ assertFalse(execution.isActive());
+ } finally {
+ notifications.shutdownNow();
+ assertTrue(notifications.awaitTermination(10, TimeUnit.SECONDS));
+ }
+ }
+
+ private static IPlanner planner() {
+ IPlanner planner = mock(IPlanner.class);
+ when(planner.analyze(any())).thenReturn(mock(IAnalysis.class));
+ return planner;
+ }
+
+ private static MPPQueryContext context() {
+ MPPQueryContext context = new MPPQueryContext(new QueryId("is_active"));
+ context.setQueryType(QueryType.READ);
+ return context;
+ }
+}