JackieTien97 commented on code in PR #19224: URL: https://github.com/apache/iotdb/pull/19224#discussion_r4537107498
########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryAttemptRegistry.java: ########## @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.queryengine.execution.fragment; + +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; +import org.apache.iotdb.db.queryengine.common.QueryAttemptId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; + +import static com.google.common.math.LongMath.saturatedAdd; + +/** + * Owns shared DataNode state for one execution attempt. All registration/closing bookkeeping is + * serialized by compute on the attempt key; planning and state transitions run outside that lock. + * Closed entries retain no query data after their last actual user releases it. They only fence + * late dispatch until the request's immutable admission deadline, after which the request itself is + * rejected, even if the entry has been removed. + */ +class QueryAttemptRegistry { + private static final Logger LOGGER = LoggerFactory.getLogger(QueryAttemptRegistry.class); + + private final Map<QueryAttemptId, Entry> attempts = new ConcurrentHashMap<>(); + private final LongSupplier clock; + // Expiring a fence must remain irreversible even if the wall clock subsequently goes backwards. + private final AtomicLong observedTime = new AtomicLong(); + + QueryAttemptRegistry() { + this(System::currentTimeMillis); + } + + QueryAttemptRegistry(LongSupplier clock) { + this.clock = clock; + } + + private long now() { + return observedTime.accumulateAndGet(clock.getAsLong(), Math::max); + } + + Registration register( + FragmentInstance instance, FragmentInstanceStateMachine stateMachine, boolean dataQuery) { + QueryAttemptId key = + new QueryAttemptId( + instance.getId().getQueryId(), + instance.hasQueryAttempt() + ? instance.getQueryAttemptId() + : QueryAttemptId.LEGACY_ATTEMPT); + long executionDeadline = + instance.hasQueryAttempt() + ? instance.getQueryDeadline() + : saturatedAdd(now(), instance.getTimeOut()); + long admissionDeadline = + instance.hasQueryAttempt() + ? Math.min(executionDeadline, instance.getAdmissionDeadline()) + : executionDeadline; + AtomicReference<Registration> result = new AtomicReference<>(); + attempts.compute( + key, + (ignored, existing) -> { + long currentTime = now(); + if (admissionDeadline <= currentTime + || (existing != null + && (existing.closed || existing.executionDeadline <= currentTime))) { + throw rejected(key, admissionDeadline <= currentTime); Review Comment: [P2] Avoid comparing coordinator deadlines directly with receiver wall time `queryDeadline` and `admissionDeadline` are produced using the coordinator's wall clock, whereas `now()` reads the receiving DataNode's clock. With the default 60-second admission window, a receiver whose clock is 61 seconds ahead rejects a freshly issued FI immediately, even when its query timeout is one hour and there is no network delay. A clock-injected test reproduced this; the same request using the legacy relative-timeout path is admitted. This introduces a cluster clock-synchronization requirement that the repository's clock assumptions explicitly do not provide (`THREAT_MODEL.md`, section 5). Deadline comparisons need a common time basis or an explicit clock-offset-safe admission protocol. ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryAttemptRegistry.java: ########## @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.queryengine.execution.fragment; + +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; +import org.apache.iotdb.db.queryengine.common.QueryAttemptId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; + +import static com.google.common.math.LongMath.saturatedAdd; + +/** + * Owns shared DataNode state for one execution attempt. All registration/closing bookkeeping is + * serialized by compute on the attempt key; planning and state transitions run outside that lock. + * Closed entries retain no query data after their last actual user releases it. They only fence + * late dispatch until the request's immutable admission deadline, after which the request itself is + * rejected, even if the entry has been removed. + */ +class QueryAttemptRegistry { + private static final Logger LOGGER = LoggerFactory.getLogger(QueryAttemptRegistry.class); + + private final Map<QueryAttemptId, Entry> attempts = new ConcurrentHashMap<>(); + private final LongSupplier clock; + // Expiring a fence must remain irreversible even if the wall clock subsequently goes backwards. + private final AtomicLong observedTime = new AtomicLong(); + + QueryAttemptRegistry() { + this(System::currentTimeMillis); + } + + QueryAttemptRegistry(LongSupplier clock) { + this.clock = clock; + } + + private long now() { + return observedTime.accumulateAndGet(clock.getAsLong(), Math::max); Review Comment: [P2] Do not apply the historical maximum clock to unrelated new queries `observedTime` is shared by every attempt and never decreases. After a wall-clock rollback, even a coordinator and receiver in the same JVM generate fresh request deadlines from the adjusted clock while registration continues comparing against the old maximum. For example, after a 120-second rollback, a new query's default 60-second admission deadline is already below `observedTime`, so new queries are rejected for approximately a minute despite having a one-hour execution budget. This was reproduced by calling removeExpired(), moving the injected clock backwards, and registering a completely new queryId. Keep the protection against admitting old requests scoped to the relevant attempt so that it does not reject fresh query IDs after clock correction. ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/ClusterScheduler.java: ########## @@ -114,17 +133,29 @@ && isQuery() @Override public void start() { + synchronized (this) { + if (started || stopped || stateMachine.getState().isDone()) { + return; + } + started = true; + } stateMachine.transitionToDispatching(); long startTime = System.nanoTime(); - Future<FragInstanceDispatchResult> dispatchResultFuture = - dispatcher.dispatch(rootSubPlan, instances); // NOTICE: the FragmentInstance may be dispatched to another Host due to consensus redirect. // So we need to start the state fetcher after the dispatching stage. try { + Future<FragInstanceDispatchResult> dispatchResultFuture = + dispatcher.dispatch(rootSubPlan, instances); FragInstanceDispatchResult result = dispatchResultFuture.get(); + synchronized (this) { + if (stopped) { + return; Review Comment: [P1] Complete the query state when terminal cleanup stops dispatch `stopped` does not imply that the query state machine is terminal: `Coordinator.cleanUpStaleQueries()` calls `cleanupQueryExecution()` -> `stopAndCleanup(timeout)` directly, without calling `cancel()` or transitioning the query to FAILED. If that cleanup runs while dispatch is blocked, this branch discards the dispatch result when it eventually returns, leaving the query in DISPATCHING. The state tracker has also been aborted, so the subsequent `QueryExecution.getStatus()` waits indefinitely and retains the RPC thread. A latch-controlled test with a READ query reproduced this: stopAndCleanup during dispatch, let dispatch return, then getStatus never completes. Terminal cleanup needs to complete the query state or otherwise wake its status waiters, while preserving the separate stop semantics used for retries. ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java: ########## @@ -767,7 +797,37 @@ private void dispatchLocally(final FragmentInstance instance) } @Override - public void abort() {} + public synchronized void abort() { + aborted = true; + LinkedBlockingQueue<Pair<SubPlan, Boolean>> queue = dispatchQueue; + if (queue != null) { + queue.offer(new Pair<>(null, false)); + } + } + + private synchronized void prepareAdmission(FragmentInstance instance) + throws FragmentInstanceDispatchException { + checkNotAborted(); + long deadline = instance.getAdmissionDeadline(); + if (deadline == 0) { + deadline = + Math.min( + queryDeadline, LongMath.saturatedAdd(System.currentTimeMillis(), admissionWindowMs)); + instance.setAdmissionDeadline(deadline); Review Comment: [P2] Preserve recovery after the first RPC exhausts its socket timeout The admission window equals `getDnConnectionTimeoutInMS()`, but this branch only initializes the deadline once. When the first `sendFragmentInstance` call fails after waiting for its socket timeout, `dispatchRemote()` reconnects and resends the FI with an already expired admission deadline. The receiver then returns QUERY_TIMEOUT (720), which `ClusterScheduler.needRetry()` does not retry, even if the query still has ample execution budget and the first request never reached the receiver. This was reproduced with a 20 ms socket timeout and a one-hour query timeout. Preserve the immutable cutoff for an old request, but distinguish admission expiry from execution timeout and allow a fresh query attempt, or otherwise budget admission for the supported transport retry. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
