This is an automated email from the ASF dual-hosted git repository. JackieTien97 pushed a commit to branch rc/2.0.12 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3cdfe4352a436d93e19c4a9b73320d2e50a93bf3 Author: Caideyipi <[email protected]> AuthorDate: Tue Oct 27 15:44:35 2026 +0800 Fix subscription commit recovery after RPC timeouts (#19023) Update ConsensusSubscriptionBroker.java * Retry explicit subscription commits after connection failures * Preserve completed subscription commit batches during retries * Preserve newer subscription checkpoints across commit retries * Retry transient subscription poll cleanup NACKs * Retry only unconfirmed subscription acknowledgements * Preserve subscription ACK progress across concurrent commits and server errors (cherry picked from commit 07dc28102b8840d82a7b7d7e47530f0660b4005f) --- .../base/AbstractSubscriptionConsumer.java | 182 +++++++++++-- .../base/AbstractSubscriptionProvider.java | 40 ++- .../base/SubscriptionConsumerLifecycleTest.java | 2 +- .../SubscriptionConsumerMultiProviderPollTest.java | 281 --------------------- .../db/pipe/resource/memory/PipeMemoryManager.java | 8 +- .../dataregion/memtable/TsFileProcessor.java | 3 +- .../broker/ConsensusSubscriptionBroker.java | 2 +- .../memory/PipeMemoryManagerResizeTest.java | 4 +- .../storageengine/dataregion/DataRegionTest.java | 13 +- .../rescon/memory/SystemInfoTest.java | 24 +- 10 files changed, 219 insertions(+), 340 deletions(-) diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java index 122b4138ee7..391c4d56e2e 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java @@ -110,6 +110,9 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { private static final long SLEEP_MS = 100L; private static final long SLEEP_DELTA_MS = 50L; private static final long TIMER_DELTA_MS = 250L; + private static final int MAX_COMMIT_RETRIES = 3; + private static final long COMMIT_RETRY_INTERVAL_MS = + ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_MIN_VALUE; private static final AtomicLong LAST_CONSUMER_INSTANCE_EPOCH = new AtomicLong(); private final String username; @@ -1060,9 +1063,7 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { inFlightFilesCommitContextSet.remove(commitContext); } // construct temporary message to nack - nack( - Collections.singletonList( - new SubscriptionMessage(commitContext, file.getAbsolutePath(), null))); + nackAfterPollFailure(new SubscriptionMessage(commitContext, file.getAbsolutePath(), null), e); throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); } } @@ -1271,13 +1272,25 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { throw (SubscriptionConsumerFencedException) e; } // construct temporary message to nack - nack( - Collections.singletonList( - new SubscriptionMessage(response.getCommitContext(), Collections.emptyMap()))); + nackAfterPollFailure( + new SubscriptionMessage(response.getCommitContext(), Collections.emptyMap()), e); throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e); } } + private void nackAfterPollFailure(final SubscriptionMessage message, final Exception pollFailure) + throws SubscriptionException { + try { + nack(Collections.singletonList(message)); + } catch (final SubscriptionConnectionException nackFailure) { + // Recovery was scheduled by commitInternal. A NACK replay can increment the server's poison + // message counter again. Preserve the poll failure and let the server recycle the event. + if (nackFailure != pollFailure) { + pollFailure.addSuppressed(nackFailure); + } + } + } + private Optional<SubscriptionMessage> pollTabletsInternal( final SubscriptionPollResponse initialResponse, final PollTimer timer) { final Map<String, List<Tablet>> tablets = @@ -1536,15 +1549,9 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { dataNodeIdToSubscriptionCommitContexts.entrySet()) { final List<SubscriptionCommitContext> groupedCommitContexts = entry.getValue(); requestedCount += groupedCommitContexts.size(); - final AbstractSubscriptionProvider.CommitResult commitResp = - commitInternal(entry.getKey(), groupedCommitContexts, nack); final List<SubscriptionCommitContext> acceptedCommitContexts = - commitResp.getAcceptedCommitContexts(); + commitWithRetry(entry.getKey(), groupedCommitContexts, nack); acceptedCount += acceptedCommitContexts.size(); - if (!nack) { - overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); - onAckedCommitContexts(acceptedCommitContexts); - } if (acceptedCommitContexts.size() != groupedCommitContexts.size()) { final List<SubscriptionCommitContext> failedInGroup = new ArrayList<>(groupedCommitContexts); @@ -1565,6 +1572,88 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { } } + private List<SubscriptionCommitContext> commitWithRetry( + final int dataNodeId, + final List<SubscriptionCommitContext> commitContexts, + final boolean nack) + throws SubscriptionException { + final List<SubscriptionCommitContext> remainingCommitContexts = new ArrayList<>(commitContexts); + final List<SubscriptionCommitContext> acceptedCommitContexts = new ArrayList<>(); + for (int retries = 0; ; retries++) { + checkIfFenced(); + AbstractSubscriptionProvider.CommitResult result = + AbstractSubscriptionProvider.CommitResult.empty(); + SubscriptionConnectionException connectionFailure = null; + SubscriptionException failure = null; + try { + result = commitInternal(dataNodeId, remainingCommitContexts, nack); + } catch (final SubscriptionException e) { + failure = getCommitFailure(e); + result = getCompletedCommitResult(e); + if (failure instanceof SubscriptionConnectionException) { + connectionFailure = (SubscriptionConnectionException) failure; + } + } + + final List<SubscriptionCommitContext> acceptedInAttempt = result.getAcceptedCommitContexts(); + acceptedCommitContexts.addAll(acceptedInAttempt); + if (!nack) { + // Apply each response once, before waiting. Reapplying older progress at retry completion + // could overwrite a concurrent commit; accepted contexts must also leave the buffer now. + overlayCommittedPositions(result.getCommittedProgressByTopic()); + onAckedCommitContexts(acceptedInAttempt); + } + if (failure != null && connectionFailure == null) { + throw failure; + } + remainingCommitContexts.removeAll(new HashSet<>(acceptedInAttempt)); + if (nack + || remainingCommitContexts.isEmpty() + || retries >= MAX_COMMIT_RETRIES + || isClosed() + || Thread.currentThread().isInterrupted()) { + if (connectionFailure != null) { + throw connectionFailure; + } + return acceptedCommitContexts; + } + checkIfFenced(); + // A partial response has no permanent-rejection reason. Retry only its rejected contexts + // and any batches whose response was lost, using the same budget as connection recovery. + // commitInternal has released the provider read lock before we wait for recovery. + try { + Thread.sleep(COMMIT_RETRY_INTERVAL_MS); + } catch (final InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + if (connectionFailure != null) { + connectionFailure.addSuppressed(interruptedException); + throw connectionFailure; + } + return acceptedCommitContexts; + } + checkIfFenced(); + if (isClosed()) { + if (connectionFailure != null) { + throw connectionFailure; + } + return acceptedCommitContexts; + } + } + } + + private SubscriptionException getCommitFailure(final SubscriptionException e) { + return e instanceof AbstractSubscriptionProvider.CommitException + ? ((AbstractSubscriptionProvider.CommitException) e).getFailure() + : e; + } + + private AbstractSubscriptionProvider.CommitResult getCompletedCommitResult( + final SubscriptionException e) { + return e instanceof AbstractSubscriptionProvider.CommitException + ? ((AbstractSubscriptionProvider.CommitException) e).getCompletedResult() + : AbstractSubscriptionProvider.CommitResult.empty(); + } + /** Invoked after the server accepts commit contexts from an acknowledgement request. */ protected void onAckedCommitContexts( final Collection<SubscriptionCommitContext> acceptedCommitContexts) { @@ -1610,10 +1699,26 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { commitInternal(entry.getKey(), groupedCommitContexts, false); overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); removableCommitContexts.addAll(commitResp.getAcceptedCommitContexts()); - } catch (final SubscriptionConnectionException e) { + } catch (final SubscriptionException e) { + final SubscriptionException failure = getCommitFailure(e); + final AbstractSubscriptionProvider.CommitResult completedResult = + getCompletedCommitResult(e); + overlayCommittedPositions(completedResult.getCommittedProgressByTopic()); + final Set<SubscriptionCommitContext> acceptedBeforeFailure = + new HashSet<>(completedResult.getAcceptedCommitContexts()); + removableCommitContexts.addAll(acceptedBeforeFailure); + if (!(failure instanceof SubscriptionConnectionException)) { + // The caller cannot receive the removable set when an error propagates. Remove confirmed + // contexts from every completed group before leaving the automatic/close-time ACK path. + onAckedCommitContexts(removableCommitContexts); + throw failure; + } int stagedCount = 0; int retainedCount = 0; for (final SubscriptionCommitContext commitContext : groupedCommitContexts) { + if (acceptedBeforeFailure.contains(commitContext)) { + continue; + } if (isConsensusCommitContext(commitContext)) { stagePendingRedirectAck(commitContext); removableCommitContexts.add(commitContext); @@ -1637,7 +1742,7 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { this, retainedCount, entry.getKey(), - e); + failure); } } } @@ -1676,6 +1781,7 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { final List<SubscriptionCommitContext> subscriptionCommitContexts, final boolean nack) throws SubscriptionException { + checkIfFenced(); providers.acquireReadLock(); try { final AbstractSubscriptionProvider provider = providers.getProvider(dataNodeId); @@ -1694,11 +1800,13 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { } try { return provider.commit(subscriptionCommitContexts, nack); - } catch (final SubscriptionConsumerFencedException e) { - fence(e); - throw e; - } catch (final SubscriptionConnectionException e) { - providers.syncIfUnavailable(this); + } catch (final SubscriptionException e) { + final SubscriptionException failure = getCommitFailure(e); + if (failure instanceof SubscriptionConsumerFencedException) { + fence((SubscriptionConsumerFencedException) failure); + } else if (failure instanceof SubscriptionConnectionException) { + providers.syncIfUnavailable(this); + } throw e; } } finally { @@ -2090,15 +2198,11 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { } final List<SubscriptionCommitContext> contextsToRedirect = new ArrayList<>(pendingContexts); + AbstractSubscriptionProvider.CommitResult commitResp; try { - final AbstractSubscriptionProvider.CommitResult commitResp = - commitInternal(entry.getValue(), contextsToRedirect, false); - overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); - commitResp.getAcceptedCommitContexts().forEach(pendingContexts::remove); - if (pendingContexts.isEmpty()) { - pendingRedirectAcksByTopicRegion.remove(entry.getKey(), pendingContexts); - } + commitResp = commitInternal(entry.getValue(), contextsToRedirect, false); } catch (final SubscriptionException e) { + commitResp = getCompletedCommitResult(e); LOGGER.warn( SubscriptionMessages .LOG_ARG_FAILED_REDIRECT_ARG_PENDING_CONSENSUS_ACK_S_ARG_VIA_E6A10AC5, @@ -2108,6 +2212,11 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { entry.getValue(), e); } + overlayCommittedPositions(commitResp.getCommittedProgressByTopic()); + commitResp.getAcceptedCommitContexts().forEach(pendingContexts::remove); + if (pendingContexts.isEmpty()) { + pendingRedirectAcksByTopicRegion.remove(entry.getKey(), pendingContexts); + } } } @@ -2162,7 +2271,24 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { if (Objects.isNull(progressByTopic) || progressByTopic.isEmpty()) { return; } - progressByTopic.forEach(this::overlayCommittedPositions); + // ACK responses from different providers can arrive out of order, including completed batches + // carried by a failure. Preserve the newest position for each writer. Explicit seek operations + // use the topic-scoped overlay above and can still move a checkpoint backward. + for (final Entry<String, TopicProgress> entry : progressByTopic.entrySet()) { + if (Objects.isNull(entry.getValue())) { + continue; + } + for (final RegionProgress regionProgress : entry.getValue().getRegionProgress().values()) { + if (Objects.nonNull(regionProgress)) { + regionProgress + .getWriterPositions() + .forEach( + (writerId, writerProgress) -> + mergeTopicProgress( + committedPositionsByTopic, entry.getKey(), writerId, writerProgress)); + } + } + } } private void overlayTopicProgress( diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java index 56f73249103..125005763dc 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java @@ -804,20 +804,32 @@ public abstract class AbstractSubscriptionProvider { final List<SubscriptionCommitContext> acceptedCommitContexts = new ArrayList<>(); final Map<String, TopicProgress> committedProgressByTopic = new LinkedHashMap<>(); + int completedCommitContextCount = 0; rpcLock.lock(); try { for (final List<SubscriptionCommitContext> batch : batches) { final CommitResult result = commitOnce(batch, nack); + completedCommitContextCount += batch.size(); acceptedCommitContexts.addAll(result.getAcceptedCommitContexts()); mergeTopicProgress(committedProgressByTopic, result.getCommittedProgressByTopic()); } + } catch (final SubscriptionException e) { + if (nack || completedCommitContextCount == 0) { + throw e; + } + // Preserve every completed response even if a later batch fails with a server error. The + // consumer applies these results before propagating the original failure or retrying. + throw new CommitException( + e, + completedCommitContextCount, + new CommitResult(acceptedCommitContexts, committedProgressByTopic)); } finally { rpcLock.unlock(); } return new CommitResult(acceptedCommitContexts, committedProgressByTopic); } - private CommitResult commitOnce( + CommitResult commitOnce( final List<SubscriptionCommitContext> subscriptionCommitContexts, final boolean nack) throws SubscriptionException { final PipeSubscribeCommitReq req; @@ -898,6 +910,32 @@ public abstract class AbstractSubscriptionProvider { TPipeSubscribeResp send(TPipeSubscribeReq req) throws TException; } + static final class CommitException extends SubscriptionException { + private final int completedCommitContextCount; + private final CommitResult completedResult; + + private CommitException( + final SubscriptionException cause, + final int completedCommitContextCount, + final CommitResult completedResult) { + super(cause.getMessage(), cause); + this.completedCommitContextCount = completedCommitContextCount; + this.completedResult = completedResult; + } + + int getCompletedCommitContextCount() { + return completedCommitContextCount; + } + + CommitResult getCompletedResult() { + return completedResult; + } + + SubscriptionException getFailure() { + return (SubscriptionException) getCause(); + } + } + static final class CommitResult { private static final CommitResult EMPTY = new CommitResult(Collections.emptyList(), Collections.emptyMap()); diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java index 51f35c5d820..7a05318acf8 100644 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java @@ -436,7 +436,7 @@ public class SubscriptionConsumerLifecycleTest { Assert.assertEquals(1, uncommittedCommitContexts.size()); Assert.assertEquals( Collections.singleton(rejectedCommitContext), uncommittedCommitContexts.get(0L)); - Assert.assertEquals(1, consumer.commitRequestCount); + Assert.assertEquals(4, consumer.commitRequestCount); } finally { consumer.close(); } diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java deleted file mode 100644 index 34ef7a8fc48..00000000000 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.session.subscription.consumer.base; - -import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.rpc.subscription.config.TopicConfig; -import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse; -import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType; -import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload; -import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress; -import org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHeartbeatResp; -import org.apache.iotdb.session.AbstractSessionBuilder; -import org.apache.iotdb.session.subscription.SubscriptionTreeSessionBuilder; -import org.apache.iotdb.session.subscription.payload.SubscriptionMessage; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.IMeasurementSchema; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -public class SubscriptionConsumerMultiProviderPollTest { - - private static final String HOST = "127.0.0.1"; - private static final int FIRST_PORT = 10_001; - private static final String TOPIC = "topic1"; - private static final String CONSUMER_ID = "test_consumer"; - private static final String CONSUMER_GROUP_ID = "test_consumer_group"; - private static final long LONG_INTERVAL_MS = 86_400_000L; - - @Test - public void testPollTriesNextProviderBeforeBackingOff() throws SubscriptionException { - final TestPullConsumer consumer = new TestPullConsumer(2, 0); - try { - consumer.open(); - consumer.subscribeTopic(); - - final List<SubscriptionMessage> messages = consumer.pollForTest(1_000L); - - Assert.assertEquals(1, messages.size()); - Assert.assertEquals(1, consumer.getPollCount(1)); - Assert.assertEquals(1, consumer.getPollCount(2)); - Assert.assertEquals(0, consumer.getPollCount(3)); - Assert.assertEquals(0, consumer.getBackoffCount()); - } finally { - consumer.close(); - } - } - - @Test - public void testPollBacksOffAfterAllProvidersAreEmpty() throws SubscriptionException { - final TestPullConsumer consumer = new TestPullConsumer(1, 1); - try { - consumer.open(); - consumer.subscribeTopic(); - - final List<SubscriptionMessage> messages = consumer.pollForTest(1_000L); - - Assert.assertEquals(1, messages.size()); - Assert.assertEquals(2, consumer.getPollCount(1)); - Assert.assertEquals(1, consumer.getPollCount(2)); - Assert.assertEquals(1, consumer.getPollCount(3)); - Assert.assertEquals(1, consumer.getBackoffCount()); - } finally { - consumer.close(); - } - } - - private static class TestPullConsumer extends AbstractSubscriptionPullConsumer { - - private final Map<Integer, Integer> pollCounts = new HashMap<>(); - private final int dataProviderId; - private final int emptyPollsBeforeData; - private int backoffCount; - - private TestPullConsumer(final int dataProviderId, final int emptyPollsBeforeData) { - super( - new AbstractSubscriptionPullConsumerBuilder() - .host(HOST) - .port(FIRST_PORT) - .consumerId(CONSUMER_ID) - .consumerGroupId(CONSUMER_GROUP_ID) - .heartbeatIntervalMs(LONG_INTERVAL_MS) - .endpointsSyncIntervalMs(LONG_INTERVAL_MS) - .autoCommit(false)); - this.dataProviderId = dataProviderId; - this.emptyPollsBeforeData = emptyPollsBeforeData; - } - - @Override - protected AbstractSubscriptionProvider constructSubscriptionProvider( - final TEndPoint endPoint, - final String username, - final String password, - final String encryptedPassword, - final String consumerId, - final String consumerGroupId, - final String ownerId, - final Long ownerEpoch, - final int thriftMaxFrameSize, - final long heartbeatIntervalMs, - final int connectionTimeoutInMs) { - return new TestSubscriptionProvider( - endPoint, - username, - password, - encryptedPassword, - consumerId, - consumerGroupId, - ownerId, - ownerEpoch, - thriftMaxFrameSize, - heartbeatIntervalMs, - connectionTimeoutInMs, - pollCounts, - dataProviderId, - emptyPollsBeforeData); - } - - private void subscribeTopic() { - subscribedTopics = Collections.singletonMap(TOPIC, new TopicConfig()); - } - - private List<SubscriptionMessage> pollForTest(final long timeoutMs) - throws SubscriptionException { - return poll(timeoutMs); - } - - private int getPollCount(final int dataNodeId) { - return pollCounts.getOrDefault(dataNodeId, 0); - } - - private int getBackoffCount() { - return backoffCount; - } - - @Override - void sleepAfterEmptyPollRound() { - backoffCount++; - } - } - - private static class TestSubscriptionProvider extends AbstractSubscriptionProvider { - - private final int dataNodeId; - private final Map<Integer, Integer> pollCounts; - private final int dataProviderId; - private final int emptyPollsBeforeData; - - private TestSubscriptionProvider( - final TEndPoint endPoint, - final String username, - final String password, - final String encryptedPassword, - final String consumerId, - final String consumerGroupId, - final String ownerId, - final Long ownerEpoch, - final int thriftMaxFrameSize, - final long heartbeatIntervalMs, - final int connectionTimeoutInMs, - final Map<Integer, Integer> pollCounts, - final int dataProviderId, - final int emptyPollsBeforeData) { - super( - endPoint, - username, - password, - encryptedPassword, - consumerId, - consumerGroupId, - ownerId, - ownerEpoch, - thriftMaxFrameSize, - heartbeatIntervalMs, - connectionTimeoutInMs); - this.dataNodeId = endPoint.port - FIRST_PORT + 1; - this.pollCounts = pollCounts; - this.dataProviderId = dataProviderId; - this.emptyPollsBeforeData = emptyPollsBeforeData; - } - - @Override - protected AbstractSessionBuilder constructSubscriptionSessionBuilder( - final String host, - final int port, - final String username, - final String password, - final String encryptedPassword, - final int thriftMaxFrameSize, - final int connectionTimeoutInMs) { - final boolean useEncryptedPassword = Objects.nonNull(encryptedPassword); - return new SubscriptionTreeSessionBuilder() - .host(host) - .port(port) - .username(username) - .password(useEncryptedPassword ? encryptedPassword : password) - .useEncryptedPassword(useEncryptedPassword) - .thriftMaxFrameSize(thriftMaxFrameSize) - .connectionTimeoutInMs(connectionTimeoutInMs); - } - - @Override - synchronized void handshake() { - setAvailable(); - } - - @Override - synchronized void close() { - setUnavailable(); - } - - @Override - int getDataNodeId() { - return dataNodeId; - } - - @Override - PipeSubscribeHeartbeatResp heartbeat( - final List<SubscriptionCommitContext> processorBufferedCommitContexts) { - final PipeSubscribeHeartbeatResp response = new PipeSubscribeHeartbeatResp(); - response.getTopics().put(TOPIC, new TopicConfig()); - response.getEndPoints().put(1, new TEndPoint(HOST, FIRST_PORT)); - response.getEndPoints().put(2, new TEndPoint(HOST, FIRST_PORT + 1)); - response.getEndPoints().put(3, new TEndPoint(HOST, FIRST_PORT + 2)); - return response; - } - - @Override - List<SubscriptionPollResponse> poll( - final Set<String> topicNames, - final long timeoutMs, - final Map<String, TopicProgress> progressByTopic) - throws SubscriptionException { - pollCounts.merge(dataNodeId, 1, Integer::sum); - if (dataNodeId != dataProviderId || pollCounts.get(dataNodeId) <= emptyPollsBeforeData) { - return Collections.emptyList(); - } - final SubscriptionCommitContext commitContext = - new SubscriptionCommitContext(dataNodeId, 0, TOPIC, CONSUMER_GROUP_ID, 0L); - final List<IMeasurementSchema> schemas = - Collections.singletonList(new MeasurementSchema("s1", TSDataType.INT64)); - final Tablet tablet = new Tablet("root.sg.d1", schemas, 1); - tablet.setTimestamps(new long[] {1L}); - ((long[]) tablet.getValues()[0])[0] = 1L; - tablet.setRowSize(1); - return Collections.singletonList( - new SubscriptionPollResponse( - SubscriptionPollResponseType.TABLETS.getType(), - new TabletsPayload(Collections.singletonList(tablet), -1), - commitContext)); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java index 32c3eb9ed11..11c5d14aaae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java @@ -855,13 +855,7 @@ public class PipeMemoryManager { if (force) { throw new PipeRuntimeOutOfMemoryCriticalException( - String.format( - DataNodePipeMessages - .PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC, - memoryAllocateMaxRetries, - getTotalNonFloatingMemorySizeInBytes(), - memoryBlock.getUsedMemoryInBytes(), - sizeInBytes)); + getResizeFailureMessage(memoryAllocateMaxRetries, sizeInBytes)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 3601f433c25..1c0dd008dfe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -1475,7 +1475,8 @@ public class TsFileProcessor { break; } // The asynchronous submitter needs the region write lock held by this insertion. - // Submit other already-selected MemTables here so they can release memory while we wait. + // Submit other already-selected MemTables here so they can release memory while we + // wait. submitOtherSelectedMemTablesToFlush(); try { TimeUnit.MILLISECONDS.sleep(config.getCheckPeriodWhenInsertBlocked()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java index 8bb3b3ad084..2d9d7a58c02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java @@ -792,7 +792,7 @@ public class ConsensusSubscriptionBroker implements ISubscriptionBroker { topicOwnershipSnapshots.remove(topicName); } } - queuesToClose.forEach(queue -> queue.closeAsync(false, false)); + queuesToClose.forEach(queue -> queue.closeAsync(false)); } public void wakeUpPrefetchingQueues(final String topicName, final boolean isTableModel) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java index 298647b8eb3..6de1776e8be 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java @@ -259,7 +259,9 @@ public class PipeMemoryManagerResizeTest { Assert.assertThrows( PipeRuntimeOutOfMemoryCriticalException.class, () -> manager.forceAllocateForTsFileParserWithRetry(40)); - Assert.assertTrue(allocationException.getMessage().contains("usedTsFiles=1100 bytes")); + Assert.assertTrue( + allocationException.getMessage(), + allocationException.getMessage().contains("usedTsFiles=1100 bytes")); Assert.assertTrue(allocationException.getMessage().contains("tabletLimit=900 bytes")); Assert.assertTrue( allocationException.getMessage().contains("tabletAndTsFileLimit=1200 bytes")); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 70810de95fa..d652b831b1d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -67,6 +67,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.constant import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.storageengine.dataregion.flush.FlushManager; import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy; +import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; @@ -79,7 +80,6 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; -import org.apache.iotdb.db.storageengine.rescon.memory.WriteMemoryBlock; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.iotdb.rpc.RpcUtils; @@ -241,6 +241,8 @@ public class DataRegionTest { final boolean originalSeparateData = config.isEnableSeparateData(); final long partitionInterval = TimePartitionUtils.getTimePartitionInterval(); try { + systemInfo.close(); + systemInfo.loadWriteMemory(); config.setEnableSeparateData(true); for (int i = 1; i <= 384; i++) { insertMemoryAdmissionTestRow(i, i); @@ -275,8 +277,7 @@ public class DataRegionTest { final DataRegionInfo regionInfo = (DataRegionInfo) regionInfoField.get(dataRegion); final long memoryBefore = regionInfo.getMemCost(); setMemoryAdmissionTestThreshold("memorySizeForMemtable", 2 * memoryBefore); - setMemoryAdmissionTestThreshold( - "FLUSH_THRESHOLD", (double) memoryBefore - selected.getWorkMemTableRamCost() / 2); + setMemoryAdmissionTestThreshold("FLUSH_THRESHOLD", (double) memoryBefore - 1); setMemoryAdmissionTestThreshold("REJECT_THRESHOLD", (double) memoryBefore - 1); config.setMaxWaitingTimeWhenInsertBlocked(5_000); config.setCheckPeriodWhenInsertBlocked(10); @@ -309,9 +310,6 @@ public class DataRegionTest { Assert.assertFalse(currentMemTable.shouldFlush()); Assert.assertFalse(unselectedMemTable.shouldFlush()); Assert.assertEquals(129, currentMemTable.getTotalPointsNum()); - for (WriteMemoryBlock block : regionInfo.getWriteMemoryManager().getMemoryBlocks()) { - Assert.assertEquals(0, block.getReservedMemoryInBytes()); - } Assert.assertFalse(systemInfo.isRejected()); } finally { dataRegion.writeUnlock(); @@ -321,7 +319,7 @@ public class DataRegionTest { insertMemoryAdmissionTestRow(partitionInterval + 130, 130); Assert.assertSame(currentMemTable, current.getWorkMemTable()); Assert.assertEquals(3, dataRegion.getTsFileManager().getTsFileList(true).size()); - dataRegion.syncCloseAllWorkingAndClosingTsFileProcessors(); + dataRegion.syncCloseAllWorkingTsFileProcessors(); int points = 0; for (TsFileResource resource : dataRegion.getTsFileManager().getTsFileList(true)) { try (TsFileSequenceReader reader = new TsFileSequenceReader(resource.getTsFilePath()); @@ -340,6 +338,7 @@ public class DataRegionTest { } Assert.assertEquals(515, points); } finally { + systemInfo.close(); systemInfo.loadWriteMemory(); config.setMaxWaitingTimeWhenInsertBlocked(originalWait); config.setCheckPeriodWhenInsertBlocked(originalPeriod); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java index d330af1777e..8c5e3e362d5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java @@ -86,7 +86,7 @@ public class SystemInfoTest { @Test public void testRejectedWriterCannotFlushSmallerCurrentProcessor() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); Assert.assertTrue(systemInfo.isRejected()); @@ -97,7 +97,7 @@ public class SystemInfoTest { @Test public void testRejectedWriterFirstTabletCannotBypassSelection() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); verify(currentProcessor, never()).setWorkMemTableShouldFlush(); @@ -106,7 +106,7 @@ public class SystemInfoTest { @Test public void testRepeatedReportsSubmitEachMemtableOnce() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); for (int i = 0; i < 10; i++) { Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); @@ -120,7 +120,7 @@ public class SystemInfoTest { @Test public void testRepeatedReportsForFirstTabletSubmitOnce() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); for (int i = 0; i < 10; i++) { Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); @@ -134,7 +134,7 @@ public class SystemInfoTest { public void testNewMemtableCanBeScheduledAfterPreviousFlush() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L); when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); mockWorkMemTable(currentProcessor); @@ -148,9 +148,9 @@ public class SystemInfoTest { public void testMarkedCurrentProcessorCannotReachHardMemoryLimit() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L); when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); - dataRegionInfo.getWriteMemoryManager().addMemory(100); + dataRegionInfo.addStorageGroupMemCost(100); try { systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor); @@ -166,7 +166,7 @@ public class SystemInfoTest { public void testNoCurrentMemtableCannotBypassRejection() throws Exception { when(currentProcessor.getWorkMemTable()).thenReturn(null); when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L); - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); Assert.assertTrue(systemInfo.isRejected()); @@ -176,10 +176,10 @@ public class SystemInfoTest { @Test public void testRejectionClearsAfterMemoryIsReleased() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(900); + dataRegionInfo.addStorageGroupMemCost(900); Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); - dataRegionInfo.getWriteMemoryManager().releaseMemory(500); + dataRegionInfo.releaseStorageGroupMemCost(500); systemInfo.resetStorageGroupStatus(dataRegionInfo); Assert.assertFalse(systemInfo.isRejected()); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); @@ -190,7 +190,7 @@ public class SystemInfoTest { public void testCurrentProcessorCannotExceedHardMemoryLimit() throws Exception { when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L); when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L); - dataRegionInfo.getWriteMemoryManager().addMemory(1100); + dataRegionInfo.addStorageGroupMemCost(1100); try { systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor); @@ -202,7 +202,7 @@ public class SystemInfoTest { @Test public void testNormalFlushStillSelectsLargestMemtable() throws Exception { - dataRegionInfo.getWriteMemoryManager().addMemory(700); + dataRegionInfo.addStorageGroupMemCost(700); Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor)); Assert.assertFalse(systemInfo.isRejected());
