This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new b4f60f64b8a Fix pipe TsFile transfer when mod file is missing (#18756)
(#18767)
b4f60f64b8a is described below
commit b4f60f64b8a98e5af42aa68e58242ae0794b8ceb
Author: Caideyipi <[email protected]>
AuthorDate: Wed Nov 4 17:30:20 2026 +0800
Fix pipe TsFile transfer when mod file is missing (#18756) (#18767)
---
.../thrift/async/IoTDBDataRegionAsyncSink.java | 102 +++++++++++++++++++-
.../async/handler/PipeTransferTsFileHandler.java | 106 +++++++++++++++++----
.../thrift/async/IoTDBDataRegionAsyncSinkTest.java | 49 ++++++++++
.../PipeTransferTsFileHandlerCleanupTest.java | 56 +++++++++++
4 files changed, 290 insertions(+), 23 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java
index bb405c3966e..36d72b2d5e8 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.client.ThriftClient;
import
org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient;
import
org.apache.iotdb.commons.exception.pipe.PipeRuntimeOutOfMemoryCriticalException;
+import
org.apache.iotdb.commons.exception.pipe.PipeRuntimeSinkCriticalException;
import
org.apache.iotdb.commons.exception.pipe.PipeRuntimeSinkNonReportTimeConfigurableException;
import
org.apache.iotdb.commons.exception.pipe.PipeRuntimeSinkResourceException;
import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey;
@@ -119,6 +120,7 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
"Failed to retry transferring events in the retry queue. Remaining
events: %d (tablet events: %d, tsfile events: %d). Last failure: %s.";
private static final boolean isSplitTSFileBatchModeEnabled = true;
+ private static final int MAX_FILE_NOT_FOUND_RETRY_TIMES = 5;
private final IoTDBDataRegionSyncSink syncConnector = new
IoTDBDataRegionSyncSink();
private final BlockingQueue<Event> retryEventQueue = new
LinkedBlockingQueue<>();
@@ -129,6 +131,10 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
// semantics because the same payload may compare equal.
private final Map<Event, PipeResourceFailureType>
retryEvent2ResourceFailureType =
new IdentityHashMap<>();
+ // File disappearance is permanent for a given TsFile event. Keep its retry
count separate from
+ // transient receiver failures so one missing file cannot retry forever in
the async queue.
+ private final Map<Event, Integer> missingFileRetryTimes = new
IdentityHashMap<>();
+ private volatile PipeRuntimeSinkCriticalException
missingFileRetryLimitException;
// Keep only the latest text to avoid retaining the complete exception chain
for every event.
private volatile String lastRetryFailureMessage;
// Events removed from the retry queue remain here while their next transfer
is being started.
@@ -511,6 +517,21 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
throw new
FileNotFoundException(pipeTsFileInsertionEvent.getTsFile().getAbsolutePath());
}
+ final boolean transferMod =
+ pipeTsFileInsertionEvent.isWithMod()
+ && clientManager.supportModsIfIsDataNodeReceiver()
+ && pipeTsFileInsertionEvent.getModFile() != null
+ && pipeTsFileInsertionEvent.getModFile().exists();
+ if (pipeTsFileInsertionEvent.isWithMod()
+ && clientManager.supportModsIfIsDataNodeReceiver()
+ && pipeTsFileInsertionEvent.getModFile() != null
+ && !pipeTsFileInsertionEvent.getModFile().exists()) {
+ LOGGER.warn(
+ "TsFile {}: modification file {} is missing, transfer the TsFile
without modifications.",
+ pipeTsFileInsertionEvent.getTsFile(),
+ pipeTsFileInsertionEvent.getModFile());
+ }
+
pipeTransferTsFileHandler =
new PipeTransferTsFileHandler(
this,
@@ -524,8 +545,7 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
new AtomicBoolean(false),
pipeTsFileInsertionEvent.getTsFile(),
pipeTsFileInsertionEvent.getModFile(),
- pipeTsFileInsertionEvent.isWithMod()
- && clientManager.supportModsIfIsDataNodeReceiver(),
+ transferMod,
pipeTsFileInsertionEvent.getDatabaseName());
trackRetryHandler(
pipeTransferTsFileHandler,
Collections.singletonList(pipeTsFileInsertionEvent));
@@ -662,6 +682,7 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
* @see PipeConnector#transfer(TsFileInsertionEvent) for more details.
*/
private void transferQueuedEventsIfNecessary(final boolean forced) {
+ throwIfMissingFileRetryLimitExceeded();
throwIfReceiverProbeIsDelayed();
if ((retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty())
@@ -727,6 +748,7 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
}
}
+ throwIfMissingFileRetryLimitExceeded();
throwIfReceiverProbeIsDelayed();
// Stop retrying if the execution time exceeds the threshold for better
realtime performance
@@ -869,10 +891,12 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
final EnrichedEvent enrichedEvent = (EnrichedEvent) event;
if (enrichedEvent.isReleased()) {
retryingEvent2Handlers.remove(event);
+ missingFileRetryTimes.remove(event);
return;
}
if (isDroppedPipe(enrichedEvent)) {
retryingEvent2Handlers.remove(event);
+ missingFileRetryTimes.remove(event);
enrichedEvent.clearReferenceCount(IoTDBDataRegionAsyncSink.class.getName());
return;
}
@@ -880,6 +904,7 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
if (isClosed.get()) {
retryingEvent2Handlers.remove(event);
+ missingFileRetryTimes.remove(event);
if (event instanceof EnrichedEvent) {
((EnrichedEvent)
event).clearReferenceCount(IoTDBDataRegionAsyncSink.class.getName());
}
@@ -888,6 +913,7 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
if (retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty()) {
lastRetryFailureMessage = null;
+ missingFileRetryLimitException = null;
}
if (e != null) {
@@ -909,6 +935,14 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
final PipeResourceFailureType previousResourceFailureType =
retryEvent2ResourceFailureType.get(event);
+ if (!alreadyInRetryQueue && isMissingFileFailure(event, e)) {
+ final int failureCount = missingFileRetryTimes.merge(event, 1,
Integer::sum);
+ if (failureCount > MAX_FILE_NOT_FOUND_RETRY_TIMES) {
+ reportMissingFileEventAfterRetryLimit(event, e, failureCount - 1);
+ return;
+ }
+ }
+
if (resourceFailureType != null
&& event instanceof EnrichedEvent
&& (!alreadyInRetryQueue || previousResourceFailureType !=
resourceFailureType)) {
@@ -946,6 +980,64 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
}
}
+ private static boolean isMissingFileFailure(final Event event, final
Exception exception) {
+ if (!(event instanceof PipeTsFileInsertionEvent) || exception == null) {
+ return false;
+ }
+ return ErrorHandlingCommonUtils.getRootCause(exception) instanceof
FileNotFoundException;
+ }
+
+ private void reportMissingFileEventAfterRetryLimit(
+ final Event event, final Exception cause, final int retryTimes) {
+ final String missingFile = getMissingFile(cause);
+ final File tsFile = ((PipeTsFileInsertionEvent) event).getTsFile();
+ final String message =
+ String.format(
+ "Failed to transfer TsFile %s because file %s is missing after %s
retries.",
+ tsFile, missingFile, retryTimes);
+ final PipeRuntimeSinkCriticalException criticalException =
+ new PipeRuntimeSinkCriticalException(message, cause);
+
+ retryingEvent2Handlers.remove(event);
+ retryEvent2ResourceFailureType.put(event, null);
+ missingFileRetryTimes.remove(event);
+ lastRetryFailureMessage = message;
+ missingFileRetryLimitException = criticalException;
+ retryTsFileQueue.offer((PipeTsFileInsertionEvent) event);
+ retryEventQueueEventCounter.increaseEventCount(event);
+ if (event instanceof EnrichedEvent) {
+ final EnrichedEvent enrichedEvent = (EnrichedEvent) event;
+ PipeLogger.log(
+ LOGGER::error,
+ criticalException,
+ "Failed to transfer TsFile %s because file %s is missing after %s
retries.",
+ tsFile,
+ missingFile,
+ retryTimes);
+ PipeDataNodeAgent.runtime().report(enrichedEvent, criticalException);
+ } else {
+ LOGGER.error(message, cause);
+ }
+ }
+
+ private void throwIfMissingFileRetryLimitExceeded() {
+ final PipeRuntimeSinkCriticalException exception =
missingFileRetryLimitException;
+ if (exception != null) {
+ throw exception;
+ }
+ }
+
+ private static String getMissingFile(final Exception exception) {
+ final Throwable rootCause =
ErrorHandlingCommonUtils.getRootCause(exception);
+ return hasText(rootCause.getMessage()) ? rootCause.getMessage() :
rootCause.toString();
+ }
+
+ public synchronized void clearFileNotFoundRetryTimes(final Iterable<?
extends Event> events) {
+ for (final Event event : events) {
+ missingFileRetryTimes.remove(event);
+ }
+ }
+
/**
* Add failure {@link EnrichedEvent}s to retry queue.
*
@@ -1151,6 +1243,7 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
((EnrichedEvent)
event).clearReferenceCount(IoTDBDataRegionAsyncSink.class.getName());
retryEventQueueEventCounter.decreaseEventCount(event);
retryEvent2ResourceFailureType.remove(event);
+ missingFileRetryTimes.remove(event);
retryingEvent2Handlers.remove(event);
return true;
}
@@ -1164,6 +1257,7 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
((EnrichedEvent)
event).clearReferenceCount(IoTDBDataRegionAsyncSink.class.getName());
retryEventQueueEventCounter.decreaseEventCount(event);
retryEvent2ResourceFailureType.remove(event);
+ missingFileRetryTimes.remove(event);
retryingEvent2Handlers.remove(event);
return true;
}
@@ -1179,6 +1273,7 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
if (retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty()) {
lastRetryFailureMessage = null;
+ missingFileRetryLimitException = null;
}
}
@@ -1231,12 +1326,15 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink
{
retryTsFileQueue.isEmpty() ? retryEventQueue.poll() :
retryTsFileQueue.poll();
retryEventQueueEventCounter.decreaseEventCount(event);
retryEvent2ResourceFailureType.remove(event);
+ missingFileRetryTimes.remove(event);
retryingEvent2Handlers.remove(event);
if (event instanceof EnrichedEvent) {
((EnrichedEvent)
event).clearReferenceCount(IoTDBDataRegionAsyncSink.class.getName());
}
}
retryEvent2ResourceFailureType.clear();
+ missingFileRetryTimes.clear();
+ missingFileRetryLimitException = null;
lastRetryFailureMessage = null;
retryingEvent2Handlers.clear();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
index b8b757250f8..44273fc60c6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
@@ -51,6 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.Arrays;
@@ -79,13 +80,20 @@ public class PipeTransferTsFileHandler extends
PipeTransferTrackableHandler {
private File currentFile;
private final boolean transferMod;
+ // A mod file can be cleaned up after the event is created. Keep the
requested mode immutable,
+ // but downgrade the requests sent by this handler when the sidecar is no
longer available.
+ private boolean effectiveTransferMod;
+
private final String dataBaseName;
- private final String conversionTaskId;
+ private String conversionTaskId;
+ private final int outputIndex;
private final int readFileBufferSize;
private PipeTsFileMemoryBlock memoryBlock;
private byte[] readBuffer;
private long position;
+ private boolean transferringModFile;
+ private boolean transferringTsFile;
private RandomAccessFile reader;
@@ -140,13 +148,13 @@ public class PipeTransferTsFileHandler extends
PipeTransferTrackableHandler {
this.tsFile = tsFile;
this.modFile = modFile;
this.transferMod = transferMod;
+ this.effectiveTransferMod = transferMod && modFile != null &&
modFile.exists();
this.dataBaseName = dataBaseName;
- conversionTaskId =
- connector.shouldAsyncLoadTsFileOnTypeMismatch()
- ? PipeTransferTsFileSealWithModReq.generateConversionTaskId(
- connector.getSinkTaskId(), events, dataBaseName, outputIndex,
transferMod)
- : null;
- currentFile = transferMod ? modFile : tsFile;
+ this.outputIndex = outputIndex;
+ this.transferringModFile = false;
+ this.transferringTsFile = false;
+ conversionTaskId = generateConversionTaskId();
+ currentFile = effectiveTransferMod ? modFile : tsFile;
// NOTE: Waiting for resource enough for slicing here may cause deadlock!
// TsFile events are producing and consuming at the same time, and the
memory of a TsFile
@@ -156,7 +164,7 @@ public class PipeTransferTsFileHandler extends
PipeTransferTrackableHandler {
// will cause a deadlock.
waitForResourceEnough4Slicing((long) ((1 + Math.random()) * 20 * 1000));
// 20 - 40 seconds
final long maxFileLength =
- transferMod && Objects.nonNull(modFile)
+ effectiveTransferMod && Objects.nonNull(modFile)
? Math.max(tsFile.length(), modFile.length())
: tsFile.length();
readFileBufferSize =
@@ -198,7 +206,7 @@ public class PipeTransferTsFileHandler extends
PipeTransferTrackableHandler {
}
if (reader == null) {
- reader = transferMod ? new RandomAccessFile(modFile, "r") : new
RandomAccessFile(tsFile, "r");
+ openNextFile();
}
client.setShouldReturnSelf(false);
@@ -207,21 +215,15 @@ public class PipeTransferTsFileHandler extends
PipeTransferTrackableHandler {
final int readLength = readNextFilePiece(reader, readBuffer);
if (readLength == -1) {
- if (currentFile == modFile) {
- currentFile = tsFile;
- position = 0;
- try {
- reader.close();
- } catch (final IOException e) {
- LOGGER.warn("Failed to close file reader when successfully
transferred mod file.", e);
- }
- reader = new RandomAccessFile(tsFile, "r");
+ reader.close();
+ reader = null;
+ if (openNextFile()) {
transfer(clientManager, client);
- } else if (currentFile == tsFile) {
+ } else {
isSealSignalSent.set(true);
final TPipeTransferReq uncompressedReq;
- if (transferMod) {
+ if (effectiveTransferMod) {
uncompressedReq =
PipeTransferTsFileSealWithModReq.toTPipeTransferReq(
modFile.getName(),
@@ -269,7 +271,7 @@ public class PipeTransferTsFileHandler extends
PipeTransferTrackableHandler {
? readBuffer
: Arrays.copyOfRange(readBuffer, 0, readLength);
final TPipeTransferReq uncompressedReq =
- transferMod
+ effectiveTransferMod
? PipeTransferTsFilePieceWithModReq.toTPipeTransferReq(
currentFile.getName(), position, payload)
: PipeTransferTsFilePieceReq.toTPipeTransferReq(
@@ -343,6 +345,7 @@ public class PipeTransferTsFileHandler extends
PipeTransferTrackableHandler {
} finally {
final int referenceCount = eventsReferenceCount.decrementAndGet();
if (referenceCount <= 0) {
+ sink.clearFileNotFoundRetryTimes(events);
events.forEach(
event ->
event.decreaseReferenceCount(PipeTransferTsFileHandler.class.getName(), true));
@@ -495,6 +498,67 @@ public class PipeTransferTsFileHandler extends
PipeTransferTrackableHandler {
events.forEach(event ->
event.clearReferenceCount(PipeTransferTsFileHandler.class.getName()));
}
+ private boolean advanceToNextFile() {
+ if (transferMod && !transferringModFile) {
+ transferringModFile = true;
+ if (modFile != null && modFile.exists()) {
+ effectiveTransferMod = true;
+ conversionTaskId = generateConversionTaskId();
+ currentFile = modFile;
+ position = 0;
+ return true;
+ }
+
+ // The mod file is an optional sidecar. It may have been cleaned up
after the event was
+ // created, so transfer the TsFile without modifications instead of
creating a WithMod
+ // request that the receiver can never finish.
+ effectiveTransferMod = false;
+ conversionTaskId = generateConversionTaskId();
+ if (modFile != null) {
+ LOGGER.warn(
+ "TsFile {}: modification file {} is missing, transfer the TsFile
without modifications.",
+ tsFile,
+ modFile);
+ }
+ }
+ if (!transferringTsFile) {
+ transferringTsFile = true;
+ currentFile = tsFile;
+ position = 0;
+ return true;
+ }
+ return false;
+ }
+
+ private boolean openNextFile() throws IOException {
+ while (advanceToNextFile()) {
+ try {
+ reader = new RandomAccessFile(currentFile, "r");
+ return true;
+ } catch (final FileNotFoundException e) {
+ if (currentFile == modFile && effectiveTransferMod) {
+ // Re-check after advanceToNextFile(). The sidecar can disappear in
this small window.
+ effectiveTransferMod = false;
+ conversionTaskId = generateConversionTaskId();
+ LOGGER.warn(
+ "TsFile {}: modification file {} is missing, transfer the TsFile
without modifications.",
+ tsFile,
+ modFile);
+ continue;
+ }
+ throw e;
+ }
+ }
+ return false;
+ }
+
+ private String generateConversionTaskId() {
+ return sink.shouldAsyncLoadTsFileOnTypeMismatch()
+ ? PipeTransferTsFileSealWithModReq.generateConversionTaskId(
+ sink.getSinkTaskId(), events, dataBaseName, outputIndex,
effectiveTransferMod)
+ : null;
+ }
+
@Override
public synchronized void close() {
try {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSinkTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSinkTest.java
index cd0213bc735..fb924b31e30 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSinkTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSinkTest.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.db.pipe.sink.protocol.thrift.async;
+import
org.apache.iotdb.commons.exception.pipe.PipeRuntimeSinkCriticalException;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.exception.PipeException;
@@ -26,6 +28,11 @@ import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.lang.reflect.Field;
+import java.util.Map;
+
public class IoTDBDataRegionAsyncSinkTest {
@Test
@@ -63,4 +70,46 @@ public class IoTDBDataRegionAsyncSinkTest {
Assert.assertEquals("java.lang.NullPointerException",
sink.getLastRetryFailureMessage());
}
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testMissingTsFileRetryLimitKeepsEventQueuedAndStopsSink() throws
Exception {
+ final IoTDBDataRegionAsyncSink sink = new IoTDBDataRegionAsyncSink();
+ final PipeTsFileInsertionEvent event =
Mockito.mock(PipeTsFileInsertionEvent.class);
+ final File missingTsFile = new
File("missing-retry-limit.tsfile").getAbsoluteFile();
+ Mockito.when(event.getTsFile()).thenReturn(missingTsFile);
+
+ final Field retryLimitField =
+
IoTDBDataRegionAsyncSink.class.getDeclaredField("MAX_FILE_NOT_FOUND_RETRY_TIMES");
+ retryLimitField.setAccessible(true);
+ final int retryLimit = retryLimitField.getInt(null);
+
+ final Field retryTimesField =
+
IoTDBDataRegionAsyncSink.class.getDeclaredField("missingFileRetryTimes");
+ retryTimesField.setAccessible(true);
+ ((Map<Event, Integer>) retryTimesField.get(sink)).put(event, retryLimit);
+
+ try {
+ sink.addFailureEventToRetryQueue(
+ event, new FileNotFoundException(missingTsFile.getAbsolutePath()));
+
+ Assert.assertEquals(1, sink.getRetryEventQueueSize());
+ Assert.assertTrue(
+
sink.getLastRetryFailureMessage().contains(missingTsFile.getAbsolutePath()));
+
Assert.assertTrue(sink.getLastRetryFailureMessage().contains(String.valueOf(retryLimit)));
+ Mockito.verify(event, Mockito.never())
+ .clearReferenceCount(IoTDBDataRegionAsyncSink.class.getName());
+
+ try {
+ sink.transfer(Mockito.mock(Event.class));
+ Assert.fail("The sink should stop transferring after the missing-file
retry limit.");
+ } catch (final PipeRuntimeSinkCriticalException e) {
+
Assert.assertTrue(e.getMessage().contains(missingTsFile.getAbsolutePath()));
+ }
+ } finally {
+ sink.clearRetryEventsReferenceCount();
+ }
+
+
Mockito.verify(event).clearReferenceCount(IoTDBDataRegionAsyncSink.class.getName());
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandlerCleanupTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandlerCleanupTest.java
index 3cca3285cca..563961d983d 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandlerCleanupTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandlerCleanupTest.java
@@ -36,6 +36,7 @@ import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.lang.reflect.Field;
+import java.lang.reflect.Method;
import java.nio.file.Files;
import java.util.Collections;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -95,6 +96,47 @@ public class PipeTransferTsFileHandlerCleanupTest {
}
}
+ @Test
+ public void testModFileDisappearingBeforeOpenDowngradesToTsFileOnly() throws
Exception {
+ final File tsFile = Files.createTempFile("pipe-transfer-source",
".tsfile").toFile();
+ final File modFile =
+ new DisappearingFile(
+ Files.createTempDirectory("pipe-transfer-missing-mod")
+ .resolve("source.tsfile.mods2")
+ .toString());
+ final PipeTsFileInsertionEvent event =
Mockito.mock(PipeTsFileInsertionEvent.class);
+ final PipeTransferTsFileHandler handler =
+ new PipeTransferTsFileHandler(
+ Mockito.mock(IoTDBDataRegionAsyncSink.class),
+ Collections.emptyMap(),
+ Collections.singletonList(event),
+ new AtomicInteger(1),
+ new AtomicBoolean(false),
+ tsFile,
+ modFile,
+ true,
+ null);
+
+ try {
+ final Method openNextFile =
PipeTransferTsFileHandler.class.getDeclaredMethod("openNextFile");
+ openNextFile.setAccessible(true);
+ Assert.assertTrue((Boolean) openNextFile.invoke(handler));
+
+ final Field currentFile =
PipeTransferTsFileHandler.class.getDeclaredField("currentFile");
+ currentFile.setAccessible(true);
+ Assert.assertSame(tsFile, currentFile.get(handler));
+
+ final Field effectiveTransferMod =
+
PipeTransferTsFileHandler.class.getDeclaredField("effectiveTransferMod");
+ effectiveTransferMod.setAccessible(true);
+ Assert.assertFalse(effectiveTransferMod.getBoolean(handler));
+ } finally {
+ handler.close();
+ Assert.assertTrue(tsFile.delete());
+ Assert.assertTrue(modFile.getParentFile().delete());
+ }
+ }
+
@Test
public void testSealFailurePassesNestedReceiverMessageToRetryQueue() throws
Exception {
final File file = Files.createTempFile("pipe-transfer-seal-failure",
".tsfile").toFile();
@@ -160,4 +202,18 @@ public class PipeTransferTsFileHandlerCleanupTest {
false,
null);
}
+
+ private static final class DisappearingFile extends File {
+
+ private final AtomicInteger existenceCheckCount = new AtomicInteger();
+
+ private DisappearingFile(final String pathname) {
+ super(pathname);
+ }
+
+ @Override
+ public boolean exists() {
+ return existenceCheckCount.incrementAndGet() <= 2;
+ }
+ }
}