This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 14d9c0c1562 Preserve failed active LOAD file groups (#19253) (#19285)
14d9c0c1562 is described below
commit 14d9c0c156281e43b6912399d027184bf3806f1f
Author: Caideyipi <[email protected]>
AuthorDate: Mon Nov 16 14:08:01 2026 +0800
Preserve failed active LOAD file groups (#19253) (#19285)
* Fix active LOAD failed-file recovery
* Fix active LOAD recovery IT time-column queries
* Fix active LOAD recovery test database level
(cherry picked from commit ed9504f3aeb8cef403b736dd773f95091b2acad9)
---
.../iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java | 439 +++++++++++++++++++++
.../iotdb/db/i18n/StorageEngineMessages.java | 5 +
.../iotdb/db/i18n/StorageEngineMessages.java | 5 +
.../load/active/ActiveLoadPendingQueue.java | 18 +-
.../load/active/ActiveLoadTsFileLoader.java | 293 +++++++++++++-
.../load/active/ActiveLoadTsFileLoaderTest.java | 135 ++++++-
6 files changed, 870 insertions(+), 25 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java
index 3827615fe24..f0a39e0da1a 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileActiveRetryIT.java
@@ -19,6 +19,13 @@
package org.apache.iotdb.db.it;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
+import
org.apache.iotdb.db.storageengine.dataregion.wal.recover.file.SealedTsFileRecoverPerformer;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
@@ -37,10 +44,20 @@ import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import java.io.File;
+import java.io.InputStream;
import java.nio.file.Files;
import java.sql.Connection;
+import java.sql.ResultSet;
import java.sql.Statement;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
@RunWith(IoTDBTestRunner.class)
@@ -50,6 +67,8 @@ public class IoTDBLoadTsFileActiveRetryIT {
private static final String DATABASE = "root.sg.test_0";
private static final String DEVICE = DATABASE + ".d_0";
private static final String MEASUREMENT = "sensor_00";
+ private static final String COLLISION_DATABASE = "root.load_p118";
+ private static final String COLLISION_DEVICE = COLLISION_DATABASE + ".d0";
private static final long
UNALLOCATABLE_TABLET_CONVERSION_BATCH_MEMORY_SIZE_IN_BYTES =
Long.MAX_VALUE / 4;
private static final MeasurementSchema TSFILE_SCHEMA =
@@ -129,6 +148,426 @@ public class IoTDBLoadTsFileActiveRetryIT {
}
}
+ @Test
+ public void testPermanentFailureKeepsSameNamedTsFileGroupsRecoverable()
throws Exception {
+ final DataNodeWrapper dataNodeWrapper =
EnvFactory.getEnv().getDataNodeWrapper(0);
+ final File sourceRoot = new File(tmpDir, "collision-sources");
+ Assert.assertTrue(sourceRoot.mkdir());
+
+ final List<File> sourceTsFiles = new ArrayList<>();
+ final File identicalFirst =
+ createFailureGroup(new File(sourceRoot, "identical-first"), 1, 10,
false, 1, 1);
+ sourceTsFiles.add(identicalFirst);
+
+ final File identicalSecondDir = new File(sourceRoot, "identical-second");
+ Assert.assertTrue(identicalSecondDir.mkdir());
+ final File identicalSecond = new File(identicalSecondDir,
identicalFirst.getName());
+ Files.copy(identicalFirst.toPath(), identicalSecond.toPath());
+ writeResource(identicalSecond);
+ writeModification(identicalSecond, 2);
+ sourceTsFiles.add(identicalSecond);
+
+ final File sameSizeDifferentContent =
+ createFailureGroup(
+ new File(sourceRoot, "same-size-different-content"), 2, 10, false,
101, 103);
+ sourceTsFiles.add(sameSizeDifferentContent);
+
+ final File differentSize =
+ createFailureGroup(new File(sourceRoot, "different-size"), 3, 50,
true, 201, 205);
+ sourceTsFiles.add(differentSize);
+
+ final List<File> expectedTsFiles =
+ snapshotTsFileGroups(sourceTsFiles, new File(tmpDir,
"collision-expected"));
+
+ Assert.assertEquals(identicalFirst.length(),
sameSizeDifferentContent.length());
+ Assert.assertNotEquals(
+ -1L, Files.mismatch(identicalFirst.toPath(),
sameSizeDifferentContent.toPath()));
+ Assert.assertNotEquals(identicalFirst.length(), differentSize.length());
+
+ try (Connection connection =
+
EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(dataNodeWrapper);
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database " + COLLISION_DATABASE);
+ statement.execute(
+ String.format(
+ "create timeseries %s.%s %s",
+ COLLISION_DEVICE, MEASUREMENT, TSDataType.INT64.name()));
+ for (final File tsFile : sourceTsFiles) {
+ statement.execute(
+ String.format(
+ "load \"%s\" with
('database-level'='1','async'='true','on-success'='none',"
+ +
"'auto-create-schema'='false','convert-on-type-mismatch'='false')",
+ tsFile.getAbsolutePath()));
+ }
+ }
+
+ final File failDir = getActiveLoadFailDir(dataNodeWrapper);
+ final List<File> failedGroups =
+ waitForCompleteFailedGroups(failDir, sourceTsFiles.size(), 60_000L);
+ Assert.assertEquals(sourceTsFiles.size(), failedGroups.size());
+ final Map<File, File> failedGroupsBySource =
+ assertFailedGroupsMatchSources(
+ expectedTsFiles, sourceTsFiles, getActiveLoadDir(dataNodeWrapper),
failedGroups);
+ final File recoveryInputRoot = new File(tmpDir, "recovery-inputs");
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ try (ResultSet resultSet =
+ statement.executeQuery("select count(" + MEASUREMENT + ") from " +
COLLISION_DEVICE)) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals("Schema mismatch failures must not load data", 0,
resultSet.getLong(1));
+ }
+ final long[] startTimes = {1, 1, 101, 201};
+ final long[] endTimes = {10, 10, 110, 250};
+ final long[] deletedTimes = {1, 2, 103, 205};
+ final long[] expectedCounts = {9, 9, 9, 49};
+ final long[] seeds = {1, 1, 2, 3};
+ for (int i = 0; i < expectedTsFiles.size(); i++) {
+ statement.execute("delete database " + COLLISION_DATABASE);
+ statement.execute("create database " + COLLISION_DATABASE);
+ statement.execute(
+ String.format(
+ "create timeseries %s.%s %s",
+ COLLISION_DEVICE, MEASUREMENT, TSDataType.INT32.name()));
+ if (i == 3) {
+ statement.execute("create timeseries " + COLLISION_DEVICE + ".extra
INT32");
+ }
+
+ final File failedGroup =
failedGroupsBySource.get(expectedTsFiles.get(i));
+ Assert.assertNotNull("Failed source group could not be located",
failedGroup);
+ final File failedTsFile = copyFailedGroup(failedGroup,
recoveryInputRoot, i);
+ statement.execute(
+ String.format(
+ "load \"%s\" with ('database-level'='1','on-success'='none',"
+ +
"'auto-create-schema'='false','convert-on-type-mismatch'='false')",
+ failedTsFile.getAbsolutePath()));
+ assertCount(
+ statement,
+ COLLISION_DEVICE,
+ String.format("time >= %d and time <= %d", startTimes[i],
endTimes[i]),
+ expectedCounts[i]);
+ assertCount(statement, COLLISION_DEVICE, "time = " + deletedTimes[i],
0);
+ assertRecoveredRows(
+ statement, startTimes[i], endTimes[i], deletedTimes[i], seeds[i],
i == 3);
+ }
+
+ final File normalTsFile = new File(tmpDir,
"normal-after-failure.tsfile");
+ try (final TsFileGenerator generator = new
TsFileGenerator(normalTsFile)) {
+ generator.registerTimeseries(COLLISION_DEVICE,
Collections.singletonList(TSFILE_SCHEMA));
+ generator.generateData(COLLISION_DEVICE, 10, 1, false, 300);
+ }
+ try (Connection activeConnection =
+
EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(dataNodeWrapper);
+ Statement activeStatement = activeConnection.createStatement()) {
+ activeStatement.execute(
+ String.format(
+ "load \"%s\" with
('database-level'='1','async'='true','on-success'='none',"
+ +
"'auto-create-schema'='false','convert-on-type-mismatch'='false')",
+ normalTsFile.getAbsolutePath()));
+ }
+ TestUtils.assertDataEventuallyOnEnv(
+ EnvFactory.getEnv(),
+ String.format(
+ "select count(%s) from %s where time >= 301 and time <= 310",
+ MEASUREMENT, COLLISION_DEVICE),
+ "count(" + COLLISION_DEVICE + "." + MEASUREMENT + "),",
+ Collections.singleton("10,"));
+ }
+ }
+
+ private File createFailureGroup(
+ final File groupDir,
+ final long randomSeed,
+ final int rowCount,
+ final boolean withExtraMeasurement,
+ final long startTimestamp,
+ final long deletionTimestamp)
+ throws Exception {
+ Assert.assertTrue(groupDir.mkdir());
+ final File tsFile = new File(groupDir, "1-0-0-0.tsfile");
+ final List<MeasurementSchema> schemas = new ArrayList<>();
+ schemas.add(TSFILE_SCHEMA);
+ if (withExtraMeasurement) {
+ schemas.add(new MeasurementSchema("extra", TSDataType.INT32,
TSEncoding.RLE));
+ }
+ try (final TsFileGenerator generator = new TsFileGenerator(tsFile)) {
+ generator.resetRandom(randomSeed);
+ generator.registerTimeseries(COLLISION_DEVICE, schemas);
+ generator.generateData(COLLISION_DEVICE, rowCount, 1, false,
startTimestamp - 1);
+ }
+ writeResource(tsFile);
+ writeModification(tsFile, deletionTimestamp);
+ return tsFile;
+ }
+
+ private void writeResource(final File tsFile) throws Exception {
+ final TsFileResource resource = new TsFileResource(tsFile);
+ try (final SealedTsFileRecoverPerformer performer =
+ new SealedTsFileRecoverPerformer(resource)) {
+ performer.recover();
+ }
+ resource.setStatusForTest(TsFileResourceStatus.NORMAL);
+ resource.deserialize();
+ resource.serialize();
+ }
+
+ private void writeModification(final File tsFile, final long timestamp)
throws Exception {
+ try (final ModificationFile modificationFile =
+ new ModificationFile(tsFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX)) {
+ modificationFile.write(
+ new Deletion(
+ new PartialPath(COLLISION_DEVICE + "." + MEASUREMENT),
+ Long.MAX_VALUE,
+ timestamp,
+ timestamp));
+ }
+ }
+
+ private List<File> waitForCompleteFailedGroups(
+ final File failDir, final int expectedCount, final long timeoutMs)
+ throws InterruptedException {
+ final long deadline = System.currentTimeMillis() + timeoutMs;
+ List<File> groups;
+ do {
+ groups = findFailedGroups(failDir);
+ if (groups.size() >= expectedCount
+ && groups.stream().allMatch(this::hasCompleteFailedGroup)) {
+ return groups;
+ }
+ if (System.currentTimeMillis() >= deadline) {
+ return groups;
+ }
+ Thread.sleep(250L);
+ } while (true);
+ }
+
+ private boolean hasCompleteFailedGroup(final File groupDir) {
+ final File tsFile = findTsFile(groupDir);
+ return tsFile != null
+ && new File(tsFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX).isFile()
+ && new File(tsFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX).isFile();
+ }
+
+ private List<File> findFailedGroups(final File failDir) {
+ final List<File> groups = new ArrayList<>();
+ if (findTsFile(failDir) != null) {
+ groups.add(failDir);
+ }
+ if (failDir == null || !failDir.isDirectory()) {
+ return groups;
+ }
+ final File[] children = failDir.listFiles();
+ if (children == null) {
+ return groups;
+ }
+ for (final File child : children) {
+ if (child.isDirectory()
+ && !child.getName().startsWith(".active-load-staging-")
+ && findTsFile(child) != null) {
+ groups.add(child);
+ }
+ }
+ return groups;
+ }
+
+ private File findTsFile(final File directory) {
+ if (directory == null || !directory.isDirectory()) {
+ return null;
+ }
+ final File[] children = directory.listFiles();
+ if (children == null) {
+ return null;
+ }
+ for (final File child : children) {
+ if (child.isFile() && child.getName().endsWith(".tsfile")) {
+ return child;
+ }
+ }
+ return null;
+ }
+
+ private Map<File, File> assertFailedGroupsMatchSources(
+ final List<File> expectedTsFiles,
+ final List<File> submittedTsFiles,
+ final File activeLoadDir,
+ final List<File> failedGroups)
+ throws Exception {
+ Assert.assertEquals(expectedTsFiles.size(), submittedTsFiles.size());
+ final Set<File> unmatchedGroups = new HashSet<>(failedGroups);
+ final Map<File, File> groupsBySource = new HashMap<>();
+ final Set<String> activeLoadFiles = new HashSet<>();
+ for (int sourceIndex = 0; sourceIndex < expectedTsFiles.size();
sourceIndex++) {
+ final File expectedTsFile = expectedTsFiles.get(sourceIndex);
+ final File submittedTsFile = submittedTsFiles.get(sourceIndex);
+ Assert.assertTrue(
+ "Expected TsFile snapshot differs from submitted source",
+ sameBytes(expectedTsFile, submittedTsFile));
+ Assert.assertTrue(
+ "Expected resource snapshot differs from submitted source",
+ sameBytes(
+ new File(expectedTsFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX),
+ new File(submittedTsFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX)));
+ Assert.assertTrue(
+ "Expected mods snapshot differs from submitted source",
+ sameBytes(
+ new File(expectedTsFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX),
+ new File(submittedTsFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX)));
+ File matchedGroup = null;
+ for (final File groupDir : unmatchedGroups) {
+ final File failedTsFile = findTsFile(groupDir);
+ if (failedTsFile != null && sameBytes(expectedTsFile, failedTsFile)) {
+ final File sourceResource =
+ new File(expectedTsFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX);
+ final File failedResource =
+ new File(failedTsFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX);
+ final File sourceMods =
+ new File(expectedTsFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX);
+ final File failedMods =
+ new File(failedTsFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX);
+ if (sourceResource.exists()
+ && failedResource.exists()
+ && sourceMods.exists()
+ && failedMods.exists()
+ && sameBytes(sourceResource, failedResource)
+ && sameBytes(sourceMods, failedMods)) {
+ final File failureInfoFile =
+ new File(groupDir, failedTsFile.getName() + ".load-failure");
+ if (!failureInfoFile.isFile()) {
+ continue;
+ }
+ final Properties failureInfo = new Properties();
+ try (InputStream inputStream =
Files.newInputStream(failureInfoFile.toPath())) {
+ failureInfo.load(inputStream);
+ }
+ final String activeLoadFile =
failureInfo.getProperty("activeLoadFile");
+ final String pendingDir = failureInfo.getProperty("pendingDir");
+ if (activeLoadFile == null
+ || !new
File(activeLoadFile).getName().equals(failedTsFile.getName())
+ || !new File(activeLoadFile)
+ .toPath()
+ .toAbsolutePath()
+ .normalize()
+
.startsWith(activeLoadDir.toPath().toAbsolutePath().normalize())
+ || pendingDir == null
+ || !new File(pendingDir)
+ .toPath()
+ .toAbsolutePath()
+ .normalize()
+
.equals(activeLoadDir.toPath().toAbsolutePath().normalize())
+ ||
Boolean.parseBoolean(failureInfo.getProperty("generatedByPipe", "true"))
+ || Boolean.parseBoolean(failureInfo.getProperty("tableModel",
"true"))
+ || !activeLoadFiles.add(activeLoadFile)) {
+ continue;
+ }
+ matchedGroup = groupDir;
+ break;
+ }
+ }
+ }
+ Assert.assertNotNull("Failed directory lost or cross-paired a TsFile
group", matchedGroup);
+ unmatchedGroups.remove(matchedGroup);
+ groupsBySource.put(expectedTsFile, matchedGroup);
+ }
+ Assert.assertTrue("Unexpected failed file groups remain",
unmatchedGroups.isEmpty());
+ return groupsBySource;
+ }
+
+ private boolean sameBytes(final File left, final File right) throws
Exception {
+ return Files.mismatch(left.toPath(), right.toPath()) == -1;
+ }
+
+ private List<File> snapshotTsFileGroups(final List<File> sourceTsFiles,
final File snapshotRoot)
+ throws Exception {
+ final List<File> snapshots = new ArrayList<>();
+ for (int i = 0; i < sourceTsFiles.size(); i++) {
+ final File sourceTsFile = sourceTsFiles.get(i);
+ final File groupDir = new File(snapshotRoot, Integer.toString(i));
+ Assert.assertTrue(groupDir.mkdirs());
+ final File snapshotTsFile = new File(groupDir, sourceTsFile.getName());
+ Files.copy(sourceTsFile.toPath(), snapshotTsFile.toPath());
+ final File sourceResource =
+ new File(sourceTsFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX);
+ final File sourceMods =
+ new File(sourceTsFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX);
+ Files.copy(
+ sourceResource.toPath(),
+ new File(snapshotTsFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX).toPath());
+ Files.copy(
+ sourceMods.toPath(),
+ new File(snapshotTsFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX).toPath());
+ snapshots.add(snapshotTsFile);
+ }
+ return snapshots;
+ }
+
+ private File copyFailedGroup(final File failedGroup, final File
recoveryRoot, final int index)
+ throws Exception {
+ final File recoveryGroup = new File(recoveryRoot, Integer.toString(index));
+ Assert.assertTrue(recoveryGroup.mkdirs());
+ final File[] files = failedGroup.listFiles(File::isFile);
+ Assert.assertNotNull(files);
+ for (final File file : files) {
+ Files.copy(file.toPath(), new File(recoveryGroup,
file.getName()).toPath());
+ }
+ final File recoveryTsFile = findTsFile(recoveryGroup);
+ Assert.assertNotNull("Failed group copy must contain its TsFile",
recoveryTsFile);
+ return recoveryTsFile;
+ }
+
+ private void assertCount(
+ final Statement statement, final String device, final String filter,
final long expected)
+ throws Exception {
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ String.format("select count(%s) from %s where %s", MEASUREMENT,
device, filter))) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(expected, resultSet.getLong(1));
+ }
+ }
+
+ private void assertRecoveredRows(
+ final Statement statement,
+ final long startTime,
+ final long endTime,
+ final long deletedTime,
+ final long seed,
+ final boolean withExtraMeasurement)
+ throws Exception {
+ final Random expectedValues = new Random(seed);
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ String.format("select %s from %s order by time", MEASUREMENT,
COLLISION_DEVICE))) {
+ for (long time = startTime; time <= endTime; time++) {
+ final int value = expectedValues.nextInt();
+ if (withExtraMeasurement) {
+ expectedValues.nextInt();
+ }
+ if (time == deletedTime) {
+ continue;
+ }
+ Assert.assertTrue("Recovered group lost a surviving key",
resultSet.next());
+ Assert.assertEquals(time, resultSet.getLong(1));
+ Assert.assertEquals("Recovered value differs from its source", value,
resultSet.getInt(2));
+ Assert.assertFalse(resultSet.wasNull());
+ }
+ Assert.assertFalse(resultSet.next());
+ }
+ if (withExtraMeasurement) {
+ final Random expectedExtraValues = new Random(seed);
+ try (ResultSet resultSet =
+ statement.executeQuery("select extra from " + COLLISION_DEVICE + "
order by time")) {
+ for (long time = startTime; time <= endTime; time++) {
+ expectedExtraValues.nextInt();
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(time, resultSet.getLong(1));
+ Assert.assertEquals(expectedExtraValues.nextInt(),
resultSet.getInt(2));
+ Assert.assertFalse(resultSet.wasNull());
+ }
+ Assert.assertFalse(resultSet.next());
+ }
+ }
+ }
+
private void generateTsFile(final File tsFile) throws Exception {
try (final TsFileGenerator generator = new TsFileGenerator(tsFile)) {
generator.registerTimeseries(DEVICE,
Collections.singletonList(TSFILE_SCHEMA));
diff --git
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java
index d799152d9b5..fda7df243c5 100644
---
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java
@@ -23,6 +23,11 @@ public final class StorageEngineMessages {
private StorageEngineMessages() {}
+ public static final String
LOG_ERROR_OCCURRED_DURING_MOVING_FILE_ARG_TO_FAIL_DIRECTORY_F40610CE =
+ "Error occurred during moving file {} to fail directory.";
+ public static final String
LOG_FAILED_TO_READ_ACTIVE_LOAD_FAILURE_METADATA_ARG_E152544E =
+ "Failed to read active LOAD failure metadata {}";
+
public static final String
LOG_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_ARG_TARGET_ARG_13205CEC =
"File renaming failed when loading tsfile. Origin: {}, Target: {}";
diff --git
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java
index 6f43d0b34c8..6e20a9ede2c 100644
---
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java
@@ -23,6 +23,11 @@ public final class StorageEngineMessages {
private StorageEngineMessages() {}
+ public static final String
LOG_ERROR_OCCURRED_DURING_MOVING_FILE_ARG_TO_FAIL_DIRECTORY_F40610CE =
+ "将文件 {} 移动到失败目录时发生错误。";
+ public static final String
LOG_FAILED_TO_READ_ACTIVE_LOAD_FAILURE_METADATA_ARG_E152544E =
+ "读取主动 LOAD 失败元数据 {} 失败";
+
public static final String
LOG_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_ARG_TARGET_ARG_13205CEC =
"加载 TsFile 时重命名文件失败。源文件:{},目标文件:{}";
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPendingQueue.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPendingQueue.java
index 3275ba8368a..718d99d9e75 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPendingQueue.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPendingQueue.java
@@ -32,6 +32,7 @@ public class ActiveLoadPendingQueue {
private final Queue<ActiveLoadEntry> pendingFileQueue = new
ConcurrentLinkedQueue<>();
private final Set<String> loadingFileSet = new HashSet<>();
+ private final Set<String> failedFileSet = new HashSet<>();
public synchronized boolean enqueue(
final String file, final String pendingDir, final boolean
isGeneratedByPipe) {
@@ -43,7 +44,9 @@ public class ActiveLoadPendingQueue {
final String pendingDir,
final boolean isGeneratedByPipe,
final String conversionTaskId) {
- if (!loadingFileSet.contains(file) && pendingFileSet.add(file)) {
+ if (!loadingFileSet.contains(file)
+ && !failedFileSet.contains(file)
+ && pendingFileSet.add(file)) {
pendingFileQueue.offer(
new ActiveLoadEntry(file, pendingDir, isGeneratedByPipe,
conversionTaskId));
@@ -72,7 +75,17 @@ public class ActiveLoadPendingQueue {
}
public synchronized boolean isFilePendingOrLoading(final String file) {
- return loadingFileSet.contains(file) || pendingFileSet.contains(file);
+ return loadingFileSet.contains(file)
+ || pendingFileSet.contains(file)
+ || failedFileSet.contains(file);
+ }
+
+ public synchronized void markFileFailed(final String file) {
+ failedFileSet.add(file);
+ }
+
+ public synchronized void clearFailedFile(final String file) {
+ failedFileSet.remove(file);
}
public synchronized int size() {
@@ -94,6 +107,7 @@ public class ActiveLoadPendingQueue {
final int pendingFileCount = pendingFileSet.size();
pendingFileSet.clear();
pendingFileQueue.clear();
+ failedFileSet.clear();
ActiveLoadingFilesNumberMetricsSet.getInstance().increaseQueuingFileCounter(-pendingFileCount);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java
index 2c98f51b64e..3ec85f87c2e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.commons.utils.RetryUtils;
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.protocol.session.IClientSession;
import org.apache.iotdb.db.protocol.session.InternalClientSession;
import org.apache.iotdb.db.protocol.session.SessionManager;
@@ -38,6 +39,8 @@ import
org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher;
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
import
org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.load.converter.PipeTsFileConversionTaskManager;
import
org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesNumberMetricsSet;
import
org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesSizeMetricsSet;
@@ -50,15 +53,24 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.ZoneId;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@@ -80,6 +92,8 @@ public class ActiveLoadTsFileLoader {
// Guarded by activeLoadExecutor, including tasks submitted but not yet
started.
private int activeLoadTasks;
private final AtomicReference<String> failDir = new AtomicReference<>();
+ private final AtomicReference<String> indexedFailDir = new
AtomicReference<>();
+ private final Map<String, FailedFileSnapshot> failedFileSnapshots = new
ConcurrentHashMap<>();
private final boolean isVerify =
IOTDB_CONFIG.isLoadActiveListeningVerifyEnable();
public int getCurrentAllowedPendingSize() {
@@ -173,6 +187,7 @@ public class ActiveLoadTsFileLoader {
}
if (executor == null) {
pendingQueue.clearPending();
+ clearFailedFileIndex();
return;
}
@@ -194,6 +209,7 @@ public class ActiveLoadTsFileLoader {
} else {
pendingQueue.clearPending();
}
+ clearFailedFileIndex();
}
}
@@ -211,6 +227,11 @@ public class ActiveLoadTsFileLoader {
}
}
+ private void clearFailedFileIndex() {
+ failedFileSnapshots.clear();
+ indexedFailDir.set(null);
+ }
+
private void tryLoadPendingTsFiles() {
final IClientSession session =
new InternalClientSession(
@@ -350,7 +371,7 @@ public class ActiveLoadTsFileLoader {
entry.getFile(),
entry.isGeneratedByPipe(),
status);
- removeFileAndResourceAndModsToFailDir(entry.getFile());
+ removeFileAndResourceAndModsToFailDir(entry);
}
}
@@ -361,7 +382,7 @@ public class ActiveLoadTsFileLoader {
"Failed to auto load tsfile {} (isGeneratedByPipe = {}) due to file
not found, will skip this file.",
entry.getFile(),
entry.isGeneratedByPipe());
- removeFileAndResourceAndModsToFailDir(entry.getFile());
+ removeFileAndResourceAndModsToFailDir(entry);
}
private void handleOtherException(
@@ -380,32 +401,146 @@ public class ActiveLoadTsFileLoader {
entry.getFile(),
entry.isGeneratedByPipe(),
e);
- removeFileAndResourceAndModsToFailDir(entry.getFile());
+ removeFileAndResourceAndModsToFailDir(entry);
}
}
- private void removeFileAndResourceAndModsToFailDir(final String filePath) {
- removeToFailDir(filePath);
- removeToFailDir(filePath + ".resource");
- removeToFailDir(filePath + ".mods");
- }
+ private synchronized void removeFileAndResourceAndModsToFailDir(
+ final ActiveLoadPendingQueue.ActiveLoadEntry entry) {
+ final String filePath = entry.getFile();
+ final File tsFile = new File(filePath);
+ final List<File> sourceFiles =
+ List.of(
+ tsFile,
+ new File(filePath + TsFileResource.RESOURCE_SUFFIX),
+ new File(filePath + ModificationFile.FILE_SUFFIX));
+ final boolean hasSourceFiles = sourceFiles.stream().anyMatch(File::exists);
+ if (!hasSourceFiles) {
+ return;
+ }
+
+ final File failDirectory = new File(failDir.get());
+ final String failureInfoSuffix = ".load-failure";
+ final String failureInfoFileName =
+ tsFile.getName().getBytes(StandardCharsets.UTF_8).length +
failureInfoSuffix.length() > 180
+ ? "active-load-" + UUID.randomUUID() + failureInfoSuffix
+ : tsFile.getName() + failureInfoSuffix;
+ File stagingDirectory = null;
+ final File targetDirectory = new File(failDirectory, "active-load-" +
UUID.randomUUID());
+ try {
+ FileUtils.forceMkdir(failDirectory);
+ stagingDirectory =
+ Files.createTempDirectory(failDirectory.toPath(),
".active-load-staging-").toFile();
+
+ for (final File sourceFile : sourceFiles) {
+ if (!sourceFile.exists()) {
+ continue;
+ }
+ Files.copy(
+ sourceFile.toPath(),
+ new File(stagingDirectory, sourceFile.getName()).toPath(),
+ StandardCopyOption.COPY_ATTRIBUTES);
+ }
+ writeFailureInfo(new File(stagingDirectory, failureInfoFileName), entry,
true);
- private void removeToFailDir(final String filePath) {
- final File sourceFile = new File(filePath);
- // prevent the resource or mods not exist
- if (!sourceFile.exists()) {
+ try {
+ Files.move(
+ stagingDirectory.toPath(), targetDirectory.toPath(),
StandardCopyOption.ATOMIC_MOVE);
+ } catch (final AtomicMoveNotSupportedException e) {
+ Files.move(stagingDirectory.toPath(), targetDirectory.toPath());
+ }
+ stagingDirectory = null;
+ } catch (final IOException e) {
+ if (targetDirectory.exists()) {
+ try {
+ FileUtils.deleteDirectory(targetDirectory);
+ } catch (final IOException cleanupException) {
+ e.addSuppressed(cleanupException);
+ }
+ }
+ if (stagingDirectory != null && stagingDirectory.exists()) {
+ try {
+ FileUtils.deleteDirectory(stagingDirectory);
+ } catch (final IOException cleanupException) {
+ e.addSuppressed(cleanupException);
+ }
+ }
+ LOGGER.warn(
+ StorageEngineMessages
+
.LOG_ERROR_OCCURRED_DURING_MOVING_FILE_ARG_TO_FAIL_DIRECTORY_F40610CE,
+ filePath,
+ e);
return;
}
- final File targetDir = new File(failDir.get());
+ deletePublishedSourceFiles(filePath, tsFile, sourceFiles, targetDirectory);
+ }
+
+ private void deletePublishedSourceFiles(
+ final String filePath,
+ final File tsFile,
+ final List<File> sourceFiles,
+ final File publishedGroup) {
try {
- RetryUtils.retryOnException(
- () -> {
-
org.apache.iotdb.commons.utils.FileUtils.moveFileWithMD5Check(sourceFile,
targetDir);
- return null;
- });
+ Files.deleteIfExists(tsFile.toPath());
} catch (final IOException e) {
- LOGGER.warn("Error occurred during moving file {} to fail directory.",
filePath, e);
+ if (tsFile.exists()) {
+ pendingQueue.markFileFailed(filePath);
+ failedFileSnapshots.put(filePath, new
FailedFileSnapshot(publishedGroup));
+ }
+ LOGGER.warn(
+ StorageEngineMessages
+
.LOG_ERROR_OCCURRED_DURING_MOVING_FILE_ARG_TO_FAIL_DIRECTORY_F40610CE,
+ filePath,
+ e);
+ return;
+ }
+
+ IOException cleanupException = null;
+ for (final File sourceFile : sourceFiles) {
+ if (sourceFile.equals(tsFile)) {
+ continue;
+ }
+ try {
+ Files.deleteIfExists(sourceFile.toPath());
+ } catch (final IOException e) {
+ if (cleanupException == null) {
+ cleanupException = e;
+ } else {
+ cleanupException.addSuppressed(e);
+ }
+ }
+ }
+ if (cleanupException != null) {
+ LOGGER.warn(
+ StorageEngineMessages
+
.LOG_ERROR_OCCURRED_DURING_MOVING_FILE_ARG_TO_FAIL_DIRECTORY_F40610CE,
+ filePath,
+ cleanupException);
+ }
+ }
+
+ private void writeFailureInfo(
+ final File failureInfoFile,
+ final ActiveLoadPendingQueue.ActiveLoadEntry entry,
+ final boolean hasFiles)
+ throws IOException {
+ if (!hasFiles) {
+ return;
+ }
+
+ final Properties properties = new Properties();
+ properties.setProperty("activeLoadFile", entry.getFile());
+ properties.setProperty("generatedByPipe",
Boolean.toString(entry.isGeneratedByPipe()));
+ properties.setProperty("tableModel", "false");
+ if (entry.getConversionTaskId() != null) {
+ properties.setProperty("conversionTaskId", entry.getConversionTaskId());
+ }
+ if (entry.getPendingDir() != null) {
+ properties.setProperty("pendingDir", entry.getPendingDir());
+ }
+ try (final OutputStream outputStream =
Files.newOutputStream(failureInfoFile.toPath())) {
+ properties.store(outputStream, "Active LOAD failure source");
}
}
@@ -436,7 +571,125 @@ public class ActiveLoadTsFileLoader {
}
public boolean isFilePendingOrLoading(final File file) {
- return pendingQueue.isFilePendingOrLoading(file.getAbsolutePath());
+ final String filePath = file.getAbsolutePath();
+ if (!file.exists()) {
+ pendingQueue.clearFailedFile(filePath);
+ failedFileSnapshots.remove(filePath);
+ } else {
+ indexPublishedFailedFilesIfNecessary();
+ final FailedFileSnapshot failedFileSnapshot =
failedFileSnapshots.get(filePath);
+ if (failedFileSnapshot != null && !failedFileSnapshot.matches(file)) {
+ pendingQueue.clearFailedFile(filePath);
+ failedFileSnapshots.remove(filePath, failedFileSnapshot);
+ }
+ }
+ return pendingQueue.isFilePendingOrLoading(filePath);
+ }
+
+ private void indexPublishedFailedFilesIfNecessary() {
+ final String configuredFailDir =
IOTDB_CONFIG.getLoadActiveListeningFailDir();
+ if (configuredFailDir == null || configuredFailDir.isEmpty()) {
+ return;
+ }
+ if (Objects.equals(indexedFailDir.get(), configuredFailDir)) {
+ return;
+ }
+
+ synchronized (indexedFailDir) {
+ if (Objects.equals(indexedFailDir.get(), configuredFailDir)) {
+ return;
+ }
+
+ final File[] failedGroups = new
File(configuredFailDir).listFiles(File::isDirectory);
+ if (failedGroups == null) {
+ return;
+ }
+
+ boolean scanCompleted = true;
+ for (final File failedGroup : failedGroups) {
+ if (!failedGroup.getName().startsWith("active-load-")) {
+ continue;
+ }
+ final File[] failureInfoFiles =
+ failedGroup.listFiles((dir, name) ->
name.endsWith(".load-failure"));
+ if (failureInfoFiles == null) {
+ scanCompleted = false;
+ continue;
+ }
+ for (final File failureInfoFile : failureInfoFiles) {
+ final Properties failureInfo = new Properties();
+ try (final InputStream inputStream =
Files.newInputStream(failureInfoFile.toPath())) {
+ failureInfo.load(inputStream);
+ final String activeLoadFile =
failureInfo.getProperty("activeLoadFile");
+ if (activeLoadFile == null) {
+ continue;
+ }
+ final String absoluteSourceFile = new
File(activeLoadFile).getAbsolutePath();
+ final File sourceTsFile = new File(absoluteSourceFile);
+ if (sourceTsFile.isFile() &&
matchesPublishedFileGroup(sourceTsFile, failedGroup)) {
+ pendingQueue.markFileFailed(absoluteSourceFile);
+ failedFileSnapshots.put(absoluteSourceFile, new
FailedFileSnapshot(failedGroup));
+ }
+ } catch (final IOException e) {
+ scanCompleted = false;
+ LOGGER.debug(
+
StorageEngineMessages.LOG_FAILED_TO_READ_ACTIVE_LOAD_FAILURE_METADATA_ARG_E152544E,
+ failureInfoFile,
+ e);
+ } catch (final RuntimeException e) {
+ LOGGER.debug(
+
StorageEngineMessages.LOG_FAILED_TO_READ_ACTIVE_LOAD_FAILURE_METADATA_ARG_E152544E,
+ failureInfoFile,
+ e);
+ }
+ }
+ }
+ if (scanCompleted) {
+ indexedFailDir.set(configuredFailDir);
+ }
+ }
+ }
+
+ private static boolean matchesPublishedFileGroup(
+ final File sourceTsFile, final File failedGroup) {
+ for (final File sourceFile : getTsFileGroupFiles(sourceTsFile)) {
+ final File failedFile = new File(failedGroup, sourceFile.getName());
+ if (sourceFile.exists() != failedFile.exists()) {
+ return false;
+ }
+ if (sourceFile.exists()) {
+ try {
+ if (Files.mismatch(sourceFile.toPath(), failedFile.toPath()) != -1) {
+ return false;
+ }
+ } catch (final IOException e) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ private static List<File> getTsFileGroupFiles(final File tsFile) {
+ final String filePath = tsFile.getAbsolutePath();
+ return List.of(
+ tsFile,
+ new File(filePath + TsFileResource.RESOURCE_SUFFIX),
+ new File(filePath + ModificationFile.FILE_SUFFIX));
+ }
+
+ private static class FailedFileSnapshot {
+ private final File publishedGroup;
+
+ private FailedFileSnapshot(final File publishedGroup) {
+ this.publishedGroup = publishedGroup;
+ }
+
+ private boolean matches(final File tsFile) {
+ // A replacement can preserve both size and mtime (for example, a
restored sidecar).
+ // Only unchanged contents should remain suppressed.
+ return matchesPublishedFileGroup(tsFile, publishedGroup);
+ }
}
// Metrics
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoaderTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoaderTest.java
index 34986ebbc36..8f8671071ec 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoaderTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoaderTest.java
@@ -40,8 +40,10 @@ import java.io.File;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.file.Files;
+import java.nio.file.attribute.FileTime;
import java.util.ArrayList;
import java.util.List;
+import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -121,10 +123,61 @@ public class ActiveLoadTsFileLoaderTest {
final File failDir = new File(config.getLoadActiveListeningFailDir());
Assert.assertFalse(tsFile.exists());
- Assert.assertTrue(new File(failDir, tsFile.getName()).exists());
+ final File[] failedGroups = failDir.listFiles(File::isDirectory);
+ Assert.assertNotNull(failedGroups);
+ Assert.assertEquals(1, failedGroups.length);
+ final File failedGroup = failedGroups[0];
+ Assert.assertTrue(new File(failedGroup, tsFile.getName()).exists());
Assert.assertTrue(
- new File(failDir, tsFile.getName() +
TsFileResource.RESOURCE_SUFFIX).exists());
- Assert.assertTrue(new File(failDir, tsFile.getName() +
ModificationFile.FILE_SUFFIX).exists());
+ new File(failedGroup, tsFile.getName() +
TsFileResource.RESOURCE_SUFFIX).exists());
+ Assert.assertTrue(
+ new File(failedGroup, tsFile.getName() +
ModificationFile.FILE_SUFFIX).exists());
+ Assert.assertTrue(new File(failedGroup, tsFile.getName() +
".load-failure").exists());
+ final Properties failureInfo = new Properties();
+ try (java.io.InputStream inputStream =
+ Files.newInputStream(new File(failedGroup, tsFile.getName() +
".load-failure").toPath())) {
+ failureInfo.load(inputStream);
+ }
+ Assert.assertEquals(tsFile.getAbsolutePath(),
failureInfo.getProperty("activeLoadFile"));
+ Assert.assertEquals(tempDir.getAbsolutePath(),
failureInfo.getProperty("pendingDir"));
+ Assert.assertEquals("false", failureInfo.getProperty("generatedByPipe"));
+ Assert.assertEquals("false", failureInfo.getProperty("tableModel"));
+ }
+
+ @Test
+ public void
testFailurePublicationPreservesSourcesWhenFailDirectoryIsUnavailable()
+ throws Exception {
+ final ActiveLoadTsFileLoader loader = new ActiveLoadTsFileLoader();
+ final File tsFile = createTsFileWithCompanionFiles("unpublished.tsfile");
+ final ActiveLoadPendingQueue.ActiveLoadEntry entry =
+ new ActiveLoadPendingQueue.ActiveLoadEntry(
+ tsFile.getAbsolutePath(), tempDir.getAbsolutePath(), false);
+ invokeInitFailDirIfNecessary(loader);
+ final File failDir = new File(config.getLoadActiveListeningFailDir());
+ Files.delete(failDir.toPath());
+ Assert.assertTrue(failDir.createNewFile());
+
+ invokeHandleLoadFailure(
+ loader,
+ entry,
+ new
TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage("permanent
error"));
+ for (final String suffix :
+ new String[] {"", TsFileResource.RESOURCE_SUFFIX,
ModificationFile.FILE_SUFFIX}) {
+ Assert.assertTrue(
+ "An unpublished group must retain every source",
+ new File(tsFile.getAbsolutePath() + suffix).exists());
+ }
+ Assert.assertFalse(
+ "Publication failure must remain retryable",
loader.isFilePendingOrLoading(tsFile));
+
+ Files.delete(failDir.toPath());
+ Assert.assertTrue(failDir.mkdir());
+ invokeHandleLoadFailure(
+ loader,
+ entry,
+ new
TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage("permanent
error"));
+ Assert.assertFalse(tsFile.exists());
+ Assert.assertEquals(1, failDir.listFiles(File::isDirectory).length);
}
@Test
@@ -146,6 +199,82 @@ public class ActiveLoadTsFileLoaderTest {
Assert.assertTrue(pendingQueue.isEmpty());
}
+ @Test
+ public void testFailedFileIsNotRequeuedUntilPendingStateIsCleared() {
+ final ActiveLoadPendingQueue pendingQueue = new ActiveLoadPendingQueue();
+ final String tsFilePath = new File(tempDir,
"failed-pending.tsfile").getAbsolutePath();
+ Assert.assertTrue(pendingQueue.enqueue(tsFilePath,
tempDir.getAbsolutePath(), false));
+ Assert.assertNotNull(pendingQueue.dequeueFromPending());
+
+ pendingQueue.markFileFailed(tsFilePath);
+ pendingQueue.removeFromLoading(tsFilePath);
+
+ Assert.assertTrue(pendingQueue.isFilePendingOrLoading(tsFilePath));
+ Assert.assertFalse(pendingQueue.enqueue(tsFilePath,
tempDir.getAbsolutePath(), false));
+ pendingQueue.clearPending();
+ Assert.assertFalse(pendingQueue.isFilePendingOrLoading(tsFilePath));
+ }
+
+ @Test
+ public void
testPublishedFailureGroupSuppressesUnchangedSourceAfterLoaderRestart()
+ throws Exception {
+ final File tsFile =
createTsFileWithCompanionFiles("restart-failed.tsfile");
+ final File resourceFile = new File(tsFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX);
+ Files.writeString(resourceFile.toPath(), "original resource");
+ final File failedGroup = writePublishedFailureGroup(tsFile,
"active-load-recovered");
+
+ final ActiveLoadTsFileLoader restartedLoader = new
ActiveLoadTsFileLoader();
+ Assert.assertTrue(
+ "A published failure group must prevent an unchanged active source
from being loaded again",
+ restartedLoader.isFilePendingOrLoading(tsFile));
+
+ final FileTime originalModifiedTime =
Files.getLastModifiedTime(resourceFile.toPath());
+ Files.writeString(resourceFile.toPath(), "replaced resource");
+ Files.setLastModifiedTime(resourceFile.toPath(), originalModifiedTime);
+ final File publishedResource = new File(failedGroup,
resourceFile.getName());
+ Assert.assertNotEquals(-1L, Files.mismatch(resourceFile.toPath(),
publishedResource.toPath()));
+ Assert.assertEquals(publishedResource.length(), resourceFile.length());
+ Assert.assertEquals(originalModifiedTime,
Files.getLastModifiedTime(resourceFile.toPath()));
+ Assert.assertFalse(
+ "A same-sized replacement preserving mtime must be eligible for
loading",
+ restartedLoader.isFilePendingOrLoading(tsFile));
+ }
+
+ @Test
+ public void testFailureIndexRetriesWhenFailDirectoryBecomesAvailable()
throws Exception {
+ final File tsFile = createTsFileWithCompanionFiles("late-fail-dir.tsfile");
+ final File failDir = new File(config.getLoadActiveListeningFailDir());
+ final ActiveLoadTsFileLoader restartedLoader = new
ActiveLoadTsFileLoader();
+
+ Assert.assertFalse("The fail directory is absent at the first scan",
failDir.exists());
+ Assert.assertFalse(restartedLoader.isFilePendingOrLoading(tsFile));
+
+ Assert.assertTrue(failDir.mkdirs());
+ writePublishedFailureGroup(tsFile, "active-load-late");
+
+ Assert.assertTrue(
+ "A failed index scan must retry after the fail directory becomes
available",
+ restartedLoader.isFilePendingOrLoading(tsFile));
+ }
+
+ private File writePublishedFailureGroup(final File tsFile, final String
groupName)
+ throws Exception {
+ final File failedGroup = new File(config.getLoadActiveListeningFailDir(),
groupName);
+ Assert.assertTrue(failedGroup.mkdirs());
+ for (final String suffix :
+ new String[] {"", TsFileResource.RESOURCE_SUFFIX,
ModificationFile.FILE_SUFFIX}) {
+ final File sourceFile = new File(tsFile.getAbsolutePath() + suffix);
+ Files.copy(sourceFile.toPath(), new File(failedGroup,
sourceFile.getName()).toPath());
+ }
+ final Properties failureInfo = new Properties();
+ failureInfo.setProperty("activeLoadFile", tsFile.getAbsolutePath());
+ try (java.io.OutputStream outputStream =
+ Files.newOutputStream(new File(failedGroup, tsFile.getName() +
".load-failure").toPath())) {
+ failureInfo.store(outputStream, "test failure metadata");
+ }
+ return failedGroup;
+ }
+
@Test
public void testPendingEntryRetainsConversionTaskId() {
final ActiveLoadPendingQueue pendingQueue = new ActiveLoadPendingQueue();