This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch rc/1.3.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rc/1.3.3 by this push:
new 8ca5cf6a2e8 [to rc/1.3.3] Skip the file timestamp check of tsfiles
generated by repair task (#13582)
8ca5cf6a2e8 is described below
commit 8ca5cf6a2e85e4837f3098bcc98ce89084a75c71
Author: shuwenwei <[email protected]>
AuthorDate: Tue Sep 24 15:50:18 2024 +0800
[to rc/1.3.3] Skip the file timestamp check of tsfiles generated by repair
task (#13582)
* skip the check of tsfiles generated by repair task
* add more compaction logs for debug
---
.../apache/iotdb/db/storageengine/dataregion/DataRegion.java | 6 +++++-
.../compaction/execute/task/AbstractCompactionTask.java | 2 +-
.../execute/task/InsertionCrossSpaceCompactionTask.java | 2 ++
.../execute/task/RepairUnsortedFileCompactionTask.java | 4 ++++
.../dataregion/compaction/repair/RepairDataFileScanUtil.java | 10 +++++++++-
.../compaction/repair/RepairTimePartitionScanTask.java | 2 +-
.../db/storageengine/dataregion/tsfile/TsFileResource.java | 2 +-
7 files changed, 23 insertions(+), 5 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 8b7f919c998..fd2d8c63fd7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -767,7 +767,11 @@ public class DataRegion implements IDataRegionForQuery {
private void checkTsFileTime(File tsFile, long currentTime) throws
DataRegionException {
String[] items = tsFile.getName().replace(TSFILE_SUFFIX,
"").split(FILE_NAME_SEPARATOR);
long fileTime = Long.parseLong(items[0]);
- if (fileTime > currentTime) {
+ // skip files generated by repair compaction task
+ long version = Long.parseLong(items[1]);
+ if (version > 0
+ && fileTime > currentTime
+ && fileTime <
RepairUnsortedFileCompactionTask.getInitialAllocatedFileTimestamp()) {
throw new DataRegionException(
String.format(
"data region %s[%s] is down, because the time of tsfile %s is
larger than system current time, "
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
index 75556001fd6..7173167541f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
@@ -474,7 +474,7 @@ public abstract class AbstractCompactionTask {
: timeDiff;
});
List<TsFileResource> overlapFilesInTimePartition =
-
RepairDataFileScanUtil.checkTimePartitionHasOverlap(timePartitionSeqFiles);
+
RepairDataFileScanUtil.checkTimePartitionHasOverlap(timePartitionSeqFiles,
true);
if (!overlapFilesInTimePartition.isEmpty()) {
LOGGER.error(
"Failed to pass compaction validation, source seq files: {},
source unseq files: {}, target files: {}",
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
index e2f31a28ffa..1f7e2b9c018 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
@@ -121,11 +121,13 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
recoverMemoryStatus = true;
LOGGER.info(
"{}-{} [Compaction] InsertionCrossSpaceCompaction task starts with
unseq file {}, "
+ + "nearest seq files are {}, "
+ "target file name timestamp is {}, "
+ "file size is {} MB.",
storageGroupName,
dataRegionId,
unseqFileToInsert,
+ selectedSeqFiles,
timestamp,
unseqFileToInsert.getTsFileSize() / 1024 / 1024);
boolean isSuccess = true;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
index ccc02e9f5ed..7f1e23c956a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
@@ -54,6 +54,10 @@ public class RepairUnsortedFileCompactionTask extends
InnerSpaceCompactionTask {
}
}
+ public static long getInitialAllocatedFileTimestamp() {
+ return Long.MAX_VALUE / 2;
+ }
+
private final TsFileResource sourceFile;
private final boolean rewriteFile;
private CountDownLatch latch;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java
index b0409b913a9..8c224cb18f2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java
@@ -231,7 +231,8 @@ public class RepairDataFileScanUtil {
return isBrokenFile;
}
- public static List<TsFileResource>
checkTimePartitionHasOverlap(List<TsFileResource> resources) {
+ public static List<TsFileResource> checkTimePartitionHasOverlap(
+ List<TsFileResource> resources, boolean printOverlappedDevices) {
List<TsFileResource> overlapResources = new ArrayList<>();
Map<IDeviceID, Long> deviceEndTimeMap = new HashMap<>();
for (TsFileResource resource : resources) {
@@ -259,6 +260,13 @@ public class RepairDataFileScanUtil {
}
long deviceEndTimeInPreviousFile = deviceEndTimeMap.get(device);
if (deviceStartTimeInCurrentFile <= deviceEndTimeInPreviousFile) {
+ if (printOverlappedDevices) {
+ logger.error(
+ "Device {} has overlapped data, start time in current file is
{}, end time in previous file is {}",
+ device,
+ deviceStartTimeInCurrentFile,
+ deviceEndTimeInPreviousFile);
+ }
fileHasOverlap = true;
overlapResources.add(resource);
break;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartitionScanTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartitionScanTask.java
index 828307e6397..0a263ca6ce3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartitionScanTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartitionScanTask.java
@@ -142,7 +142,7 @@ public class RepairTimePartitionScanTask implements
Callable<Void> {
List<TsFileResource> seqList =
tsFileManager.getTsFileListSnapshot(timePartition.getTimePartitionId(), true);
List<TsFileResource> overlapFiles =
- RepairDataFileScanUtil.checkTimePartitionHasOverlap(seqList);
+ RepairDataFileScanUtil.checkTimePartitionHasOverlap(seqList, false);
for (TsFileResource overlapFile : overlapFiles) {
if (!timePartition.getTsFileManager().isAllowCompaction()) {
LOGGER.info(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index bdcdfb2f75b..10d9a89ff85 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -646,7 +646,7 @@ public class TsFileResource {
@Override
public String toString() {
- return String.format("file is %s, status: %s", file.toString(),
getStatus());
+ return String.format("{file: %s, status: %s}", file.toString(),
getStatus());
}
@Override
