This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 197965e9ae9 Skip the file timestamp check of tsfiles generated by
repair task (#13581)
197965e9ae9 is described below
commit 197965e9ae998b5a0a19a7a366c06eb31d6671d5
Author: shuwenwei <[email protected]>
AuthorDate: Tue Sep 24 15:50:08 2024 +0800
Skip the file timestamp check of tsfiles generated by repair task (#13581)
* skip the check of tsfiles generated by repair task
* add more compaction logs for debug
---
.../apache/iotdb/db/storageengine/dataregion/DataRegion.java | 6 +++++-
.../compaction/execute/task/AbstractCompactionTask.java | 2 +-
.../execute/task/InsertionCrossSpaceCompactionTask.java | 2 ++
.../execute/task/RepairUnsortedFileCompactionTask.java | 4 ++++
.../dataregion/compaction/repair/RepairDataFileScanUtil.java | 10 +++++++++-
.../compaction/repair/RepairTimePartitionScanTask.java | 2 +-
.../db/storageengine/dataregion/tsfile/TsFileResource.java | 2 +-
7 files changed, 23 insertions(+), 5 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 53da04da2cc..dd599218612 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -779,7 +779,11 @@ public class DataRegion implements IDataRegionForQuery {
private void checkTsFileTime(File tsFile, long currentTime) throws
DataRegionException {
String[] items = tsFile.getName().replace(TSFILE_SUFFIX,
"").split(FILE_NAME_SEPARATOR);
long fileTime = Long.parseLong(items[0]);
- if (fileTime > currentTime) {
+ // skip files generated by repair compaction task
+ long version = Long.parseLong(items[1]);
+ if (version > 0
+ && fileTime > currentTime
+ && fileTime <
RepairUnsortedFileCompactionTask.getInitialAllocatedFileTimestamp()) {
throw new DataRegionException(
String.format(
"data region %s[%s] is down, because the time of tsfile %s is
larger than system current time, "
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
index 75556001fd6..7173167541f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
@@ -474,7 +474,7 @@ public abstract class AbstractCompactionTask {
: timeDiff;
});
List<TsFileResource> overlapFilesInTimePartition =
-
RepairDataFileScanUtil.checkTimePartitionHasOverlap(timePartitionSeqFiles);
+
RepairDataFileScanUtil.checkTimePartitionHasOverlap(timePartitionSeqFiles,
true);
if (!overlapFilesInTimePartition.isEmpty()) {
LOGGER.error(
"Failed to pass compaction validation, source seq files: {},
source unseq files: {}, target files: {}",
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
index e2f31a28ffa..1f7e2b9c018 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
@@ -121,11 +121,13 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
recoverMemoryStatus = true;
LOGGER.info(
"{}-{} [Compaction] InsertionCrossSpaceCompaction task starts with
unseq file {}, "
+ + "nearest seq files are {}, "
+ "target file name timestamp is {}, "
+ "file size is {} MB.",
storageGroupName,
dataRegionId,
unseqFileToInsert,
+ selectedSeqFiles,
timestamp,
unseqFileToInsert.getTsFileSize() / 1024 / 1024);
boolean isSuccess = true;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
index e2ac24b0dba..d9ec119bea3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
@@ -54,6 +54,10 @@ public class RepairUnsortedFileCompactionTask extends
InnerSpaceCompactionTask {
}
}
+ public static long getInitialAllocatedFileTimestamp() {
+ return Long.MAX_VALUE / 2;
+ }
+
private final TsFileResource sourceFile;
private final boolean rewriteFile;
private CountDownLatch latch;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java
index 64439887ae8..d977ec01b5c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java
@@ -242,7 +242,8 @@ public class RepairDataFileScanUtil {
return isBrokenFile;
}
- public static List<TsFileResource>
checkTimePartitionHasOverlap(List<TsFileResource> resources) {
+ public static List<TsFileResource> checkTimePartitionHasOverlap(
+ List<TsFileResource> resources, boolean printOverlappedDevices) {
List<TsFileResource> overlapResources = new ArrayList<>();
Map<IDeviceID, Long> deviceEndTimeMap = new HashMap<>();
for (TsFileResource resource : resources) {
@@ -270,6 +271,13 @@ public class RepairDataFileScanUtil {
}
long deviceEndTimeInPreviousFile = deviceEndTimeMap.get(device);
if (deviceStartTimeInCurrentFile <= deviceEndTimeInPreviousFile) {
+ if (printOverlappedDevices) {
+ logger.error(
+ "Device {} has overlapped data, start time in current file is
{}, end time in previous file is {}",
+ device,
+ deviceStartTimeInCurrentFile,
+ deviceEndTimeInPreviousFile);
+ }
fileHasOverlap = true;
overlapResources.add(resource);
break;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartitionScanTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartitionScanTask.java
index 8bbae0af103..cba0fe26b0e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartitionScanTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartitionScanTask.java
@@ -143,7 +143,7 @@ public class RepairTimePartitionScanTask implements
Callable<Void> {
List<TsFileResource> seqList =
tsFileManager.getTsFileListSnapshot(timePartition.getTimePartitionId(), true);
List<TsFileResource> overlapFiles =
- RepairDataFileScanUtil.checkTimePartitionHasOverlap(seqList);
+ RepairDataFileScanUtil.checkTimePartitionHasOverlap(seqList, false);
for (TsFileResource overlapFile : overlapFiles) {
if (!timePartition.getTsFileManager().isAllowCompaction()) {
LOGGER.info(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index fd83f497301..27e81ac93eb 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -648,7 +648,7 @@ public class TsFileResource {
@Override
public String toString() {
- return String.format("file is %s, status: %s", file.toString(),
getStatus());
+ return String.format("{file: %s, status: %s}", file.toString(),
getStatus());
}
@Override
