This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 43c4981 [ISSUE-2730] Add the number of unseq merge times in TsFile
name. (#2939)
43c4981 is described below
commit 43c4981ea0cb3f936f56af8256b08ea06af4e156
Author: WilliamSong11 <[email protected]>
AuthorDate: Wed Mar 31 10:23:53 2021 +0800
[ISSUE-2730] Add the number of unseq merge times in TsFile name. (#2939)
---
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 8 ++
.../db/engine/compaction/TsFileManagement.java | 18 +--
.../level/LevelCompactionTsFileManagement.java | 20 ++--
.../iotdb/db/engine/merge/task/MergeFileTask.java | 33 ++++--
.../engine/storagegroup/StorageGroupProcessor.java | 5 +-
.../db/engine/storagegroup/TsFileResource.java | 132 +++++++++++++++++++++
.../org/apache/iotdb/db/utils/FilePathUtils.java | 2 +-
.../db/engine/compaction/CompactionChunkTest.java | 4 +
.../compaction/LevelCompactionMoreDataTest.java | 6 +
.../compaction/LevelCompactionRecoverTest.java | 12 ++
.../db/engine/compaction/LevelCompactionTest.java | 10 +-
.../LevelCompactionTsFileManagementTest.java | 8 ++
.../NoCompactionTsFileManagementTest.java | 8 ++
.../engine/merge/MaxFileMergeFileSelectorTest.java | 16 ++-
.../iotdb/db/engine/merge/MergeOverLapTest.java | 6 +
.../iotdb/db/engine/merge/MergeTaskTest.java | 1 +
.../apache/iotdb/db/engine/merge/MergeTest.java | 8 ++
.../db/integration/IoTDBRemovePartitionIT.java | 1 +
18 files changed, 260 insertions(+), 38 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
index 5aaf98f..dd9422d 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
@@ -143,4 +143,12 @@ public class IoTDBConstant {
// thrift
public static final int LEFT_SIZE_IN_REQUEST = 4 * 1024 * 1024;
+
+ // change tsFile name
+ public static final int FILE_NAME_SUFFIX_INDEX = 0;
+ public static final int FILE_NAME_SUFFIX_TIME_INDEX = 0;
+ public static final int FILE_NAME_SUFFIX_VERSION_INDEX = 1;
+ public static final int FILE_NAME_SUFFIX_MERGECNT_INDEX = 2;
+ public static final int FILE_NAME_SUFFIX_UNSEQMERGECNT_INDEX = 3;
+ public static final String FILE_NAME_SUFFIX_SEPARATOR = "\\.";
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
index b176365..6225735 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
@@ -392,20 +392,20 @@ public abstract class TsFileManagement {
mergedFile.delete();
}
updateMergeModification(seqFile);
- if (i == seqFiles.size() - 1) {
- // FIXME if there is an exception, the the modification file will be
not closed.
- removeMergingModification();
- isUnseqMerging = false;
- Files.delete(mergeLog.toPath());
- }
- } catch (IOException e) {
- logger.error(
- "{} a merge task ends but cannot delete log {}", storageGroupName,
mergeLog.toPath());
} finally {
doubleWriteUnlock(seqFile);
}
}
+ try {
+ removeMergingModification();
+ isUnseqMerging = false;
+ Files.delete(mergeLog.toPath());
+ } catch (IOException e) {
+ logger.error(
+ "{} a merge task ends but cannot delete log {}", storageGroupName,
mergeLog.toPath());
+ }
+
logger.info("{} a merge task ends", storageGroupName);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index 8dfbd2b..9284dfc 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -589,6 +589,7 @@ public class LevelCompactionTsFileManagement extends
TsFileManagement {
}
isSeqMerging = true;
long startTimeMillis = System.currentTimeMillis();
+ CompactionLogger compactionLogger = null;
try {
logger.info("{} start to filter compaction condition", storageGroupName);
for (int i = 0; i < currMaxLevel - 1; i++) {
@@ -600,14 +601,13 @@ public class LevelCompactionTsFileManagement extends
TsFileManagement {
isSeqMerging = false;
merge(isForceFullMerge, getTsFileList(true),
mergeResources.get(i), Long.MAX_VALUE);
} else {
- CompactionLogger compactionLogger =
- new CompactionLogger(storageGroupDir, storageGroupName);
+ compactionLogger = new CompactionLogger(storageGroupDir,
storageGroupName);
// log source file list and target file for recover
for (TsFileResource mergeResource : mergeResources.get(i)) {
compactionLogger.logFile(SOURCE_NAME, mergeResource.getTsFile());
}
File newLevelFile =
- createNewTsFileName(mergeResources.get(i).get(0).getTsFile(),
i + 1);
+
TsFileResource.modifyTsFileNameMergeCnt(mergeResources.get(i).get(0).getTsFile());
compactionLogger.logSequence(sequence);
compactionLogger.logFile(TARGET_NAME, newLevelFile);
List<TsFileResource> toMergeTsFiles = mergeResources.get(i);
@@ -664,6 +664,13 @@ public class LevelCompactionTsFileManagement extends
TsFileManagement {
}
}
} catch (Exception e) {
+ if (compactionLogger != null) {
+ try {
+ compactionLogger.close();
+ } catch (IOException ioException) {
+ logger.error("{} Compaction log close fail", storageGroupName +
COMPACTION_LOG_NAME);
+ }
+ }
restoreCompaction();
logger.error("Error occurred in Compaction Merge thread", e);
} finally {
@@ -677,13 +684,6 @@ public class LevelCompactionTsFileManagement extends
TsFileManagement {
}
}
- /** if level < maxLevel-1, the file need compaction else, the file can be
merged later */
- private File createNewTsFileName(File sourceFile, int level) {
- String path = sourceFile.getPath();
- String prefixPath = path.substring(0,
path.lastIndexOf(FILE_NAME_SEPARATOR) + 1);
- return new File(prefixPath + level + TSFILE_SUFFIX);
- }
-
private List<SortedSet<TsFileResource>> newSequenceTsFileResources(Long k) {
List<SortedSet<TsFileResource>> newSequenceTsFileResources = new
CopyOnWriteArrayList<>();
for (int i = 0; i < seqLevelNum; i++) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
index c1ac878..15b5176 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
@@ -49,12 +49,14 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import static
org.apache.iotdb.db.engine.storagegroup.TsFileResource.modifyTsFileNameUnseqMergCnt;
+
/**
* MergeFileTask merges the merge temporary files with the seqFiles, either
move the merged chunks
* in the temp files into the seqFiles or move the unmerged chunks into the
merge temp files,
* depending on which one is the majority.
*/
-class MergeFileTask {
+public class MergeFileTask {
private static final Logger logger =
LoggerFactory.getLogger(MergeFileTask.class);
@@ -196,11 +198,20 @@ class MergeFileTask {
}
updateStartTimeAndEndTime(seqFile, oldFileWriter);
oldFileWriter.endFile();
-
updatePlanIndexes(seqFile);
seqFile.serialize();
mergeLogger.logFileMergeEnd();
logger.debug("{} moved merged chunks of {} to the old file", taskName,
seqFile);
+
+ newFileWriter.getFile().delete();
+ // change tsFile name
+ File nextMergeVersionFile =
modifyTsFileNameUnseqMergCnt(seqFile.getTsFile());
+ fsFactory.moveFile(seqFile.getTsFile(), nextMergeVersionFile);
+ fsFactory.moveFile(
+ fsFactory.getFile(seqFile.getTsFile().getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX),
+ fsFactory.getFile(
+ nextMergeVersionFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX));
+ seqFile.setFile(nextMergeVersionFile);
} catch (Exception e) {
restoreOldFile(seqFile);
throw e;
@@ -327,20 +338,26 @@ class MergeFileTask {
}
updateStartTimeAndEndTime(seqFile, fileWriter);
resource.removeFileReader(seqFile);
+
FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath());
fileWriter.endFile();
updatePlanIndexes(seqFile);
- seqFile.serialize();
mergeLogger.logFileMergeEnd();
logger.debug("{} moved unmerged chunks of {} to the new file", taskName,
seqFile);
seqFile.writeLock();
try {
-
FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath());
- File newMergeFile = seqFile.getTsFile();
- newMergeFile.delete();
- fsFactory.moveFile(fileWriter.getFile(), newMergeFile);
- seqFile.setFile(newMergeFile);
+ seqFile.serialize();
+
+ // change tsFile name
+ seqFile.getTsFile().delete();
+ File nextMergeVersionFile =
modifyTsFileNameUnseqMergCnt(seqFile.getTsFile());
+ fsFactory.moveFile(fileWriter.getFile(), nextMergeVersionFile);
+ fsFactory.moveFile(
+ fsFactory.getFile(seqFile.getTsFile().getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX),
+ fsFactory.getFile(
+ nextMergeVersionFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX));
+ seqFile.setFile(nextMergeVersionFile);
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index b5b76df..aee66f6 100755
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1239,7 +1239,7 @@ public class StorageGroupProcessor {
}
private String getNewTsFileName(long time, long version, int mergeCnt) {
- return time + FILE_NAME_SEPARATOR + version + FILE_NAME_SEPARATOR +
mergeCnt + TSFILE_SUFFIX;
+ return TsFileResource.getNewTsFileName(System.currentTimeMillis(),
version, 0, 0);
}
public void syncCloseOneTsFileProcessor(boolean sequence, TsFileProcessor
tsFileProcessor) {
@@ -2466,7 +2466,8 @@ public class StorageGroupProcessor {
return tsfileName;
}
- return getNewTsFileName(preTime + ((subsequenceTime - preTime) >> 1),
subsequenceVersion, 0);
+ return TsFileResource.getNewTsFileName(
+ preTime + ((subsequenceTime - preTime) >> 1), subsequenceVersion, 0,
0);
}
/**
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 407dc9c..56ddd78 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -59,6 +59,15 @@ import java.util.Objects;
import java.util.Random;
import java.util.Set;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_INDEX;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_MERGECNT_INDEX;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_SEPARATOR;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_TIME_INDEX;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_UNSEQMERGECNT_INDEX;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FILE_NAME_SUFFIX_VERSION_INDEX;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
+
@SuppressWarnings("java:S1135") // ignore todos
public class TsFileResource {
@@ -753,4 +762,127 @@ public class TsFileResource {
public void setTimeIndex(ITimeIndex timeIndex) {
this.timeIndex = timeIndex;
}
+
+ // change tsFile name
+
+ public static String getNewTsFileName(long time, long version, int mergeCnt,
int unSeqMergeCnt) {
+ return time
+ + FILE_NAME_SEPARATOR
+ + version
+ + FILE_NAME_SEPARATOR
+ + mergeCnt
+ + FILE_NAME_SEPARATOR
+ + unSeqMergeCnt
+ + TSFILE_SUFFIX;
+ }
+
+ public static TsFileName getTsFileName(String FileName) {
+ String[] fileName =
+
FileName.split(FILE_NAME_SUFFIX_SEPARATOR)[FILE_NAME_SUFFIX_INDEX].split(
+ FILE_NAME_SEPARATOR);
+ TsFileName tsFileName =
+ new TsFileName(
+ Long.parseLong(fileName[FILE_NAME_SUFFIX_TIME_INDEX]),
+ Long.parseLong(fileName[FILE_NAME_SUFFIX_VERSION_INDEX]),
+ Integer.parseInt(fileName[FILE_NAME_SUFFIX_MERGECNT_INDEX]),
+ Integer.parseInt(fileName[FILE_NAME_SUFFIX_UNSEQMERGECNT_INDEX]));
+ return tsFileName;
+ }
+
+ public static TsFileResource modifyTsFileNameUnseqMergCnt(TsFileResource
tsFileResource) {
+ File tsFile = tsFileResource.getTsFile();
+ String path = tsFile.getParent();
+ TsFileName tsFileName =
getTsFileName(tsFileResource.getTsFile().getName());
+ tsFileName.setUnSeqMergeCnt(tsFileName.getUnSeqMergeCnt() + 1);
+ tsFileResource.setFile(
+ new File(
+ path,
+ tsFileName.time
+ + FILE_NAME_SEPARATOR
+ + tsFileName.version
+ + FILE_NAME_SEPARATOR
+ + tsFileName.mergeCnt
+ + FILE_NAME_SEPARATOR
+ + tsFileName.unSeqMergeCnt
+ + TSFILE_SUFFIX));
+ return tsFileResource;
+ }
+
+ public static File modifyTsFileNameUnseqMergCnt(File tsFile) {
+ String path = tsFile.getParent();
+ TsFileName tsFileName = getTsFileName(tsFile.getName());
+ tsFileName.setUnSeqMergeCnt(tsFileName.getUnSeqMergeCnt() + 1);
+ return new File(
+ path,
+ tsFileName.time
+ + FILE_NAME_SEPARATOR
+ + tsFileName.version
+ + FILE_NAME_SEPARATOR
+ + tsFileName.mergeCnt
+ + FILE_NAME_SEPARATOR
+ + tsFileName.unSeqMergeCnt
+ + TSFILE_SUFFIX);
+ }
+
+ public static File modifyTsFileNameMergeCnt(File tsFile) {
+ String path = tsFile.getParent();
+ TsFileName tsFileName = getTsFileName(tsFile.getName());
+ tsFileName.setMergeCnt(tsFileName.getMergeCnt() + 1);
+ return new File(
+ path,
+ tsFileName.time
+ + FILE_NAME_SEPARATOR
+ + tsFileName.version
+ + FILE_NAME_SEPARATOR
+ + tsFileName.mergeCnt
+ + FILE_NAME_SEPARATOR
+ + tsFileName.unSeqMergeCnt
+ + TSFILE_SUFFIX);
+ }
+
+ public static class TsFileName {
+ private long time;
+ private long version;
+ private int mergeCnt;
+ private int unSeqMergeCnt;
+
+ public TsFileName(long time, long version, int mergeCnt, int
unSeqMergeCnt) {
+ this.time = time;
+ this.version = version;
+ this.mergeCnt = mergeCnt;
+ this.unSeqMergeCnt = unSeqMergeCnt;
+ }
+
+ public long getTime() {
+ return time;
+ }
+
+ public long getVersion() {
+ return version;
+ }
+
+ public int getMergeCnt() {
+ return mergeCnt;
+ }
+
+ public int getUnSeqMergeCnt() {
+ return unSeqMergeCnt;
+ }
+
+ public void setTime(long time) {
+ this.time = time;
+ }
+
+ public void setVersion(long version) {
+ this.version = version;
+ }
+
+ public void setMergeCnt(int mergeCnt) {
+ this.mergeCnt = mergeCnt;
+ }
+
+ public void setUnSeqMergeCnt(int unSeqMergeCnt) {
+ this.unSeqMergeCnt = unSeqMergeCnt;
+ }
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FilePathUtils.java
b/server/src/main/java/org/apache/iotdb/db/utils/FilePathUtils.java
index 9555e9c..e7038c3 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FilePathUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FilePathUtils.java
@@ -251,7 +251,7 @@ public class FilePathUtils {
public static long splitAndGetTsFileVersion(String tsFileName) {
String[] names = tsFileName.split(FILE_NAME_SEPARATOR);
- if (names.length != 3) {
+ if (names.length != 4) {
return 0;
}
return Long.parseLong(names[1]);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java
index e073539..74f2aa5 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java
@@ -85,6 +85,8 @@ public class CompactionChunkTest extends LevelCompactionTest {
+ 0
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 1
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource targetTsfileResource = new TsFileResource(file);
RateLimiter compactionWriteRateLimiter =
MergeManager.getINSTANCE().getMergeWriteRateLimiter();
@@ -168,6 +170,8 @@ public class CompactionChunkTest extends
LevelCompactionTest {
+ 0
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 1
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource targetTsfileResource = new TsFileResource(file);
RateLimiter compactionWriteRateLimiter =
MergeManager.getINSTANCE().getMergeWriteRateLimiter();
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionMoreDataTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionMoreDataTest.java
index f2b985c..55df625 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionMoreDataTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionMoreDataTest.java
@@ -101,6 +101,8 @@ public class LevelCompactionMoreDataTest extends
LevelCompactionTest {
+ i
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
@@ -117,6 +119,8 @@ public class LevelCompactionMoreDataTest extends
LevelCompactionTest {
+ (10000 + i)
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
@@ -133,6 +137,8 @@ public class LevelCompactionMoreDataTest extends
LevelCompactionTest {
+ unseqFileNum
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java
index 6305aa5..7adffc2 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java
@@ -126,6 +126,8 @@ public class LevelCompactionRecoverTest extends
LevelCompactionTest {
+ 0
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 1
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile")));
compactionLogger.logFile(TARGET_NAME, targetTsFileResource.getTsFile());
CompactionUtils.merge(
@@ -215,6 +217,8 @@ public class LevelCompactionRecoverTest extends
LevelCompactionTest {
+ 0
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 1
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile")));
compactionLogger.logFile(TARGET_NAME, targetTsFileResource.getTsFile());
CompactionUtils.merge(
@@ -328,6 +332,8 @@ public class LevelCompactionRecoverTest extends
LevelCompactionTest {
+ 0
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 1
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile")));
compactionLogger.logFile(TARGET_NAME, targetTsFileResource.getTsFile());
CompactionUtils.merge(
@@ -447,6 +453,8 @@ public class LevelCompactionRecoverTest extends
LevelCompactionTest {
+ 0
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 1
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile")));
compactionLogger.logFile(TARGET_NAME, targetTsFileResource.getTsFile());
CompactionUtils.merge(
@@ -594,6 +602,8 @@ public class LevelCompactionRecoverTest extends
LevelCompactionTest {
+ 0
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 1
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile")));
compactionLogger.logFile(TARGET_NAME, targetTsFileResource.getTsFile());
levelCompactionTsFileManagement.add(targetTsFileResource, true);
@@ -649,6 +659,8 @@ public class LevelCompactionRecoverTest extends
LevelCompactionTest {
+ 0
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 1
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile")));
compactionLogger.logFile(TARGET_NAME, targetTsFileResource.getTsFile());
CompactionUtils.merge(
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java
index c34328f..94d496d 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java
@@ -132,6 +132,8 @@ abstract class LevelCompactionTest {
+ i
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
@@ -148,10 +150,12 @@ abstract class LevelCompactionTest {
+ (10000 + i)
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
- tsFileResource.updatePlanIndexes((long) (i + seqFileNum));
+ tsFileResource.updatePlanIndexes(i + seqFileNum);
unseqResources.add(tsFileResource);
prepareFile(tsFileResource, i * ptNum, ptNum * (i + 1) / unseqFileNum,
10000);
}
@@ -164,10 +168,12 @@ abstract class LevelCompactionTest {
+ unseqFileNum
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
- tsFileResource.updatePlanIndexes((long) (seqFileNum + unseqFileNum));
+ tsFileResource.updatePlanIndexes(seqFileNum + unseqFileNum);
unseqResources.add(tsFileResource);
prepareFile(tsFileResource, 0, ptNum * unseqFileNum, 20000);
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTsFileManagementTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTsFileManagementTest.java
index 10ed211..4dc40b4 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTsFileManagementTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTsFileManagementTest.java
@@ -83,6 +83,8 @@ public class LevelCompactionTsFileManagementTest extends
LevelCompactionTest {
+ 10
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"))),
false));
assertTrue(levelCompactionTsFileManagement.contains(seqResources.get(0),
false));
@@ -96,6 +98,8 @@ public class LevelCompactionTsFileManagementTest extends
LevelCompactionTest {
+ 10
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"))),
false));
assertFalse(levelCompactionTsFileManagement.isEmpty(true));
@@ -129,6 +133,8 @@ public class LevelCompactionTsFileManagementTest extends
LevelCompactionTest {
+ 10
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 10
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"))),
true);
levelCompactionTsFileManagement.add(
@@ -140,6 +146,8 @@ public class LevelCompactionTsFileManagementTest extends
LevelCompactionTest {
+ 10
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 10
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"))),
false);
assertEquals(1, levelCompactionTsFileManagement.size(true));
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/NoCompactionTsFileManagementTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/NoCompactionTsFileManagementTest.java
index 931743c..2c14215 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/NoCompactionTsFileManagementTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/NoCompactionTsFileManagementTest.java
@@ -84,6 +84,8 @@ public class NoCompactionTsFileManagementTest extends
LevelCompactionTest {
+ 10
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"))),
false));
assertTrue(noCompactionTsFileManagement.contains(seqResources.get(0),
false));
@@ -97,6 +99,8 @@ public class NoCompactionTsFileManagementTest extends
LevelCompactionTest {
+ 10
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"))),
false));
assertFalse(noCompactionTsFileManagement.isEmpty(true));
@@ -129,6 +133,8 @@ public class NoCompactionTsFileManagementTest extends
LevelCompactionTest {
+ 10
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 10
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"))),
true);
noCompactionTsFileManagement.add(
@@ -140,6 +146,8 @@ public class NoCompactionTsFileManagementTest extends
LevelCompactionTest {
+ 10
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 10
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"))),
false);
noCompactionTsFileManagement.forkCurrentFileList(0);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
index 7275a01..b77f2ce 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
@@ -35,6 +35,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -115,6 +116,7 @@ public class MaxFileMergeFileSelectorTest extends MergeTest
{
+ 0
+ ".tsfile"));
TsFileResource largeUnseqTsFileResource = new TsFileResource(file);
+ unseqResources.add(largeUnseqTsFileResource);
largeUnseqTsFileResource.setClosed(true);
largeUnseqTsFileResource.setMinPlanIndex(10);
largeUnseqTsFileResource.setMaxPlanIndex(10);
@@ -135,10 +137,10 @@ public class MaxFileMergeFileSelectorTest extends
MergeTest {
newTimeIndex.updateStartTime(device, timeIndex.getStartTime(device));
}
secondTsFileResource.setTimeIndex(newTimeIndex);
- unseqResources.clear();
- unseqResources.add(largeUnseqTsFileResource);
- MergeResource resource = new MergeResource(seqResources, unseqResources);
+ List<TsFileResource> newUnseqResources = new ArrayList<>();
+ newUnseqResources.add(largeUnseqTsFileResource);
+ MergeResource resource = new MergeResource(seqResources,
newUnseqResources);
IMergeFileSelector mergeFileSelector = new
MaxFileMergeFileSelector(resource, Long.MAX_VALUE);
List[] result = mergeFileSelector.select();
assertEquals(0, result.length);
@@ -166,6 +168,7 @@ public class MaxFileMergeFileSelectorTest extends MergeTest
{
+ 0
+ ".tsfile"));
TsFileResource largeUnseqTsFileResource = new TsFileResource(file);
+ unseqResources.add(largeUnseqTsFileResource);
largeUnseqTsFileResource.setClosed(true);
largeUnseqTsFileResource.setMinPlanIndex(10);
largeUnseqTsFileResource.setMaxPlanIndex(10);
@@ -186,11 +189,12 @@ public class MaxFileMergeFileSelectorTest extends
MergeTest {
newTimeIndex.updateStartTime(device, timeIndex.getStartTime(device));
}
secondTsFileResource.setTimeIndex(newTimeIndex);
- unseqResources.clear();
- unseqResources.add(largeUnseqTsFileResource);
+ List<TsFileResource> newUnseqResources = new ArrayList<>();
+ newUnseqResources.add(largeUnseqTsFileResource);
long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE;
- MergeResource mergeResource = new MergeResource(seqResources,
unseqResources, timeLowerBound);
+ MergeResource mergeResource =
+ new MergeResource(seqResources, newUnseqResources, timeLowerBound);
assertEquals(5, mergeResource.getSeqFiles().size());
assertEquals(1, mergeResource.getUnseqFiles().size());
mergeResource.clear();
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
index c95fb14..6c70fe7 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
@@ -83,6 +83,8 @@ public class MergeOverLapTest extends MergeTest {
+ i
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
@@ -100,6 +102,8 @@ public class MergeOverLapTest extends MergeTest {
+ (10000 + i)
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
@@ -116,6 +120,8 @@ public class MergeOverLapTest extends MergeTest {
+ unseqFileNum
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
index f4dab57..25c4bef 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
@@ -146,6 +146,7 @@ public class MergeTaskTest extends MergeTest {
smallUnseqTsFileResource.setMaxPlanIndex(10);
smallUnseqTsFileResource.setVersion(10);
prepareFile(smallUnseqTsFileResource, 0, 50, 0);
+ unseqResources.add(smallUnseqTsFileResource);
// remove all data of first file
for (String deviceId : deviceIds) {
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
index 7eb1d62..d6e6735 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
@@ -134,6 +134,8 @@ abstract class MergeTest {
+ i
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
@@ -152,6 +154,8 @@ abstract class MergeTest {
+ (10000 + i)
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
@@ -170,6 +174,8 @@ abstract class MergeTest {
+ unseqFileNum
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
tsFileResource.setClosed(true);
@@ -183,9 +189,11 @@ abstract class MergeTest {
private void removeFiles() throws IOException {
for (TsFileResource tsFileResource : seqResources) {
tsFileResource.remove();
+ tsFileResource.getModFile().remove();
}
for (TsFileResource tsFileResource : unseqResources) {
tsFileResource.remove();
+ tsFileResource.getModFile().remove();
}
FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRemovePartitionIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRemovePartitionIT.java
index 149744f..09a2c43 100644
---
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRemovePartitionIT.java
+++
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRemovePartitionIT.java
@@ -64,6 +64,7 @@ public class IoTDBRemovePartitionIT {
StorageEngine.setEnablePartition(false);
StorageEngine.setTimePartitionInterval(-1);
EnvironmentUtils.cleanEnv();
+
ch.qos.logback.classic.Logger rootLogger =
(ch.qos.logback.classic.Logger)
LoggerFactory.getLogger(ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME);
