This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch overlap_check_tool
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/overlap_check_tool by this
push:
new 3fcff18b35a add chunkgroup level judgement
3fcff18b35a is described below
commit 3fcff18b35a436655dce4e948336027527da65fd
Author: Jinrui.Zhang <[email protected]>
AuthorDate: Wed Aug 9 11:22:05 2023 +0800
add chunkgroup level judgement
---
.../compaction/tool/OverlapStatisticTool.java | 5 ++--
.../compaction/tool/TsFileStatisticReader.java | 3 ++
.../compaction/tool/UnseqSpaceStatistics.java | 16 ++++++++--
.../compaction/tools/UnseqSpaceStatisticsTest.java | 35 ++++++++++++----------
4 files changed, 40 insertions(+), 19 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/OverlapStatisticTool.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/OverlapStatisticTool.java
index 9b104da84cd..3774119c30e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/OverlapStatisticTool.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/OverlapStatisticTool.java
@@ -231,12 +231,13 @@ public class OverlapStatisticTool {
Interval interval =
new Interval(chunkMetadata.getStartTime(),
chunkMetadata.getEndTime());
String measurementId = chunkMetadata.getMeasurementUid();
- if (unseqSpaceStatistics.hasOverlap(deviceId, measurementId,
interval)) {
+ if (unseqSpaceStatistics.chunkHasOverlap(deviceId, measurementId,
interval)) {
isFileOverlap = true;
overlapChunkNum++;
}
}
overlapStatistic.overlappedChunks += overlapChunkNum;
+
overlapStatistic.overlappedChunkGroups += overlapChunkNum == 0 ? 0 :
1;
}
overlapStatistic.totalChunkGroups += chunkGroupStatisticsList.size();
@@ -258,7 +259,7 @@ public class OverlapStatisticTool {
List<ChunkGroupStatistics> chunkGroupStatisticsList =
reader.getChunkGroupStatistics();
for (ChunkGroupStatistics statistics : chunkGroupStatisticsList) {
for (ChunkMetadata chunkMetadata :
statistics.getChunkMetadataList()) {
- unseqSpaceStatistics.update(
+ unseqSpaceStatistics.updateMeasurement(
statistics.getDeviceID(),
chunkMetadata.getMeasurementUid(),
new Interval(chunkMetadata.getStartTime(),
chunkMetadata.getEndTime()));
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/TsFileStatisticReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/TsFileStatisticReader.java
index 0fa935ae18f..cd88972310d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/TsFileStatisticReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/TsFileStatisticReader.java
@@ -80,6 +80,9 @@ public class TsFileStatisticReader implements Closeable {
private final boolean isAligned;
private int totalChunkNum = 0;
+ private long startTime;
+ private long endTime;
+
private ChunkGroupStatistics(String deviceId, boolean isAligned) {
this.deviceID = deviceId;
this.isAligned = isAligned;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/UnseqSpaceStatistics.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/UnseqSpaceStatistics.java
index 0028ea452cb..06e85fbbe22 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/UnseqSpaceStatistics.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/UnseqSpaceStatistics.java
@@ -26,15 +26,23 @@ public class UnseqSpaceStatistics {
// 设备 -> 序列 -> 时间范围
private Map<String, Map<String, ITimeRange>> deviceStatisticMap = new
HashMap<>();
+ private Map<String, ITimeRange> chunkGroupStatisticMap = new HashMap<>();
+
// 更新某个设备的某个序列的时间范围
- public void update(String device, String measurementUID, Interval interval) {
+ public void updateMeasurement(String device, String measurementUID, Interval
interval) {
deviceStatisticMap
.computeIfAbsent(device, key -> new HashMap<>())
.computeIfAbsent(measurementUID, key -> new ListTimeRangeImpl())
.addInterval(interval);
}
- public boolean hasOverlap(String device, String measurementUID, Interval
interval) {
+ public void updateDevice(String device, Interval interval) {
+ chunkGroupStatisticMap
+ .computeIfAbsent(device, key -> new ListTimeRangeImpl())
+ .addInterval(interval);
+ }
+
+ public boolean chunkHasOverlap(String device, String measurementUID,
Interval interval) {
if (!deviceStatisticMap.containsKey(device)) {
return false;
}
@@ -44,6 +52,10 @@ public class UnseqSpaceStatistics {
return
deviceStatisticMap.get(device).get(measurementUID).isOverlapped(interval);
}
+ public boolean chunkGroupHasOverlap(String device, Interval interval) {
+ return false;
+ }
+
public Map<String, Map<String, ITimeRange>> getDeviceStatisticMap() {
return deviceStatisticMap;
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tools/UnseqSpaceStatisticsTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tools/UnseqSpaceStatisticsTest.java
index af7213cac7a..4d6f012da5d 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tools/UnseqSpaceStatisticsTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tools/UnseqSpaceStatisticsTest.java
@@ -11,10 +11,10 @@ public class UnseqSpaceStatisticsTest {
@Test
public void test01() {
UnseqSpaceStatistics unseqSpaceStatistics = new UnseqSpaceStatistics();
- unseqSpaceStatistics.update("root.db.d1", "s1", new Interval(1, 10));
- unseqSpaceStatistics.update("root.db.d1", "s1", new Interval(5, 15));
- unseqSpaceStatistics.update("root.db.d1", "s2", new Interval(1, 10));
- unseqSpaceStatistics.update("root.db.d2", "s2", new Interval(1, 10));
+ unseqSpaceStatistics.updateMeasurement("root.db.d1", "s1", new Interval(1,
10));
+ unseqSpaceStatistics.updateMeasurement("root.db.d1", "s1", new Interval(5,
15));
+ unseqSpaceStatistics.updateMeasurement("root.db.d1", "s2", new Interval(1,
10));
+ unseqSpaceStatistics.updateMeasurement("root.db.d2", "s2", new Interval(1,
10));
Assert.assertEquals(2,
unseqSpaceStatistics.getDeviceStatisticMap().size());
Assert.assertEquals(2,
unseqSpaceStatistics.getDeviceStatisticMap().get("root.db.d1").size());
@@ -24,16 +24,21 @@ public class UnseqSpaceStatisticsTest {
@Test
public void test02() {
UnseqSpaceStatistics unseqSpaceStatistics = new UnseqSpaceStatistics();
- unseqSpaceStatistics.update("root.db.d1", "s1", new Interval(1, 10));
- unseqSpaceStatistics.update("root.db.d1", "s1", new Interval(5, 15));
- unseqSpaceStatistics.update("root.db.d1", "s2", new Interval(1, 10));
- unseqSpaceStatistics.update("root.db.d2", "s2", new Interval(1, 10));
-
- Assert.assertTrue(unseqSpaceStatistics.hasOverlap("root.db.d1", "s1", new
Interval(1, 10)));
- Assert.assertFalse(unseqSpaceStatistics.hasOverlap("root.db.d1", "s4", new
Interval(1, 10)));
- Assert.assertFalse(unseqSpaceStatistics.hasOverlap("root.db.d2", "s1", new
Interval(1, 10)));
-
- Assert.assertFalse(unseqSpaceStatistics.hasOverlap("root.db.d3", "s1", new
Interval(1, 10)));
- Assert.assertFalse(unseqSpaceStatistics.hasOverlap("root.db.d1", "s1", new
Interval(21, 30)));
+ unseqSpaceStatistics.updateMeasurement("root.db.d1", "s1", new Interval(1,
10));
+ unseqSpaceStatistics.updateMeasurement("root.db.d1", "s1", new Interval(5,
15));
+ unseqSpaceStatistics.updateMeasurement("root.db.d1", "s2", new Interval(1,
10));
+ unseqSpaceStatistics.updateMeasurement("root.db.d2", "s2", new Interval(1,
10));
+
+ Assert.assertTrue(
+ unseqSpaceStatistics.chunkHasOverlap("root.db.d1", "s1", new
Interval(1, 10)));
+ Assert.assertFalse(
+ unseqSpaceStatistics.chunkHasOverlap("root.db.d1", "s4", new
Interval(1, 10)));
+ Assert.assertFalse(
+ unseqSpaceStatistics.chunkHasOverlap("root.db.d2", "s1", new
Interval(1, 10)));
+
+ Assert.assertFalse(
+ unseqSpaceStatistics.chunkHasOverlap("root.db.d3", "s1", new
Interval(1, 10)));
+ Assert.assertFalse(
+ unseqSpaceStatistics.chunkHasOverlap("root.db.d1", "s1", new
Interval(21, 30)));
}
}
