This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch overlap_check_tool
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/overlap_check_tool by this
push:
new 2a70a046373 Add Overlap check tool (#10811)
2a70a046373 is described below
commit 2a70a04637398fcb691c7bbcb15f6f7aa9f7d117
Author: shuwenwei <[email protected]>
AuthorDate: Wed Aug 9 01:17:19 2023 +0800
Add Overlap check tool (#10811)
* add overlap_check_tools
* fix some bugs
* fix imports of ListTimeRangeImpl
* mvn spotless:apply
* add scripts
* print final result
---
.../tools/tsfile/overlap-statistic-tool.bat | 62 ++++++
.../tools/tsfile/overlap-statistic-tool.sh | 51 +++++
.../compaction/tool/ListTimeRangeImpl.java | 4 +-
.../compaction/tool/OverlapStatisticTool.java | 216 ++++++++++++++++++---
.../compaction/tool/TsFileStatisticReader.java | 72 +++++--
5 files changed, 366 insertions(+), 39 deletions(-)
diff --git
a/iotdb-core/datanode/src/assembly/resources/tools/tsfile/overlap-statistic-tool.bat
b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/overlap-statistic-tool.bat
new file mode 100644
index 00000000000..c098685e111
--- /dev/null
+++
b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/overlap-statistic-tool.bat
@@ -0,0 +1,62 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+
+@echo off
+echo ````````````````````````
+echo Starting Validating the TsFile
+echo ````````````````````````
+
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..\..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
+popd
+
+if NOT DEFINED MAIN_CLASS set
MAIN_CLASS=org.apache.iotdb.db.storageengine.dataregion.compaction.tool.OverlapStatisticTool
+if NOT DEFINED JAVA_HOME goto :err
+
+@REM
-----------------------------------------------------------------------------
+@REM ***** CLASSPATH library setting *****
+@REM Ensure that any user defined CLASSPATH variables are not used on startup
+set CLASSPATH="%IOTDB_HOME%\lib\*"
+
+goto okClasspath
+
+:append
+set CLASSPATH=%CLASSPATH%;%1
+goto :eof
+
+@REM
-----------------------------------------------------------------------------
+:okClasspath
+
+"%JAVA_HOME%\bin\java" -cp "%CLASSPATH%" %MAIN_CLASS% %*
+
+goto finally
+
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+
+@REM
-----------------------------------------------------------------------------
+:finally
+
+ENDLOCAL
diff --git
a/iotdb-core/datanode/src/assembly/resources/tools/tsfile/overlap-statistic-tool.sh
b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/overlap-statistic-tool.sh
new file mode 100644
index 00000000000..7b4cb59f8fd
--- /dev/null
+++
b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/overlap-statistic-tool.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ---------------------
+echo Starting Validating the TsFile
+echo ---------------------
+
+source "$(dirname "$0")/../../sbin/iotdb-common.sh"
+#get_iotdb_include and checkAllVariables is in iotdb-common.sh
+VARS=$(get_iotdb_include "$*")
+checkAllVariables
+export IOTDB_HOME="${IOTDB_HOME}/.."
+eval set -- "$VARS"
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}":"$f
+done
+
+MAIN_CLASS=org.apache.iotdb.db.storageengine.dataregion.compaction.tool.OverlapStatisticTool
+
+"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+exit $?
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/ListTimeRangeImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/ListTimeRangeImpl.java
index 12a3b490b22..9b3991f58e1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/ListTimeRangeImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/ListTimeRangeImpl.java
@@ -19,7 +19,9 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.tool;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
public class ListTimeRangeImpl implements ITimeRange {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/OverlapStatisticTool.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/OverlapStatisticTool.java
index 7a05ba9c9a4..9b104da84cd 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/OverlapStatisticTool.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/OverlapStatisticTool.java
@@ -20,79 +20,239 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.tool;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.tool.TsFileStatisticReader.ChunkGroupStatistics;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.utils.Pair;
+import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
public class OverlapStatisticTool {
- private List<Long> timePartitions;
private long seqFileCount;
private long processedTimePartitionCount;
private long processedSeqFileCount;
+ private final Map<String, Pair<List<String>, List<String>>>
timePartitionFileMap =
+ new HashMap<>();
- public static void main(String args[]) {
-
+ public static void main(String[] args) {
+ if (args.length == 0) {
+ System.out.println("Please input data dir paths.");
+ return;
+ }
+ OverlapStatisticTool tool = new OverlapStatisticTool();
// 1. 处理参数，从输入中获取数据目录的路径
-
+ List<String> dataDirs = tool.getDataDirsFromArgs(args);
// 2. 进行计算
- OverlapStatisticTool tool = new OverlapStatisticTool();
- tool.process(null);
+ tool.process(dataDirs);
+ }
+
+ private List<String> getDataDirsFromArgs(String[] args) {
+ return new ArrayList<>(Arrays.asList(args));
}
public void process(List<String> dataDirs) {
// 0. 预处理
- processDataDirs();
+ processDataDirs(dataDirs);
// 1. 构造最终结果集
OverlapStatistic statistic = new OverlapStatistic();
+ for (Map.Entry<String, Pair<List<String>, List<String>>>
timePartitionFilesEntry :
+ timePartitionFileMap.entrySet()) {
+ String timePartition = timePartitionFilesEntry.getKey();
+ Pair<List<String>, List<String>> timePartitionFiles =
timePartitionFilesEntry.getValue();
+ OverlapStatistic partialRet =
+ processOneTimePartition(timePartitionFiles.left,
timePartitionFiles.right);
- // 2. 根据时间分区的信息
- for (Long timePartition : timePartitions) {
- OverlapStatistic partialRet = processOneTimePartiton(timePartition,
dataDirs);
+ // 2. 根据时间分区的信息
// 将该时间分区的结果集更新到最终结果集
-
+ statistic.merge(partialRet);
// 更新并打印进度
-
+ updateProcessAndPrint(timePartition, partialRet);
}
+ System.out.println("--------------------" + "final result" +
"--------------------");
+ printOneStatistics(statistic);
}
- private void updateProcessAndPrint(OverlapStatistic partialRet) {
+ private void updateProcessAndPrint(String timePartition, OverlapStatistic
partialRet) {
processedTimePartitionCount += 1;
processedSeqFileCount += partialRet.totalFiles;
// 打印进度
+
+ System.out.println("--------------------" + timePartition +
"--------------------");
+ printOneStatistics(partialRet);
}
- private void processDataDirs() {
- // 1. 遍历所有的时间分区，构造 timePartitions
+ private void printOneStatistics(OverlapStatistic overlapStatistic) {
+ double overlappedSeqFilePercentage;
+ if (overlapStatistic.totalFiles == 0) {
+ overlappedSeqFilePercentage = 0;
+ } else {
+ overlappedSeqFilePercentage =
+ (double) overlapStatistic.overlappedFiles /
overlapStatistic.totalFiles * 100;
+ }
+
+ double overlappedChunkGroupPercentage;
+ if (overlapStatistic.totalChunkGroups == 0) {
+ overlappedChunkGroupPercentage = 0;
+ } else {
+ overlappedChunkGroupPercentage =
+ (double) overlapStatistic.overlappedChunkGroups /
overlapStatistic.totalChunkGroups * 100;
+ }
+
+ double overlappedChunkPercentage;
+ if (overlapStatistic.totalChunks == 0) {
+ overlappedChunkPercentage = 0;
+ } else {
+ overlappedChunkPercentage =
+ (double) overlapStatistic.overlappedChunks /
overlapStatistic.totalChunks * 100;
+ }
+ System.out.printf(
+ "overlapped_seq_file is %d, total seq file is %d,
overlapped_seq_file_percentage is %.2f%%\n",
+ overlapStatistic.overlappedFiles, overlapStatistic.totalFiles,
overlappedSeqFilePercentage);
+ System.out.printf(
+ "overlapped_chunk_group is %d, total chunk group is %d,
overlapped_chunk_group_percentage is %.2f%%\n",
+ overlapStatistic.overlappedChunkGroups,
+ overlapStatistic.totalChunkGroups,
+ overlappedChunkGroupPercentage);
+ System.out.printf(
+ "overlapped_chunk is %d, total chunk is %d,
overlapped_chunk_percentage is %.2f%%\n",
+ overlapStatistic.overlappedChunks, overlapStatistic.totalChunks,
overlappedChunkPercentage);
+ System.out.printf("processed time partition count: %d\n",
processedTimePartitionCount);
+ System.out.printf(
+ "processed seq file count: %d, total seq file count: %d\n",
+ processedSeqFileCount, seqFileCount);
+ }
+ private void processDataDirs(List<String> dataDirs) {
+ // 1. 遍历所有的时间分区，构造 timePartitions
// 2. 统计顺序文件的总数
+ for (String dataDirPath : dataDirs) {
+ File dataDir = new File(dataDirPath);
+ if (!dataDir.exists() || !dataDir.isDirectory()) {
+ continue;
+ }
+ processDataDirWithIsSeq(dataDirPath, true);
+ processDataDirWithIsSeq(dataDirPath, false);
+ }
+ }
+
+ private void processDataDirWithIsSeq(String dataDirPath, boolean isSeq) {
+ String dataDirWithIsSeq;
+ if (isSeq) {
+ dataDirWithIsSeq = dataDirPath + "/sequence";
+ } else {
+ dataDirWithIsSeq = dataDirPath + "/unsequence";
+ }
+ File dataDirWithIsSequence = new File(dataDirWithIsSeq);
+ if (!dataDirWithIsSequence.exists() ||
!dataDirWithIsSequence.isDirectory()) {
+ System.out.println(dataDirWithIsSequence + " is not a correct path");
+ return;
+ }
+
+ for (File storageGroupDir :
Objects.requireNonNull(dataDirWithIsSequence.listFiles())) {
+ if (!storageGroupDir.isDirectory()) {
+ continue;
+ }
+ String storageGroup = storageGroupDir.getName();
+ for (File dataRegionDir :
Objects.requireNonNull(storageGroupDir.listFiles())) {
+ if (!dataRegionDir.isDirectory()) {
+ continue;
+ }
+ String dataRegion = dataRegionDir.getName();
+ for (File timePartitionDir :
Objects.requireNonNull(dataRegionDir.listFiles())) {
+ if (!timePartitionDir.isDirectory()) {
+ continue;
+ }
+
+ String timePartitionKey =
+ calculateTimePartitionKey(storageGroup, dataRegion,
timePartitionDir.getName());
+ Pair<List<String>, List<String>> timePartitionFiles =
+ timePartitionFileMap.computeIfAbsent(
+ timePartitionKey, v -> new Pair<>(new ArrayList<>(), new
ArrayList<>()));
+ for (File file :
Objects.requireNonNull(timePartitionDir.listFiles())) {
+ if (!file.isFile()) {
+ continue;
+ }
+ if (!file.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
+ continue;
+ }
+ String resourceFilePath = file.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX;
+ if (!new File(resourceFilePath).exists()) {
+ System.out.println(
+ resourceFilePath
+ + " is not exist, the tsfile is skipped because it is
not closed.");
+ continue;
+ }
+ String filePath = file.getAbsolutePath();
+ if (isSeq) {
+ timePartitionFiles.left.add(filePath);
+ seqFileCount++;
+ } else {
+ timePartitionFiles.right.add(filePath);
+ }
+ }
+ }
+ }
+ }
}
- private OverlapStatistic processOneTimePartiton(long timePartition,
List<String> dataDirs) {
+ private String calculateTimePartitionKey(
+ String storageGroup, String dataRegion, String timePartition) {
+ return storageGroup + "-" + dataRegion + "-" + timePartition;
+ }
+
+ private OverlapStatistic processOneTimePartition(List<String> seqFiles,
List<String> unseqFiles) {
// 1. 根据 timePartition，获取所有数据目录下的的乱序文件，构造 UnseqSpaceStatistics
- UnseqSpaceStatistics unseqSpaceStatistics =
buildUnseqSpaceStatistics(timePartition, dataDirs);
+ UnseqSpaceStatistics unseqSpaceStatistics =
buildUnseqSpaceStatistics(unseqFiles);
// 2. 遍历该时间分区下的所有顺序文件，获取每一个 chunk 的信息，依次进行 overlap 检查，并更新统计信息
OverlapStatistic overlapStatistic = new OverlapStatistic();
- List<String> seqFiles = getFilesInOnePartition(timePartition, dataDirs,
true);
+ overlapStatistic.totalFiles += seqFiles.size();
for (String seqFile : seqFiles) {
+ boolean isFileOverlap = false;
try (TsFileStatisticReader reader = new TsFileStatisticReader(seqFile)) {
// 统计顺序文件的信息并更新到 overlapStatistic
+ List<ChunkGroupStatistics> chunkGroupStatisticsList =
reader.getChunkGroupStatistics();
+ for (ChunkGroupStatistics chunkGroupStatistics :
chunkGroupStatisticsList) {
+ overlapStatistic.totalChunks +=
chunkGroupStatistics.getTotalChunkNum();
+ String deviceId = chunkGroupStatistics.getDeviceID();
+ int overlapChunkNum = 0;
+
+ for (ChunkMetadata chunkMetadata :
chunkGroupStatistics.getChunkMetadataList()) {
+ Interval interval =
+ new Interval(chunkMetadata.getStartTime(),
chunkMetadata.getEndTime());
+ String measurementId = chunkMetadata.getMeasurementUid();
+ if (unseqSpaceStatistics.hasOverlap(deviceId, measurementId,
interval)) {
+ isFileOverlap = true;
+ overlapChunkNum++;
+ }
+ }
+ overlapStatistic.overlappedChunks += overlapChunkNum;
+ overlapStatistic.overlappedChunkGroups += overlapChunkNum == 0 ? 0 :
1;
+ }
+ overlapStatistic.totalChunkGroups += chunkGroupStatisticsList.size();
} catch (IOException e) {
throw new RuntimeException(e);
}
+ if (isFileOverlap) {
+ overlapStatistic.overlappedFiles += 1;
+ }
}
return overlapStatistic;
}
- private UnseqSpaceStatistics buildUnseqSpaceStatistics(
- long timePartition, List<String> dataDirs) {
+ private UnseqSpaceStatistics buildUnseqSpaceStatistics(List<String>
unseqFiles) {
UnseqSpaceStatistics unseqSpaceStatistics = new UnseqSpaceStatistics();
- List<String> unseqFiles = getFilesInOnePartition(timePartition, dataDirs,
false);
for (String unseqFile : unseqFiles) {
try (TsFileStatisticReader reader = new
TsFileStatisticReader(unseqFile)) {
List<ChunkGroupStatistics> chunkGroupStatisticsList =
reader.getChunkGroupStatistics();
@@ -111,11 +271,6 @@ public class OverlapStatisticTool {
return unseqSpaceStatistics;
}
- private List<String> getFilesInOnePartition(
- long timePartition, List<String> dataDirs, boolean isSeq) {
- return null;
- }
-
private static class OverlapStatistic {
private long totalFiles;
private long totalChunkGroups;
@@ -124,5 +279,14 @@ public class OverlapStatisticTool {
private long overlappedFiles;
private long overlappedChunkGroups;
private long overlappedChunks;
+
+ private void merge(OverlapStatistic other) {
+ this.totalFiles += other.totalFiles;
+ this.totalChunkGroups += other.totalChunkGroups;
+ this.totalChunks += other.totalChunks;
+ this.overlappedFiles += other.overlappedFiles;
+ this.overlappedChunkGroups += other.overlappedChunkGroups;
+ this.overlappedChunks += other.overlappedChunks;
+ }
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/TsFileStatisticReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/TsFileStatisticReader.java
index f7cbc96b625..0fa935ae18f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/TsFileStatisticReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tool/TsFileStatisticReader.java
@@ -19,39 +19,87 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.tool;
-import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.read.TsFileDeviceIterator;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.utils.Pair;
import java.io.Closeable;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
public class TsFileStatisticReader implements Closeable {
- public TsFileStatisticReader(String filePath) {}
+ private final TsFileSequenceReader reader;
- public List<ChunkGroupStatistics> getChunkGroupStatistics() {
- return null;
+ private final List<ChunkGroupStatistics> chunkGroupStatisticsList;
+
+ public TsFileStatisticReader(String filePath) throws IOException {
+ reader = new TsFileSequenceReader(filePath);
+ chunkGroupStatisticsList = new ArrayList<>();
+ }
+
+ public List<ChunkGroupStatistics> getChunkGroupStatistics() throws
IOException {
+ TsFileDeviceIterator allDevicesIteratorWithIsAligned =
+ reader.getAllDevicesIteratorWithIsAligned();
+ while (allDevicesIteratorWithIsAligned.hasNext()) {
+ Pair<String, Boolean> deviceWithIsAligned =
allDevicesIteratorWithIsAligned.next();
+ String deviceId = deviceWithIsAligned.left;
+ boolean isAligned = deviceWithIsAligned.right;
+
+ ChunkGroupStatistics chunkGroupStatistics = new
ChunkGroupStatistics(deviceId, isAligned);
+ Iterator<Map<String, List<ChunkMetadata>>>
measurementChunkMetadataListMapIterator =
+ reader.getMeasurementChunkMetadataListMapIterator(deviceId);
+
+ while (measurementChunkMetadataListMapIterator.hasNext()) {
+ Map<String, List<ChunkMetadata>> measurementChunkMetadataListMap =
+ measurementChunkMetadataListMapIterator.next();
+ for (Map.Entry<String, List<ChunkMetadata>>
measurementChunkMetadataList :
+ measurementChunkMetadataListMap.entrySet()) {
+ List<ChunkMetadata> chunkMetadataList =
measurementChunkMetadataList.getValue();
+ chunkGroupStatistics.chunkMetadataList.addAll(chunkMetadataList);
+ chunkGroupStatistics.totalChunkNum += chunkMetadataList.size();
+ }
+ }
+ chunkGroupStatisticsList.add(chunkGroupStatistics);
+ }
+ return chunkGroupStatisticsList;
}
@Override
- public void close() throws IOException {}
+ public void close() throws IOException {
+ this.reader.close();
+ }
public static class ChunkGroupStatistics {
- private String deviceID;
- private ChunkGroupMetadata chunkGroupMetadata;
- private List<ChunkMetadata> chunkMetadataList;
+ private final String deviceID;
+ private final List<ChunkMetadata> chunkMetadataList;
+ private final boolean isAligned;
+ private int totalChunkNum = 0;
- public String getDeviceID() {
- return deviceID;
+ private ChunkGroupStatistics(String deviceId, boolean isAligned) {
+ this.deviceID = deviceId;
+ this.isAligned = isAligned;
+ this.chunkMetadataList = new ArrayList<>();
}
- public ChunkGroupMetadata getChunkGroupMetadata() {
- return chunkGroupMetadata;
+ public String getDeviceID() {
+ return deviceID;
}
public List<ChunkMetadata> getChunkMetadataList() {
return chunkMetadataList;
}
+
+ public int getTotalChunkNum() {
+ return totalChunkNum;
+ }
+
+ public boolean isAligned() {
+ return isAligned;
+ }
}
}
