This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch pr_1758 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9d3e49b0b1337df2db54b8551b0ed08b5d450347 Author: 张凌哲 <[email protected]> AuthorDate: Thu Sep 24 14:40:02 2020 +0800 remove count by chunk point --- .../db/engine/storagegroup/TsFileProcessor.java | 2 +- .../level/LevelTsFileManagement.java | 58 +++++++++++++++------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 136b979..327d9f3 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -673,7 +673,7 @@ public class TsFileProcessor { private void endFile() throws IOException, TsFileProcessorException { long closeStartTime = System.currentTimeMillis(); - updateDeviceChunkPointSizeCache(); +// updateDeviceChunkPointSizeCache(); tsFileResource.serialize(); writer.endFile(); tsFileResource.cleanCloseFlag(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tsfilemanagement/level/LevelTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/tsfilemanagement/level/LevelTsFileManagement.java index c9173f7..d5ac754 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/tsfilemanagement/level/LevelTsFileManagement.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/tsfilemanagement/level/LevelTsFileManagement.java @@ -416,6 +416,43 @@ public class LevelTsFileManagement extends TsFileManagement { forkedUnSeqListDeviceSet = unSeqResult.right; } +// private Pair<Long, Set<String>> forkTsFileList( +// List<List<TsFileResource>> forkedTsFileResources, +// List rawTsFileResources) { +// forkedTsFileResources.clear(); +// // just fork part of the TsFile list, controlled by max_merge_chunk_point +// long pointNum = 0; +// // all flush to target file +// Set<String> deviceSet = new HashSet<>(); +// for (int i = 0; i < maxLevelNum - 1; i++) { +// List<TsFileResource> forkedLevelTsFileResources = new ArrayList<>(); +// Collection<TsFileResource> levelRawTsFileResources = (Collection<TsFileResource>) rawTsFileResources +// .get(i); +// synchronized (levelRawTsFileResources) { +// for (TsFileResource tsFileResource : levelRawTsFileResources) { +// if (tsFileResource.isClosed()) { +// Map<String, Long> chunkPointMap = FileChunkPointSizeCache.getInstance() +// .get(tsFileResource.getTsFile()); +// for (Entry<String, Long> deviceChunkPoint : chunkPointMap.entrySet()) { +// deviceSet.add(deviceChunkPoint.getKey()); +// pointNum += deviceChunkPoint.getValue(); +// } +// } +// if (deviceSet.size() > 0 +// && pointNum / deviceSet.size() >= maxChunkPointNum) { +// break; +// } +// } +// } +// if (deviceSet.size() > 0 +// && pointNum / deviceSet.size() >= maxChunkPointNum) { +// break; +// } +// forkedTsFileResources.add(forkedLevelTsFileResources); +// } +// return new Pair<>(pointNum, deviceSet); +// } + private Pair<Long, Set<String>> forkTsFileList( List<List<TsFileResource>> forkedTsFileResources, List rawTsFileResources) { @@ -431,23 +468,10 @@ public class LevelTsFileManagement extends TsFileManagement { synchronized (levelRawTsFileResources) { for (TsFileResource tsFileResource : levelRawTsFileResources) { if (tsFileResource.isClosed()) { - Map<String, Long> chunkPointMap = FileChunkPointSizeCache.getInstance() - .get(tsFileResource.getTsFile()); - for (Entry<String, Long> deviceChunkPoint : chunkPointMap.entrySet()) { - deviceSet.add(deviceChunkPoint.getKey()); - pointNum += deviceChunkPoint.getValue(); - } - } - if (deviceSet.size() > 0 - && pointNum / deviceSet.size() >= maxChunkPointNum) { - break; + forkedLevelTsFileResources.add(tsFileResource); } } } - if (deviceSet.size() > 0 - && pointNum / deviceSet.size() >= maxChunkPointNum) { - break; - } forkedTsFileResources.add(forkedLevelTsFileResources); } return new Pair<>(pointNum, deviceSet); @@ -468,9 +492,9 @@ public class LevelTsFileManagement extends TsFileManagement { long pointNum = sequence ? forkedSeqListPointNum : forkedUnSeqListPointNum; Set<String> deviceSet = sequence ? forkedSeqListDeviceSet : forkedUnSeqListDeviceSet; - logger - .info("{} current sg subLevel point num: {}, device num: {}", storageGroupName, pointNum, - deviceSet.size()); +// logger +// .info("{} current sg subLevel point num: {}, device num: {}", storageGroupName, pointNum, +// deviceSet.size()); HotCompactionLogger hotCompactionLogger = new HotCompactionLogger(storageGroupDir, storageGroupName); if (deviceSet.size() > 0 && pointNum / deviceSet.size() > IoTDBDescriptor.getInstance()
