zhanglingzhe0820 commented on a change in pull request #1758:
URL: https://github.com/apache/iotdb/pull/1758#discussion_r508538584
##########
File path:
server/src/main/java/org/apache/iotdb/db/engine/tsfilemanagement/level/LevelTsFileManagement.java
##########
@@ -407,135 +430,152 @@ public void recover() {
}
@Override
- public void forkCurrentFileList(long timePartition) throws IOException {
- Pair<Long, Map<Path, MeasurementSchema>> seqResult = forkTsFileList(
+ public void forkCurrentFileList(long timePartition) {
+ Pair<Double, Double> seqStatisticsPair = forkTsFileList(
forkedSequenceTsFileResources,
- sequenceTsFileResources.computeIfAbsent(timePartition,
this::newSequenceTsFileResources));
- forkedSeqListPointNum = seqResult.left;
- forkedSeqListPathMeasurementSchemaMap = seqResult.right;
- Pair<Long, Map<Path, MeasurementSchema>> unSeqResult = forkTsFileList(
+ sequenceTsFileResources.computeIfAbsent(timePartition,
this::newSequenceTsFileResources),
+ maxLevelNum);
+ forkedSeqListPointNum = seqStatisticsPair.left;
+ forkedSeqListMeasurementSize = seqStatisticsPair.right;
+ Pair<Double, Double> unSeqStatisticsPair = forkTsFileList(
forkedUnSequenceTsFileResources,
unSequenceTsFileResources
- .computeIfAbsent(timePartition,
this::newUnSequenceTsFileResources));
- forkedUnSeqListPointNum = unSeqResult.left;
- forkedUnSeqListPathMeasurementSchemaMap = unSeqResult.right;
+ .computeIfAbsent(timePartition,
this::newUnSequenceTsFileResources), maxUnseqLevelNum);
+ forkedUnSeqListPointNum = unSeqStatisticsPair.left;
+ forkedUnSeqListMeasurementSize = unSeqStatisticsPair.right;
}
- private Pair<Long, Map<Path, MeasurementSchema>> forkTsFileList(
+ private Pair<Double, Double> forkTsFileList(
List<List<TsFileResource>> forkedTsFileResources,
- List rawTsFileResources) throws IOException {
+ List rawTsFileResources, int currMaxLevel) {
forkedTsFileResources.clear();
// just fork part of the TsFile list, controlled by max_merge_chunk_point
long pointNum = 0;
// all flush to target file
- Map<Path, MeasurementSchema> pathMeasurementSchemaMap = new HashMap<>();
- for (int i = 0; i < maxLevelNum - 1; i++) {
+ ICardinality measurementSet = new HyperLogLog(13);
+ for (int i = 0; i < currMaxLevel - 1; i++) {
List<TsFileResource> forkedLevelTsFileResources = new ArrayList<>();
Collection<TsFileResource> levelRawTsFileResources =
(Collection<TsFileResource>) rawTsFileResources
.get(i);
- for (TsFileResource tsFileResource : levelRawTsFileResources) {
- if (tsFileResource.isClosed()) {
- RestorableTsFileIOWriter writer;
- try {
- writer = new RestorableTsFileIOWriter(
- tsFileResource.getTsFile());
- } catch (Exception e) {
- logger.error("[Hot Compaction] {} open writer failed",
- tsFileResource.getTsFile().getPath(), e);
- continue;
- }
- Map<String, Map<String, List<ChunkMetadata>>> schemaMap = writer
- .getMetadatasForQuery();
- for (Entry<String, Map<String, List<ChunkMetadata>>> schemaMapEntry
: schemaMap
- .entrySet()) {
- String device = schemaMapEntry.getKey();
- for (Entry<String, List<ChunkMetadata>> entry :
schemaMapEntry.getValue()
- .entrySet()) {
- String measurement = entry.getKey();
- List<ChunkMetadata> chunkMetadataList = entry.getValue();
- for (ChunkMetadata chunkMetadata : chunkMetadataList) {
- pointNum += chunkMetadata.getNumOfPoints();
+ synchronized (levelRawTsFileResources) {
+ for (TsFileResource tsFileResource : levelRawTsFileResources) {
+ if (tsFileResource.isClosed()) {
+ String path = tsFileResource.getTsFile().getAbsolutePath();
+ try {
+ if (tsFileResource.getTsFile().exists()) {
+ TsFileSequenceReader reader = new TsFileSequenceReader(path);
Review comment:
close it afterward
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
