This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch NewTsFileV2 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c76d13519e284f02d56d4aacc5a7056396aa678e Author: JackieTien97 <[email protected]> AuthorDate: Wed Jan 20 18:22:31 2021 +0800 wait for upgrade tool --- .../resources/conf/iotdb-engine.properties | 6 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 14 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +- .../db/engine/cache/CacheHitRatioMonitor.java | 25 --- .../engine/cache/CacheHitRatioMonitorMXBean.java | 11 - .../apache/iotdb/db/engine/cache/ChunkCache.java | 12 +- .../iotdb/db/engine/cache/ChunkMetadataCache.java | 222 --------------------- .../db/engine/cache/TimeSeriesMetadataCache.java | 12 +- .../db/engine/compaction/TsFileManagement.java | 2 - .../level/LevelCompactionTsFileManagement.java | 2 - .../iotdb/db/engine/merge/task/MergeFileTask.java | 4 - .../apache/iotdb/db/qp/executor/PlanExecutor.java | 2 - .../db/query/reader/chunk/DiskChunkLoader.java | 11 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 33 ++- .../chunk/metadata/MemChunkMetadataLoader.java | 6 - .../org/apache/iotdb/db/service/TSServiceImpl.java | 6 +- .../db/engine/cache/ChunkMetadataCacheTest.java | 172 ---------------- .../db/engine/compaction/LevelCompactionTest.java | 3 - .../apache/iotdb/db/engine/merge/MergeTest.java | 2 - .../iotdb/db/integration/IoTDBClearCacheIT.java | 3 - .../query/reader/series/SeriesReaderTestUtil.java | 2 - .../apache/iotdb/db/utils/EnvironmentUtils.java | 2 - .../apache/iotdb/spark/db/EnvironmentUtils.java | 21 +- .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 19 ++ .../tsfile/file/metadata/TimeseriesMetadata.java | 27 ++- .../iotdb/tsfile/read/TsFileSequenceReader.java | 143 +++++-------- .../read/controller/IChunkMetadataLoader.java | 8 - .../file/metadata/TimeSeriesMetadataTest.java | 2 +- 28 files changed, 134 insertions(+), 648 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 8201e9d..73fa1da 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -380,9 +380,9 @@ query_time_threshold=60000 # whether to cache meta data(ChunkMetadata and TimeSeriesMetadata) or not. meta_data_cache_enable=true -# Read memory Allocation Ratio: ChunkMetadataCache, ChunkCache, TimeSeriesMetadataCache, memory used for constructing QueryDataSet and Free Memory Used in Query. -# The parameter form is a:b:c:d:e, where a, b, c, d and e are integers. for example: 1:1:1:1:1 , 1:1:1:3:4 -chunkmeta_chunk_timeseriesmeta_free_memory_proportion=1:1:1:3:4 +# Read memory Allocation Ratio: ChunkCache, TimeSeriesMetadataCache, memory used for constructing QueryDataSet and Free Memory Used in Query. +# The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 1:1:1:1 , 1:2:3:4 +chunk_timeseriesmeta_free_memory_proportion=1:2:3:4 # cache size for MManager. # This cache is used to improve insert speed where all path check and TSDataType will be cached in MManager with corresponding Path. diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 0871744..7463691 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -409,12 +409,8 @@ public class IoTDBConfig { /** * Memory allocated for timeSeriesMetaData cache in read process */ - private long allocateMemoryForTimeSeriesMetaDataCache = allocateMemoryForRead / 10; + private long allocateMemoryForTimeSeriesMetaDataCache = allocateMemoryForRead / 5; - /** - * Memory allocated for chunkMetaData cache in read process - */ - private long allocateMemoryForChunkMetaDataCache = allocateMemoryForRead / 10; /** * Memory allocated for chunk cache in read process @@ -1683,14 +1679,6 @@ public class IoTDBConfig { this.allocateMemoryForTimeSeriesMetaDataCache = allocateMemoryForTimeSeriesMetaDataCache; } - public long getAllocateMemoryForChunkMetaDataCache() { - return allocateMemoryForChunkMetaDataCache; - } - - public void setAllocateMemoryForChunkMetaDataCache(long allocateMemoryForChunkMetaDataCache) { - this.allocateMemoryForChunkMetaDataCache = allocateMemoryForChunkMetaDataCache; - } - public long getAllocateMemoryForChunkCache() { return allocateMemoryForChunkCache; } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index b11f9d3..0c5c19e 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -835,7 +835,7 @@ public class IoTDBDescriptor { } String queryMemoryAllocateProportion = properties - .getProperty("chunkmeta_chunk_timeseriesmeta_free_memory_proportion"); + .getProperty("chunk_timeseriesmeta_free_memory_proportion"); if (queryMemoryAllocateProportion != null) { String[] proportions = queryMemoryAllocateProportion.split(":"); int proportionSum = 0; @@ -845,14 +845,12 @@ public class IoTDBDescriptor { long maxMemoryAvailable = conf.getAllocateMemoryForRead(); if (proportionSum != 0) { try { - conf.setAllocateMemoryForChunkMetaDataCache( - maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum); conf.setAllocateMemoryForChunkCache( - maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum); + maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum); conf.setAllocateMemoryForTimeSeriesMetaDataCache( - maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum); + maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum); conf.setAllocateMemoryForReadWithoutCache( - maxMemoryAvailable * Integer.parseInt(proportions[3].trim()) / proportionSum); + maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum); } catch (Exception e) { throw new RuntimeException( "Each subsection of configuration item chunkmeta_chunk_timeseriesmeta_free_memory_proportion" diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java index 2c411ac..df078ec 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java @@ -53,31 +53,6 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic } @Override - public double getChunkMetaDataHitRatio() { - return ChunkMetadataCache.getInstance().calculateChunkMetaDataHitRatio(); - } - - @Override - public long getChunkMetaDataCacheUsedMemory() { - return ChunkMetadataCache.getInstance().getUsedMemory(); - } - - @Override - public long getChunkMetaDataCacheMaxMemory() { - return ChunkMetadataCache.getInstance().getMaxMemory(); - } - - @Override - public double getChunkMetaDataCacheUsedMemoryProportion() { - return ChunkMetadataCache.getInstance().getUsedMemoryProportion(); - } - - @Override - public long getChunkMetaDataCacheAverageSize() { - return ChunkMetadataCache.getInstance().getAverageSize(); - } - - @Override public double getChunkHitRatio() { return ChunkCache.getInstance().calculateChunkHitRatio(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java index 0514481..ae2697b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java @@ -20,17 +20,6 @@ package org.apache.iotdb.db.engine.cache; public interface CacheHitRatioMonitorMXBean { - double getChunkMetaDataHitRatio(); - - long getChunkMetaDataCacheUsedMemory(); - - long getChunkMetaDataCacheMaxMemory(); - - double getChunkMetaDataCacheUsedMemoryProportion(); - - long getChunkMetaDataCacheAverageSize(); - - double getChunkHitRatio(); long getChunkCacheUsedMemory(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java index c621a26..345d2e0 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java @@ -25,6 +25,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; @@ -83,8 +84,11 @@ public class ChunkCache { return ChunkCacheHolder.INSTANCE; } - public Chunk get(ChunkMetadata chunkMetaData, TsFileSequenceReader reader) throws IOException { + public Chunk get(ChunkMetadata chunkMetaData) throws IOException { if (!CACHE_ENABLE) { + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(chunkMetaData.getFilePath(), chunkMetaData.isClosed()); Chunk chunk = reader.readMemChunk(chunkMetaData); return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeleteIntervalList(), chunkMetaData.getStatistics()); @@ -104,6 +108,8 @@ public class ChunkCache { printCacheLog(true); } else { printCacheLog(false); + TsFileSequenceReader reader = FileReaderManager.getInstance() + .get(chunkMetaData.getFilePath(), chunkMetaData.isClosed()); try { chunk = reader.readMemChunk(chunkMetaData); } catch (IOException e) { @@ -112,7 +118,9 @@ public class ChunkCache { } lock.writeLock().lock(); try { - lruCache.put(chunkMetaData, chunk); + if (!lruCache.containsKey(chunkMetaData)) { + lruCache.put(chunkMetaData, chunk); + } } finally { lock.writeLock().unlock(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java deleted file mode 100644 index f8fd5f3..0000000 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.engine.cache; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBConstant; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.query.control.FileReaderManager; -import org.apache.iotdb.db.utils.TestOnly; -import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; -import org.apache.iotdb.tsfile.read.TsFileSequenceReader; -import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.utils.RamUsageEstimator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class is used to cache <code>List<ChunkMetaData></code> of tsfile in IoTDB. The caching - * strategy is LRU. - */ -public class ChunkMetadataCache { - - private static final Logger logger = LoggerFactory.getLogger(ChunkMetadataCache.class); - private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("QUERY_DEBUG"); - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForChunkMetaDataCache(); - private static final boolean CACHE_ENABLE = config.isMetaDataCacheEnable(); - - /** - * key: file path dot deviceId dot sensorId. - * <p> - * value: chunkMetaData list of one timeseries in the file. - */ - private final LRULinkedHashMap<AccountableString, List<ChunkMetadata>> lruCache; - - private final ReadWriteLock lock = new ReentrantReadWriteLock(); - - private final AtomicLong cacheHitNum = new AtomicLong(); - private final AtomicLong cacheRequestNum = new AtomicLong(); - - - private ChunkMetadataCache(long memoryThreshold) { - if (CACHE_ENABLE) { - logger.info("ChunkMetadataCache size = " + memoryThreshold); - } - lruCache = new LRULinkedHashMap<AccountableString, List<ChunkMetadata>>(memoryThreshold) { - @Override - protected long calEntrySize(AccountableString key, List<ChunkMetadata> value) { - if (value.isEmpty()) { - return RamUsageEstimator.sizeOf(key) + RamUsageEstimator.shallowSizeOf(value); - } - long entrySize; - if (count < 10) { - long currentSize = value.get(0).calculateRamSize(); - averageSize = ((averageSize * count) + currentSize) / (++count); - entrySize = RamUsageEstimator.sizeOf(key) - + (currentSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * value.size() - + RamUsageEstimator.shallowSizeOf(value); - } else if (count < 100000) { - count++; - entrySize = RamUsageEstimator.sizeOf(key) - + (averageSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * value.size() - + RamUsageEstimator.shallowSizeOf(value); - } else { - averageSize = value.get(0).calculateRamSize(); - count = 1; - entrySize = RamUsageEstimator.sizeOf(key) - + (averageSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * value.size() - + RamUsageEstimator.shallowSizeOf(value); - } - return entrySize; - } - }; - } - - public static ChunkMetadataCache getInstance() { - return ChunkMetadataCacheSingleton.INSTANCE; - } - - /** - * get {@link ChunkMetadata}. THREAD SAFE. - */ - public List<ChunkMetadata> get(String filePath, Path seriesPath, - TimeseriesMetadata timeseriesMetadata) throws IOException { - if (timeseriesMetadata == null) { - return Collections.emptyList(); - } - if (!CACHE_ENABLE) { - // bloom filter part - TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true); - // If timeseries isn't included in the tsfile, empty list is returned. - return tsFileReader.readChunkMetaDataList(timeseriesMetadata); - } - - AccountableString key = new AccountableString(filePath + IoTDBConstant.PATH_SEPARATOR - + seriesPath.getDevice() + IoTDBConstant.PATH_SEPARATOR + seriesPath.getMeasurement()); - - cacheRequestNum.incrementAndGet(); - - lock.readLock().lock(); - List<ChunkMetadata> chunkMetadataList; - try { - chunkMetadataList = lruCache.get(key); - } finally { - lock.readLock().unlock(); - } - - if (chunkMetadataList != null) { - printCacheLog(true); - cacheHitNum.incrementAndGet(); - } else { - printCacheLog(false); - TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true); - chunkMetadataList = tsFileReader.readChunkMetaDataList(timeseriesMetadata); - lock.writeLock().lock(); - try { - lruCache.put(key, chunkMetadataList); - } finally { - lock.writeLock().unlock(); - } - } - if (config.isDebugOn()) { - DEBUG_LOGGER.info( - "Chunk meta data list size: " + chunkMetadataList.size() + " key is: " + key.getString()); - chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString())); - } - return new ArrayList<>(chunkMetadataList); - } - - private void printCacheLog(boolean isHit) { - if (!logger.isDebugEnabled()) { - return; - } - logger.debug( - "[ChunkMetaData cache {}hit] The number of requests for cache is {}, hit rate is {}.", - isHit ? "" : "didn't ", cacheRequestNum.get(), - cacheHitNum.get() * 1.0 / cacheRequestNum.get()); - } - - double calculateChunkMetaDataHitRatio() { - if (cacheRequestNum.get() != 0) { - return cacheHitNum.get() * 1.0 / cacheRequestNum.get(); - } else { - return 0; - } - } - - public long getUsedMemory() { - return lruCache.getUsedMemory(); - } - - public long getMaxMemory() { - return lruCache.getMaxMemory(); - } - - public double getUsedMemoryProportion() { - return lruCache.getUsedMemoryProportion(); - } - - public long getAverageSize() { - return lruCache.getAverageSize(); - } - - /** - * clear LRUCache. - */ - public void clear() { - lock.writeLock().lock(); - if (lruCache != null) { - lruCache.clear(); - } - lock.writeLock().unlock(); - } - - public void remove(TsFileResource resource) { - lock.writeLock().lock(); - if (resource != null) { - lruCache.entrySet() - .removeIf(e -> e.getKey().getString().startsWith(resource.getTsFilePath())); - } - lock.writeLock().unlock(); - } - - @TestOnly - public boolean isEmpty() { - return lruCache.isEmpty(); - } - - /** - * singleton pattern. - */ - private static class ChunkMetadataCacheSingleton { - - private static final ChunkMetadataCache INSTANCE = new - ChunkMetadataCache(MEMORY_THRESHOLD_IN_B); - } -} \ No newline at end of file diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java index a0cf793..b329aa5 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java @@ -83,7 +83,10 @@ public class TimeSeriesMetadataCache { currentSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(key.device) + RamUsageEstimator.sizeOf(key.measurement) + RamUsageEstimator.shallowSizeOf(value) + RamUsageEstimator.sizeOf(value.getMeasurementId()) + RamUsageEstimator - .shallowSizeOf(value.getStatistics()); + .shallowSizeOf(value.getStatistics()) + + (value.getChunkMetadataList().get(0).calculateRamSize() + + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * value.getChunkMetadataList().size() + + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList()); averageSize = ((averageSize * count) + currentSize) / (++count); } else if (count < 100000) { count++; @@ -92,7 +95,10 @@ public class TimeSeriesMetadataCache { averageSize = RamUsageEstimator.shallowSizeOf(key) + RamUsageEstimator.sizeOf(key.device) + RamUsageEstimator.sizeOf(key.measurement) + RamUsageEstimator.shallowSizeOf(value) + RamUsageEstimator.sizeOf(value.getMeasurementId()) + RamUsageEstimator - .shallowSizeOf(value.getStatistics()); + .shallowSizeOf(value.getStatistics()) + + (value.getChunkMetadataList().get(0).calculateRamSize() + + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * value.getChunkMetadataList().size() + + RamUsageEstimator.shallowSizeOf(value.getChunkMetadataList()); count = 1; currentSize = averageSize; } @@ -193,7 +199,7 @@ public class TimeSeriesMetadataCache { "Get timeseries: {}.{} metadata in file: {} from cache: {}.", key.device, key.measurement, key.filePath, timeseriesMetadata); } - return timeseriesMetadata; + return new TimeseriesMetadata(timeseriesMetadata); } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java index 3d8e755..3d88e70 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java @@ -32,7 +32,6 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.cache.ChunkCache; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.merge.manage.MergeManager; import org.apache.iotdb.db.engine.merge.manage.MergeResource; @@ -322,7 +321,6 @@ public abstract class TsFileManagement { // clean cache if (IoTDBDescriptor.getInstance().getConfig().isMetaDataCacheEnable()) { ChunkCache.getInstance().clear(); - ChunkMetadataCache.getInstance().clear(); TimeSeriesMetadataCache.getInstance().clear(); } } finally { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java index ec1ebdd..35c7f7d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java @@ -41,7 +41,6 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.CopyOnWriteArrayList; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.compaction.TsFileManagement; import org.apache.iotdb.db.engine.compaction.utils.CompactionLogAnalyzer; import org.apache.iotdb.db.engine.compaction.utils.CompactionLogger; @@ -123,7 +122,6 @@ public class LevelCompactionTsFileManagement extends TsFileManagement { private void deleteLevelFile(TsFileResource seqFile) { seqFile.writeLock(); try { - ChunkMetadataCache.getInstance().remove(seqFile); FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath()); seqFile.setDeleted(true); seqFile.delete(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java index a21e1a0..fbf6bc1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.cache.ChunkCache; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.merge.manage.MergeContext; import org.apache.iotdb.db.engine.merge.manage.MergeResource; @@ -143,7 +142,6 @@ class MergeFileTask { seqFile.writeLock(); try { - ChunkMetadataCache.getInstance().remove(seqFile); FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath()); resource.removeFileReader(seqFile); @@ -294,7 +292,6 @@ class MergeFileTask { seqFile.writeLock(); try { resource.removeFileReader(seqFile); - ChunkMetadataCache.getInstance().remove(seqFile); File newMergeFile = seqFile.getTsFile(); newMergeFile.delete(); @@ -306,7 +303,6 @@ class MergeFileTask { // clean cache if (IoTDBDescriptor.getInstance().getConfig().isMetaDataCacheEnable()) { ChunkCache.getInstance().clear(); - ChunkMetadataCache.getInstance().clear(); TimeSeriesMetadataCache.getInstance().clear(); FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath()); } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java index b3416a3..0ee2ce8 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java @@ -71,7 +71,6 @@ import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.cache.ChunkCache; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.flush.pool.FlushTaskPoolManager; import org.apache.iotdb.db.engine.merge.manage.MergeManager; @@ -337,7 +336,6 @@ public class PlanExecutor implements IPlanExecutor { private void operateClearCache() { ChunkCache.getInstance().clear(); - ChunkMetadataCache.getInstance().clear(); TimeSeriesMetadataCache.getInstance().clear(); } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java index 60ceab0..4e946ce 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java @@ -21,10 +21,7 @@ package org.apache.iotdb.db.query.reader.chunk; import java.io.IOException; import org.apache.iotdb.db.engine.cache.ChunkCache; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; -import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.controller.IChunkLoader; @@ -33,17 +30,13 @@ import org.apache.iotdb.tsfile.read.controller.IChunkLoader; */ public class DiskChunkLoader implements IChunkLoader { - private final TsFileResource resource; + public DiskChunkLoader() { - public DiskChunkLoader(TsFileResource resource) { - this.resource = resource; } @Override public Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException { - TsFileSequenceReader tsFileSequenceReader = - FileReaderManager.getInstance().get(resource.getTsFilePath(), resource.isClosed()); - return ChunkCache.getInstance().get(chunkMetaData, tsFileSequenceReader); + return ChunkCache.getInstance().get(chunkMetaData); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java index 2b55d3b..09a7e0e 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.query.reader.chunk.metadata; -import java.io.IOException; import java.util.List; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -33,11 +32,11 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter; public class DiskChunkMetadataLoader implements IChunkMetadataLoader { - private TsFileResource resource; - private PartialPath seriesPath; - private QueryContext context; + private final TsFileResource resource; + private final PartialPath seriesPath; + private final QueryContext context; // time filter or value filter, only used to check time range - private Filter filter; + private final Filter filter; public DiskChunkMetadataLoader(TsFileResource resource, PartialPath seriesPath, QueryContext context, Filter filter) { @@ -69,18 +68,6 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { return chunkMetadataList; } - /** - * For query v0.9/v1 tsfile only When generate temporary timeseriesMetadata set DiskChunkLoader to - * each chunkMetadata in the List - * - * @param chunkMetadataList - * @throws IOException - */ - @Override - public void setDiskChunkLoader(List<ChunkMetadata> chunkMetadataList) { - setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context); - } - public static void setDiskChunkLoader(List<ChunkMetadata> chunkMetadataList, TsFileResource resource, PartialPath seriesPath, QueryContext context) { List<Modification> pathModifications = @@ -89,10 +76,14 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { if (!pathModifications.isEmpty()) { QueryUtils.modifyChunkMetaData(chunkMetadataList, pathModifications); } - - for (ChunkMetadata data : chunkMetadataList) { - data.setChunkLoader(new DiskChunkLoader(resource)); - } + // it is ok, even if it is not thread safe, because the cost of creating a DiskChunkLoader is very cheap. + chunkMetadataList.forEach(chunkMetadata -> { + if (chunkMetadata.getChunkLoader() == null) { + chunkMetadata.setFilePath(resource.getTsFilePath()); + chunkMetadata.setClosed(resource.isClosed()); + chunkMetadata.setChunkLoader(new DiskChunkLoader()); + } + }); } } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java index 396cb24..843a982 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.query.reader.chunk.metadata; -import java.io.IOException; import java.util.List; import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -69,9 +68,4 @@ public class MemChunkMetadataLoader implements IChunkMetadataLoader { } return chunkMetadataList; } - - @Override - public void setDiskChunkLoader(List<ChunkMetadata> chunkMetadataList) throws IOException { - // DO NOTHING - } } diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java index 261d0f5..d7a4739 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java @@ -49,7 +49,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.cost.statistic.Measurement; import org.apache.iotdb.db.cost.statistic.Operation; import org.apache.iotdb.db.engine.cache.ChunkCache; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.exception.BatchProcessException; import org.apache.iotdb.db.exception.IoTDBException; @@ -664,11 +663,10 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } if (config.isDebugOn()) { SLOW_SQL_LOGGER.info( - "ChunkCache used memory proportion: {}\nChunkMetadataCache used memory proportion: {}\n" + "ChunkCache used memory proportion: {}\n" + "TimeSeriesMetadataCache used memory proportion: {}", ChunkCache.getInstance() - .getUsedMemoryProportion(), - ChunkMetadataCache.getInstance().getUsedMemoryProportion(), TimeSeriesMetadataCache + .getUsedMemoryProportion(), TimeSeriesMetadataCache .getInstance().getUsedMemoryProportion()); } } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java deleted file mode 100644 index 7241405..0000000 --- a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.engine.cache; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.constant.TestConstant; -import org.apache.iotdb.db.engine.MetadataManagerHelper; -import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy.DirectFlushPolicy; -import org.apache.iotdb.db.engine.querycontext.QueryDataSource; -import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor; -import org.apache.iotdb.db.engine.storagegroup.TsFileProcessor; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.exception.WriteProcessException; -import org.apache.iotdb.db.exception.metadata.IllegalPathException; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.metadata.PartialPath; -import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; -import org.apache.iotdb.db.query.context.QueryContext; -import org.apache.iotdb.db.query.control.FileReaderManager; -import org.apache.iotdb.db.utils.EnvironmentUtils; -import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.write.record.TSRecord; -import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class ChunkMetadataCacheTest { - - private QueryContext context = EnvironmentUtils.TEST_QUERY_CONTEXT; - - private String storageGroup = "root.vehicle.d0"; - private String measurementId0 = "s0"; - private String measurementId1 = "s1"; - private String measurementId2 = "s2"; - private String measurementId3 = "s3"; - private String measurementId4 = "s4"; - private String measurementId5 = "s5"; - private StorageGroupProcessor storageGroupProcessor; - private String systemDir = TestConstant.BASE_OUTPUT_PATH.concat("data") - .concat(File.separator).concat("info"); - - private int prevUnseqLevelNum = 0; - - @Before - public void setUp() throws Exception { - prevUnseqLevelNum = IoTDBDescriptor.getInstance().getConfig().getUnseqLevelNum(); - IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(2); - EnvironmentUtils.envSetUp(); - MetadataManagerHelper.initMetadata(); - storageGroupProcessor = new StorageGroupProcessor(systemDir, storageGroup, - new DirectFlushPolicy()); - insertData(); - } - - @After - public void tearDown() throws Exception { - FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders(); - storageGroupProcessor.syncDeleteDataFiles(); - EnvironmentUtils.cleanEnv(); - EnvironmentUtils.cleanDir(systemDir); - IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(prevUnseqLevelNum); - } - - private void insertOneRecord(long time, int num) - throws WriteProcessException, IllegalPathException { - TSRecord record = new TSRecord(time, storageGroup); - record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId0, String.valueOf(num))); - record.addTuple(DataPoint.getDataPoint(TSDataType.INT64, measurementId1, String.valueOf(num))); - record.addTuple(DataPoint.getDataPoint(TSDataType.FLOAT, measurementId2, String.valueOf(num))); - record.addTuple(DataPoint.getDataPoint(TSDataType.DOUBLE, measurementId3, String.valueOf(num))); - record.addTuple(DataPoint.getDataPoint(TSDataType.BOOLEAN, measurementId4, "True")); - InsertRowPlan insertRowPlan = new InsertRowPlan(record); - storageGroupProcessor.insert(insertRowPlan); - } - - protected void insertData() throws IOException, WriteProcessException, IllegalPathException { - for (int j = 1; j <= 100; j++) { - insertOneRecord(j, j); - } - for (TsFileProcessor tsFileProcessor : storageGroupProcessor - .getWorkSequenceTsFileProcessors()) { - tsFileProcessor.syncFlush(); - } - - for (int j = 10; j >= 1; j--) { - insertOneRecord(j, j); - } - for (int j = 11; j <= 20; j++) { - insertOneRecord(j, j); - } - storageGroupProcessor.syncCloseAllWorkingTsFileProcessors(); - - for (int j = 21; j <= 30; j += 2) { - insertOneRecord(j, 0); // will be covered when read - } - storageGroupProcessor.syncCloseAllWorkingTsFileProcessors(); - - for (int j = 21; j <= 30; j += 2) { - insertOneRecord(j, j); - } - storageGroupProcessor.syncCloseAllWorkingTsFileProcessors(); - - insertOneRecord(2, 100); - } - - @Test - public void test1() throws IOException, QueryProcessException, IllegalPathException { - IoTDBDescriptor.getInstance().getConfig().setMetaDataCacheEnable(false); - QueryDataSource queryDataSource = storageGroupProcessor - .query(new PartialPath(storageGroup), measurementId5, context, null, null); - - List<TsFileResource> seqResources = queryDataSource.getSeqResources(); - List<TsFileResource> unseqResources = queryDataSource.getUnseqResources(); - - Assert.assertEquals(1, seqResources.size()); - Assert.assertEquals(3, unseqResources.size()); - Assert.assertTrue(seqResources.get(0).isClosed()); - Assert.assertTrue(unseqResources.get(0).isClosed()); - Assert.assertTrue(unseqResources.get(1).isClosed()); - Assert.assertTrue(unseqResources.get(2).isClosed()); - - List<ChunkMetadata> metaDataList = ChunkMetadataCache.getInstance() - .get(seqResources.get(0).getTsFilePath(), new Path(storageGroup, measurementId5), null); - Assert.assertEquals(0, metaDataList.size()); - } - - @Test - public void test2() throws IOException, QueryProcessException, IllegalPathException { - IoTDBDescriptor.getInstance().getConfig().setMetaDataCacheEnable(true); - QueryDataSource queryDataSource = storageGroupProcessor - .query(new PartialPath(storageGroup), measurementId5, context, null, null); - - List<TsFileResource> seqResources = queryDataSource.getSeqResources(); - List<TsFileResource> unseqResources = queryDataSource.getUnseqResources(); - - Assert.assertEquals(1, seqResources.size()); - Assert.assertEquals(3, unseqResources.size()); - Assert.assertTrue(seqResources.get(0).isClosed()); - Assert.assertTrue(unseqResources.get(0).isClosed()); - Assert.assertTrue(unseqResources.get(1).isClosed()); - Assert.assertTrue(unseqResources.get(2).isClosed()); - - List<ChunkMetadata> metaDataList = ChunkMetadataCache.getInstance() - .get(seqResources.get(0).getTsFilePath(), new Path(storageGroup, measurementId5), null); - Assert.assertEquals(0, metaDataList.size()); - } - - -} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java index 7fbc509..1747971 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionTest.java @@ -30,9 +30,7 @@ import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.cache.ChunkCache; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; -import org.apache.iotdb.db.engine.merge.manage.MergeManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.MetadataException; @@ -91,7 +89,6 @@ abstract class LevelCompactionTest { IoTDBDescriptor.getInstance().getConfig() .setMergeChunkPointNumberThreshold(prevMergeChunkThreshold); ChunkCache.getInstance().clear(); - ChunkMetadataCache.getInstance().clear(); TimeSeriesMetadataCache.getInstance().clear(); IoTDB.metaManager.clear(); EnvironmentUtils.cleanAllDir(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java index e296014..6746aec 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java @@ -30,7 +30,6 @@ import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.cache.ChunkCache; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.merge.manage.MergeManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -90,7 +89,6 @@ abstract class MergeTest { unseqResources.clear(); IoTDBDescriptor.getInstance().getConfig().setMergeChunkPointNumberThreshold(prevMergeChunkThreshold); ChunkCache.getInstance().clear(); - ChunkMetadataCache.getInstance().clear(); TimeSeriesMetadataCache.getInstance().clear(); IoTDB.metaManager.clear(); EnvironmentUtils.cleanAllDir(); diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBClearCacheIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBClearCacheIT.java index 74a20dc..578f298a 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBClearCacheIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBClearCacheIT.java @@ -28,7 +28,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import org.apache.iotdb.db.engine.cache.ChunkCache; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.jdbc.Config; @@ -157,13 +156,11 @@ public class IoTDBClearCacheIT { Assert.assertEquals(10, cnt); } assertFalse(ChunkCache.getInstance().isEmpty()); - assertFalse(ChunkMetadataCache.getInstance().isEmpty()); assertFalse(TimeSeriesMetadataCache.getInstance().isEmpty()); statement.execute("CLEAR CACHE"); assertTrue(ChunkCache.getInstance().isEmpty()); - assertTrue(ChunkMetadataCache.getInstance().isEmpty()); assertTrue(TimeSeriesMetadataCache.getInstance().isEmpty()); } catch (Exception e) { diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java index 6bff6c1..b86e766 100644 --- a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java +++ b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java @@ -30,7 +30,6 @@ import java.util.Map; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.cache.ChunkCache; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.merge.manage.MergeManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -74,7 +73,6 @@ public class SeriesReaderTestUtil { seqResources.clear(); unseqResources.clear(); ChunkCache.getInstance().clear(); - ChunkMetadataCache.getInstance().clear(); TimeSeriesMetadataCache.getInstance().clear(); IoTDB.metaManager.clear(); EnvironmentUtils.cleanAllDir(); diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java index 2dafafd..4420922 100644 --- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java @@ -37,7 +37,6 @@ import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.cache.ChunkCache; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.compaction.CompactionMergeTaskPoolManager; import org.apache.iotdb.db.exception.StorageEngineException; @@ -128,7 +127,6 @@ public class EnvironmentUtils { // clean cache if (config.isMetaDataCacheEnable()) { ChunkCache.getInstance().clear(); - ChunkMetadataCache.getInstance().clear(); TimeSeriesMetadataCache.getInstance().clear(); } // close metadata diff --git a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java index ed458f2..90c596e 100644 --- a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java +++ b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java @@ -18,6 +18,13 @@ */ package org.apache.iotdb.spark.db; +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Locale; import org.apache.commons.io.FileUtils; import org.apache.iotdb.db.auth.AuthException; import org.apache.iotdb.db.auth.authorizer.BasicAuthorizer; @@ -26,7 +33,8 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.engine.StorageEngine; -import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; +import org.apache.iotdb.db.engine.cache.ChunkCache; +import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.flush.FlushManager; import org.apache.iotdb.db.exception.StartupException; import org.apache.iotdb.db.exception.StorageEngineException; @@ -42,14 +50,6 @@ import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Locale; - /** * <p> * This class is used for cleaning test environment in unit test and integration test @@ -120,7 +120,8 @@ public class EnvironmentUtils { MultiFileLogNodeManager.getInstance().stop(); // clean cache if (config.isMetaDataCacheEnable()) { - ChunkMetadataCache.getInstance().clear(); + ChunkCache.getInstance().clear(); + TimeSeriesMetadataCache.getInstance().clear(); } // close metadata IoTDB.metaManager.clear(); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java index 083c9e1..981a764 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java @@ -76,6 +76,9 @@ public class ChunkMetadata implements Accountable { // used for SeriesReader to indicate whether it is a seq/unseq timeseries metadata private boolean isSeq = true; + private boolean isClosed; + private String filePath; + private ChunkMetadata() { } @@ -289,4 +292,20 @@ public class ChunkMetadata implements Accountable { public boolean isSeq() { return isSeq; } + + public boolean isClosed() { + return isClosed; + } + + public void setClosed(boolean closed) { + isClosed = closed; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } } \ No newline at end of file diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java index a8c30dd..13ff2a0 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java @@ -58,8 +58,7 @@ public class TimeseriesMetadata implements Accountable { // used for SeriesReader to indicate whether it is a seq/unseq timeseries metadata private boolean isSeq = true; - private ByteBuffer chunkMetadataListByteBuffer; - + // used to save chunk metadata list while serializing private PublicBAOS chunkMetadataListBuffer; private ArrayList<ChunkMetadata> chunkMetadataList; @@ -87,9 +86,10 @@ public class TimeseriesMetadata implements Accountable { this.dataType = timeseriesMetadata.dataType; this.statistics = timeseriesMetadata.statistics; this.modified = timeseriesMetadata.modified; + this.chunkMetadataList = new ArrayList<>(timeseriesMetadata.chunkMetadataList); } - public static TimeseriesMetadata deserializeFrom(ByteBuffer buffer) { + public static TimeseriesMetadata deserializeFrom(ByteBuffer buffer, boolean needChunkMetadata) { TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata(); timeseriesMetaData.setTimeSeriesMetadataType(ReadWriteIOUtils.readByte(buffer)); timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readVarIntString(buffer)); @@ -99,8 +99,16 @@ public class TimeseriesMetadata implements Accountable { timeseriesMetaData .setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize); timeseriesMetaData.setStatistics(Statistics.deserialize(buffer, timeseriesMetaData.dataType)); - timeseriesMetaData.chunkMetadataListByteBuffer = buffer.slice(); - timeseriesMetaData.chunkMetadataListByteBuffer.limit(chunkMetaDataListDataSize); + if (needChunkMetadata) { + ByteBuffer byteBuffer = buffer.slice(); + byteBuffer.limit(chunkMetaDataListDataSize); + timeseriesMetaData.chunkMetadataList = new ArrayList<>(); + while (byteBuffer.hasRemaining()) { + timeseriesMetaData.chunkMetadataList.add(ChunkMetadata.deserializeFrom(byteBuffer, timeseriesMetaData)); + } + // minimize the storage of an ArrayList instance. + timeseriesMetaData.chunkMetadataList.trimToSize(); + } buffer.position(buffer.position() + chunkMetaDataListDataSize); return timeseriesMetaData; } @@ -183,15 +191,6 @@ public class TimeseriesMetadata implements Accountable { } public List<ChunkMetadata> getChunkMetadataList() { - if (chunkMetadataList == null) { - chunkMetadataList = new ArrayList<>(); - while (chunkMetadataListByteBuffer.hasRemaining()) { - chunkMetadataList.add(ChunkMetadata.deserializeFrom(chunkMetadataListByteBuffer, this)); - } - // minimize the storage of an ArrayList instance. - chunkMetadataList.trimToSize(); - chunkMetadataListByteBuffer = null; - } return chunkMetadataList; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index 1661423..e059751 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -308,7 +308,8 @@ public class TsFileSequenceReader implements AutoCloseable { private Map<String, TimeseriesMetadata> readDeviceMetadataFromDisk(String device) throws IOException { readFileMetadata(); - List<TimeseriesMetadata> timeseriesMetadataList = getDeviceTimeseriesMetadata(device); + List<TimeseriesMetadata> timeseriesMetadataList = getDeviceTimeseriesMetadataWithoutChunkMetadata( + device); Map<String, TimeseriesMetadata> deviceMetadata = new HashMap<>(); for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { deviceMetadata.put(timeseriesMetadata.getMeasurementId(), timeseriesMetadata); @@ -343,7 +344,7 @@ public class TsFileSequenceReader implements AutoCloseable { buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right); while (buffer.hasRemaining()) { try { - timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer)); + timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, true)); } catch (BufferOverflowException e) { logger.error("Something error happened while deserializing TimeseriesMetadata of file {}", file); @@ -389,7 +390,7 @@ public class TsFileSequenceReader implements AutoCloseable { while (buffer.hasRemaining()) { TimeseriesMetadata timeseriesMetadata; try { - timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer); + timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer, true); } catch (BufferOverflowException e) { logger.error("Something error happened while deserializing TimeseriesMetadata of file {}", file); @@ -439,7 +440,7 @@ public class TsFileSequenceReader implements AutoCloseable { measurementMetadataIndexPair.right); while (buffer.hasRemaining()) { try { - timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer)); + timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, true)); } catch (BufferOverflowException e) { logger.error("Something error happened while deserializing TimeseriesMetadata of file {}", file); @@ -463,51 +464,6 @@ public class TsFileSequenceReader implements AutoCloseable { return resultTimeseriesMetadataList; } - /** - * Traverse and read TimeseriesMetadata of specific measurements in one device. This method need - * to deserialize all TimeseriesMetadata and then judge, in order to avoid frequent I/O when the - * number of queried measurements is too large. Attention: This method is not used currently - * - * @param timeseriesMetadataList TimeseriesMetadata list, to store the result - * @param type MetadataIndexNode type - * @param metadataIndexPair <MetadataIndexEntry, offset> pair - * @param measurements measurements to be queried - * @throws IOException io error - */ - private void traverseAndReadTimeseriesMetadataInOneDevice( - List<TimeseriesMetadata> timeseriesMetadataList, MetadataIndexNodeType type, - Pair<MetadataIndexEntry, Long> metadataIndexPair, Set<String> measurements) - throws IOException { - ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right); - switch (type) { - case LEAF_DEVICE: - case INTERNAL_MEASUREMENT: - MetadataIndexNode metadataIndexNode = MetadataIndexNode.deserializeFrom(buffer); - int metadataIndexListSize = metadataIndexNode.getChildren().size(); - for (int i = 0; i < metadataIndexListSize; i++) { - long endOffset = metadataIndexNode.getEndOffset(); - if (i != metadataIndexListSize - 1) { - endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset(); - } - traverseAndReadTimeseriesMetadataInOneDevice(timeseriesMetadataList, - metadataIndexNode.getNodeType(), - new Pair<>(metadataIndexNode.getChildren().get(i), endOffset), measurements); - } - break; - case LEAF_MEASUREMENT: - while (buffer.hasRemaining()) { - TimeseriesMetadata timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer); - if (measurements.contains(timeseriesMetadata.getMeasurementId())) { - timeseriesMetadataList.add(timeseriesMetadata); - } - } - break; - default: - throw new IOException("Failed to traverse and read TimeseriesMetadata in device: " + - metadataIndexPair.left.getName() + ". Wrong MetadataIndexEntry type."); - } - } - protected int binarySearchInTimeseriesMetadataList( List<TimeseriesMetadata> timeseriesMetadataList, String key) { int low = 0; @@ -577,35 +533,15 @@ public class TsFileSequenceReader implements AutoCloseable { public Map<String, List<ChunkMetadata>> readChunkMetadataInDevice(String device) throws IOException { readFileMetadata(); - - long start = 0; - int size = 0; List<TimeseriesMetadata> timeseriesMetadataMap = getDeviceTimeseriesMetadata(device); if (timeseriesMetadataMap.isEmpty()) { return new HashMap<>(); } - for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataMap) { - if (start == 0) { - start = timeseriesMetadata.getOffsetOfChunkMetaDataList(); - } - size += timeseriesMetadata.getDataSizeOfChunkMetaDataList(); - } - // read buffer of all ChunkMetadatas of this device - ByteBuffer buffer = readData(start, size); Map<String, List<ChunkMetadata>> seriesMetadata = new HashMap<>(); - int index = 0; - int curSize = timeseriesMetadataMap.get(index).getDataSizeOfChunkMetaDataList(); - while (buffer.hasRemaining()) { - if (buffer.position() >= curSize) { - index++; - curSize += timeseriesMetadataMap.get(index).getDataSizeOfChunkMetaDataList(); - } - ChunkMetadata chunkMetadata = ChunkMetadata - .deserializeFrom(buffer, timeseriesMetadataMap.get(index)); - seriesMetadata.computeIfAbsent(chunkMetadata.getMeasurementUid(), key -> new ArrayList<>()) - .add(chunkMetadata); + for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataMap) { + seriesMetadata + .put(timeseriesMetadata.getMeasurementId(), timeseriesMetadata.getChunkMetadataList()); } - return seriesMetadata; } @@ -633,15 +569,17 @@ public class TsFileSequenceReader implements AutoCloseable { * @param buffer byte buffer * @param deviceId String * @param timeseriesMetadataMap map: deviceId -> timeseriesMetadata list + * @param needChunkMetadata deserialize chunk metadata list or not */ private void generateMetadataIndex(MetadataIndexEntry metadataIndex, ByteBuffer buffer, String deviceId, MetadataIndexNodeType type, - Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap) throws IOException { + Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap, boolean needChunkMetadata) + throws IOException { try { if (type.equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>(); while (buffer.hasRemaining()) { - timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer)); + timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, needChunkMetadata)); } timeseriesMetadataMap.computeIfAbsent(deviceId, k -> new ArrayList<>()) .addAll(timeseriesMetadataList); @@ -657,7 +595,7 @@ public class TsFileSequenceReader implements AutoCloseable { ByteBuffer nextBuffer = readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset); generateMetadataIndex(metadataIndexNode.getChildren().get(i), nextBuffer, deviceId, - metadataIndexNode.getNodeType(), timeseriesMetadataMap); + metadataIndexNode.getNodeType(), timeseriesMetadataMap, needChunkMetadata); } } } catch (BufferOverflowException e) { @@ -666,6 +604,9 @@ public class TsFileSequenceReader implements AutoCloseable { } } + /** + * TimeseriesMetadata don't need deserialize chunk metadata list + */ public Map<String, List<TimeseriesMetadata>> getAllTimeseriesMetadata() throws IOException { if (tsFileMetaData == null) { readFileMetadata(); @@ -681,11 +622,37 @@ public class TsFileSequenceReader implements AutoCloseable { } ByteBuffer buffer = readData(metadataIndexEntry.getOffset(), endOffset); generateMetadataIndex(metadataIndexEntry, buffer, null, - metadataIndexNode.getNodeType(), timeseriesMetadataMap); + metadataIndexNode.getNodeType(), timeseriesMetadataMap, false); } return timeseriesMetadataMap; } + /** + * This method will only deserialize the TimeseriesMetadata, not including chunk metadata list + */ + private List<TimeseriesMetadata> getDeviceTimeseriesMetadataWithoutChunkMetadata(String device) + throws IOException { + MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex(); + Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset( + metadataIndexNode, device, true, true); + if (metadataIndexPair == null) { + return Collections.emptyList(); + } + ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right); + Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>(); + generateMetadataIndex(metadataIndexPair.left, buffer, device, + MetadataIndexNodeType.INTERNAL_MEASUREMENT, timeseriesMetadataMap, false); + List<TimeseriesMetadata> deviceTimeseriesMetadata = new ArrayList<>(); + for (List<TimeseriesMetadata> timeseriesMetadataList : timeseriesMetadataMap.values()) { + deviceTimeseriesMetadata.addAll(timeseriesMetadataList); + } + return deviceTimeseriesMetadata; + } + + /** + * This method will not only deserialize the TimeseriesMetadata, but also all the chunk metadata + * list meanwhile. + */ private List<TimeseriesMetadata> getDeviceTimeseriesMetadata(String device) throws IOException { MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex(); Pair<MetadataIndexEntry, Long> metadataIndexPair = getMetadataAndEndOffset( @@ -696,7 +663,7 @@ public class TsFileSequenceReader implements AutoCloseable { ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right); Map<String, List<TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>(); generateMetadataIndex(metadataIndexPair.left, buffer, device, - MetadataIndexNodeType.INTERNAL_MEASUREMENT, timeseriesMetadataMap); + MetadataIndexNodeType.INTERNAL_MEASUREMENT, timeseriesMetadataMap, true); List<TimeseriesMetadata> deviceTimeseriesMetadata = new ArrayList<>(); for (List<TimeseriesMetadata> timeseriesMetadataList : timeseriesMetadataMap.values()) { deviceTimeseriesMetadata.addAll(timeseriesMetadataList); @@ -729,7 +696,8 @@ public class TsFileSequenceReader implements AutoCloseable { Pair<MetadataIndexEntry, Long> childIndexEntry = metadataIndex .getChildIndexEntry(name, false); ByteBuffer buffer = readData(childIndexEntry.left.getOffset(), childIndexEntry.right); - return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name, isDeviceLevel, + return getMetadataAndEndOffset(MetadataIndexNode.deserializeFrom(buffer), name, + isDeviceLevel, false); } } catch (BufferOverflowException e) { @@ -1145,24 +1113,7 @@ public class TsFileSequenceReader implements AutoCloseable { */ public List<ChunkMetadata> readChunkMetaDataList(TimeseriesMetadata timeseriesMetaData) throws IOException { - try { - readFileMetadata(); - ArrayList<ChunkMetadata> chunkMetadataList = new ArrayList<>(); - long startOffsetOfChunkMetadataList = timeseriesMetaData.getOffsetOfChunkMetaDataList(); - int dataSizeOfChunkMetadataList = timeseriesMetaData.getDataSizeOfChunkMetaDataList(); - - ByteBuffer buffer = readData(startOffsetOfChunkMetadataList, dataSizeOfChunkMetadataList); - while (buffer.hasRemaining()) { - chunkMetadataList.add(ChunkMetadata.deserializeFrom(buffer, timeseriesMetaData)); - } - - // minimize the storage of an ArrayList instance. - chunkMetadataList.trimToSize(); - return chunkMetadataList; - } catch (BufferOverflowException e) { - logger.error("Something error happened while reading ChunkMetaDataList of file {}", file); - throw e; - } + return timeseriesMetaData.getChunkMetadataList(); } /** diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java index bbda4b8..95da50f 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java @@ -31,12 +31,4 @@ public interface IChunkMetadataLoader { */ List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) throws IOException; - - /** - * For query 0.9/v1 tsfile only - * @param chunkMetadataList - * @throws IOException - */ - void setDiskChunkLoader(List<ChunkMetadata> chunkMetadataList) throws IOException; - } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeSeriesMetadataTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeSeriesMetadataTest.java index b5ba185..05bdc3e 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeSeriesMetadataTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/TimeSeriesMetadataTest.java @@ -67,7 +67,7 @@ public class TimeSeriesMetadataTest { ByteBuffer buffer = ByteBuffer.allocate((int) fch.size()); fch.read(buffer); buffer.flip(); - metaData = TimeseriesMetadata.deserializeFrom(buffer); + metaData = TimeseriesMetadata.deserializeFrom(buffer, true); return metaData; } catch (IOException e) { e.printStackTrace();
