This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch CachedChunk in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit a74ce816788ae1424e4aee18a69f5d77d3c1da06 Author: JackieTien97 <[email protected]> AuthorDate: Fri Feb 14 11:24:41 2020 +0800 add cached chunk in server --- .../resources/conf/iotdb-engine.properties | 6 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 25 +++- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 17 ++- .../db/engine/cache/CacheHitRatioMonitor.java | 5 + .../engine/cache/CacheHitRatioMonitorMXBean.java | 2 + .../apache/iotdb/db/engine/cache/ChunkCache.java | 153 +++++++++++++++++++++ .../iotdb/db/engine/cache/DeviceMetaDataCache.java | 13 +- .../iotdb/db/query/control/FileReaderManager.java | 15 -- .../db/query/reader/chunk/DiskChunkLoader.java | 43 ++---- .../iotdb/db/query/reader/series/SeriesReader.java | 4 +- .../writelog/recover/UnseqTsFileRecoverTest.java | 4 +- .../apache/iotdb/tsfile/read/ReadOnlyTsFile.java | 4 +- ...kLoaderImpl.java => CachedChunkLoaderImpl.java} | 6 +- .../tsfile/read/controller/ChunkLoaderTest.java | 2 +- .../read/query/executor/QueryExecutorTest.java | 4 +- .../query/timegenerator/ReaderByTimestampTest.java | 4 +- .../query/timegenerator/TimeGeneratorTest.java | 4 +- .../iotdb/tsfile/read/reader/ReaderTest.java | 6 +- 18 files changed, 224 insertions(+), 93 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 64a1185..59b8172 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -275,9 +275,9 @@ chunk_merge_point_threshold=20480 # whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. meta_data_cache_enable=true -# Read memory Allocation Ratio: FileMetaDataCache, ChunkMetaDataCache, and Free Memory Used in Query. -# The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 3:6:10 -filemeta_chunkmeta_free_memory_proportion=3:6:10 +# Read memory Allocation Ratio: FileMetaDataCache, ChunkMetaDataCache, ChunkCache and Free Memory Used in Query. +# The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 1:1:1:1 , 3:6:10:20 +filemeta_chunkmeta_free_memory_proportion=3:6:10:20 #################### diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index cedbec2..251c11c 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -188,12 +188,17 @@ public class IoTDBConfig { /** * Memory allocated for fileMetaData cache in read process */ - private long allocateMemoryForFileMetaDataCache = allocateMemoryForRead * 3 / 19; + private long allocateMemoryForFileMetaDataCache = allocateMemoryForRead * 3 / 39; /** * Memory allocated for chunkMetaData cache in read process */ - private long allocateMemoryForChumkMetaDataCache = allocateMemoryForRead * 6 / 19; + private long allocateMemoryForChunkMetaDataCache = allocateMemoryForRead * 6 / 39; + + /** + * Memory allocated for chunk cache in read process + */ + private long allocateMemoryForChunkCache = allocateMemoryForRead * 10 / 39; /** * The statMonitor writes statistics info into IoTDB every backLoopPeriodSec secs. The default @@ -1047,12 +1052,20 @@ public class IoTDBConfig { this.allocateMemoryForFileMetaDataCache = allocateMemoryForFileMetaDataCache; } - public long getAllocateMemoryForChumkMetaDataCache() { - return allocateMemoryForChumkMetaDataCache; + public long getAllocateMemoryForChunkMetaDataCache() { + return allocateMemoryForChunkMetaDataCache; + } + + public void setAllocateMemoryForChunkMetaDataCache(long allocateMemoryForChunkMetaDataCache) { + this.allocateMemoryForChunkMetaDataCache = allocateMemoryForChunkMetaDataCache; + } + + public long getAllocateMemoryForChunkCache() { + return allocateMemoryForChunkCache; } - public void setAllocateMemoryForChumkMetaDataCache(long allocateMemoryForChumkMetaDataCache) { - this.allocateMemoryForChumkMetaDataCache = allocateMemoryForChumkMetaDataCache; + public void setAllocateMemoryForChunkCache(long allocateMemoryForChunkCache) { + this.allocateMemoryForChunkCache = allocateMemoryForChunkCache; } public boolean isEnableWatermark() { diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index e5b07bc..b10a96e 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -18,14 +18,6 @@ */ package org.apache.iotdb.db.conf; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.time.ZoneId; -import java.util.Properties; import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.utils.FilePathUtils; @@ -33,6 +25,11 @@ import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.*; +import java.net.URL; +import java.time.ZoneId; +import java.util.Properties; + public class IoTDBDescriptor { private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class); @@ -527,8 +524,10 @@ public class IoTDBDescriptor { try { conf.setAllocateMemoryForFileMetaDataCache( maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum); - conf.setAllocateMemoryForChumkMetaDataCache( + conf.setAllocateMemoryForChunkMetaDataCache( maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum); + conf.setAllocateMemoryForChunkCache( + maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum); } catch (Exception e) { throw new RuntimeException( "Each subsection of configuration item filemeta_chunkmeta_free_memory_proportion should be an integer, which is " diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java index 09b0419..0d06b9b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java @@ -66,6 +66,11 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic return tsfileMetaDataHitRatio; } + @Override + public double getChunkHitRatio() { + return ChunkCache.getInstance().calculateChunkHitRatio(); + } + public static CacheHitRatioMonitor getInstance() { return instance; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java index 9cd10c5..5bc02e7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java @@ -23,4 +23,6 @@ public interface CacheHitRatioMonitorMXBean { double getChunkMetaDataHitRatio(); double getTsfileMetaDataHitRatio(); + + double getChunkHitRatio(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java new file mode 100644 index 0000000..cfb9d63 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.cache; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.Chunk; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * This class is used to cache <code>Chunk</code> of <code>ChunkMetaData</code> in IoTDB. The caching + * strategy is LRU. + */ +public class ChunkCache { + + private static final Logger logger = LoggerFactory.getLogger(ChunkCache.class); + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final long MEMORY_THRESHOLD_IN_CHUNK_CACHE = config.getAllocateMemoryForChunkCache(); + private static boolean cacheEnable = config.isMetaDataCacheEnable(); + + private final LRULinkedHashMap<ChunkMetaData, Chunk> lruCache; + + private AtomicLong cacheHitNum = new AtomicLong(); + private AtomicLong cacheRequestNum = new AtomicLong(); + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final Lock readLock = lock.readLock(); + private final Lock writeLock = lock.writeLock(); + + + private ChunkCache() { + lruCache = new LRULinkedHashMap<ChunkMetaData, Chunk>(MEMORY_THRESHOLD_IN_CHUNK_CACHE, true) { + @Override + protected long calEntrySize(ChunkMetaData key, Chunk value) { + return RamUsageEstimator.sizeOf(value) + RamUsageEstimator.sizeOf(value); + } + }; + } + + public static ChunkCache getInstance() { + return ChunkCacheHolder.INSTANCE; + } + + public Chunk get(ChunkMetaData chunkMetaData, TsFileSequenceReader reader) throws IOException { + if (!cacheEnable) { + Chunk chunk = reader.readMemChunk(chunkMetaData); + return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType()); + } + + cacheRequestNum.incrementAndGet(); + + try { + readLock.lock(); + if (lruCache.containsKey(chunkMetaData)) { + cacheHitNum.incrementAndGet(); + printCacheLog(true); + Chunk chunk = lruCache.get(chunkMetaData); + return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType()); + } + } finally { + readLock.unlock(); + } + + writeLock.lock(); + if (lruCache.containsKey(chunkMetaData)) { + writeLock.unlock(); + readLock.lock(); + cacheHitNum.incrementAndGet(); + printCacheLog(true); + Chunk chunk = lruCache.get(chunkMetaData); + readLock.unlock(); + return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType()); + } + printCacheLog(false); + Chunk chunk = reader.readMemChunk(chunkMetaData); + lruCache.put(chunkMetaData, chunk); + writeLock.unlock(); + return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType()); + + } + + private void printCacheLog(boolean isHit) { + if (!logger.isDebugEnabled()) { + return; + } + logger.debug( + "[ChunkMetaData cache {}hit] The number of requests for cache is {}, hit rate is {}.", + isHit ? "" : "didn't ", cacheRequestNum.get(), + cacheHitNum.get() * 1.0 / cacheRequestNum.get()); + } + + public double calculateChunkHitRatio() { + if (cacheRequestNum.get() != 0) { + return cacheHitNum.get() * 1.0 / cacheRequestNum.get(); + } else { + return 0; + } + } + + + /** + * clear LRUCache. + */ + public void clear() { + writeLock.lock(); + if (lruCache != null) { + lruCache.clear(); + } + writeLock.unlock(); + } + + public void remove(ChunkMetaData chunkMetaData) { + writeLock.lock(); + if (chunkMetaData != null) { + lruCache.remove(chunkMetaData); + } + writeLock.unlock(); + } + + /** + * singleton pattern. + */ + private static class ChunkCacheHolder { + + private static final ChunkCache INSTANCE = new ChunkCache(); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java index e5b749f..6ae882a 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java @@ -18,13 +18,6 @@ */ package org.apache.iotdb.db.engine.cache; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngine; @@ -37,6 +30,10 @@ import org.apache.iotdb.tsfile.utils.BloomFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; + /** * This class is used to cache <code>List<ChunkMetaData></code> of tsfile in IoTDB. The caching * strategy is LRU. @@ -45,7 +42,7 @@ public class DeviceMetaDataCache { private static final Logger logger = LoggerFactory.getLogger(DeviceMetaDataCache.class); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForChumkMetaDataCache(); + private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForChunkMetaDataCache(); private static StorageEngine storageEngine = StorageEngine.getInstance(); private static boolean cacheEnable = config.isMetaDataCacheEnable(); /** diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java index 850d444..8c86de2 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java +++ b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java @@ -25,13 +25,10 @@ import org.apache.iotdb.db.service.IService; import org.apache.iotdb.db.service.ServiceType; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.UnClosedTsFileReader; -import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; -import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; @@ -73,8 +70,6 @@ public class FileReaderManager implements IService { */ private Map<TsFileResource, AtomicInteger> unclosedReferenceMap; - private final Map<TsFileSequenceReader, IChunkLoader> chunkLoaderMap; - private ScheduledExecutorService executorService; private FileReaderManager() { @@ -82,7 +77,6 @@ public class FileReaderManager implements IService { unclosedFileReaderMap = new ConcurrentHashMap<>(); closedReferenceMap = new ConcurrentHashMap<>(); unclosedReferenceMap = new ConcurrentHashMap<>(); - chunkLoaderMap = new HashMap<>(); executorService = IoTDBThreadPoolFactory.newScheduledThreadPool(1, "open-files-manager"); @@ -126,9 +120,6 @@ public class FileReaderManager implements IService { if (refAtom != null && refAtom.get() == 0) { try { - synchronized (chunkLoaderMap) { - chunkLoaderMap.remove(reader); - } reader.close(); } catch (IOException e) { logger.error("Can not close TsFileSequenceReader {} !", reader.getFileName(), e); @@ -171,12 +162,6 @@ public class FileReaderManager implements IService { } - public IChunkLoader get(TsFileSequenceReader reader) { - synchronized (chunkLoaderMap) { - return chunkLoaderMap.computeIfAbsent(reader, ChunkLoaderImpl::new); - } - } - /** * Increase the reference count of the reader specified by filePath. Only when the reference count * of a reader equals zero, the reader can be closed and removed. diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java similarity index 51% copy from tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java copy to server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java index a1935d5..7206f6e 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java @@ -16,51 +16,28 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.tsfile.read.controller; -import org.apache.iotdb.tsfile.common.cache.LRUCache; +package org.apache.iotdb.db.query.reader.chunk; + +import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Chunk; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import java.io.IOException; -/** - * Read one Chunk and cache it into a LRUCache. - */ -public class ChunkLoaderImpl implements IChunkLoader { - - private static final int DEFAULT_CHUNK_CACHE_SIZE = 1000; - private TsFileSequenceReader reader; - private LRUCache<ChunkMetaData, Chunk> chunkCache; - - public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader) { - this(fileSequenceReader, DEFAULT_CHUNK_CACHE_SIZE); - } - - /** - * constructor of ChunkLoaderImpl. - * - * @param fileSequenceReader file sequence reader - * @param cacheSize cache size - */ - public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader, int cacheSize) { - - this.reader = fileSequenceReader; +public class DiskChunkLoader implements IChunkLoader { - chunkCache = new LRUCache<ChunkMetaData, Chunk>(cacheSize) { + private final TsFileSequenceReader reader; - @Override - public Chunk loadObjectByKey(ChunkMetaData metaData) throws IOException { - return reader.readMemChunk(metaData); - } - }; + public DiskChunkLoader(TsFileSequenceReader reader) { + this.reader = reader; } @Override public Chunk getChunk(ChunkMetaData chunkMetaData) throws IOException { - Chunk chunk = chunkCache.get(chunkMetaData); - return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt(), reader.getEndianType()); + return ChunkCache.getInstance().get(chunkMetaData, reader); } @Override @@ -70,6 +47,6 @@ public class ChunkLoaderImpl implements IChunkLoader { @Override public void clear() { - chunkCache.clear(); + // no cache need clear } } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java index 5140dbb..0158cfc 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.FileReaderManager; +import org.apache.iotdb.db.query.reader.chunk.DiskChunkLoader; import org.apache.iotdb.db.query.reader.chunk.MemChunkLoader; import org.apache.iotdb.db.query.reader.chunk.MemChunkReader; import org.apache.iotdb.db.query.reader.universal.PriorityMergeReader; @@ -38,7 +39,6 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter; @@ -366,7 +366,7 @@ public class SeriesReader { if (data.getChunkLoader() == null) { TsFileSequenceReader tsFileSequenceReader = FileReaderManager.getInstance() .get(resource, resource.isClosed()); - data.setChunkLoader(new ChunkLoaderImpl(tsFileSequenceReader)); + data.setChunkLoader(new DiskChunkLoader(tsFileSequenceReader)); } } List<ReadOnlyMemChunk> memChunks = resource.getReadOnlyMemChunk(); diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java index f961299..6fe01ff 100644 --- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java @@ -39,7 +39,7 @@ import org.apache.iotdb.tsfile.read.TimeValuePair; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; +import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier; import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; @@ -161,7 +161,7 @@ public class UnseqTsFileRecoverTest { TsFileSequenceReader fileReader = new TsFileSequenceReader(tsF.getPath(), true); IMetadataQuerier metadataQuerier = new MetadataQuerierByFileImpl(fileReader); - IChunkLoader chunkLoader = new ChunkLoaderImpl(fileReader); + IChunkLoader chunkLoader = new CachedChunkLoaderImpl(fileReader); Path path = new Path("device1", "sensor1"); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java index 89b64db..c4e9fbb 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java @@ -20,7 +20,7 @@ package org.apache.iotdb.tsfile.read; import java.io.IOException; import org.apache.iotdb.tsfile.read.controller.IChunkLoader; -import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; +import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier; import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.iotdb.tsfile.read.expression.QueryExpression; @@ -40,7 +40,7 @@ public class ReadOnlyTsFile implements AutoCloseable { public ReadOnlyTsFile(TsFileSequenceReader fileReader) throws IOException { this.fileReader = fileReader; this.metadataQuerier = new MetadataQuerierByFileImpl(fileReader); - this.chunkLoader = new ChunkLoaderImpl(fileReader); + this.chunkLoader = new CachedChunkLoaderImpl(fileReader); tsFileExecutor = new TsFileExecutor(metadataQuerier, chunkLoader); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java similarity index 90% rename from tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java rename to tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java index a1935d5..fbe7d92 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/CachedChunkLoaderImpl.java @@ -28,13 +28,13 @@ import java.io.IOException; /** * Read one Chunk and cache it into a LRUCache. */ -public class ChunkLoaderImpl implements IChunkLoader { +public class CachedChunkLoaderImpl implements IChunkLoader { private static final int DEFAULT_CHUNK_CACHE_SIZE = 1000; private TsFileSequenceReader reader; private LRUCache<ChunkMetaData, Chunk> chunkCache; - public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader) { + public CachedChunkLoaderImpl(TsFileSequenceReader fileSequenceReader) { this(fileSequenceReader, DEFAULT_CHUNK_CACHE_SIZE); } @@ -44,7 +44,7 @@ public class ChunkLoaderImpl implements IChunkLoader { * @param fileSequenceReader file sequence reader * @param cacheSize cache size */ - public ChunkLoaderImpl(TsFileSequenceReader fileSequenceReader, int cacheSize) { + public CachedChunkLoaderImpl(TsFileSequenceReader fileSequenceReader, int cacheSize) { this.reader = fileSequenceReader; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java index 19a8416..5094bd2 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderTest.java @@ -54,7 +54,7 @@ public class ChunkLoaderTest { List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile .getChunkMetaDataList(new Path("d2.s1")); - ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader); + CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader); for (ChunkMetaData chunkMetaData : chunkMetaDataList) { Chunk chunk = seriesChunkLoader.getChunk(chunkMetaData); ChunkHeader chunkHeader = chunk.getHeader(); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java index 54eacc0..28d772c 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java @@ -25,7 +25,7 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.common.RowRecord; import org.apache.iotdb.tsfile.read.controller.IChunkLoader; -import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; +import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.iotdb.tsfile.read.expression.IExpression; import org.apache.iotdb.tsfile.read.expression.QueryExpression; @@ -59,7 +59,7 @@ public class QueryExecutorTest { TsFileGeneratorForTest.generateFile(rowCount, 16 * 1024 * 1024, 10000); fileReader = new TsFileSequenceReader(FILE_PATH); metadataQuerierByFile = new MetadataQuerierByFileImpl(fileReader); - chunkLoader = new ChunkLoaderImpl(fileReader); + chunkLoader = new CachedChunkLoaderImpl(fileReader); queryExecutorWithQueryFilter = new TsFileExecutor(metadataQuerierByFile, chunkLoader); } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java index e5818eb..3045dc8 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/ReaderByTimestampTest.java @@ -27,7 +27,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; +import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.iotdb.tsfile.read.reader.series.AbstractFileSeriesReader; import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader; @@ -61,7 +61,7 @@ public class ReaderByTimestampTest { @Test public void readByTimestamp() throws IOException { - ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader); + CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader); List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile .getChunkMetaDataList(new Path("d1.s1")); AbstractFileSeriesReader seriesReader = new FileSeriesReader(seriesChunkLoader, diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java index e1b9687..38bc71d 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java @@ -24,7 +24,7 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.controller.IChunkLoader; -import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; +import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.iotdb.tsfile.read.expression.IExpression; import org.apache.iotdb.tsfile.read.expression.impl.BinaryExpression; @@ -53,7 +53,7 @@ public class TimeGeneratorTest { TsFileGeneratorForTest.generateFile(1000, 10 * 1024 * 1024, 10000); fileReader = new TsFileSequenceReader(FILE_PATH); metadataQuerierByFile = new MetadataQuerierByFileImpl(fileReader); - chunkLoader = new ChunkLoaderImpl(fileReader); + chunkLoader = new CachedChunkLoaderImpl(fileReader); } @After diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java index 93c72dc..2622ca8 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/ReaderTest.java @@ -23,7 +23,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; +import org.apache.iotdb.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression; import org.apache.iotdb.tsfile.read.filter.TimeFilter; @@ -65,7 +65,7 @@ public class ReaderTest { @Test public void readTest() throws IOException { int count = 0; - ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader); + CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader); List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile .getChunkMetaDataList(new Path("d1.s1")); @@ -101,7 +101,7 @@ public class ReaderTest { @Test public void readWithFilterTest() throws IOException { - ChunkLoaderImpl seriesChunkLoader = new ChunkLoaderImpl(fileReader); + CachedChunkLoaderImpl seriesChunkLoader = new CachedChunkLoaderImpl(fileReader); List<ChunkMetaData> chunkMetaDataList = metadataQuerierByFile .getChunkMetaDataList(new Path("d1.s1"));
