This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch CacheImprove in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4c587ae0d8701aff5125f7b7355293204c918ffd Author: JackieTien97 <[email protected]> AuthorDate: Tue Oct 27 20:21:02 2020 +0800 improve the chunk metadata cache process --- .../iotdb/db/engine/cache/ChunkMetadataCache.java | 33 ++++++++-------------- .../chunk/metadata/DiskChunkMetadataLoader.java | 21 ++++++++------ .../chunk/metadata/MemChunkMetadataLoader.java | 3 +- .../db/engine/cache/ChunkMetadataCacheTest.java | 6 ++-- .../apache/iotdb/db/engine/merge/MergeTest.java | 19 +++++++------ .../query/reader/series/SeriesReaderTestUtil.java | 21 ++++++++------ .../apache/iotdb/db/utils/EnvironmentUtils.java | 23 +++++++-------- .../tsfile/file/metadata/TimeseriesMetadata.java | 2 +- .../read/controller/IChunkMetadataLoader.java | 4 ++- 9 files changed, 66 insertions(+), 66 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java index aca5ad2..40c69b7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.engine.cache; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; @@ -30,12 +31,11 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.query.control.FileReaderManager; -import org.apache.iotdb.db.utils.FileLoaderUtils; import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.utils.BloomFilter; import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,22 +106,16 @@ public class ChunkMetadataCache { /** * get {@link ChunkMetadata}. THREAD SAFE. */ - public List<ChunkMetadata> get(String filePath, Path seriesPath) - throws IOException { + public List<ChunkMetadata> get(String filePath, Path seriesPath, + TimeseriesMetadata timeseriesMetadata) throws IOException { + if (timeseriesMetadata == null) { + return Collections.emptyList(); + } if (!CACHE_ENABLE) { // bloom filter part TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true); - BloomFilter bloomFilter = tsFileReader.readBloomFilter(); - if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath())) { - if (logger.isDebugEnabled()) { - logger.debug(String - .format("path not found by bloom filter, file is: %s, path is: %s", filePath, - seriesPath)); - } - return new ArrayList<>(); - } // If timeseries isn't included in the tsfile, empty list is returned. - return tsFileReader.getChunkMetadataList(seriesPath); + return tsFileReader.readChunkMetaDataList(timeseriesMetadata); } AccountableString key = new AccountableString(filePath + IoTDBConstant.PATH_SEPARATOR @@ -137,19 +131,13 @@ public class ChunkMetadataCache { lock.readLock().unlock(); } - if (chunkMetadataList != null) { printCacheLog(true); cacheHitNum.incrementAndGet(); } else { printCacheLog(false); - // bloom filter part TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true); - BloomFilter bloomFilter = tsFileReader.readBloomFilter(); - if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath())) { - return new ArrayList<>(); - } - chunkMetadataList = FileLoaderUtils.getChunkMetadataList(seriesPath, filePath); + chunkMetadataList = tsFileReader.readChunkMetaDataList(timeseriesMetadata); lock.writeLock().lock(); try { lruCache.put(key, chunkMetadataList); @@ -208,7 +196,8 @@ public class ChunkMetadataCache { public void remove(TsFileResource resource) { lock.writeLock().lock(); if (resource != null) { - lruCache.entrySet().removeIf(e -> e.getKey().getString().startsWith(resource.getTsFilePath())); + lruCache.entrySet() + .removeIf(e -> e.getKey().getString().startsWith(resource.getTsFilePath())); } lock.writeLock().unlock(); } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java index 5e9de76..95466e9 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.reader.chunk.DiskChunkLoader; import org.apache.iotdb.db.utils.QueryUtils; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader; import org.apache.iotdb.tsfile.read.filter.basic.Filter; @@ -39,7 +40,8 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { // time filter or value filter, only used to check time range private Filter filter; - public DiskChunkMetadataLoader(TsFileResource resource, PartialPath seriesPath, QueryContext context, Filter filter) { + public DiskChunkMetadataLoader(TsFileResource resource, PartialPath seriesPath, + QueryContext context, Filter filter) { this.resource = resource; this.seriesPath = seriesPath; this.context = context; @@ -47,9 +49,10 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { } @Override - public List<ChunkMetadata> loadChunkMetadataList() throws IOException { + public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) + throws IOException { List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache - .getInstance().get(resource.getTsFilePath(), seriesPath); + .getInstance().get(resource.getTsFilePath(), seriesPath, timeseriesMetadata); setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context); @@ -57,15 +60,15 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { * remove not satisfied ChunkMetaData */ chunkMetadataList.removeIf(chunkMetaData -> (filter != null && !filter - .satisfyStartEndTime(chunkMetaData.getStartTime(), chunkMetaData.getEndTime())) - || chunkMetaData.getStartTime() > chunkMetaData.getEndTime()); + .satisfyStartEndTime(chunkMetaData.getStartTime(), chunkMetaData.getEndTime())) + || chunkMetaData.getStartTime() > chunkMetaData.getEndTime()); return chunkMetadataList; } /** - * For query v0.9/v1 tsfile only - * When generate temporary timeseriesMetadata - * set DiskChunkLoader to each chunkMetadata in the List + * For query v0.9/v1 tsfile only When generate temporary timeseriesMetadata set DiskChunkLoader to + * each chunkMetadata in the List + * * @param chunkMetadataList * @throws IOException */ @@ -75,7 +78,7 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader { } public static void setDiskChunkLoader(List<ChunkMetadata> chunkMetadataList, - TsFileResource resource, PartialPath seriesPath, QueryContext context) { + TsFileResource resource, PartialPath seriesPath, QueryContext context) { List<Modification> pathModifications = context.getPathModifications(resource.getModFile(), seriesPath); diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java index d71a370..ad2a4e3 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.metadata.PartialPath; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader; import org.apache.iotdb.tsfile.read.filter.basic.Filter; @@ -43,7 +44,7 @@ public class MemChunkMetadataLoader implements IChunkMetadataLoader { } @Override - public List<ChunkMetadata> loadChunkMetadataList() { + public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) { List<ChunkMetadata> chunkMetadataList = resource.getChunkMetadataList(); DiskChunkMetadataLoader.setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java index 1b1135e..6608b95 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java @@ -34,8 +34,6 @@ import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.metadata.IllegalPathException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.PartialPath; -import org.apache.iotdb.db.metadata.mnode.MNode; -import org.apache.iotdb.db.metadata.mnode.MeasurementMNode; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.FileReaderManager; @@ -142,7 +140,7 @@ public class ChunkMetadataCacheTest { Assert.assertTrue(unseqResources.get(2).isClosed()); List<ChunkMetadata> metaDataList = ChunkMetadataCache.getInstance() - .get(seqResources.get(0).getTsFilePath(), new Path(storageGroup, measurementId5)); + .get(seqResources.get(0).getTsFilePath(), new Path(storageGroup, measurementId5), null); Assert.assertEquals(0, metaDataList.size()); } @@ -163,7 +161,7 @@ public class ChunkMetadataCacheTest { Assert.assertTrue(unseqResources.get(2).isClosed()); List<ChunkMetadata> metaDataList = ChunkMetadataCache.getInstance() - .get(seqResources.get(0).getTsFilePath(), new Path(storageGroup, measurementId5)); + .get(seqResources.get(0).getTsFilePath(), new Path(storageGroup, measurementId5), null); Assert.assertEquals(0, metaDataList.size()); } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java index 9baeb48..ec0f773 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java @@ -19,10 +19,19 @@ package org.apache.iotdb.db.engine.merge; +import static org.apache.iotdb.db.conf.IoTDBConstant.PATH_SEPARATOR; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.constant.TestConstant; +import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; +import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.merge.manage.MergeManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StorageEngineException; @@ -43,14 +52,6 @@ import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static org.apache.iotdb.db.conf.IoTDBConstant.PATH_SEPARATOR; - abstract class MergeTest { static final String MERGE_TEST_SG = "root.mergeTest"; @@ -88,7 +89,9 @@ abstract class MergeTest { seqResources.clear(); unseqResources.clear(); IoTDBDescriptor.getInstance().getConfig().setChunkMergePointThreshold(prevMergeChunkThreshold); + ChunkCache.getInstance().clear(); ChunkMetadataCache.getInstance().clear(); + TimeSeriesMetadataCache.getInstance().clear(); IoTDB.metaManager.clear(); EnvironmentUtils.cleanAllDir(); MergeManager.getINSTANCE().stop(); diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java index b434f2f..6d70b47 100644 --- a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java +++ b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java @@ -19,9 +19,19 @@ package org.apache.iotdb.db.query.reader.series; +import static org.apache.iotdb.db.conf.IoTDBConstant.PATH_SEPARATOR; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.constant.TestConstant; +import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; +import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.merge.manage.MergeManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.metadata.MetadataException; @@ -38,15 +48,6 @@ import org.apache.iotdb.tsfile.write.record.TSRecord; import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; -import java.io.File; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.iotdb.db.conf.IoTDBConstant.PATH_SEPARATOR; - public class SeriesReaderTestUtil { private static int seqFileNum = 5; @@ -71,7 +72,9 @@ public class SeriesReaderTestUtil { removeFiles(seqResources, unseqResources); seqResources.clear(); unseqResources.clear(); + ChunkCache.getInstance().clear(); ChunkMetadataCache.getInstance().clear(); + TimeSeriesMetadataCache.getInstance().clear(); IoTDB.metaManager.clear(); EnvironmentUtils.cleanAllDir(); MergeManager.getINSTANCE().stop(); diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java index 616fd97..bf01c11 100644 --- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java @@ -36,7 +36,9 @@ import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.StorageEngine; +import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; +import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.FileReaderManager; @@ -58,17 +60,17 @@ public class EnvironmentUtils { private static final Logger logger = LoggerFactory.getLogger(EnvironmentUtils.class); - private static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private static DirectoryManager directoryManager = DirectoryManager.getInstance(); + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final DirectoryManager directoryManager = DirectoryManager.getInstance(); public static long TEST_QUERY_JOB_ID = 1; public static QueryContext TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID); - private static long oldTsFileThreshold = config.getTsFileSizeThreshold(); + private static final long oldTsFileThreshold = config.getTsFileSizeThreshold(); - private static int oldMaxMemTableNumber = config.getMaxMemtableNumber(); + private static final int oldMaxMemTableNumber = config.getMaxMemtableNumber(); - private static long oldGroupSizeInByte = config.getMemtableSizeThreshold(); + private static final long oldGroupSizeInByte = config.getMemtableSizeThreshold(); private static IoTDB daemon; @@ -88,7 +90,7 @@ public class EnvironmentUtils { transport.open(); logger.error("stop daemon failed. 6667 can be connected now."); transport.close(); - } catch (TTransportException e) { + } catch (TTransportException ignored) { } } //try sync service @@ -98,7 +100,7 @@ public class EnvironmentUtils { transport.open(); logger.error("stop Sync daemon failed. 5555 can be connected now."); transport.close(); - } catch (TTransportException e) { + } catch (TTransportException ignored) { } } //try jmx connection @@ -112,14 +114,11 @@ public class EnvironmentUtils { //do nothing } //try MetricService - Socket socket = new Socket(); - try { + try (Socket socket = new Socket()) { socket.connect(new InetSocketAddress("127.0.0.1", 8181)); logger.error("stop MetricService failed. 8181 can be connected now."); } catch (Exception e) { //do nothing - } finally { - socket.close(); } // clean storage group manager @@ -132,7 +131,9 @@ public class EnvironmentUtils { // clean cache if (config.isMetaDataCacheEnable()) { + ChunkCache.getInstance().clear(); ChunkMetadataCache.getInstance().clear(); + TimeSeriesMetadataCache.getInstance().clear(); } // close metadata IoTDB.metaManager.clear(); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java index ad8fc3f..0869643 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java @@ -142,7 +142,7 @@ public class TimeseriesMetadata implements Accountable { } public List<ChunkMetadata> loadChunkMetadataList() throws IOException { - return chunkMetadataLoader.loadChunkMetadataList(); + return chunkMetadataLoader.loadChunkMetadataList(this); } public boolean isModified() { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java index f80c42f..bbda4b8 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java @@ -22,13 +22,15 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import java.io.IOException; import java.util.List; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; public interface IChunkMetadataLoader { /** * read all chunk metadata of one time series in one file. */ - List<ChunkMetadata> loadChunkMetadataList() throws IOException; + List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) + throws IOException; /** * For query 0.9/v1 tsfile only
