This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch jira641 in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit ffa0084dc783e2b5d7a33bcf0cddbc8afb781919 Author: HTHou <[email protected]> AuthorDate: Wed May 6 21:56:07 2020 +0800 Remove TsFileMetadataCache --- .../db/engine/cache/CacheHitRatioMonitor.java | 6 - .../engine/cache/CacheHitRatioMonitorMXBean.java | 2 - .../iotdb/db/engine/cache/ChunkMetadataCache.java | 10 +- .../db/engine/cache/TimeSeriesMetadataCache.java | 11 +- .../iotdb/db/engine/cache/TsFileMetaDataCache.java | 174 --------------------- .../iotdb/db/engine/merge/task/MergeFileTask.java | 3 - .../apache/iotdb/db/engine/merge/MergeTest.java | 2 - .../query/reader/series/SeriesReaderTestUtil.java | 3 - .../apache/iotdb/db/utils/EnvironmentUtils.java | 3 - .../iotdb/tsfile/read/TsFileSequenceReader.java | 11 ++ 10 files changed, 19 insertions(+), 206 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java index 457cc8a..3d5f4f5 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitor.java @@ -61,12 +61,6 @@ public class CacheHitRatioMonitor implements CacheHitRatioMonitorMXBean, IServic } @Override - public double getTsfileMetaDataHitRatio() { - tsfileMetaDataHitRatio = TsFileMetaDataCache.getInstance().calculateTsFileMetaDataHitRatio(); - return tsfileMetaDataHitRatio; - } - - @Override public double getChunkHitRatio() { return ChunkCache.getInstance().calculateChunkHitRatio(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java index f546ef0..644374d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRatioMonitorMXBean.java @@ -22,8 +22,6 @@ public interface CacheHitRatioMonitorMXBean { double getChunkMetaDataHitRatio(); - double getTsfileMetaDataHitRatio(); - double getChunkHitRatio(); double getTimeSeriesMetadataHitRatio(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java index 3ed15cb..2e8f294 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java @@ -32,7 +32,6 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.db.utils.FileLoaderUtils; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.utils.BloomFilter; @@ -102,8 +101,8 @@ public class ChunkMetadataCache { throws IOException { if (!cacheEnable) { // bloom filter part - TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(filePath); - BloomFilter bloomFilter = fileMetaData.getBloomFilter(); + TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true); + BloomFilter bloomFilter = tsFileReader.readBloomFilter(); if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath())) { if (logger.isDebugEnabled()) { logger.debug(String @@ -112,7 +111,6 @@ public class ChunkMetadataCache { return new ArrayList<>(); } // If timeseries isn't included in the tsfile, empty list is returned. - TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true); return tsFileReader.getChunkMetadataList(seriesPath); } @@ -141,8 +139,8 @@ public class ChunkMetadataCache { } printCacheLog(false); // bloom filter part - TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(filePath); - BloomFilter bloomFilter = fileMetaData.getBloomFilter(); + TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true); + BloomFilter bloomFilter = tsFileReader.readBloomFilter(); if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath())) { return new ArrayList<>(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java index e620cd1..ccd430f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java @@ -31,7 +31,6 @@ import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; -import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.utils.BloomFilter; @@ -91,13 +90,12 @@ public class TimeSeriesMetadataCache { throws IOException { if (!cacheEnable) { // bloom filter part - TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(key.filePath); - BloomFilter bloomFilter = fileMetaData.getBloomFilter(); + TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true); + BloomFilter bloomFilter = reader.readBloomFilter(); if (bloomFilter != null && !bloomFilter .contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) { return null; } - TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true); return reader.readTimeseriesMetadata(new Path(key.device, key.measurement)); } @@ -123,13 +121,12 @@ public class TimeSeriesMetadataCache { } printCacheLog(false); // bloom filter part - TsFileMetadata fileMetaData = TsFileMetaDataCache.getInstance().get(key.filePath); - BloomFilter bloomFilter = fileMetaData.getBloomFilter(); + TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true); + BloomFilter bloomFilter = reader.readBloomFilter(); if (bloomFilter != null && !bloomFilter .contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) { return null; } - TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true); List<TimeseriesMetadata> timeSeriesMetadataList = reader .readTimeseriesMetadata(key.device, allSensors); // put TimeSeriesMetadata of all sensors used in this query into cache diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java deleted file mode 100644 index ec59e95..0000000 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.engine.cache; - -import java.io.IOException; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.utils.FileLoaderUtils; -import org.apache.iotdb.tsfile.common.conf.TSFileConfig; -import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class is used to cache <code>TsFileMetaData</code> of tsfile in IoTDB. - */ -public class TsFileMetaDataCache { - - private static final Logger logger = LoggerFactory.getLogger(TsFileMetaDataCache.class); - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - - private static boolean cacheEnable = config.isMetaDataCacheEnable(); - private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForFileMetaDataCache(); - - /** - * TsFile path -> TsFileMetaData - */ - private final LRULinkedHashMap<String, TsFileMetadata> cache; - private final ReadWriteLock lock = new ReentrantReadWriteLock(); - - private AtomicLong cacheHitNum = new AtomicLong(); - private AtomicLong cacheRequestNum = new AtomicLong(); - - /** - * estimated size of metadataIndex entry in TsFileMetaData. - */ - private long metadataIndexEntrySize = 0; - - private TsFileMetaDataCache() { - logger.info("TsFileMetaDataCache size = " + MEMORY_THRESHOLD_IN_B); - cache = new LRULinkedHashMap<String, TsFileMetadata>(MEMORY_THRESHOLD_IN_B, true) { - @Override - protected long calEntrySize(String key, TsFileMetadata value) { - if (metadataIndexEntrySize == 0 && value.getMetadataIndex() != null - && !value.getMetadataIndex().getChildren().isEmpty()) { - metadataIndexEntrySize = RamUsageEstimator - .sizeOf(value.getMetadataIndex().getChildren().iterator().next()); - } - // totalChunkNum, invalidChunkNum - long valueSize = 4 + 4L; - - // metadataIndex - if (value.getMetadataIndex() != null) { - valueSize += value.getMetadataIndex().getChildren().size() * metadataIndexEntrySize; - } - - // versionInfo - if (value.getVersionInfo() != null) { - valueSize += value.getVersionInfo().size() * 16; - } - return key.getBytes(TSFileConfig.STRING_CHARSET).length + valueSize; - } - }; - } - - public static TsFileMetaDataCache getInstance() { - return TsFileMetaDataCacheHolder.INSTANCE; - } - - /** - * get the TsFileMetaData for given TsFile. - * - * @param filePath -given TsFile - */ - public TsFileMetadata get(String filePath) throws IOException { - if (!cacheEnable) { - return FileLoaderUtils.getTsFileMetadata(filePath); - } - - cacheRequestNum.incrementAndGet(); - - lock.readLock().lock(); - try { - if (cache.containsKey(filePath)) { - cacheHitNum.incrementAndGet(); - printCacheLog(true); - return cache.get(filePath); - } - } finally { - lock.readLock().unlock(); - } - - lock.writeLock().lock(); - try { - if (cache.containsKey(filePath)) { - cacheHitNum.incrementAndGet(); - printCacheLog(true); - return cache.get(filePath); - } - printCacheLog(false); - TsFileMetadata fileMetaData = FileLoaderUtils.getTsFileMetadata(filePath); - cache.put(filePath, fileMetaData); - return fileMetaData; - } finally { - lock.writeLock().unlock(); - } - } - - private void printCacheLog(boolean isHit) { - if (!logger.isDebugEnabled()) { - return; - } - if (isHit) { - logger.debug( - "[TsFileMetaData cache hit] The number of requests for cache is {}, hit rate is {}.", - cacheRequestNum.get(), cacheHitNum.get() * 1.0 / cacheRequestNum.get()); - } else { - logger.debug( - "[TsFileMetaData cache didn't hit] The number of requests for cache is {}, hit rate is {}.", - cacheRequestNum.get(), cacheHitNum.get() * 1.0 / cacheRequestNum.get()); - } - } - - double calculateTsFileMetaDataHitRatio() { - if (cacheRequestNum.get() != 0) { - return cacheHitNum.get() * 1.0 / cacheRequestNum.get(); - } else { - return 0; - } - } - - public void remove(TsFileResource resource) { - synchronized (cache) { - cache.remove(resource.getPath()); - } - } - - public void clear() { - synchronized (cache) { - cache.clear(); - } - } - - /** - * Singleton pattern - */ - private static class TsFileMetaDataCacheHolder { - - private TsFileMetaDataCacheHolder() { - } - - private static final TsFileMetaDataCache INSTANCE = new TsFileMetaDataCache(); - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java index b6a5778..006ec5c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java @@ -30,7 +30,6 @@ import java.util.Map.Entry; import java.util.Set; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; -import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache; import org.apache.iotdb.db.engine.merge.manage.MergeContext; import org.apache.iotdb.db.engine.merge.manage.MergeResource; import org.apache.iotdb.db.engine.merge.recover.MergeLogger; @@ -124,7 +123,6 @@ class MergeFileTask { seqFile.getWriteQueryLock().writeLock().lock(); try { - TsFileMetaDataCache.getInstance().remove(seqFile); ChunkMetadataCache.getInstance().remove(seqFile); FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getPath()); @@ -262,7 +260,6 @@ class MergeFileTask { seqFile.getWriteQueryLock().writeLock().lock(); try { resource.removeFileReader(seqFile); - TsFileMetaDataCache.getInstance().remove(seqFile); ChunkMetadataCache.getInstance().remove(seqFile); FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getPath()); seqFile.getFile().delete(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java index f626830..55cee97 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java @@ -30,7 +30,6 @@ import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; -import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache; import org.apache.iotdb.db.engine.merge.manage.MergeManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.metadata.MetadataException; @@ -87,7 +86,6 @@ abstract class MergeTest { seqResources.clear(); unseqResources.clear(); IoTDBDescriptor.getInstance().getConfig().setChunkMergePointThreshold(prevMergeChunkThreshold); - TsFileMetaDataCache.getInstance().clear(); ChunkMetadataCache.getInstance().clear(); MManager.getInstance().clear(); EnvironmentUtils.cleanAllDir(); diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java index 74689eb..58177fd 100644 --- a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java +++ b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java @@ -31,7 +31,6 @@ import java.util.Map; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; -import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache; import org.apache.iotdb.db.engine.merge.manage.MergeManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.metadata.MetadataException; @@ -43,7 +42,6 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.write.TsFileWriter; import org.apache.iotdb.tsfile.write.record.TSRecord; import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint; @@ -73,7 +71,6 @@ public class SeriesReaderTestUtil { removeFiles(seqResources, unseqResources); seqResources.clear(); unseqResources.clear(); - TsFileMetaDataCache.getInstance().clear(); ChunkMetadataCache.getInstance().clear(); MManager.getInstance().clear(); EnvironmentUtils.cleanAllDir(); diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java index 94b5e73..d64a8b3 100644 --- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java @@ -31,14 +31,12 @@ import org.apache.commons.io.FileUtils; import org.apache.iotdb.db.auth.AuthException; import org.apache.iotdb.db.auth.authorizer.LocalFileAuthorizer; import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; -import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.metadata.MManager; import org.apache.iotdb.db.query.context.QueryContext; @@ -134,7 +132,6 @@ public class EnvironmentUtils { // clean cache if (config.isMetaDataCacheEnable()) { - TsFileMetaDataCache.getInstance().clear(); ChunkMetadataCache.getInstance().clear(); } // close metadata diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index efb5c31..0bd8e92 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -56,6 +56,7 @@ import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.iotdb.tsfile.read.reader.TsFileInput; +import org.apache.iotdb.tsfile.utils.BloomFilter; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.utils.VersionUtils; @@ -263,6 +264,16 @@ public class TsFileSequenceReader implements AutoCloseable { } /** + * this function does not modify the position of the file reader. + * + * @throws IOException io error + */ + public BloomFilter readBloomFilter() throws IOException { + readFileMetadata(); + return tsFileMetaData.getBloomFilter(); + } + + /** * this function reads measurements and TimeseriesMetaDatas in given device Thread Safe * * @param device name
