This is an automated email from the ASF dual-hosted git repository. suyue pushed a commit to branch add_cache_configuration in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit f94d42114e4e43e4bc42182a00e1391d187fb7ec Author: suyue <2335813...@qq.com> AuthorDate: Fri Aug 16 14:27:21 2019 +0800 increase configuration parameters of cache --- .../resources/conf/iotdb-engine.properties | 10 ++++++ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 40 +++++++++++++++++++++- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 24 +++++++++++++ .../iotdb/db/engine/cache/DeviceMetaDataCache.java | 20 ++++++++--- .../iotdb/db/engine/cache/TsFileMetaDataCache.java | 18 +++++++--- .../iotdb/db/engine/cache/TsFileMetadataUtils.java | 16 +++++++++ 6 files changed, 119 insertions(+), 9 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 01dea2c..7a67eeb 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -157,6 +157,16 @@ concurrent_flush_thread=0 # (i.e., whether use ChunkBufferPool), value true, false chunk_buffer_pool_enable = false +#################### +### Metadata Cache Configuration +#################### + +# whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. +meta_data_cache_enable=true +# Read memory Allocation Ratio: FileMetaDataCache, ChunkMetaDataCache, and Free Memory Used in Query. +# The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 3:6:10 +filemeta_chunkmeta_free_memory_proportion=3:6:10 + #################### ### Statistics Monitor configuration diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 86c3622..1a98a36 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -99,7 +99,7 @@ public class IoTDBConfig { private String systemDir = "data/system"; /** - * Schema directory, including storage set of values. + * Schema directory, including storage set of values. */ private String schemaDir = "data/system/schema"; @@ -157,6 +157,20 @@ public class IoTDBConfig { private long memtableSizeThreshold = 128 * 1024 * 1024L; /** + * whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. + */ + private boolean metaDataCacheEnable = true; + /** + * Memory allocated for fileMetaData cache in read process + */ + private long allocateMemoryForFileMetaDataCache = allocateMemoryForRead * 3 / 19; + + /** + * Memory allocated for chunkMetaData cache in read process + */ + private long allocateMemoryForChumkMetaDataCache = allocateMemoryForRead * 6 / 19; + + /** * The statMonitor writes statistics info into IoTDB every backLoopPeriodSec secs. The default * value is 5s. */ @@ -618,4 +632,28 @@ public class IoTDBConfig { public void setMemtableSizeThreshold(long memtableSizeThreshold) { this.memtableSizeThreshold = memtableSizeThreshold; } + + public boolean isMetaDataCacheEnable() { + return metaDataCacheEnable; + } + + public void setMetaDataCacheEnable(boolean metaDataCacheEnable) { + this.metaDataCacheEnable = metaDataCacheEnable; + } + + public long getAllocateMemoryForFileMetaDataCache() { + return allocateMemoryForFileMetaDataCache; + } + + public void setAllocateMemoryForFileMetaDataCache(long allocateMemoryForFileMetaDataCache) { + this.allocateMemoryForFileMetaDataCache = allocateMemoryForFileMetaDataCache; + } + + public long getAllocateMemoryForChumkMetaDataCache() { + return allocateMemoryForChumkMetaDataCache; + } + + public void setAllocateMemoryForChumkMetaDataCache(long allocateMemoryForChumkMetaDataCache) { + this.allocateMemoryForChumkMetaDataCache = allocateMemoryForChumkMetaDataCache; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index c916114..b255154 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -130,6 +130,10 @@ public class IoTDBDescriptor { Boolean.parseBoolean(properties.getProperty("enable_parameter_adapter", Boolean.toString(conf.isEnableParameterAdapter())))); + conf.setMetaDataCacheEnable( + Boolean.parseBoolean(properties.getProperty("meta_data_cache_enable", + Boolean.toString(conf.isMetaDataCacheEnable())))); + initMemoryAllocate(properties); conf.setEnableWal(Boolean.parseBoolean(properties.getProperty("enable_wal", @@ -263,6 +267,26 @@ public class IoTDBDescriptor { conf.setAllocateMemoryForRead( maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum); } + + if (!conf.isMetaDataCacheEnable()) { + return; + } + + String queryMemoryAllocateProportion = properties + .getProperty("filemeta_chunkmeta_free_memory_proportion"); + if (queryMemoryAllocateProportion != null) { + String[] proportions = queryMemoryAllocateProportion.split(":"); + int proportionSum = 0; + for (String proportion : proportions) { + proportionSum += Integer.parseInt(proportion.trim()); + } + long maxMemoryAvailable = conf.getAllocateMemoryForRead(); + conf.setAllocateMemoryForFileMetaDataCache( + maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum); + conf.setAllocateMemoryForChumkMetaDataCache( + maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum); + } + } private static class IoTDBDescriptorHolder { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java index 369e5e0..042769c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java @@ -46,8 +46,8 @@ public class DeviceMetaDataCache { private static StorageEngine storageEngine = StorageEngine.getInstance(); - private static final long MEMORY_THRESHOLD_IN_B = (long) (0.3 * config - .getAllocateMemoryForRead()); + private static boolean cacheEnable = config.isMetaDataCacheEnable(); + private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForChumkMetaDataCache(); /** * key: file path dot deviceId dot sensorId. * <p> @@ -64,6 +64,9 @@ public class DeviceMetaDataCache { private long chunkMetaDataSize = 0; private DeviceMetaDataCache(long memoryThreshold) { + if (!cacheEnable) { + return; + } lruCache = new LRULinkedHashMap<String, List<ChunkMetaData>>(memoryThreshold, true) { @Override protected long calEntrySize(String key, List<ChunkMetaData> value) { @@ -84,6 +87,13 @@ public class DeviceMetaDataCache { */ public List<ChunkMetaData> get(String filePath, Path seriesPath) throws IOException { + if (!cacheEnable) { + TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(filePath); + TsDeviceMetadata deviceMetaData = TsFileMetadataUtils + .getTsDeviceMetaData(filePath, seriesPath, fileMetaData); + return TsFileMetadataUtils.getChunkMetaDataList(seriesPath.getMeasurement(), deviceMetaData); + } + StringBuilder builder = new StringBuilder(filePath).append(".").append(seriesPath.getDevice()); String pathDeviceStr = builder.toString(); String key = builder.append(".").append(seriesPath.getMeasurement()).toString(); @@ -117,7 +127,7 @@ public class DeviceMetaDataCache { TsDeviceMetadata deviceMetaData = TsFileMetadataUtils .getTsDeviceMetaData(filePath, seriesPath, fileMetaData); // If measurement isn't included in the tsfile, empty list is returned. - if(deviceMetaData == null){ + if (deviceMetaData == null) { return new ArrayList<>(); } Map<Path, List<ChunkMetaData>> chunkMetaData = TsFileMetadataUtils @@ -169,7 +179,9 @@ public class DeviceMetaDataCache { */ public void clear() { synchronized (lruCache) { - lruCache.clear(); + if (lruCache != null) { + lruCache.clear(); + } } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java index c29b345..6b7e448 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java @@ -34,8 +34,8 @@ public class TsFileMetaDataCache { private static final Logger logger = LoggerFactory.getLogger(TsFileMetaDataCache.class); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private static final long MEMORY_THRESHOLD_IN_B = (long) (0.25 * config - .getAllocateMemoryForRead()); + private static boolean cacheEnable = config.isMetaDataCacheEnable(); + private static final long MEMORY_THRESHOLD_IN_B = config.getAllocateMemoryForFileMetaDataCache(); /** * key: Tsfile path. value: TsFileMetaData */ @@ -57,6 +57,9 @@ public class TsFileMetaDataCache { private long versionAndCreatebySize = 10; private TsFileMetaDataCache() { + if (!cacheEnable) { + return; + } cache = new LRULinkedHashMap<String, TsFileMetaData>(MEMORY_THRESHOLD_IN_B, true) { @Override protected long calEntrySize(String key, TsFileMetaData value) { @@ -86,6 +89,9 @@ public class TsFileMetaDataCache { * @param path -given path */ public TsFileMetaData get(String path) throws IOException { + if (!cacheEnable) { + return TsFileMetadataUtils.getTsFileMetaData(path); + } Object internPath = path.intern(); cacheRequestNum.incrementAndGet(); @@ -122,13 +128,17 @@ public class TsFileMetaDataCache { public void remove(String path) { synchronized (cache) { - cache.remove(path); + if (cache != null) { + cache.remove(path); + } } } public void clear() { synchronized (cache) { - cache.clear(); + if (cache != null) { + cache.clear(); + } } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java index 8a4424b..cd53e3b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java @@ -98,4 +98,20 @@ public class TsFileMetadataUtils { return pathToChunkMetaDataList; } + public static List<ChunkMetaData> getChunkMetaDataList(String sensor, + TsDeviceMetadata tsDeviceMetadata) { + List<ChunkMetaData> chunkMetaDataList = new ArrayList<>(); + for (ChunkGroupMetaData chunkGroupMetaData : tsDeviceMetadata.getChunkGroupMetaDataList()) { + List<ChunkMetaData> chunkMetaDataListInOneChunkGroup = chunkGroupMetaData + .getChunkMetaDataList(); + + for (ChunkMetaData chunkMetaData : chunkMetaDataListInOneChunkGroup) { + if (sensor.equals(chunkMetaData.getMeasurementUid())) { + chunkMetaData.setVersion(chunkGroupMetaData.getVersion()); + chunkMetaDataList.add(chunkMetaData); + } + } + } + return chunkMetaDataList; + } }