This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/benchant in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5d6c41e0ce5be5bdd3268a211ed265a24404e62f Author: JackieTien97 <[email protected]> AuthorDate: Mon Jun 5 17:05:59 2023 +0800 update cache in flush and compacting --- .../iotdb/db/engine/cache/BloomFilterCache.java | 4 + .../db/engine/cache/TimeSeriesMetadataCache.java | 8 + .../impl/ReadChunkCompactionPerformer.java | 3 +- .../writer/AbstractCrossCompactionWriter.java | 3 +- .../writer/AbstractInnerCompactionWriter.java | 4 +- .../db/engine/storagegroup/TsFileProcessor.java | 3 +- .../db/metadata/schemaregion/SchemaEngine.java | 2 - .../metadata/schemaregion/SchemaRegionLoader.java | 77 +---- .../UpdateCacheRestorableTsFileIOWriter.java | 58 ++++ .../db/query/reader/UpdateCacheTsFileIOWriter.java | 51 +++ .../file/AbstractTsFileRecoverPerformer.java | 3 +- .../META-INF/native-image/reflect-config.json | 364 +-------------------- .../META-INF/native-image/resource-config.json | 3 - .../utils/CompactionFileGeneratorUtils.java | 10 +- .../tsfile/file/metadata/TimeseriesMetadata.java | 4 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 11 + .../write/writer/tsmiterator/TSMIterator.java | 18 +- 17 files changed, 179 insertions(+), 447 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/BloomFilterCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/BloomFilterCache.java index b1d5b0b5b40..6a2f6b196d8 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/BloomFilterCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/BloomFilterCache.java @@ -125,6 +125,10 @@ public class BloomFilterCache { return entryAverageSize.get(); } + public void updateCache(String filePath, BloomFilter bloomFilter) { + lruCache.put(new BloomFilterCacheKey(filePath), bloomFilter); + } + /** clear LRUCache. */ public void clear() { lruCache.invalidateAll(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java index 823b47be2c8..0f4e69567d1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java @@ -238,6 +238,14 @@ public class TimeSeriesMetadataCache { return BloomFilterCache.getInstance().calculateBloomFilterHitRatio(); } + public void updateCache(String filePath, String deviceId, TimeseriesMetadata timeseriesMetadata) { + if (timeseriesMetadata.getStatistics().getCount() != 0) { + lruCache.put( + new TimeSeriesMetadataCacheKey(filePath, deviceId, timeseriesMetadata.getMeasurementId()), + timeseriesMetadata); + } + } + /** clear LRUCache. */ public void clear() { lruCache.invalidateAll(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java index da0c9c91dd8..8c0dcb463f1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.engine.compaction.execute.utils.executor.readchunk.Al import org.apache.iotdb.db.engine.compaction.execute.utils.executor.readchunk.SingleSeriesCompactionExecutor; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.query.reader.UpdateCacheTsFileIOWriter; import org.apache.iotdb.db.rescon.SystemInfo; import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; @@ -72,7 +73,7 @@ public class ReadChunkCompactionPerformer implements ISeqCompactionPerformer { * IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion()); try (MultiTsFileDeviceIterator deviceIterator = new MultiTsFileDeviceIterator(seqFiles); TsFileIOWriter writer = - new TsFileIOWriter(targetResource.getTsFile(), true, sizeForFileWriter)) { + new UpdateCacheTsFileIOWriter(targetResource.getTsFile(), true, sizeForFileWriter)) { while (deviceIterator.hasNextDevice()) { Pair<String, Boolean> deviceInfo = deviceIterator.nextDevice(); String device = deviceInfo.left; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java index 44409808daf..3e19bf05b39 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.query.reader.UpdateCacheTsFileIOWriter; import org.apache.iotdb.db.rescon.SystemInfo; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; import org.apache.iotdb.tsfile.read.TimeValuePair; @@ -77,7 +78,7 @@ public abstract class AbstractCrossCompactionWriter extends AbstractCompactionWr boolean enableMemoryControl = IoTDBDescriptor.getInstance().getConfig().isEnableMemControl(); for (int i = 0; i < targetResources.size(); i++) { this.targetFileWriters.add( - new TsFileIOWriter( + new UpdateCacheTsFileIOWriter( targetResources.get(i).getTsFile(), enableMemoryControl, memorySizeForEachWriter)); isEmptyFile[i] = true; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java index 6404730d0e4..326cf8ad8e1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.compaction.execute.utils.writer; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.query.reader.UpdateCacheTsFileIOWriter; import org.apache.iotdb.db.rescon.SystemInfo; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.read.TimeValuePair; @@ -50,7 +51,8 @@ public abstract class AbstractInnerCompactionWriter extends AbstractCompactionWr * IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion()); boolean enableMemoryControl = IoTDBDescriptor.getInstance().getConfig().isEnableMemControl(); this.fileWriter = - new TsFileIOWriter(targetFileResource.getTsFile(), enableMemoryControl, sizeForFileWriter); + new UpdateCacheTsFileIOWriter( + targetFileResource.getTsFile(), enableMemoryControl, sizeForFileWriter); this.targetResource = targetFileResource; isEmptyFile = true; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index cbc609e3605..e0649cda0ad 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -55,6 +55,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.pipe.core.collector.realtime.listener.PipeInsertionDataNodeListener; import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.reader.UpdateCacheRestorableTsFileIOWriter; import org.apache.iotdb.db.rescon.MemTableManager; import org.apache.iotdb.db.rescon.PrimitiveArrayManager; import org.apache.iotdb.db.rescon.SystemInfo; @@ -194,7 +195,7 @@ public class TsFileProcessor { this.sequence = sequence; this.tsFileResource = new TsFileResource(tsfile, this); this.dataRegionInfo = dataRegionInfo; - this.writer = new RestorableTsFileIOWriter(tsfile); + this.writer = new UpdateCacheRestorableTsFileIOWriter(tsfile); this.updateLatestFlushTimeCallback = updateLatestFlushTimeCallback; this.walNode = WALManager.getInstance() diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java index e8f3fdfbb21..8fbe4ea4dca 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java @@ -93,8 +93,6 @@ public class SchemaEngine { public void init() { logger.info("used schema engine mode: {}.", config.getSchemaEngineMode()); - schemaRegionLoader.init(config.getSchemaEngineMode()); - initSchemaEngineStatistics(); SchemaResourceManager.initSchemaResource(schemaEngineStatistics); // CachedSchemaEngineMetric depend on CacheMemoryManager, so it should be initialized after diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionLoader.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionLoader.java index 0eab047f8d3..c7a29aad07c 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionLoader.java @@ -20,18 +20,12 @@ package org.apache.iotdb.db.metadata.schemaregion; import org.apache.iotdb.commons.exception.MetadataException; -import org.apache.iotdb.db.metadata.MetadataConstant; -import org.reflections.Reflections; -import org.reflections.util.ConfigurationBuilder; -import org.reflections.util.FilterBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; class SchemaRegionLoader { @@ -43,82 +37,15 @@ class SchemaRegionLoader { private String currentMode; - private Constructor<ISchemaRegion> currentConstructor; - @SuppressWarnings("unchecked") - SchemaRegionLoader() { - Reflections reflections = - new Reflections( - new ConfigurationBuilder() - .forPackages(PACKAGE_NAME) - .filterInputsBy(new FilterBuilder().includePackage(PACKAGE_NAME))); - - Set<Class<?>> annotatedSchemaRegionSet = reflections.getTypesAnnotatedWith(SchemaRegion.class); - - for (Class<?> annotatedSchemaRegion : annotatedSchemaRegionSet) { - boolean isSchemaRegion = false; - for (Class<?> interfaces : annotatedSchemaRegion.getInterfaces()) { - if (interfaces == ISchemaRegion.class) { - isSchemaRegion = true; - break; - } - } - if (!isSchemaRegion) { - logger.warn( - String.format( - "Class %s is not a subclass of ISchemaRegion.", annotatedSchemaRegion.getName())); - continue; - } - SchemaRegion annotationInfo = annotatedSchemaRegion.getAnnotation(SchemaRegion.class); - constructorMap.compute( - annotationInfo.mode(), - (k, v) -> { - if (v == null) { - try { - return (Constructor<ISchemaRegion>) - annotatedSchemaRegion.getConstructor(ISchemaRegionParams.class); - } catch (NoSuchMethodException e) { - logger.error(e.getMessage(), e); - return null; - } - } - logger.warn( - "Duplicated SchemaRegion implementation, {} and {}, with same mode name [{}]", - v.getClass().getName(), - annotatedSchemaRegion.getName(), - k); - return v; - }); - } - } - - void init(String schemaEngineMode) { - Constructor<ISchemaRegion> constructor = constructorMap.get(schemaEngineMode); - if (constructor == null) { - logger.warn( - "There's no SchemaRegion implementation with target mode {}. Use default mode {}", - schemaEngineMode, - MetadataConstant.DEFAULT_SCHEMA_ENGINE_MODE); - currentMode = MetadataConstant.DEFAULT_SCHEMA_ENGINE_MODE; - currentConstructor = constructorMap.get(currentMode); - } else { - currentMode = schemaEngineMode; - currentConstructor = constructor; - } - } + SchemaRegionLoader() {} ISchemaRegion createSchemaRegion(ISchemaRegionParams schemaRegionParams) throws MetadataException { - try { - return currentConstructor.newInstance(schemaRegionParams); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { - logger.warn(e.getMessage(), e); - throw new MetadataException(e); - } + return new SchemaRegionMemoryImpl(schemaRegionParams); } void clear() { currentMode = null; - currentConstructor = null; } } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheRestorableTsFileIOWriter.java b/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheRestorableTsFileIOWriter.java new file mode 100644 index 00000000000..cefbb412fa9 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheRestorableTsFileIOWriter.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.query.reader; + +import org.apache.iotdb.db.engine.cache.BloomFilterCache; +import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.iotdb.tsfile.utils.BloomFilter; +import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; + +import java.io.File; +import java.io.IOException; + +public class UpdateCacheRestorableTsFileIOWriter extends RestorableTsFileIOWriter { + + private static final BloomFilterCache BLOOM_FILTER_CACHE = BloomFilterCache.getInstance(); + + private static final TimeSeriesMetadataCache TIME_SERIES_METADATA_CACHE = + TimeSeriesMetadataCache.getInstance(); + + public UpdateCacheRestorableTsFileIOWriter(File file) throws IOException { + super(file); + } + + public UpdateCacheRestorableTsFileIOWriter(File file, long maxMetadataSize) throws IOException { + super(file, maxMetadataSize); + } + + public UpdateCacheRestorableTsFileIOWriter(File file, boolean truncate) throws IOException { + super(file, truncate); + } + + @Override + protected void updateCache(String device, TimeseriesMetadata timeseriesMetadata) { + TIME_SERIES_METADATA_CACHE.updateCache(file.getPath(), device, timeseriesMetadata); + } + + @Override + protected void updateCache(BloomFilter bloomFilter) { + BLOOM_FILTER_CACHE.updateCache(file.getPath(), bloomFilter); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheTsFileIOWriter.java b/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheTsFileIOWriter.java new file mode 100644 index 00000000000..12ef2a7b79e --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/UpdateCacheTsFileIOWriter.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.query.reader; + +import org.apache.iotdb.db.engine.cache.BloomFilterCache; +import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; +import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.iotdb.tsfile.utils.BloomFilter; +import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; + +import java.io.File; +import java.io.IOException; + +public class UpdateCacheTsFileIOWriter extends TsFileIOWriter { + + public UpdateCacheTsFileIOWriter(File file, boolean enableMemoryControl, long maxMetadataSize) + throws IOException { + super(file, enableMemoryControl, maxMetadataSize); + } + + private static final BloomFilterCache BLOOM_FILTER_CACHE = BloomFilterCache.getInstance(); + + private static final TimeSeriesMetadataCache TIME_SERIES_METADATA_CACHE = + TimeSeriesMetadataCache.getInstance(); + + @Override + protected void updateCache(String device, TimeseriesMetadata timeseriesMetadata) { + TIME_SERIES_METADATA_CACHE.updateCache(file.getPath(), device, timeseriesMetadata); + } + + @Override + protected void updateCache(BloomFilter bloomFilter) { + BLOOM_FILTER_CACHE.updateCache(file.getPath(), bloomFilter); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java index aa08daf4151..e305c4def2b 100644 --- a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.wal.recover.file; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.DataRegionException; +import org.apache.iotdb.db.query.reader.UpdateCacheRestorableTsFileIOWriter; import org.apache.iotdb.db.utils.FileLoaderUtils; import org.apache.iotdb.tsfile.exception.NotCompatibleTsFileException; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; @@ -77,7 +78,7 @@ public abstract class AbstractTsFileRecoverPerformer implements Closeable { // try to remove corrupted part of the TsFile try { - writer = new RestorableTsFileIOWriter(tsFile); + writer = new UpdateCacheRestorableTsFileIOWriter(tsFile); } catch (NotCompatibleTsFileException e) { boolean result = tsFile.delete(); logger.warn( diff --git a/server/src/main/resources/META-INF/native-image/reflect-config.json b/server/src/main/resources/META-INF/native-image/reflect-config.json index 545175af084..989ba09286d 100644 --- a/server/src/main/resources/META-INF/native-image/reflect-config.json +++ b/server/src/main/resources/META-INF/native-image/reflect-config.json @@ -23,9 +23,6 @@ { "name":"[Ljavax.management.openmbean.CompositeData;" }, -{ - "name":"[Lnet.sf.cglib.proxy.Callback;" -}, { "name":"[Lorg.HdrHistogram.DoubleRecorder;" }, @@ -229,28 +226,12 @@ "name":"com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl", "methods":[{"name":"<init>","parameterTypes":[] }] }, -{ - "name":"java.io.DataInputStream" -}, -{ - "name":"java.io.DataOutputStream" -}, { "name":"java.io.FileDescriptor" }, { "name":"java.io.FilePermission" }, -{ - "name":"java.io.InputStream" -}, -{ - "name":"java.io.Serializable" -}, -{ - "name":"java.lang.AutoCloseable", - "queryAllDeclaredMethods":true -}, { "name":"java.lang.Boolean", "allDeclaredFields":true, @@ -266,16 +247,6 @@ "allDeclaredFields":true, "fields":[{"name":"TYPE"}] }, -{ - "name":"java.lang.Class" -}, -{ - "name":"java.lang.ClassLoader", - "methods":[{"name":"defineClass","parameterTypes":["java.lang.String","byte[]","int","int","java.security.ProtectionDomain"] }] -}, -{ - "name":"java.lang.Comparable" -}, { "name":"java.lang.Deprecated", "queryAllPublicMethods":true @@ -285,9 +256,6 @@ "allDeclaredFields":true, "fields":[{"name":"TYPE"}] }, -{ - "name":"java.lang.Enum" -}, { "name":"java.lang.Float", "allDeclaredFields":true, @@ -303,17 +271,13 @@ "allDeclaredFields":true, "fields":[{"name":"TYPE"}] }, -{ - "name":"java.lang.Module" -}, { "name":"java.lang.Number", "allDeclaredFields":true }, { "name":"java.lang.Object", - "allDeclaredFields":true, - "queryAllDeclaredMethods":true + "allDeclaredFields":true }, { "name":"java.lang.ProcessHandle", @@ -338,9 +302,6 @@ "name":"java.lang.String", "allDeclaredFields":true }, -{ - "name":"java.lang.StringBuilder" -}, { "name":"java.lang.Thread", "fields":[{"name":"threadLocalRandomProbe"}] @@ -353,12 +314,6 @@ "name":"java.lang.Void", "fields":[{"name":"TYPE"}] }, -{ - "name":"java.lang.annotation.Annotation" -}, -{ - "name":"java.lang.invoke.CallSite" -}, { "name":"java.lang.management.BufferPoolMXBean", "queryAllPublicMethods":true @@ -475,19 +430,12 @@ { "name":"java.security.AllPermission" }, -{ - "name":"java.security.ProtectionDomain" -}, { "name":"java.security.SecureRandomParameters" }, { "name":"java.security.SecurityPermission" }, -{ - "name":"java.util.AbstractMap", - "allDeclaredFields":true -}, { "name":"java.util.AbstractMap$SimpleEntry", "allDeclaredFields":true @@ -499,17 +447,9 @@ { "name":"java.util.Date" }, -{ - "name":"java.util.Iterator" -}, -{ - "name":"java.util.List", - "methods":[{"name":"copyOf","parameterTypes":["java.util.Collection"] }] -}, { "name":"java.util.Optional", - "allDeclaredFields":true, - "methods":[{"name":"isEmpty","parameterTypes":[] }] + "allDeclaredFields":true }, { "name":"java.util.OptionalDouble", @@ -527,22 +467,6 @@ "name":"java.util.PropertyPermission", "methods":[{"name":"<init>","parameterTypes":["java.lang.String","java.lang.String"] }] }, -{ - "name":"java.util.concurrent.ConcurrentHashMap", - "allDeclaredFields":true -}, -{ - "name":"java.util.concurrent.ConcurrentHashMap$CollectionView", - "allDeclaredFields":true -}, -{ - "name":"java.util.concurrent.ConcurrentHashMap$EntrySetView", - "allDeclaredFields":true -}, -{ - "name":"java.util.concurrent.ConcurrentHashMap$Node", - "allDeclaredFields":true -}, { "name":"java.util.concurrent.atomic.LongAdder", "queryAllPublicConstructors":true, @@ -552,9 +476,6 @@ {"name":"sum","parameterTypes":[] } ] }, -{ - "name":"java.util.function.Function" -}, { "name":"java.util.logging.LogManager", "methods":[{"name":"getLoggingMXBean","parameterTypes":[] }] @@ -563,9 +484,6 @@ "name":"java.util.logging.LoggingMXBean", "queryAllPublicMethods":true }, -{ - "name":"java.util.zip.DeflaterInputStream" -}, { "name":"javax.management.MBeanOperationInfo", "queryAllPublicMethods":true, @@ -618,62 +536,6 @@ "name":"jdk.management.jfr.SettingDescriptorInfo", "queryAllPublicMethods":true }, -{ - "name":"net.sf.cglib.core.KeyFactory" -}, -{ - "name":"net.sf.cglib.core.MethodWrapper$MethodWrapperKey", - "queryAllDeclaredMethods":true -}, -{ - "name":"net.sf.cglib.core.MethodWrapper$MethodWrapperKey$$KeyFactoryByCGLIB$$d45e49f7", - "methods":[{"name":"<init>","parameterTypes":[] }] -}, -{ - "name":"net.sf.cglib.core.ReflectUtils" -}, -{ - "name":"net.sf.cglib.core.Signature" -}, -{ - "name":"net.sf.cglib.proxy.Dispatcher" -}, -{ - "name":"net.sf.cglib.proxy.Enhancer" -}, -{ - "name":"net.sf.cglib.proxy.Enhancer$EnhancerKey", - "queryAllDeclaredMethods":true -}, -{ - "name":"net.sf.cglib.proxy.Enhancer$EnhancerKey$$KeyFactoryByCGLIB$$7fb24d72", - "methods":[{"name":"<init>","parameterTypes":[] }] -}, -{ - "name":"net.sf.cglib.proxy.Factory", - "queryAllDeclaredMethods":true -}, -{ - "name":"net.sf.cglib.proxy.FixedValue" -}, -{ - "name":"net.sf.cglib.proxy.InvocationHandler" -}, -{ - "name":"net.sf.cglib.proxy.LazyLoader" -}, -{ - "name":"net.sf.cglib.proxy.MethodInterceptor" -}, -{ - "name":"net.sf.cglib.proxy.NoOp" -}, -{ - "name":"net.sf.cglib.proxy.ProxyRefDispatcher" -}, -{ - "name":"net.sf.cglib.reflect.FastClass" -}, { "name":"org.HdrHistogram.ConcurrentHistogram", "methods":[{"name":"<init>","parameterTypes":["long","long","int"] }] @@ -702,10 +564,6 @@ "name":"org.apache.iotdb.commons.auth.authorizer.LocalFileAuthorizer", "methods":[{"name":"<init>","parameterTypes":[] }] }, -{ - "name":"org.apache.iotdb.commons.client.ThriftClient", - "queryAllDeclaredMethods":true -}, { "name":"org.apache.iotdb.commons.concurrent.threadpool.WrappedScheduledExecutorService", "queryAllPublicConstructors":true @@ -742,60 +600,6 @@ "name":"org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor", "methods":[{"name":"<init>","parameterTypes":["int"] }] }, -{ - "name":"org.apache.iotdb.commons.path.PathPatternNode$Serializer" -}, -{ - "name":"org.apache.iotdb.commons.schema.filter.SchemaFilterVisitor" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.IMNode" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.common.AbstractAboveDatabaseMNode" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.common.AbstractDatabaseDeviceMNode" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.common.AbstractDatabaseMNode" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.common.AbstractDeviceMNode" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.common.AbstractMeasurementMNode" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.info.IDatabaseDeviceInfo" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.info.IDatabaseInfo" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.info.IDeviceInfo" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.info.IMeasurementInfo" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.utils.IMNodeContainer" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.utils.IMNodeFactory" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.utils.IMNodeIterator" -}, -{ - "name":"org.apache.iotdb.commons.schema.node.visitor.MNodeVisitor" -}, -{ - "name":"org.apache.iotdb.commons.schema.tree.AbstractTreeVisitor" -}, -{ - "name":"org.apache.iotdb.commons.schema.view.viewExpression.visitor.ViewExpressionVisitor" -}, { "name":"org.apache.iotdb.commons.security.encrypt.MessageDigestEncrypt", "methods":[{"name":"<init>","parameterTypes":[] }] @@ -808,10 +612,6 @@ "name":"org.apache.iotdb.commons.service.metric.MetricServiceMBean", "queryAllPublicMethods":true }, -{ - "name":"org.apache.iotdb.confignode.rpc.thrift.IConfigNodeRPCService$Iface", - "queryAllDeclaredMethods":true -}, { "name":"org.apache.iotdb.consensus.iot.IoTConsensus", "methods":[{"name":"<init>","parameterTypes":["org.apache.iotdb.consensus.config.ConsensusConfig","org.apache.iotdb.consensus.IStateMachine$Registry"] }] @@ -828,29 +628,6 @@ "name":"org.apache.iotdb.consensus.ratis.RatisConsensus", "methods":[{"name":"<init>","parameterTypes":["org.apache.iotdb.consensus.config.ConsensusConfig","org.apache.iotdb.consensus.IStateMachine$Registry"] }] }, -{ - "name":"org.apache.iotdb.db.client.ConfigNodeClient", - "queryAllDeclaredMethods":true, - "queryAllDeclaredConstructors":true, - "methods":[{"name":"<init>","parameterTypes":["java.util.List","org.apache.iotdb.commons.client.property.ThriftClientProperty","org.apache.iotdb.commons.client.ClientManager"] }] -}, -{ - "name":"org.apache.iotdb.db.client.ConfigNodeClient$$EnhancerByCGLIB$$e2d55b5e", - "queryAllDeclaredMethods":true, - "queryAllDeclaredConstructors":true, - "methods":[ - {"name":"<init>","parameterTypes":["java.util.List","org.apache.iotdb.commons.client.property.ThriftClientProperty","org.apache.iotdb.commons.client.ClientManager"] }, - {"name":"CGLIB$SET_THREAD_CALLBACKS","parameterTypes":["net.sf.cglib.proxy.Callback[]"] } - ] -}, -{ - "name":"org.apache.iotdb.db.client.ConfigNodeClient$$EnhancerByCGLIB$$e2d55b5e$$FastClassByCGLIB$$90aef4e7", - "methods":[{"name":"<init>","parameterTypes":["java.lang.Class"] }] -}, -{ - "name":"org.apache.iotdb.db.client.ConfigNodeClient$$FastClassByCGLIB$$c29c157a", - "methods":[{"name":"<init>","parameterTypes":["java.lang.Class"] }] -}, { "name":"org.apache.iotdb.db.conf.IoTDBConfig", "allDeclaredFields":true @@ -866,6 +643,10 @@ "name":"org.apache.iotdb.db.engine.cache.CacheHitRatioMonitorMXBean", "queryAllPublicMethods":true }, +{ + "name":"org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache$TimeSeriesMetadataCacheKey", + "allDeclaredFields":true +}, { "name":"org.apache.iotdb.db.engine.flush.FlushManager", "queryAllPublicConstructors":true @@ -874,122 +655,6 @@ "name":"org.apache.iotdb.db.engine.flush.FlushManagerMBean", "queryAllPublicMethods":true }, -{ - "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.IDualKeyCache" -}, -{ - "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.IDualKeyCacheComputation" -}, -{ - "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.IDualKeyCacheStats" -}, -{ - "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.impl.ICacheEntry" -}, -{ - "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.impl.ICacheEntryGroup" -}, -{ - "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.impl.ICacheEntryManager" -}, -{ - "name":"org.apache.iotdb.db.metadata.cache.dualkeycache.impl.ICacheSizeComputer" -}, -{ - "name":"org.apache.iotdb.db.metadata.cache.lastCache.container.ILastCacheContainer" -}, -{ - "name":"org.apache.iotdb.db.metadata.cache.lastCache.container.value.ILastCacheValue" -}, -{ - "name":"org.apache.iotdb.db.metadata.idtable.IDTable" -}, -{ - "name":"org.apache.iotdb.db.metadata.idtable.IDiskSchemaManager" -}, -{ - "name":"org.apache.iotdb.db.metadata.idtable.entry.IDeviceID" -}, -{ - "name":"org.apache.iotdb.db.metadata.logfile.IDeserializer" -}, -{ - "name":"org.apache.iotdb.db.metadata.logfile.ISerializer" -}, -{ - "name":"org.apache.iotdb.db.metadata.mnode.mem.info.BasicMNodeInfo" -}, -{ - "name":"org.apache.iotdb.db.metadata.mtree.store.IMTreeStore" -}, -{ - "name":"org.apache.iotdb.db.metadata.mtree.store.disk.cache.CacheEntry" -}, -{ - "name":"org.apache.iotdb.db.metadata.mtree.store.disk.cache.ICacheManager" -}, -{ - "name":"org.apache.iotdb.db.metadata.mtree.store.disk.memcontrol.IReleaseFlushStrategy" -}, -{ - "name":"org.apache.iotdb.db.metadata.mtree.store.disk.schemafile.ISchemaFile" -}, -{ - "name":"org.apache.iotdb.db.metadata.mtree.store.disk.schemafile.ISchemaPage" -}, -{ - "name":"org.apache.iotdb.db.metadata.mtree.store.disk.schemafile.ISegment" -}, -{ - "name":"org.apache.iotdb.db.metadata.mtree.store.disk.schemafile.pagemgr.IPageManager" -}, -{ - "name":"org.apache.iotdb.db.metadata.mtree.traverser.counter.Counter" -}, -{ - "name":"org.apache.iotdb.db.metadata.mtree.traverser.updater.Updater" -}, -{ - "name":"org.apache.iotdb.db.metadata.plan.schemaregion.ISchemaRegionPlan" -}, -{ - "name":"org.apache.iotdb.db.metadata.plan.schemaregion.SchemaRegionPlanVisitor" -}, -{ - "name":"org.apache.iotdb.db.metadata.query.info.ISchemaInfo" -}, -{ - "name":"org.apache.iotdb.db.metadata.rescon.ISchemaEngineStatistics" -}, -{ - "name":"org.apache.iotdb.db.metadata.rescon.ISchemaRegionStatistics" -}, -{ - "name":"org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion" -}, -{ - "name":"org.apache.iotdb.db.metadata.schemaregion.ISchemaRegionParams" -}, -{ - "name":"org.apache.iotdb.db.metadata.schemaregion.SchemaRegionMemoryImpl", - "methods":[{"name":"<init>","parameterTypes":["org.apache.iotdb.db.metadata.schemaregion.ISchemaRegionParams"] }] -}, -{ - "name":"org.apache.iotdb.db.metadata.schemaregion.SchemaRegionSchemaFileImpl", - "methods":[{"name":"<init>","parameterTypes":["org.apache.iotdb.db.metadata.schemaregion.ISchemaRegionParams"] }] -}, -{ - "name":"org.apache.iotdb.db.metadata.template.ITemplateManager" -}, -{ - "name":"org.apache.iotdb.db.metadata.template.alter.TemplateAlterInfo" -}, -{ - "name":"org.apache.iotdb.db.metadata.utils.ResourceByPathUtils" -}, -{ - "name":"org.apache.iotdb.db.mpp.common.schematree.IMeasurementSchemaInfo" -}, { "name":"org.apache.iotdb.db.mpp.execution.exchange.MPPDataExchangeService", "queryAllPublicConstructors":true @@ -1002,9 +667,6 @@ "name":"org.apache.iotdb.db.mpp.execution.operator.process.last.LastQueryMergeOperator$Location", "allDeclaredFields":true }, -{ - "name":"org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanVisitor" -}, { "name":"org.apache.iotdb.db.query.control.SessionManager", "queryAllPublicConstructors":true @@ -1046,7 +708,16 @@ "methods":[{"name":"loadAll","parameterTypes":["java.lang.Iterable"] }] }, { - "name":"org.apache.iotdb.metrics.metricsets.IMetricSet" + "name":"org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata", + "allDeclaredFields":true +}, +{ + "name":"org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics", + "allDeclaredFields":true +}, +{ + "name":"org.apache.iotdb.tsfile.file.metadata.statistics.Statistics", + "allDeclaredFields":true }, { "name":"org.apache.iotdb.tsfile.read.common.block.TsBlock", @@ -1464,9 +1135,6 @@ {"name":"f4"} ] }, -{ - "name":"org.slf4j.impl.StaticLoggerBinder" -}, { "name":"sun.management.ClassLoadingImpl", "queryAllPublicConstructors":true diff --git a/server/src/main/resources/META-INF/native-image/resource-config.json b/server/src/main/resources/META-INF/native-image/resource-config.json index 2960797dd0a..c11048a2ab2 100644 --- a/server/src/main/resources/META-INF/native-image/resource-config.json +++ b/server/src/main/resources/META-INF/native-image/resource-config.json @@ -19,9 +19,6 @@ { "pattern":"\\Qlogback-test.xml\\E" }, - { - "pattern":"\\Qorg/apache/iotdb/db/metadata\\E" - }, { "pattern":"\\Qorg/slf4j/impl/StaticLoggerBinder.class\\E" }, diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionFileGeneratorUtils.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionFileGeneratorUtils.java index a5cd18a2974..fdfc65e750c 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionFileGeneratorUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionFileGeneratorUtils.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator.TsFileName; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.query.reader.UpdateCacheRestorableTsFileIOWriter; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -181,7 +182,8 @@ public class CompactionFileGeneratorUtils { if (!newTsFileResource.getTsFile().getParentFile().exists()) { newTsFileResource.getTsFile().getParentFile().mkdirs(); } - RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(newTsFileResource.getTsFile()); + RestorableTsFileIOWriter writer = + new UpdateCacheRestorableTsFileIOWriter(newTsFileResource.getTsFile()); Map<String, List<String>> deviceMeasurementMap = new HashMap<>(); for (String fullPath : fullPaths) { PartialPath partialPath = new PartialPath(fullPath); @@ -238,7 +240,8 @@ public class CompactionFileGeneratorUtils { TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage(); TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(Integer.MAX_VALUE); - RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(newTsFileResource.getTsFile()); + RestorableTsFileIOWriter writer = + new UpdateCacheRestorableTsFileIOWriter(newTsFileResource.getTsFile()); Map<String, List<String>> deviceMeasurementMap = new HashMap<>(); for (String fullPath : fullPaths) { PartialPath partialPath = new PartialPath(fullPath); @@ -329,7 +332,8 @@ public class CompactionFileGeneratorUtils { if (!newTsFileResource.getTsFile().getParentFile().exists()) { newTsFileResource.getTsFile().getParentFile().mkdirs(); } - RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(newTsFileResource.getTsFile()); + RestorableTsFileIOWriter writer = + new UpdateCacheRestorableTsFileIOWriter(newTsFileResource.getTsFile()); Map<String, List<String>> deviceMeasurementMap = new HashMap<>(); for (String fullPath : fullPaths) { PartialPath partialPath = new PartialPath(fullPath); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java index 2dd9b211232..f59e2de6829 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java @@ -82,13 +82,15 @@ public class TimeseriesMetadata implements ITimeSeriesMetadata { String measurementId, TSDataType dataType, Statistics<? extends Serializable> statistics, - PublicBAOS chunkMetadataListBuffer) { + PublicBAOS chunkMetadataListBuffer, + ArrayList<IChunkMetadata> chunkMetadataList) { this.timeSeriesMetadataType = timeSeriesMetadataType; this.chunkMetaDataListDataSize = chunkMetaDataListDataSize; this.measurementId = measurementId; this.dataType = dataType; this.statistics = statistics; this.chunkMetadataListBuffer = chunkMetadataListBuffer; + this.chunkMetadataList = chunkMetadataList; } public TimeseriesMetadata(TimeseriesMetadata timeseriesMetadata) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index e62aeb40c7e..57786c2f527 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -349,6 +349,14 @@ public class TsFileIOWriter implements AutoCloseable { logger.info("Time for flushing metadata is {} ms", cost); } + protected void updateCache(String device, TimeseriesMetadata timeseriesMetadata) { + // do nothing + } + + protected void updateCache(BloomFilter bloomFilter) { + // do nothing + } + private void checkInMemoryPathCount() { for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) { pathCount += chunkGroupMetadata.getChunkMetadataList().size(); @@ -426,6 +434,8 @@ public class TsFileIOWriter implements AutoCloseable { seriesIdxForCurrDevice++; // serialize the timeseries metadata to file timeseriesMetadata.serializeTo(out.wrapAsStream()); + timeseriesMetadata.setChunkMetadataListBuffer(null); + updateCache(currentDevice, timeseriesMetadata); } addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out); @@ -444,6 +454,7 @@ public class TsFileIOWriter implements AutoCloseable { int size = tsFileMetadata.serializeTo(out.wrapAsStream()); size += tsFileMetadata.serializeBloomFilter(out.wrapAsStream(), filter); + updateCache(filter); // write TsFileMetaData size ReadWriteIOUtils.write(size, out.wrapAsStream()); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java index 5d17e0ab7e9..d4c2f1ebcd7 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java @@ -95,16 +95,14 @@ public class TSMIterator { seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); } - TimeseriesMetadata timeseriesMetadata = - new TimeseriesMetadata( - (byte) - ((serializeStatistic ? (byte) 1 : (byte) 0) | chunkMetadataList.get(0).getMask()), - chunkMetadataListLength, - measurementId, - dataType, - seriesStatistics, - publicBAOS); - return timeseriesMetadata; + return new TimeseriesMetadata( + (byte) ((serializeStatistic ? (byte) 1 : (byte) 0) | chunkMetadataList.get(0).getMask()), + chunkMetadataListLength, + measurementId, + dataType, + seriesStatistics, + publicBAOS, + (ArrayList<IChunkMetadata>) chunkMetadataList); } public static List<Pair<Path, List<IChunkMetadata>>> sortChunkMetadata(
