This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 3bc92acd21 Eliminate schema region device cache (#8563)
3bc92acd21 is described below
commit 3bc92acd210eb46cc9fdc0ec935e1076349c7c63
Author: Marcos_Zyk <[email protected]>
AuthorDate: Wed Dec 21 21:09:07 2022 +0800
Eliminate schema region device cache (#8563)
Eliminate schema region device cache (#8563)
---
.../resources/conf/iotdb-common.properties | 5 --
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 ----
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 ---
.../schemaregion/SchemaRegionMemoryImpl.java | 48 +---------------
.../schemaregion/SchemaRegionSchemaFileImpl.java | 64 +---------------------
.../schemaRegion/AbstractSchemaRegionTest.java | 2 +-
6 files changed, 3 insertions(+), 135 deletions(-)
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties
b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index f2777f76a7..aaee013c98 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -244,11 +244,6 @@
### Schema Engine Configuration
####################
-# cache size for SchemaRegion.
-# This cache is used to improve insert speed where all path check and
TSDataType will be cached in SchemaRegion with corresponding Path.
-# Datatype: int
-# schema_region_device_node_cache_size=10000
-
# thread pool size for read operation in DataNode's coordinator.
# Datatype: int
# coordinator_read_executor_size=20
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index f163c7fad7..acd93765dc 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -529,9 +529,6 @@ public class IoTDBConfig {
/** Set true to enable writing monitor time series. */
private boolean enableMonitorSeriesWrite = false;
- /** Cache size of {@code checkAndGetDataTypeCache}. */
- private int schemaRegionDeviceNodeCacheSize = 10000;
-
/** Cache size of {@code checkAndGetDataTypeCache}. */
private int mRemoteSchemaCacheSize = 100000;
@@ -1576,14 +1573,6 @@ public class IoTDBConfig {
this.rpcMaxConcurrentClientNum = rpcMaxConcurrentClientNum;
}
- public int getSchemaRegionDeviceNodeCacheSize() {
- return schemaRegionDeviceNodeCacheSize;
- }
-
- void setSchemaRegionDeviceNodeCacheSize(int schemaRegionDeviceNodeCacheSize)
{
- this.schemaRegionDeviceNodeCacheSize = schemaRegionDeviceNodeCacheSize;
- }
-
public int getmRemoteSchemaCacheSize() {
return mRemoteSchemaCacheSize;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index a38db9c033..96440077ce 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -565,14 +565,6 @@ public class IoTDBDescriptor {
"raw_query_blocking_queue_capacity",
Integer.toString(conf.getRawQueryBlockingQueueCapacity()))));
- conf.setSchemaRegionDeviceNodeCacheSize(
- Integer.parseInt(
- properties
- .getProperty(
- "schema_region_device_node_cache_size",
-
Integer.toString(conf.getSchemaRegionDeviceNodeCacheSize()))
- .trim()));
-
conf.setmRemoteSchemaCacheSize(
Integer.parseInt(
properties
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java
index 47a5c94039..52c3ad704c 100644
---
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java
+++
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java
@@ -80,10 +80,6 @@ import
org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
-import org.checkerframework.checker.nullness.qual.NonNull;
-import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -157,8 +153,6 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
private MemoryStatistics memoryStatistics = MemoryStatistics.getInstance();
private MTreeBelowSGMemoryImpl mtree;
- // device -> DeviceMNode
- private LoadingCache<PartialPath, IMNode> mNodeCache;
private TagManager tagManager;
// seriesNumberMonitor may be null
@@ -177,19 +171,6 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
storageGroupDirPath = config.getSchemaDir() + File.separator +
storageGroupFullPath;
schemaRegionDirPath = storageGroupDirPath + File.separator +
schemaRegionId.getId();
- int cacheSize = config.getSchemaRegionDeviceNodeCacheSize();
- mNodeCache =
- Caffeine.newBuilder()
- .maximumSize(cacheSize)
- .build(
- new
com.github.benmanes.caffeine.cache.CacheLoader<PartialPath, IMNode>() {
- @Override
- public @Nullable IMNode load(@NonNull PartialPath
partialPath)
- throws MetadataException {
- return mtree.getNodeByPath(partialPath);
- }
- });
-
// In ratis mode, no matter create schemaRegion or recover schemaRegion,
the working dir should
// be clear first
if (config.isClusterMode()
@@ -374,10 +355,6 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
@Override
public synchronized void clear() {
try {
-
- if (this.mNodeCache != null) {
- this.mNodeCache.invalidateAll();
- }
if (this.mtree != null) {
this.mtree.clear();
}
@@ -572,9 +549,6 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
plan.getCompressor(),
plan.getProps(),
plan.getAlias());
-
- // the cached mNode may be replaced by new entityMNode in mtree
- mNodeCache.invalidate(path.getDevicePath());
} catch (Throwable t) {
if (seriesNumerMonitor != null) {
seriesNumerMonitor.deleteTimeSeries(1);
@@ -661,9 +635,6 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
plan.getEncodings(),
plan.getCompressors(),
plan.getAliasList());
-
- // the cached mNode may be replaced by new entityMNode in mtree
- mNodeCache.invalidate(prefixPath);
} catch (Throwable t) {
if (seriesNumerMonitor != null) {
seriesNumerMonitor.deleteTimeSeries(seriesCount);
@@ -840,10 +811,6 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
IMeasurementMNode measurementMNode = pair.right;
removeFromTagInvertedIndex(measurementMNode);
- IMNode node = measurementMNode.getParent();
-
- mNodeCache.invalidate(node.getPartialPath());
-
schemaStatisticsManager.deleteTimeseries(1);
if (seriesNumerMonitor != null) {
seriesNumerMonitor.deleteTimeSeries(1);
@@ -886,10 +853,6 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
removeFromTagInvertedIndex(measurementMNode);
PartialPath storageGroupPath = pair.left;
- IMNode node = measurementMNode.getParent();
-
- mNodeCache.invalidate(node.getPartialPath());
-
schemaStatisticsManager.deleteTimeseries(1);
if (seriesNumerMonitor != null) {
seriesNumerMonitor.deleteTimeSeries(1);
@@ -908,16 +871,7 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
*/
private IMNode getDeviceNodeWithAutoCreate(PartialPath path)
throws IOException, MetadataException {
- IMNode node;
- try {
- return mNodeCache.get(path);
- } catch (Exception e) {
- if (!(e.getCause() instanceof MetadataException)) {
- throw e;
- }
- }
-
- node = mtree.getDeviceNodeWithAutoCreating(path);
+ IMNode node = mtree.getDeviceNodeWithAutoCreating(path);
writeToMLog(SchemaRegionPlanFactory.getAutoCreateDeviceMNodePlan(node.getPartialPath()));
return node;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java
index a1b68f4ad3..7ce9971c04 100644
---
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java
+++
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java
@@ -30,7 +30,6 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.AliasAlreadyExistException;
import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
-import org.apache.iotdb.db.exception.metadata.PathNotExistException;
import
org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException;
import org.apache.iotdb.db.exception.metadata.SeriesNumberOverflowException;
import org.apache.iotdb.db.exception.metadata.SeriesOverflowException;
@@ -77,11 +76,6 @@ import
org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
-import com.github.benmanes.caffeine.cache.RemovalCause;
-import org.checkerframework.checker.nullness.qual.NonNull;
-import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -159,8 +153,6 @@ public class SchemaRegionSchemaFileImpl implements
ISchemaRegion {
private final MemoryStatistics memoryStatistics =
MemoryStatistics.getInstance();
private MTreeBelowSGCachedImpl mtree;
- // device -> DeviceMNode
- private final LoadingCache<PartialPath, IMNode> mNodeCache;
private TagManager tagManager;
// seriesNumberMonitor may be null
@@ -179,25 +171,6 @@ public class SchemaRegionSchemaFileImpl implements
ISchemaRegion {
storageGroupDirPath = config.getSchemaDir() + File.separator +
storageGroupFullPath;
schemaRegionDirPath = storageGroupDirPath + File.separator +
schemaRegionId.getId();
- int cacheSize = config.getSchemaRegionDeviceNodeCacheSize();
- mNodeCache =
- Caffeine.newBuilder()
- .maximumSize(cacheSize)
- .removalListener(
- (PartialPath path, IMNode node, RemovalCause cause) -> {
- if (!isClearing) {
- mtree.unPinMNode(node);
- }
- })
- .build(
- new
com.github.benmanes.caffeine.cache.CacheLoader<PartialPath, IMNode>() {
- @Override
- public @Nullable IMNode load(@NonNull PartialPath
partialPath)
- throws MetadataException {
-
- return mtree.getNodeByPath(partialPath);
- }
- });
this.seriesNumerMonitor = seriesNumerMonitor;
init();
}
@@ -368,9 +341,6 @@ public class SchemaRegionSchemaFileImpl implements
ISchemaRegion {
public synchronized void clear() {
isClearing = true;
try {
- if (this.mNodeCache != null) {
- this.mNodeCache.invalidateAll();
- }
if (this.mtree != null) {
this.mtree.clear();
}
@@ -593,8 +563,6 @@ public class SchemaRegionSchemaFileImpl implements
ISchemaRegion {
}
try {
- // the cached mNode may be replaced by new entityMNode in mtree
- mNodeCache.invalidate(path.getDevicePath());
// update statistics and schemaDataTypeNumMap
schemaStatisticsManager.addTimeseries(1);
@@ -749,8 +717,6 @@ public class SchemaRegionSchemaFileImpl implements
ISchemaRegion {
}
try {
- // the cached mNode may be replaced by new entityMNode in mtree
- mNodeCache.invalidate(prefixPath);
// update statistics and schemaDataTypeNumMap
schemaStatisticsManager.addTimeseries(seriesCount);
@@ -936,10 +902,6 @@ public class SchemaRegionSchemaFileImpl implements
ISchemaRegion {
IMeasurementMNode measurementMNode = pair.right;
removeFromTagInvertedIndex(measurementMNode);
- IMNode node = measurementMNode.getParent();
-
- mNodeCache.invalidate(node.getPartialPath());
-
schemaStatisticsManager.deleteTimeseries(1);
if (seriesNumerMonitor != null) {
seriesNumerMonitor.deleteTimeSeries(1);
@@ -977,10 +939,6 @@ public class SchemaRegionSchemaFileImpl implements
ISchemaRegion {
removeFromTagInvertedIndex(measurementMNode);
PartialPath storageGroupPath = pair.left;
- IMNode node = measurementMNode.getParent();
-
- mNodeCache.invalidate(node.getPartialPath());
-
schemaStatisticsManager.deleteTimeseries(1);
if (seriesNumerMonitor != null) {
seriesNumerMonitor.deleteTimeSeries(1);
@@ -999,27 +957,7 @@ public class SchemaRegionSchemaFileImpl implements
ISchemaRegion {
*/
private IMNode getDeviceNodeWithAutoCreate(PartialPath path)
throws IOException, MetadataException {
- IMNode node;
- try {
- node = mNodeCache.get(path);
- try {
- mtree.pinMNode(node);
- return node;
- } catch (MetadataException e) {
- // the node in mNodeCache has been evicted, thus get it via the
following progress
- return mtree.getNodeByPath(path);
- }
- } catch (Exception e) {
- if (e.getCause() instanceof MetadataException) {
- if (!config.isAutoCreateSchemaEnabled()) {
- throw new PathNotExistException(path.getFullPath());
- }
- } else {
- throw e;
- }
- }
-
- node = mtree.getDeviceNodeWithAutoCreating(path);
+ IMNode node = mtree.getDeviceNodeWithAutoCreating(path);
writeToMLog(SchemaRegionPlanFactory.getAutoCreateDeviceMNodePlan(node.getPartialPath()));
return node;
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/AbstractSchemaRegionTest.java
b/server/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/AbstractSchemaRegionTest.java
index f8b6cbbfbc..49b4b5956c 100644
---
a/server/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/AbstractSchemaRegionTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/AbstractSchemaRegionTest.java
@@ -65,7 +65,7 @@ public abstract class AbstractSchemaRegionTest {
new SchemaRegionTestParams(
"Raw-Config",
config.getSchemaEngineMode(),
- config.getSchemaRegionDeviceNodeCacheSize(),
+ config.getCachedMNodeSizeInSchemaFileMode(),
config.isClusterMode());
config.setSchemaEngineMode(testParams.schemaEngineMode);
config.setCachedMNodeSizeInSchemaFileMode(testParams.cachedMNodeSize);
