This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch rc/1.3.1 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 86f946a283b3260eb2bc542530360a1190c6d406 Author: Zhijia Cao <[email protected]> AuthorDate: Tue Feb 6 16:10:01 2024 +0800 Storage Engine: put DeviceCache into the total memory of the storage engine (#12016) --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 23 ++--- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 104 ++++++++++----------- .../cache/schema/DataNodeDevicePathCache.java | 19 +++- .../planner/plan/node/write/DeleteDataNode.java | 5 +- .../planner/plan/node/write/InsertRowNode.java | 5 +- .../planner/plan/node/write/InsertTabletNode.java | 4 +- .../db/storageengine/dataregion/DataRegion.java | 2 +- .../dataregion/modification/Deletion.java | 6 +- .../tsfile/timeindex/DeviceTimeIndex.java | 14 ++- .../db/storageengine/DevicePathCacheTests.java | 67 +++++++++++++ .../resources/conf/iotdb-common.properties | 10 +- 11 files changed, 177 insertions(+), 82 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index f70e3f8d8d1..431cfe3fdca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -166,11 +166,14 @@ public class IoTDBConfig { private double rejectProportion = 0.8; /** The proportion of write memory for memtable */ - private double writeProportionForMemtable = 0.76; + private double writeProportionForMemtable = 0.72; /** The proportion of write memory for compaction */ private double compactionProportion = 0.2; + /** The proportion of write memory for device path cache */ + private double devicePathCacheProportion = 0.04; + /** * If memory cost of data region increased more than proportion of {@linkplain * IoTDBConfig#getAllocateMemoryForStorageEngine()}*{@linkplain @@ -949,8 +952,6 @@ public class IoTDBConfig { */ private int partitionCacheSize = 1000; - private int devicePathCacheSize = 500_000; - /** Cache size of user and role */ private int authorCacheSize = 100; @@ -3097,14 +3098,6 @@ public class IoTDBConfig { this.partitionCacheSize = partitionCacheSize; } - public int getDevicePathCacheSize() { - return devicePathCacheSize; - } - - public void setDevicePathCacheSize(int devicePathCacheSize) { - this.devicePathCacheSize = devicePathCacheSize; - } - public int getAuthorCacheSize() { return authorCacheSize; } @@ -3265,6 +3258,14 @@ public class IoTDBConfig { return compactionProportion; } + public double getDevicePathCacheProportion() { + return devicePathCacheProportion; + } + + public void setDevicePathCacheProportion(double devicePathCacheProportion) { + this.devicePathCacheProportion = devicePathCacheProportion; + } + public static String getEnvironmentVariables() { return "\n\t" + IoTDBConstant.IOTDB_HOME diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 731e460333c..fc9bb6c2364 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -752,12 +752,6 @@ public class IoTDBDescriptor { conf.setKerberosPrincipal( properties.getProperty("kerberos_principal", conf.getKerberosPrincipal())); - // the size of device path cache - conf.setDevicePathCacheSize( - Integer.parseInt( - properties.getProperty( - "device_path_cache_size", String.valueOf(conf.getDevicePathCacheSize())))); - // the num of memtables in each database conf.setConcurrentWritingTimePartition( Integer.parseInt( @@ -1809,57 +1803,61 @@ public class IoTDBDescriptor { private void initStorageEngineAllocate(Properties properties) { long storageMemoryTotal = conf.getAllocateMemoryForStorageEngine(); - - int proportionSum = 10; - int writeProportion = 8; - int compactionProportion = 2; - int writeProportionSum = 20; - int memTableProportion = 19; - int timePartitionInfo = 1; - - String storageMemoryAllocatePortion = + String valueOfStorageEngineMemoryProportion = properties.getProperty("storage_engine_memory_proportion"); - if (storageMemoryAllocatePortion != null) { - String[] proportions = storageMemoryAllocatePortion.split(":"); - int loadedProportionSum = 0; - for (String proportion : proportions) { - loadedProportionSum += Integer.parseInt(proportion.trim()); - } - - if (loadedProportionSum != 0) { - proportionSum = loadedProportionSum; - writeProportion = Integer.parseInt(proportions[0].trim()); - compactionProportion = Integer.parseInt(proportions[1].trim()); - } - conf.setCompactionProportion((double) compactionProportion / (double) proportionSum); - } - - String allocationRatioForWrite = properties.getProperty("write_memory_proportion"); - if (allocationRatioForWrite != null) { - String[] proportions = allocationRatioForWrite.split(":"); - int loadedProportionSum = 0; - for (String proportion : proportions) { - loadedProportionSum += Integer.parseInt(proportion.trim()); + if (valueOfStorageEngineMemoryProportion != null) { + String[] storageProportionArray = valueOfStorageEngineMemoryProportion.split(":"); + int storageEngineMemoryProportion = 0; + for (String proportion : storageProportionArray) { + int proportionValue = Integer.parseInt(proportion.trim()); + if (proportionValue <= 0) { + LOGGER.warn( + "The value of storage_engine_memory_proportion is illegal, use default value 8:2 ."); + return; + } + storageEngineMemoryProportion += proportionValue; } + conf.setCompactionProportion( + (double) Integer.parseInt(storageProportionArray[1].trim()) + / (double) storageEngineMemoryProportion); + + String valueOfWriteMemoryProportion = properties.getProperty("write_memory_proportion"); + if (valueOfWriteMemoryProportion != null) { + String[] writeProportionArray = valueOfWriteMemoryProportion.split(":"); + int writeMemoryProportion = 0; + for (String proportion : writeProportionArray) { + int proportionValue = Integer.parseInt(proportion.trim()); + writeMemoryProportion += proportionValue; + if (proportionValue <= 0) { + LOGGER.warn( + "The value of write_memory_proportion is illegal, use default value 18:1:1 ."); + return; + } + } - if (loadedProportionSum != 0) { - writeProportionSum = loadedProportionSum; - memTableProportion = Integer.parseInt(proportions[0].trim()); - timePartitionInfo = Integer.parseInt(proportions[1].trim()); + double writeAllProportionOfStorageEngineMemory = + (double) Integer.parseInt(storageProportionArray[0].trim()) + / storageEngineMemoryProportion; + double memTableProportion = + (double) Integer.parseInt(writeProportionArray[0].trim()) / writeMemoryProportion; + double timePartitionInfoProportion = + (double) Integer.parseInt(writeProportionArray[1].trim()) / writeMemoryProportion; + double devicePathCacheProportion = + (double) Integer.parseInt(writeProportionArray[2].trim()) / writeMemoryProportion; + // writeProportionForMemtable = 8/10 * 18/20 = 0.72 default + conf.setWriteProportionForMemtable( + writeAllProportionOfStorageEngineMemory * memTableProportion); + + // allocateMemoryForTimePartitionInfo = storageMemoryTotal * 8/10 * 1/20 default + conf.setAllocateMemoryForTimePartitionInfo( + (long) + ((writeAllProportionOfStorageEngineMemory * timePartitionInfoProportion) + * storageMemoryTotal)); + + // device path cache default memory is value is 8/10 * 1/20 = 0.04 for StorageEngine + conf.setDevicePathCacheProportion( + writeAllProportionOfStorageEngineMemory * devicePathCacheProportion); } - // memtableProportionForWrite = 19/20 default - double memtableProportionForWrite = - ((double) memTableProportion / (double) writeProportionSum); - - // timePartitionInfoForWrite = 1/20 default - double timePartitionInfoForWrite = ((double) timePartitionInfo / (double) writeProportionSum); - // proportionForWrite = 8/10 default - double proportionForWrite = ((double) (writeProportion) / (double) proportionSum); - // writeProportionForMemtable = 8/10 * 19/20 = 0.76 default - conf.setWriteProportionForMemtable(proportionForWrite * memtableProportionForWrite); - // allocateMemoryForTimePartitionInfo = storageMemoryTotal * 8/10 * 1/20 default - conf.setAllocateMemoryForTimePartitionInfo( - (long) ((proportionForWrite * timePartitionInfoForWrite) * storageMemoryTotal)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java index c0ad149daaa..0a189043e11 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Weigher; /** This cache is for reducing duplicated DeviceId PartialPath initialization in write process. */ public class DataNodeDevicePathCache { @@ -35,7 +36,15 @@ public class DataNodeDevicePathCache { private final Cache<String, PartialPath> devicePathCache; private DataNodeDevicePathCache() { - devicePathCache = Caffeine.newBuilder().maximumSize(config.getDevicePathCacheSize()).build(); + devicePathCache = + Caffeine.newBuilder() + .maximumWeight( + (long) + (config.getAllocateMemoryForStorageEngine() + * config.getDevicePathCacheProportion())) + .weigher( + (Weigher<String, PartialPath>) (key, val) -> (PartialPath.estimateSize(val) + 32)) + .build(); } public static DataNodeDevicePathCache getInstance() { @@ -63,6 +72,14 @@ public class DataNodeDevicePathCache { } } + public String getDeviceId(String deviceId) { + try { + return getPartialPath(deviceId).getFullPath(); + } catch (IllegalPathException e) { + return deviceId; + } + } + public void cleanUp() { devicePathCache.cleanUp(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java index a72d8c4349c..27bd91df077 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -159,7 +160,9 @@ public class DeleteDataNode extends WritePlanNode implements WALEntryValue { List<PartialPath> pathList = new ArrayList<>(size); for (int i = 0; i < size; i++) { try { - pathList.add(new PartialPath(ReadWriteIOUtils.readString(stream))); + pathList.add( + DataNodeDevicePathCache.getInstance() + .getPartialPath(ReadWriteIOUtils.readString(stream))); } catch (IllegalPathException e) { throw new IllegalArgumentException("Cannot deserialize InsertRowNode", e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index a9e6d8eb175..ab29ddfbd31 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -622,7 +623,9 @@ public class InsertRowNode extends InsertNode implements WALEntryValue { insertNode.setSearchIndex(stream.readLong()); insertNode.setTime(stream.readLong()); try { - insertNode.setDevicePath(new PartialPath(ReadWriteIOUtils.readString(stream))); + insertNode.setDevicePath( + DataNodeDevicePathCache.getInstance() + .getPartialPath(ReadWriteIOUtils.readString(stream))); } catch (IllegalPathException e) { throw new IllegalArgumentException(DESERIALIZE_ERROR, e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 4dbacc65971..2f309e101de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -866,7 +867,8 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue { private void subDeserializeFromWAL(DataInputStream stream) throws IOException { searchIndex = stream.readLong(); try { - devicePath = new PartialPath(ReadWriteIOUtils.readString(stream)); + devicePath = + DataNodeDevicePathCache.getInstance().getPartialPath(ReadWriteIOUtils.readString(stream)); } catch (IllegalPathException e) { throw new IllegalArgumentException("Cannot deserialize InsertTabletNode", e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 301c68bddc8..a9a43ebbc87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -588,7 +588,7 @@ public class DataRegion implements IDataRegionForQuery { Map<String, Long> endTimeMap = new HashMap<>(); for (String deviceId : resource.getDevices()) { long endTime = resource.getEndTime(deviceId); - endTimeMap.put(deviceId.intern(), endTime); + endTimeMap.put(deviceId, endTime); } if (config.isEnableSeparateData()) { lastFlushTimeMap.updateMultiDeviceFlushedTime(timePartitionId, endTimeMap); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/Deletion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/Deletion.java index 67c8969dd84..747a50d2f67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/Deletion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/Deletion.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.storageengine.dataregion.modification; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -115,7 +116,10 @@ public class Deletion extends Modification implements Cloneable { long startTime = stream.readLong(); long endTime = stream.readLong(); return new Deletion( - new PartialPath(ReadWriteIOUtils.readString(stream)), 0, startTime, endTime); + DataNodeDevicePathCache.getInstance().getPartialPath(ReadWriteIOUtils.readString(stream)), + 0, + startTime, + endTime); } public long getSerializedSize() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java index 090f6998f78..30f0d7fa179 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java @@ -21,7 +21,6 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.SerializeUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.PartitionViolationException; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; @@ -117,7 +116,9 @@ public class DeviceTimeIndex implements ITimeIndex { } for (int i = 0; i < deviceNum; i++) { - String path = ReadWriteIOUtils.readString(inputStream).intern(); + String path = + DataNodeDevicePathCache.getInstance() + .getDeviceId(ReadWriteIOUtils.readString(inputStream)); int index = ReadWriteIOUtils.readInt(inputStream); deviceToIndex.put(path, index); } @@ -138,7 +139,8 @@ public class DeviceTimeIndex implements ITimeIndex { } for (int i = 0; i < deviceNum; i++) { - String path = SerializeUtils.deserializeString(buffer).intern(); + String path = + DataNodeDevicePathCache.getInstance().getDeviceId(ReadWriteIOUtils.readString(buffer)); int index = buffer.getInt(); deviceToIndex.put(path, index); } @@ -171,7 +173,9 @@ public class DeviceTimeIndex implements ITimeIndex { ReadWriteIOUtils.skip(inputStream, 2L * deviceNum * ReadWriteIOUtils.LONG_LEN); Set<String> devices = new HashSet<>(); for (int i = 0; i < deviceNum; i++) { - String path = ReadWriteIOUtils.readString(inputStream).intern(); + String path = + DataNodeDevicePathCache.getInstance() + .getDeviceId(ReadWriteIOUtils.readString(inputStream)); ReadWriteIOUtils.skip(inputStream, ReadWriteIOUtils.INT_LEN); devices.add(path); } @@ -215,7 +219,7 @@ public class DeviceTimeIndex implements ITimeIndex { index = deviceToIndex.get(deviceId); } else { index = deviceToIndex.size(); - deviceToIndex.put(deviceId.intern(), index); + deviceToIndex.put(DataNodeDevicePathCache.getInstance().getDeviceId(deviceId), index); if (startTimes.length <= index) { startTimes = enLargeArray(startTimes, Long.MAX_VALUE); endTimes = enLargeArray(endTimes, Long.MIN_VALUE); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/DevicePathCacheTests.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/DevicePathCacheTests.java new file mode 100644 index 00000000000..0492e9ea154 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/DevicePathCacheTests.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine; + +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class DevicePathCacheTests { + + @Before + public void before() {} + + @Test + public void test() { + String path1 = "root.testdb.testd1.s1"; + String deviceId1 = DataNodeDevicePathCache.getInstance().getDeviceId("root.testdb.testd1.s1"); + Assert.assertEquals(path1, deviceId1); + + String path2 = "root.testdb.testd1.select"; + String deviceId2 = + DataNodeDevicePathCache.getInstance().getDeviceId("root.testdb.testd1.select"); + Assert.assertEquals(path2, deviceId2); + + String path3 = "root.sg.`a``b`"; + String deviceId3 = DataNodeDevicePathCache.getInstance().getDeviceId("root.sg.`a``b`"); + Assert.assertEquals(path3, deviceId3); + + String path4 = "root.sg.`a.b`"; + String deviceId4 = DataNodeDevicePathCache.getInstance().getDeviceId("root.sg.`a.b`"); + Assert.assertEquals(path4, deviceId4); + + String path5 = "root.sg.`111`"; + String deviceId5 = DataNodeDevicePathCache.getInstance().getDeviceId("root.sg.`111`"); + Assert.assertEquals(path5, deviceId5); + } + + @Test + public void test02() { + String path1 = "root.testdb.testd1.s1"; + String deviceId1 = + DataNodeDevicePathCache.getInstance().getDeviceId(new String("root.testdb.testd1.s1")); + String deviceId2 = + DataNodeDevicePathCache.getInstance().getDeviceId(new String("root.testdb.testd1.s1")); + Assert.assertEquals(path1, deviceId1); + Assert.assertEquals(deviceId2, deviceId1); + } +} diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties index e6bf64a48b5..07e8848eea8 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties @@ -185,10 +185,11 @@ data_replication_factor=1 # The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 8:2 , 7:3 # storage_engine_memory_proportion=8:2 -# Memory allocation ratio in writing: Memtable, TimePartitionInfo +# Memory allocation ratio in writing: Memtable, TimePartitionInfo, DevicePathCache # Memtable is the total memory size of all memtables # TimePartitionInfo is the total memory size of last flush time of all data regions -# write_memory_proportion=19:1 +# DevicePathCache is the deviceId cache, keep only one copy of the same deviceId in memory +# write_memory_proportion=18:1:1 # Max number of concurrent writing time partitions in one database # This parameter is used to control total memTable number when memory control is disabled @@ -542,11 +543,6 @@ data_replication_factor=1 # Datatype: boolean # 0.13_data_insert_adapt=false -# The max size of the device path cache. This cache is for avoiding initialize duplicated device id object in write process. -# Datatype: int -# device_path_cache_size=500000 - - # Verify that TSfiles generated by Flush, Load, and Compaction are correct. The following is verified: # 1. Check whether the file contains a header and a tail # 2. Check whether files can be deserialized successfully
