This is an automated email from the ASF dual-hosted git repository. sunzesong pushed a commit to branch remove_dynamic_adapter in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 330473d48f51bbfc86c54d3e0c1e00ab6e9afe48 Author: samperson1997 <[email protected]> AuthorDate: Wed Aug 12 23:57:39 2020 +0800 [IOTDB-834] Remove DynamicAdapter --- .../resources/conf/iotdb-engine.properties | 2 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +- .../db/conf/adapter/ActiveTimeSeriesCounter.java | 8 +- .../iotdb/db/conf/adapter/CompressionRatio.java | 15 -- .../db/conf/adapter/IActiveTimeSeriesCounter.java | 2 +- .../iotdb/db/conf/adapter/IDynamicAdapter.java | 47 ---- .../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 297 --------------------- .../db/conf/adapter/ManageDynamicParameters.java | 125 --------- .../conf/adapter/ManageDynamicParametersMBean.java | 42 --- .../iotdb/db/engine/cache/ChunkMetadataCache.java | 2 - .../iotdb/db/engine/flush/MemTableFlushTask.java | 12 +- .../db/engine/storagegroup/TsFileProcessor.java | 5 +- .../org/apache/iotdb/db/metadata/MManager.java | 40 +-- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 6 - .../java/org/apache/iotdb/db/service/IoTDB.java | 4 - .../iotdb/db/tools/memestimation/MemEstTool.java | 93 ------- .../db/tools/memestimation/MemEstToolCmd.java | 118 -------- .../conf/adapter/ActiveTimeSeriesCounterTest.java | 20 +- .../db/conf/adapter/CompressionRatioTest.java | 2 +- .../adapter/IoTDBConfigDynamicAdapterTest.java | 133 --------- .../db/engine/cache/ChunkMetadataCacheTest.java | 5 +- .../db/engine/memtable/MemTableFlushTaskTest.java | 4 +- .../storagegroup/StorageGroupProcessorTest.java | 4 +- .../storagegroup/TsFileProcessorEnableVmTest.java | 4 +- .../engine/storagegroup/TsFileProcessorTest.java | 4 +- .../iotdb/db/metadata/MManagerBasicTest.java | 8 +- .../iotdb/db/query/reader/ReaderTestHelper.java | 69 ----- .../apache/iotdb/db/utils/EnvironmentUtils.java | 6 +- .../recover/RecoverResourceFromReaderTest.java | 4 +- .../db/writelog/recover/SeqTsFileRecoverTest.java | 6 +- .../writelog/recover/UnseqTsFileRecoverTest.java | 4 +- .../apache/iotdb/spark/db/EnvironmentUtils.java | 6 +- 33 files changed, 67 insertions(+), 1050 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 2f3b22b..0925726 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -223,7 +223,7 @@ mtree_snapshot_threshold_time=3600 # memtable_size_threshold = tsfile_size_threshold = IoTDB memory allocation in byte / 2 / num of storage group / 4 # Make sure the total num of time series in system * primitive_array_size * 16 <= IoTDB memory allocation in byte / 2 / 4 # you could reduce the primitive_array_size in very high workload. -enable_parameter_adapter=true +enable_active_timeseries_counter=true # Memory Allocation Ratio: Write, Read, and Free Memory. # The parameter form is a:b:c, where a, b and c are integers. for example: 1:1:1 , 6:3:1 diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 037356b..592f0ba 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -132,7 +132,7 @@ public class IoTDBConfig { */ //the default value of this parameter should be kept true in iotdb-engine.properties, //we set it as false here for convenient testing. - private boolean enableParameterAdapter = false; + private boolean enableActiveTimeseriesCounter = false; /** * Ratio of memory allocated for buffered arrays @@ -1129,12 +1129,12 @@ public class IoTDBConfig { this.mergeIntervalSec = mergeIntervalSec; } - public boolean isEnableParameterAdapter() { - return enableParameterAdapter; + public boolean isEnableActiveTimeseriesCounter() { + return enableActiveTimeseriesCounter; } - public void setEnableParameterAdapter(boolean enableParameterAdapter) { - this.enableParameterAdapter = enableParameterAdapter; + public void setEnableActiveTimeseriesCounter(boolean enableActiveTimeseriesCounter) { + this.enableActiveTimeseriesCounter = enableActiveTimeseriesCounter; } public double getBufferedArraysMemoryProportion() { diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 19ffc9f..ab8c31f 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -179,9 +179,9 @@ public class IoTDBDescriptor { conf.setTimestampPrecision(properties.getProperty("timestamp_precision", conf.getTimestampPrecision())); - conf.setEnableParameterAdapter( - Boolean.parseBoolean(properties.getProperty("enable_parameter_adapter", - Boolean.toString(conf.isEnableParameterAdapter())))); + conf.setEnableActiveTimeseriesCounter( + Boolean.parseBoolean(properties.getProperty("enable_active_timeseries_counter", + Boolean.toString(conf.isEnableActiveTimeseriesCounter())))); conf.setBufferedArraysMemoryProportion( Double.parseDouble(properties.getProperty("buffered_arrays_memory_proportion", @@ -650,14 +650,14 @@ public class IoTDBDescriptor { long tsfileSizeThreshold = Long.parseLong(properties .getProperty("tsfile_size_threshold", Long.toString(conf.getTsFileSizeThreshold())).trim()); - if (tsfileSizeThreshold >= 0 && !conf.isEnableParameterAdapter()) { + if (tsfileSizeThreshold >= 0 && !conf.isEnableActiveTimeseriesCounter()) { conf.setTsFileSizeThreshold(tsfileSizeThreshold); } long memTableSizeThreshold = Long.parseLong(properties .getProperty("memtable_size_threshold", Long.toString(conf.getMemtableSizeThreshold())).trim()); - if (memTableSizeThreshold > 0 && !conf.isEnableParameterAdapter()) { + if (memTableSizeThreshold > 0 && !conf.isEnableActiveTimeseriesCounter()) { conf.setMemtableSizeThreshold(memTableSizeThreshold); } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounter.java index 7251924..468a770 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounter.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounter.java @@ -26,9 +26,9 @@ import org.apache.iotdb.db.conf.IoTDBConstant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ActiveTimeSeriesCounter implements IActiveTimeSeriesCounter { +public class ActiveTimeseriesCounter implements IActiveTimeseriesCounter { - private static final Logger LOGGER = LoggerFactory.getLogger(ActiveTimeSeriesCounter.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ActiveTimeseriesCounter.class); /** * Map[StorageGroup, HyperLogLogCounter] */ @@ -141,10 +141,10 @@ public class ActiveTimeSeriesCounter implements IActiveTimeSeriesCounter { } private static class ActiveTimeSeriesCounterHolder { - private static final ActiveTimeSeriesCounter INSTANCE = new ActiveTimeSeriesCounter(); + private static final ActiveTimeseriesCounter INSTANCE = new ActiveTimeseriesCounter(); } - public static ActiveTimeSeriesCounter getInstance() { + public static ActiveTimeseriesCounter getInstance() { return ActiveTimeSeriesCounterHolder.INSTANCE; } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java index a5b8900..6867d9f 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java @@ -94,21 +94,6 @@ public class CompressionRatio { if (LOGGER.isInfoEnabled()) { LOGGER.info("Compression ratio is {}", compressionRatio.get()); } - if (CONFIG.isEnableParameterAdapter()) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info( - "After updating compression ratio, trying to adjust parameters, the original parameters: " - + "MemTableSize threshold is {}B, TsfileSize threshold is {}B", - CONFIG.getMemtableSizeThreshold(), CONFIG.getTsFileSizeThreshold()); - } - IoTDBConfigDynamicAdapter.getInstance().tryToAdaptParameters(); - if(LOGGER.isInfoEnabled()) { - LOGGER.info( - "After updating compression ratio, trying to adjust parameters, the modified parameters: " - + "MemTableSize threshold is {}B, TsfileSize threshold is {}B", - CONFIG.getMemtableSizeThreshold(), CONFIG.getTsFileSizeThreshold()); - } - } } /** diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IActiveTimeSeriesCounter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IActiveTimeSeriesCounter.java index 43bc10c..2e2518d 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IActiveTimeSeriesCounter.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IActiveTimeSeriesCounter.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.conf.adapter; -public interface IActiveTimeSeriesCounter { +public interface IActiveTimeseriesCounter { /** * Initialize the counter by adding a new HyperLogLog counter for the given storage group diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IDynamicAdapter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IDynamicAdapter.java deleted file mode 100644 index 1f6b58f..0000000 --- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IDynamicAdapter.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.conf.adapter; - -import org.apache.iotdb.db.exception.ConfigAdjusterException; - -public interface IDynamicAdapter { - - /** - * Adjust parameters of maxMemtableNumber, memtableSizeThreshold and tsFileSizeThreshold. - * @return {@code true} if it has appropriate parameters and adjust them successfully - * {@code false} adjusting parameters failed - */ - boolean tryToAdaptParameters(); - - /** - * Add or delete storage groups - * - * @param diff it's positive if add new storage groups; it's negative if delete old storage - * groups. - */ - void addOrDeleteStorageGroup(int diff) throws ConfigAdjusterException; - - /** - * Add or delete timeseries - * - * @param diff it's positive if create new timeseries; it's negative if delete old timeseris. - */ - void addOrDeleteTimeSeries(int diff) throws ConfigAdjusterException; - -} diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java deleted file mode 100644 index 9dbd0d7..0000000 --- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.conf.adapter; - -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.ConfigAdjusterException; -import org.apache.iotdb.db.rescon.PrimitiveArrayManager; -import org.apache.iotdb.db.service.IoTDB; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class is to dynamically adjust some important parameters of the system, determine the speed - * of MenTable brushing disk, the speed of file sealing and so on, with the continuous change of - * load in the process of system operation. - * <p> - * There are three dynamically adjustable parameters: maxMemTableNum, memtableSize and - * tsFileSizeThreshold. - * <p> - * 1. maxMemTableNum. This parameter represents the size of the MemTable available in the MemTable - * pool, which is closely related to the number of storage groups. When adding or deleting a storage - * group, the parameter also adds or deletes four MemTables. The reason why adding or deleting four - * MemTables is that when the system is running stably, the speed of the flush operation is faster - * than that of data writing, so one is used for the Flush process and the other is used for data - * writing. Otherwise, the system should limit the speed of data writing to maintain stability. And - * two for sequence data, two for unsequence data. - * <p> - * 2. memtableSize. This parameter determines the threshold value for the MemTable in memory to be - * flushed into disk. When the system load increases, the parameter should be set smaller so that - * the data in memory can be flushed into disk as soon as possible. - * <p> - * 3. tsFileSizeThreshold. This parameter determines the speed of the tsfile seal, and then - * determines the maximum size of metadata information maintained in memory. When the system load - * increases, the parameter should be smaller to seal the file as soon as possible, release the - * memory occupied by the corresponding metadata information as soon as possible. - * <p> - * The following equation is used to adjust the dynamic parameters of the data: - * <p> - * * - * - * Abbreviation of parameters: - * 1 memtableSize: m - * 2 maxMemTableNum: Nm - * 3 maxSeriesNumberAmongStorageGroup: Ns - * 4 tsFileSizeThreshold: Sf - * 5 CompressionRatio: c - * 6 chunk metadata size: a - * 7 static memory: b - * 8 allocate memory for write: S - * - * The equation: m * Nm + Nm * Ns * Sf * a * c / m + b = S - * Namely: MemTable data memory size + chunk metadata memory size + static memory size = memory size for write - * - */ -public class IoTDBConfigDynamicAdapter implements IDynamicAdapter { - - public static final String CREATE_STORAGE_GROUP = "create storage group"; - public static final String ADD_TIMESERIES = "add timeseries"; - /** - * Average queue length in memtable pool - */ - static final int MEM_TABLE_AVERAGE_QUEUE_LEN = 5; - private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConfigDynamicAdapter.class); - - // static parameter section - private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); - /** - * Metadata size of per timeseries, the default value is 2KB. - */ - private static final long TIMESERIES_METADATA_SIZE_IN_BYTE = 2L * 1024; - private static final double WAL_MEMORY_RATIO = 0.1; - public static final int MEMTABLE_NUM_FOR_EACH_PARTITION = 4; - /** - * Maximum amount of memory allocated for write process. - */ - private static long allocateMemoryForWrite = CONFIG.getAllocateMemoryForWrite(); - /** - * Metadata size of per chunk, the default value is 1.5 KB. - */ - private static long CHUNK_METADATA_SIZE_IN_BYTE = 1536L; - - // static memory section - /** - * Static memory, includes all timeseries metadata, which equals to - * TIMESERIES_METADATA_SIZE_IN_BYTE * totalTimeseriesNum, the unit is byte. - * <p> - * Currently, we think that static memory only consists of time series metadata information. We - * ignore the memory occupied by the tsfile information maintained in memory, because we think - * that this part occupies very little memory. - */ - private long staticMemory; - - private int totalStorageGroup; - - private int totalTimeseries; - - // MemTable section - - private int maxMemTableNum = MEM_TABLE_AVERAGE_QUEUE_LEN; - - private long currentMemTableSize; - - // Adapter section - - private boolean initialized = false; - - private IoTDBConfigDynamicAdapter() { - } - - public static void setChunkMetadataSizeInByte(long chunkMetadataSizeInByte) { - CHUNK_METADATA_SIZE_IN_BYTE = chunkMetadataSizeInByte; - } - - public static IoTDBConfigDynamicAdapter getInstance() { - return IoTDBConfigAdapterHolder.INSTANCE; - } - - @Override - public synchronized boolean tryToAdaptParameters() { - if (!CONFIG.isEnableParameterAdapter()) { - return true; - } - boolean canAdjust = true; - double ratio = CompressionRatio.getInstance().getRatio(); - long memtableSizeInByte = calcMemTableSize(ratio); - long memTableSizeFloorThreshold = getMemTableSizeFloorThreshold(); - long tsFileSizeThreshold = CONFIG.getTsFileSizeThreshold(); - if (memtableSizeInByte < memTableSizeFloorThreshold) { - if (LOGGER.isDebugEnabled() && initialized) { - LOGGER.debug("memtableSizeInByte {} is smaller than memTableSizeFloorThreshold {}", - memtableSizeInByte, memTableSizeFloorThreshold); - } - tsFileSizeThreshold = calcTsFileSizeThreshold(memTableSizeFloorThreshold, ratio); - if ((long) (tsFileSizeThreshold * ratio) < memTableSizeFloorThreshold) { - canAdjust = false; - } else { - // memtableSizeInByte need to be larger than memTableSizeFloorThreshold - memtableSizeInByte = Math.max(memTableSizeFloorThreshold, - memTableSizeFloorThreshold + ( - ((long) (tsFileSizeThreshold * ratio) - memTableSizeFloorThreshold) - >> 1)); - } - } - - if (canAdjust) { - CONFIG.setWalBufferSize( - (int) Math - .min(Integer.MAX_VALUE, allocateMemoryForWrite * WAL_MEMORY_RATIO / maxMemTableNum)); - CONFIG.setTsFileSizeThreshold(tsFileSizeThreshold); - CONFIG.setMemtableSizeThreshold(memtableSizeInByte); - if (LOGGER.isDebugEnabled() && initialized) { - LOGGER.debug( - "After adjusting, max memTable num is {}, tsFile threshold is {}, memtableSize is {}, memTableSizeFloorThreshold is {}, storage group = {}, total timeseries = {}, the max number of timeseries among storage groups = {}", - maxMemTableNum, tsFileSizeThreshold, memtableSizeInByte, memTableSizeFloorThreshold, - totalStorageGroup, totalTimeseries, - IoTDB.metaManager.getMaximalSeriesNumberAmongStorageGroups()); - } - currentMemTableSize = memtableSizeInByte; - } - if (!initialized) { - return true; - } - return canAdjust; - } - - /** - * Calculate appropriate MemTable size. Computing method refers to class annotations. - * - * @return MemTable byte size. If the value is -1, there is no valid solution. - */ - private long calcMemTableSize(double ratio) { - // when unit is byte, it's likely to cause Long type overflow. - // so when b is larger than Integer.MAC_VALUE use the unit KB. - double a = maxMemTableNum; - double b = allocateMemoryForWrite * (1 - WAL_MEMORY_RATIO) - staticMemory; - int magnification = b > Integer.MAX_VALUE ? 1024 : 1; - b /= magnification; - double c = - (double) CONFIG.getTsFileSizeThreshold() * maxMemTableNum * CHUNK_METADATA_SIZE_IN_BYTE - * IoTDB.metaManager.getMaximalSeriesNumberAmongStorageGroups() * ratio - / magnification / magnification; - double tempValue = b * b - 4 * a * c; - double memTableSize = ((b + Math.sqrt(tempValue)) / (2 * a)); - return tempValue < 0 ? -1 : (long) (memTableSize * magnification); - } - - /** - * Calculate appropriate Tsfile size based on MemTable size. Computing method refers to class - * annotations. - * - * @param memTableSize MemTable size - * @return Tsfile byte threshold - */ - private long calcTsFileSizeThreshold(long memTableSize, double ratio) { - return (long) ((allocateMemoryForWrite * (1 - WAL_MEMORY_RATIO) - maxMemTableNum * memTableSize - - staticMemory) * memTableSize / (ratio * maxMemTableNum * CHUNK_METADATA_SIZE_IN_BYTE - * IoTDB.metaManager.getMaximalSeriesNumberAmongStorageGroups())); - } - - /** - * Get the floor threshold MemTable size. For Primitive Array, we think that the maximum memory - * occupied by each value is 8 bytes. The reason for multiplying 2 is that the timestamp also - * takes 8 bytes. - */ - private long getMemTableSizeFloorThreshold() { - return IoTDB.metaManager.getMaximalSeriesNumberAmongStorageGroups() - * PrimitiveArrayManager.ARRAY_SIZE * Long.BYTES * 2; - } - - @Override - public void addOrDeleteStorageGroup(int diff) throws ConfigAdjusterException { - totalStorageGroup += diff; - maxMemTableNum += - MEMTABLE_NUM_FOR_EACH_PARTITION * IoTDBDescriptor.getInstance().getConfig().getConcurrentWritingTimePartition() * diff - + diff; - if (!CONFIG.isEnableParameterAdapter()) { - return; - } - - if (!tryToAdaptParameters()) { - totalStorageGroup -= diff; - maxMemTableNum -= - MEMTABLE_NUM_FOR_EACH_PARTITION * IoTDBDescriptor.getInstance().getConfig().getConcurrentWritingTimePartition() * diff - + diff; - throw new ConfigAdjusterException(CREATE_STORAGE_GROUP); - } - } - - @Override - public void addOrDeleteTimeSeries(int diff) throws ConfigAdjusterException { - if (!CONFIG.isEnableParameterAdapter()) { - return; - } - totalTimeseries += diff; - staticMemory += diff * TIMESERIES_METADATA_SIZE_IN_BYTE; - if (!tryToAdaptParameters()) { - totalTimeseries -= diff; - staticMemory -= diff * TIMESERIES_METADATA_SIZE_IN_BYTE; - throw new ConfigAdjusterException(ADD_TIMESERIES); - } - } - - public void setInitialized(boolean initialized) { - this.initialized = initialized; - } - - long getCurrentMemTableSize() { - return currentMemTableSize; - } - - public int getTotalTimeseries() { - return totalTimeseries; - } - - public int getTotalStorageGroup() { - return totalStorageGroup; - } - - /** - * Only for test - */ - public void reset() { - totalTimeseries = 0; - staticMemory = 0; - maxMemTableNum = MEM_TABLE_AVERAGE_QUEUE_LEN; - allocateMemoryForWrite = CONFIG.getAllocateMemoryForWrite(); - initialized = false; - } - - private static class IoTDBConfigAdapterHolder { - - private static final IoTDBConfigDynamicAdapter INSTANCE = new IoTDBConfigDynamicAdapter(); - - private IoTDBConfigAdapterHolder() { - - } - - } - -} diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/ManageDynamicParameters.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/ManageDynamicParameters.java deleted file mode 100644 index 4021776..0000000 --- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/ManageDynamicParameters.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.conf.adapter; - -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBConstant; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.StartupException; -import org.apache.iotdb.db.service.IService; -import org.apache.iotdb.db.service.IoTDB; -import org.apache.iotdb.db.service.JMXService; -import org.apache.iotdb.db.service.ServiceType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class is to get and set dynamic parameters through JMX. - */ -public class ManageDynamicParameters implements ManageDynamicParametersMBean, IService { - - private static final Logger LOGGER = LoggerFactory.getLogger(ManageDynamicParameters.class); - - private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); - - private final String mbeanName = String - .format("%s:%s=%s", IoTDBConstant.IOTDB_PACKAGE, IoTDBConstant.JMX_TYPE, - getID().getJmxName()); - - private ManageDynamicParameters() { - - } - - public static ManageDynamicParameters getInstance() { - return ManageDynamicParametersHolder.INSTANCE; - } - - @Override - public void showDynamicParameters() { - LOGGER.info( - "Memtable size threshold: {}B, Tsfile size threshold: {}B, Compression ratio: {}, " - + "Storage group number: {}, Timeseries number: {}, Maximal timeseries number among storage groups: {}", - CONFIG.getMemtableSizeThreshold(), - CONFIG.getTsFileSizeThreshold(), CompressionRatio.getInstance().getRatio(), - IoTDBConfigDynamicAdapter.getInstance().getTotalStorageGroup(), - IoTDBConfigDynamicAdapter.getInstance().getTotalTimeseries(), - IoTDB.metaManager.getMaximalSeriesNumberAmongStorageGroups()); - } - - @Override - public boolean isEnableDynamicAdapter() { - return CONFIG.isEnableParameterAdapter(); - } - - @Override - public void setEnableDynamicAdapter(boolean enableDynamicAdapter) { - CONFIG.setEnableParameterAdapter(enableDynamicAdapter); - } - - @Override - public long getMemTableSizeThreshold() { - return CONFIG.getMemtableSizeThreshold(); - } - - @Override - public void setMemTableSizeThreshold(long memTableSizeThreshold) { - CONFIG.setMemtableSizeThreshold(memTableSizeThreshold); - } - - @Override - public long getTsfileSizeThreshold() { - return CONFIG.getTsFileSizeThreshold(); - } - - @Override - public void setTsfileSizeThreshold(long tsfileSizeThreshold) { - CONFIG.setTsFileSizeThreshold(tsfileSizeThreshold); - } - - @Override - public void start() throws StartupException { - try { - JMXService.registerMBean(getInstance(), mbeanName); - LOGGER.info("{}: start {}...", IoTDBConstant.GLOBAL_DB_NAME, this.getID().getName()); - } catch (Exception e) { - LOGGER.error("Failed to start {} because: ", this.getID().getName(), e); - throw new StartupException(e); - } - } - - @Override - public void stop() { - JMXService.deregisterMBean(mbeanName); - LOGGER.info("{}: stop {}...", IoTDBConstant.GLOBAL_DB_NAME, this.getID().getName()); - } - - @Override - public ServiceType getID() { - return ServiceType.MANAGE_DYNAMIC_PARAMETERS_SERVICE; - } - - private static class ManageDynamicParametersHolder { - - private static final ManageDynamicParameters INSTANCE = new ManageDynamicParameters(); - - private ManageDynamicParametersHolder() { - - } - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/ManageDynamicParametersMBean.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/ManageDynamicParametersMBean.java deleted file mode 100644 index 08c5391..0000000 --- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/ManageDynamicParametersMBean.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.conf.adapter; - -public interface ManageDynamicParametersMBean { - - /** - * Show all dynamic parameters, including memtable size threshold, memtable number, tsfile size - * threshold, compression ratio, storage group number, timeseries number, max timeseries number - * among storage groups. - */ - void showDynamicParameters(); - - boolean isEnableDynamicAdapter(); - - void setEnableDynamicAdapter(boolean enableDynamicAdapter); - - long getMemTableSizeThreshold(); - - void setMemTableSizeThreshold(long memTableSizeThreshold); - - long getTsfileSizeThreshold(); - - void setTsfileSizeThreshold(long tsfileSizeThreshold); - -} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java index aca5ad2..6359ae9 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java @@ -27,7 +27,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.db.utils.FileLoaderUtils; @@ -78,7 +77,6 @@ public class ChunkMetadataCache { if (count < 10) { long currentSize = value.get(0).calculateRamSize(); averageSize = ((averageSize * count) + currentSize) / (++count); - IoTDBConfigDynamicAdapter.setChunkMetadataSizeInByte(averageSize); entrySize = RamUsageEstimator.sizeOf(key) + (currentSize + RamUsageEstimator.NUM_BYTES_OBJECT_REF) * value.size() + RamUsageEstimator.shallowSizeOf(value); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java index 25c9878..695b632 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java @@ -28,7 +28,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; +import org.apache.iotdb.db.conf.adapter.ActiveTimeseriesCounter; import org.apache.iotdb.db.engine.flush.pool.FlushSubTaskPoolManager; import org.apache.iotdb.db.engine.memtable.IMemTable; import org.apache.iotdb.db.engine.memtable.IWritableMemChunk; @@ -104,15 +104,15 @@ public class MemTableFlushTask { TVList tvList = series.getSortedTVList(); sortTime += System.currentTimeMillis() - startTime; encodingTaskQueue.add(new Pair<>(tvList, desc)); - // register active time series to the ActiveTimeSeriesCounter - if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) { - ActiveTimeSeriesCounter.getInstance().offer(storageGroup, deviceId, measurementId); + // register active time series to the ActiveTimeseriesCounter + if (IoTDBDescriptor.getInstance().getConfig().isEnableActiveTimeseriesCounter()) { + ActiveTimeseriesCounter.getInstance().offer(storageGroup, deviceId, measurementId); } } encodingTaskQueue.add(new EndChunkGroupIoTask()); } - if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) { - ActiveTimeSeriesCounter.getInstance().updateActiveRatio(storageGroup); + if (IoTDBDescriptor.getInstance().getConfig().isEnableActiveTimeseriesCounter()) { + ActiveTimeseriesCounter.getInstance().updateActiveRatio(storageGroup); } noMoreEncodingTask = true; logger.debug( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index da73f57..12a228b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -45,7 +45,6 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.conf.adapter.CompressionRatio; -import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; import org.apache.iotdb.db.engine.flush.FlushManager; import org.apache.iotdb.db.engine.flush.MemTableFlushTask; @@ -468,9 +467,7 @@ public class TsFileProcessor { } /** - * <p>In the dynamic parameter adjustment module{@link IoTDBConfigDynamicAdapter}, it calculated - * the average size of each metatable{@link IoTDBConfigDynamicAdapter#tryToAdaptParameters()}. - * However, considering that the number of timeseries between storage groups may vary greatly, + * Considering that the number of timeseries between storage groups may vary greatly, * it's inappropriate to judge whether to flush the memtable according to the average memtable * size. We need to adjust it according to the number of timeseries in a specific storage group. */ diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index 469584e..4ffca32 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -46,11 +46,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; -import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; +import org.apache.iotdb.db.conf.adapter.ActiveTimeseriesCounter; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; -import org.apache.iotdb.db.exception.ConfigAdjusterException; import org.apache.iotdb.db.exception.metadata.DeleteFailedException; import org.apache.iotdb.db.exception.metadata.IllegalPathException; import org.apache.iotdb.db.exception.metadata.MetadataException; @@ -212,7 +210,7 @@ public class MManager { isRecovering = true; int lineNumber = initFromLog(logFile); - if (config.isEnableParameterAdapter()) { + if (config.isEnableActiveTimeseriesCounter()) { List<String> storageGroups = mtree.getAllStorageGroupNames(); for (String sg : storageGroups) { MNode node = mtree.getNodeByPath(sg); @@ -397,9 +395,6 @@ public class MManager { setStorageGroup(storageGroupName); } - // check memory - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1); - TSDataType type = plan.getDataType(); // create time series in MTree MeasurementMNode leafMNode = mtree @@ -416,7 +411,7 @@ public class MManager { } // update statistics - if (config.isEnableParameterAdapter()) { + if (config.isEnableActiveTimeseriesCounter()) { int size = seriesNumberInStorageGroups.get(storageGroupName); seriesNumberInStorageGroups.put(storageGroupName, size + 1); if (size + 1 > maxSeriesNumberAmongStorageGroup) { @@ -438,7 +433,7 @@ public class MManager { } leafMNode.setOffset(offset); - } catch (IOException | ConfigAdjusterException e) { + } catch (IOException e) { throw new MetadataException(e.getMessage()); } finally { lock.writeLock().unlock(); @@ -473,7 +468,7 @@ public class MManager { if (isStorageGroup(prefixPath)) { - if (config.isEnableParameterAdapter()) { + if (config.isEnableActiveTimeseriesCounter()) { int size = seriesNumberInStorageGroups.get(prefixPath); seriesNumberInStorageGroups.put(prefixPath, 0); if (size == maxSeriesNumberAmongStorageGroup) { @@ -568,13 +563,7 @@ public class MManager { // TODO: delete the path node and all its ancestors mNodeCache.clear(); - try { - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(-1); - } catch (ConfigAdjusterException e) { - throw new MetadataException(e); - } - - if (config.isEnableParameterAdapter()) { + if (config.isEnableActiveTimeseriesCounter()) { String storageGroup = getStorageGroupName(path); int size = seriesNumberInStorageGroups.get(storageGroup); seriesNumberInStorageGroups.put(storageGroup, size - 1); @@ -598,10 +587,8 @@ public class MManager { lock.writeLock().lock(); try { mtree.setStorageGroup(storageGroup); - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(1); - - if (config.isEnableParameterAdapter()) { - ActiveTimeSeriesCounter.getInstance().init(storageGroup); + if (config.isEnableActiveTimeseriesCounter()) { + ActiveTimeseriesCounter.getInstance().init(storageGroup); seriesNumberInStorageGroups.put(storageGroup, 0); } if (!isRecovering) { @@ -609,9 +596,6 @@ public class MManager { } } catch (IOException e) { throw new MetadataException(e.getMessage()); - } catch (ConfigAdjusterException e) { - mtree.deleteStorageGroup(storageGroup); - throw new MetadataException(e); } finally { lock.writeLock().unlock(); } @@ -638,11 +622,9 @@ public class MManager { updateSchemaDataTypeNumMap(leafMNode.getSchema().getType(), -1); } - if (config.isEnableParameterAdapter()) { - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(-1); + if (config.isEnableActiveTimeseriesCounter()) { int size = seriesNumberInStorageGroups.get(storageGroup); - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(size * -1); - ActiveTimeSeriesCounter.getInstance().delete(storageGroup); + ActiveTimeseriesCounter.getInstance().delete(storageGroup); seriesNumberInStorageGroups.remove(storageGroup); if (size == maxSeriesNumberAmongStorageGroup) { maxSeriesNumberAmongStorageGroup = @@ -654,8 +636,6 @@ public class MManager { logWriter.deleteStorageGroup(storageGroup); } } - } catch (ConfigAdjusterException e) { - throw new MetadataException(e); } catch (IOException e) { throw new MetadataException(e.getMessage()); } finally { diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java index c6cfa2a..325b8a8 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java @@ -58,7 +58,6 @@ import org.apache.iotdb.db.auth.entity.User; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.conf.adapter.CompressionRatio; -import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; @@ -597,11 +596,6 @@ public class PlanExecutor implements IPlanExecutor { Integer.toString(IoTDB.metaManager.getAllStorageGroupNames().size())); addRowRecordForShowQuery( listDataSet, - timestamp++, - "timeseries number", - Integer.toString(IoTDBConfigDynamicAdapter.getInstance().getTotalTimeseries())); - addRowRecordForShowQuery( - listDataSet, timestamp, "maximal timeseries number among storage groups", Long.toString(IoTDB.metaManager.getMaximalSeriesNumberAmongStorageGroups())); diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java index 4b2af1c..b6d15b0 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java +++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java @@ -22,8 +22,6 @@ import org.apache.iotdb.db.concurrent.IoTDBDefaultThreadExceptionHandler; import org.apache.iotdb.db.conf.IoTDBConfigCheck; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; -import org.apache.iotdb.db.conf.adapter.ManageDynamicParameters; import org.apache.iotdb.db.cost.statistic.Measurement; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.cache.CacheHitRatioMonitor; @@ -102,7 +100,6 @@ public class IoTDB implements IoTDBMBean { registerManager.register(Monitor.getInstance()); registerManager.register(StatMonitor.getInstance()); registerManager.register(Measurement.INSTANCE); - registerManager.register(ManageDynamicParameters.getInstance()); registerManager.register(TVListAllocator.getInstance()); registerManager.register(CacheHitRatioMonitor.getInstance()); JMXService.registerMBean(getInstance(), mbeanName); @@ -153,7 +150,6 @@ public class IoTDB implements IoTDBMBean { IoTDB.metaManager.init(); long end = System.currentTimeMillis() - time; logger.info("spend {}ms to recover schema.", end); - IoTDBConfigDynamicAdapter.getInstance().setInitialized(true); logger.info( "After initializing, tsFile threshold is {}, memtableSize is {}", IoTDBDescriptor.getInstance().getConfig().getTsFileSizeThreshold(), diff --git a/server/src/main/java/org/apache/iotdb/db/tools/memestimation/MemEstTool.java b/server/src/main/java/org/apache/iotdb/db/tools/memestimation/MemEstTool.java deleted file mode 100644 index f765cbb..0000000 --- a/server/src/main/java/org/apache/iotdb/db/tools/memestimation/MemEstTool.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.tools.memestimation; - -import com.google.common.base.Throwables; -import com.google.common.collect.Lists; -import io.airlift.airline.Cli; -import io.airlift.airline.Help; -import io.airlift.airline.ParseArgumentsMissingException; -import io.airlift.airline.ParseArgumentsUnexpectedException; -import io.airlift.airline.ParseCommandMissingException; -import io.airlift.airline.ParseCommandUnrecognizedException; -import io.airlift.airline.ParseOptionConversionException; -import io.airlift.airline.ParseOptionMissingException; -import io.airlift.airline.ParseOptionMissingValueException; - -import java.io.IOException; -import java.util.List; -import org.apache.commons.io.FileUtils; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; - -public class MemEstTool { - - public static void main(String... args) throws IOException { - List<Class<? extends Runnable>> commands = Lists.newArrayList( - Help.class, - MemEstToolCmd.class - ); - Cli.CliBuilder<Runnable> builder = Cli.builder("memory-tool"); - - builder.withDescription("Estimate memory for writing") - .withDefaultCommand(Help.class) - .withCommands(commands); - - Cli<Runnable> parser = builder.build(); - - int status = 0; - try { - Runnable parse = parser.parse(args); - parse.run(); - } catch (IllegalArgumentException | - IllegalStateException | - ParseArgumentsMissingException | - ParseArgumentsUnexpectedException | - ParseOptionConversionException | - ParseOptionMissingException | - ParseOptionMissingValueException | - ParseCommandMissingException | - ParseCommandUnrecognizedException e) { - badUse(e); - status = 1; - } catch (Exception e) { - err(Throwables.getRootCause(e)); - status = 2; - } - - FileUtils.deleteDirectory(SystemFileFactory.INSTANCE.getFile(IoTDBDescriptor.getInstance().getConfig().getSystemDir())); - FileUtils.deleteDirectory(SystemFileFactory.INSTANCE.getFile(IoTDBDescriptor.getInstance().getConfig().getWalDir())); - for(int i=0; i < IoTDBDescriptor.getInstance().getConfig().getDataDirs().length; i++){ - FileUtils.deleteDirectory(SystemFileFactory.INSTANCE.getFile(IoTDBDescriptor.getInstance().getConfig().getDataDirs()[i])); - } - - System.exit(status); - } - - private static void badUse(Exception e) { - System.out.println("memory-tool: " + e.getMessage()); - System.out.println("See 'memory-tool help' or 'memory-tool help <command>'."); - } - - private static void err(Throwable e) { - System.err.println("error: " + e.getMessage()); - System.err.println("-- StackTrace --"); - System.err.println(Throwables.getStackTraceAsString(e)); - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/tools/memestimation/MemEstToolCmd.java b/server/src/main/java/org/apache/iotdb/db/tools/memestimation/MemEstToolCmd.java deleted file mode 100644 index a7ea25d..0000000 --- a/server/src/main/java/org/apache/iotdb/db/tools/memestimation/MemEstToolCmd.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.tools.memestimation; - -import io.airlift.airline.Command; -import io.airlift.airline.Option; -import org.apache.iotdb.db.concurrent.WrappedRunnable; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBConstant; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; -import org.apache.iotdb.db.exception.ConfigAdjusterException; -import org.apache.iotdb.db.service.IoTDB; - -@Command(name = "calmem", description = "calculate minimum memory required for writing based on the number of storage groups and timeseries") -public class MemEstToolCmd extends WrappedRunnable { - - @Option(title = "storage group number", name = {"-sg", - "--storagegroup"}, description = "Storage group number") - private String sgNumString = "10"; - - @Option(title = "total timeseries number", name = {"-ts", - "--timeseries"}, description = "Total timeseries number") - private String tsNumString = "1000"; - - @Option(title = "max timeseries", name = {"-mts", - "--maxtimeseries"}, description = "Maximum timeseries number among storage groups, make sure that it's smaller than total timeseries number") - private String maxTsNumString = "0"; - - @Override - public void runMayThrow() { - // backup origin config parameters - IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - long memTableSize = config.getMemtableSizeThreshold(); - long tsFileSize = config.getTsFileSizeThreshold(); - - // parse input parameters - long sgNum = Long.parseLong(sgNumString); - long tsNum = Long.parseLong(tsNumString); - long maxTsNum = Long.parseLong(maxTsNumString); - - // tool parameters - long stepMemory = calStepMemory(tsNum) * IoTDBConstant.GB; - long currentMemory = stepMemory; - long maxTsNumValid = maxTsNum; - long maxProcess = 0; - long start = System.currentTimeMillis(); - while (true) { - - // recover config parameter - config.setAllocateMemoryForWrite(currentMemory); - config.setMemtableSizeThreshold(memTableSize); - config.setTsFileSizeThreshold(tsFileSize); - IoTDBConfigDynamicAdapter.getInstance().reset(); - IoTDBConfigDynamicAdapter.getInstance().setInitialized(true); - IoTDB.metaManager.clear(); - - long sgCnt = 1; - long tsCnt = 0; - try { - for (; sgCnt <= sgNum; sgCnt++) { - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(1); - } - for (; tsCnt < tsNum; tsCnt++) { - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1); - if (maxTsNum == 0) { - maxTsNumValid = tsCnt / sgNum + 1; - } else { - maxTsNumValid = Math.min(tsCnt, maxTsNum); - maxTsNumValid = Math.max(maxTsNumValid, tsCnt / sgNum + 1); - } - IoTDB.metaManager.setMaxSeriesNumberAmongStorageGroup(maxTsNumValid); - } - - } catch (ConfigAdjusterException e) { - if (sgCnt > sgNum) { - maxProcess = Math.max(maxProcess, (tsCnt + 1) * 100 / tsNum); - System.out - .print(String.format("Memory estimation progress : %d%%\r", maxProcess)); - } - currentMemory += stepMemory; - continue; - } - break; - } - System.out.println(String - .format("Memory for writing: %dGB, SG: %d, TS: %d, MTS: %d", currentMemory / IoTDBConstant.GB, - sgNum, tsNum, maxTsNumValid)); - System.out.println(String.format("Calculating memory for writing consumes: %dms", - (System.currentTimeMillis() - start))); - } - - private long calStepMemory(long maxTimeseriesNumber) { - maxTimeseriesNumber /= 10000000; - int step = 1; - while (maxTimeseriesNumber > 0) { - maxTimeseriesNumber /= 10; - step *= 10; - } - return step; - } -} diff --git a/server/src/test/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounterTest.java b/server/src/test/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounterTest.java index 0cd020f..2651c0c 100644 --- a/server/src/test/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounterTest.java +++ b/server/src/test/java/org/apache/iotdb/db/conf/adapter/ActiveTimeSeriesCounterTest.java @@ -30,9 +30,9 @@ import org.junit.Before; import org.junit.Test; /** - * ActiveTimeSeriesCounter Tester. + * ActiveTimeseriesCounter Tester. */ -public class ActiveTimeSeriesCounterTest { +public class ActiveTimeseriesCounterTest { private static final String TEST_SG_PREFIX = "root.sg_"; private static int testStorageGroupNum = 10; @@ -50,16 +50,16 @@ public class ActiveTimeSeriesCounterTest { @Before public void before() throws Exception { - ActiveTimeSeriesCounter.clear(); + ActiveTimeseriesCounter.clear(); for (String storageGroup : storageGroups) { - ActiveTimeSeriesCounter.getInstance().init(storageGroup); + ActiveTimeseriesCounter.getInstance().init(storageGroup); } } @After public void after() throws Exception { for (String storageGroup : storageGroups) { - ActiveTimeSeriesCounter.getInstance().delete(storageGroup); + ActiveTimeseriesCounter.getInstance().delete(storageGroup); } } @@ -69,7 +69,7 @@ public class ActiveTimeSeriesCounterTest { @Test public void testInit() throws Exception { for (int i = 0; i < testStorageGroupNum; i++) { - assertEquals(0D, ActiveTimeSeriesCounter.getInstance().getActiveRatio(storageGroups[i]), 0.0); + assertEquals(0D, ActiveTimeseriesCounter.getInstance().getActiveRatio(storageGroups[i]), 0.0); } } @@ -85,15 +85,15 @@ public class ActiveTimeSeriesCounterTest { } finished.await(); for (String storageGroup : storageGroups) { - ActiveTimeSeriesCounter.getInstance().updateActiveRatio(storageGroup); + ActiveTimeseriesCounter.getInstance().updateActiveRatio(storageGroup); double sum = 0; for (String s : storageGroups) { - sum += ActiveTimeSeriesCounter.getInstance().getActiveRatio(s); + sum += ActiveTimeseriesCounter.getInstance().getActiveRatio(s); } assertEquals(1.0, sum, 0.001); } for (int i = 0; i < storageGroups.length; i++) { - double r = ActiveTimeSeriesCounter.getInstance().getActiveRatio(storageGroups[i]); + double r = ActiveTimeseriesCounter.getInstance().getActiveRatio(storageGroups[i]); assertEquals(measurementNum[i] / totalSeriesNum, r, 0.001); } } @@ -113,7 +113,7 @@ public class ActiveTimeSeriesCounterTest { public void runMayThrow() { try { for (int j = 0; j < sensorNum; j++) { - ActiveTimeSeriesCounter.getInstance().offer(storageGroup, "device_0", "sensor_" + j); + ActiveTimeseriesCounter.getInstance().offer(storageGroup, "device_0", "sensor_" + j); } }finally { finished.countDown(); diff --git a/server/src/test/java/org/apache/iotdb/db/conf/adapter/CompressionRatioTest.java b/server/src/test/java/org/apache/iotdb/db/conf/adapter/CompressionRatioTest.java index 4269f8f..7bb382e 100644 --- a/server/src/test/java/org/apache/iotdb/db/conf/adapter/CompressionRatioTest.java +++ b/server/src/test/java/org/apache/iotdb/db/conf/adapter/CompressionRatioTest.java @@ -49,7 +49,7 @@ public class CompressionRatioTest { FileUtils.forceMkdir(new File(directory)); compressionRatio.reset(); compressionRatio.restore(); - IoTDBDescriptor.getInstance().getConfig().setEnableParameterAdapter(true); + IoTDBDescriptor.getInstance().getConfig().setEnableActiveTimeseriesCounter(true); } @After diff --git a/server/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java b/server/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java deleted file mode 100644 index 2999b1a..0000000 --- a/server/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.conf.adapter; - -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.ConfigAdjusterException; -import org.apache.iotdb.db.service.IoTDB; -import org.apache.iotdb.db.utils.EnvironmentUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import static org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter.MEMTABLE_NUM_FOR_EACH_PARTITION; -import static org.junit.Assert.assertEquals; - -public class IoTDBConfigDynamicAdapterTest { - - private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); - - private long oldTsFileThreshold = CONFIG.getTsFileSizeThreshold(); - - private long oldGroupSizeInByte = CONFIG.getMemtableSizeThreshold(); - - @Before - public void setUp() throws Exception { - EnvironmentUtils.closeStatMonitor(); - EnvironmentUtils.envSetUp(); - IoTDBDescriptor.getInstance().getConfig().setEnableParameterAdapter(true); - } - - @After - public void tearDown() throws Exception { - EnvironmentUtils.cleanEnv(); - CONFIG.setTsFileSizeThreshold(oldTsFileThreshold); - CONFIG.setMemtableSizeThreshold(oldGroupSizeInByte); - IoTDB.metaManager.setMaxSeriesNumberAmongStorageGroup(0); - IoTDBConfigDynamicAdapter.getInstance().reset(); - } - - @Test - public void addOrDeleteStorageGroup() throws ConfigAdjusterException { - int memTableNum = IoTDBConfigDynamicAdapter.MEM_TABLE_AVERAGE_QUEUE_LEN; - for (int i = 1; i < 100; i++) { - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1); - } - IoTDB.metaManager.setMaxSeriesNumberAmongStorageGroup(100); - for (int i = 1; i < 1000000; i++) { - try { - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(1); - memTableNum += IoTDBDescriptor.getInstance().getConfig().getConcurrentWritingTimePartition() * MEMTABLE_NUM_FOR_EACH_PARTITION + 1; - assertEquals(IoTDBConfigDynamicAdapter.getInstance().getCurrentMemTableSize(), - CONFIG.getMemtableSizeThreshold()); - } catch (ConfigAdjusterException e) { - assertEquals(String.format(ConfigAdjusterException.ERROR_MSG_FORMAT, - IoTDBConfigDynamicAdapter.CREATE_STORAGE_GROUP), e.getMessage()); - break; - } - } - } - - @Test - public void addOrDeleteTimeSeries() throws ConfigAdjusterException { - int totalTimeseries = 0; - for (int i = 1; i < 100; i++) { - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(1); - } - IoTDB.metaManager.setMaxSeriesNumberAmongStorageGroup(100); - for (int i = 1; i < 1000000; i++) { - try { - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1); - - if (i % 10 == 0) { - IoTDB.metaManager.setMaxSeriesNumberAmongStorageGroup(i); - } - totalTimeseries += 1; - assertEquals(IoTDBConfigDynamicAdapter.getInstance().getCurrentMemTableSize(), - CONFIG.getMemtableSizeThreshold()); - assertEquals(IoTDBConfigDynamicAdapter.getInstance().getTotalTimeseries(), - totalTimeseries); - } catch (ConfigAdjusterException e) { - assertEquals(String.format(ConfigAdjusterException.ERROR_MSG_FORMAT, - IoTDBConfigDynamicAdapter.ADD_TIMESERIES), e.getMessage()); - assertEquals(IoTDBConfigDynamicAdapter.getInstance().getTotalTimeseries(), - totalTimeseries); - break; - } - } - } - - @Test - public void addOrDeleteTimeSeriesSyso() throws ConfigAdjusterException { - int sgNum = 1; - for (int i = 0; i < 30; i++) { - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(sgNum); - } - int i = 1; - try { - for (; i <= 280 * 3200; i++) { - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1); - IoTDB.metaManager.setMaxSeriesNumberAmongStorageGroup(i / 30 + 1); - } - } catch (ConfigAdjusterException e) { - assertEquals(String.format(ConfigAdjusterException.ERROR_MSG_FORMAT, - IoTDBConfigDynamicAdapter.ADD_TIMESERIES), e.getMessage()); - } - try { - while (true) { - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1); - IoTDB.metaManager.setMaxSeriesNumberAmongStorageGroup(IoTDB.metaManager.getMaximalSeriesNumberAmongStorageGroups() + 1); - } - } catch (ConfigAdjusterException e ) { - assertEquals(String.format(ConfigAdjusterException.ERROR_MSG_FORMAT, - IoTDBConfigDynamicAdapter.ADD_TIMESERIES), e.getMessage()); - } - } -} \ No newline at end of file diff --git a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java index a4b5825..f1ef552 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java @@ -22,7 +22,7 @@ import java.io.File; import java.io.IOException; import java.util.List; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; +import org.apache.iotdb.db.conf.adapter.ActiveTimeseriesCounter; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.MetadataManagerHelper; import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy.DirectFlushPolicy; @@ -34,7 +34,6 @@ import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.mnode.MNode; import org.apache.iotdb.db.metadata.mnode.MeasurementMNode; -import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.FileReaderManager; @@ -68,7 +67,7 @@ public class ChunkMetadataCacheTest { public void setUp() throws Exception { EnvironmentUtils.envSetUp(); MetadataManagerHelper.initMetadata(); - ActiveTimeSeriesCounter.getInstance().init(storageGroup); + ActiveTimeseriesCounter.getInstance().init(storageGroup); storageGroupProcessor = new StorageGroupProcessor(systemDir, storageGroup, new DirectFlushPolicy()); insertData(); } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java index d1ce17d..4ba2337 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java @@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.concurrent.ExecutionException; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; +import org.apache.iotdb.db.conf.adapter.ActiveTimeseriesCounter; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.MetadataManagerHelper; import org.apache.iotdb.db.engine.flush.MemTableFlushTask; @@ -51,7 +51,7 @@ public class MemTableFlushTaskTest { public void setUp() throws Exception { EnvironmentUtils.envSetUp(); MetadataManagerHelper.initMetadata(); - ActiveTimeSeriesCounter.getInstance().init(storageGroup); + ActiveTimeseriesCounter.getInstance().init(storageGroup); writer = new RestorableTsFileIOWriter(FSFactoryProducer.getFSFactory().getFile(filePath)); memTable = new PrimitiveMemTable(); IoTDBDescriptor.getInstance().getConfig().setEnableVm(false); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java index c2352d8..980e8e4 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java @@ -27,7 +27,7 @@ import java.util.Collections; import java.util.List; import java.util.Random; import java.util.concurrent.atomic.AtomicLong; -import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; +import org.apache.iotdb.db.conf.adapter.ActiveTimeseriesCounter; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.MetadataManagerHelper; import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy; @@ -72,7 +72,7 @@ public class StorageGroupProcessorTest { public void setUp() throws Exception { MetadataManagerHelper.initMetadata(); EnvironmentUtils.envSetUp(); - ActiveTimeSeriesCounter.getInstance().init(storageGroup); + ActiveTimeseriesCounter.getInstance().init(storageGroup); processor = new DummySGP(systemDir, storageGroup); MergeManager.getINSTANCE().start(); deviceMNode = new MNode(null, deviceId); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorEnableVmTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorEnableVmTest.java index 237efe7..6a94704 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorEnableVmTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorEnableVmTest.java @@ -30,7 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; +import org.apache.iotdb.db.conf.adapter.ActiveTimeseriesCounter; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.MetadataManagerHelper; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; @@ -73,7 +73,7 @@ public class TsFileProcessorEnableVmTest { public void setUp() throws Exception { EnvironmentUtils.envSetUp(); MetadataManagerHelper.initMetadata(); - ActiveTimeSeriesCounter.getInstance().init(storageGroup); + ActiveTimeseriesCounter.getInstance().init(storageGroup); context = EnvironmentUtils.TEST_QUERY_CONTEXT; IoTDBDescriptor.getInstance().getConfig().setEnableVm(true); } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java index 19975aa..a5a8a1b 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; +import org.apache.iotdb.db.conf.adapter.ActiveTimeseriesCounter; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.MetadataManagerHelper; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; @@ -73,7 +73,7 @@ public class TsFileProcessorTest { public void setUp() throws Exception { EnvironmentUtils.envSetUp(); MetadataManagerHelper.initMetadata(); - ActiveTimeSeriesCounter.getInstance().init(storageGroup); + ActiveTimeseriesCounter.getInstance().init(storageGroup); context = EnvironmentUtils.TEST_QUERY_CONTEXT; IoTDBDescriptor.getInstance().getConfig().setEnableVm(false); } diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java index ea00cf1..b5b5bdf 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java @@ -38,19 +38,19 @@ import static org.junit.Assert.*; public class MManagerBasicTest { private CompressionType compressionType; - private boolean canAdjust = IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter(); + private boolean canAdjust = IoTDBDescriptor.getInstance().getConfig().isEnableActiveTimeseriesCounter(); @Before public void setUp() throws Exception { - canAdjust = IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter(); + canAdjust = IoTDBDescriptor.getInstance().getConfig().isEnableActiveTimeseriesCounter(); compressionType = TSFileDescriptor.getInstance().getConfig().getCompressor(); EnvironmentUtils.envSetUp(); - IoTDBDescriptor.getInstance().getConfig().setEnableParameterAdapter(true); + IoTDBDescriptor.getInstance().getConfig().setEnableActiveTimeseriesCounter(true); } @After public void tearDown() throws Exception { - IoTDBDescriptor.getInstance().getConfig().setEnableParameterAdapter(canAdjust); + IoTDBDescriptor.getInstance().getConfig().setEnableActiveTimeseriesCounter(canAdjust); EnvironmentUtils.cleanEnv(); } diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/ReaderTestHelper.java b/server/src/test/java/org/apache/iotdb/db/query/reader/ReaderTestHelper.java deleted file mode 100644 index 28135c1..0000000 --- a/server/src/test/java/org/apache/iotdb/db/query/reader/ReaderTestHelper.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.query.reader; - -import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; -import org.apache.iotdb.db.constant.TestConstant; -import org.apache.iotdb.db.engine.MetadataManagerHelper; -import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy.DirectFlushPolicy; -import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor; -import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.query.control.FileReaderManager; -import org.apache.iotdb.db.service.IoTDB; -import org.apache.iotdb.db.utils.EnvironmentUtils; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; - -public abstract class ReaderTestHelper { - - private String storageGroup = "root.vehicle"; - protected String deviceId = "root.vehicle.d0"; - protected String measurementId = "s0"; - protected TSDataType dataType = TSDataType.INT32; - protected StorageGroupProcessor storageGroupProcessor; - private String systemDir = TestConstant.OUTPUT_DATA_DIR.concat("info"); - - static { - IoTDB.metaManager.init(); - } - - @Before - public void setUp() throws Exception { - EnvironmentUtils.envSetUp(); - MetadataManagerHelper.initMetadata(); - ActiveTimeSeriesCounter.getInstance().init(storageGroup); - storageGroupProcessor = new StorageGroupProcessor(systemDir, storageGroup, new DirectFlushPolicy()); - insertData(); - } - - @After - public void tearDown() throws Exception { - FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders(); - storageGroupProcessor.syncDeleteDataFiles(); - EnvironmentUtils.cleanEnv(); - EnvironmentUtils.cleanDir(systemDir); - } - - abstract protected void insertData() throws IOException, QueryProcessException; - -} \ No newline at end of file diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java index 68b55b0..73f4b81 100644 --- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java @@ -32,7 +32,6 @@ import org.apache.iotdb.db.auth.AuthException; import org.apache.iotdb.db.auth.authorizer.BasicAuthorizer; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.StorageEngine; @@ -150,7 +149,6 @@ public class EnvironmentUtils { cleanAllDir(); config.setTsFileSizeThreshold(oldTsFileThreshold); config.setMemtableSizeThreshold(oldGroupSizeInByte); - IoTDBConfigDynamicAdapter.getInstance().reset(); } public static void cleanAllDir() throws IOException { @@ -203,9 +201,7 @@ public class EnvironmentUtils { fail(e.getMessage()); } - IoTDBDescriptor.getInstance().getConfig().setEnableParameterAdapter(false); - IoTDBConfigDynamicAdapter.getInstance().setInitialized(true); - + IoTDBDescriptor.getInstance().getConfig().setEnableActiveTimeseriesCounter(false); createAllDir(); // disable the system monitor config.setEnableStatMonitor(false); diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java index fad1beb..1a56615 100644 --- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java +++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/RecoverResourceFromReaderTest.java @@ -26,7 +26,7 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; import org.apache.commons.io.FileUtils; -import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; +import org.apache.iotdb.db.conf.adapter.ActiveTimeseriesCounter; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -182,7 +182,7 @@ public class RecoverResourceFromReaderTest { TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController, resource, false, false, Collections.singletonList(new ArrayList<>())); - ActiveTimeSeriesCounter.getInstance() + ActiveTimeseriesCounter.getInstance() .init(resource.getTsFile().getParentFile().getParentFile().getName()); performer.recover().left.close(); assertEquals(1, resource.getStartTime("root.sg.device99")); diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java index d6b4fef..5676e2f 100644 --- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java @@ -32,7 +32,7 @@ import java.util.List; import java.util.Map; import org.apache.commons.io.FileUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; +import org.apache.iotdb.db.conf.adapter.ActiveTimeseriesCounter; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -174,7 +174,7 @@ public class SeqTsFileRecoverTest { public void testNonLastRecovery() throws StorageGroupProcessorException, IOException { TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController, resource, false, false, Collections.singletonList(new ArrayList<>())); - ActiveTimeSeriesCounter.getInstance().init(storageGroup); + ActiveTimeseriesCounter.getInstance().init(storageGroup); RestorableTsFileIOWriter writer = performer.recover().left; assertFalse(writer.canWrite()); writer.close(); @@ -224,7 +224,7 @@ public class SeqTsFileRecoverTest { public void testLastRecovery() throws StorageGroupProcessorException, IOException { TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController, resource, false, true, Collections.singletonList(new ArrayList<>())); - ActiveTimeSeriesCounter.getInstance().init(storageGroup); + ActiveTimeseriesCounter.getInstance().init(storageGroup); RestorableTsFileIOWriter writer = performer.recover().left; writer.makeMetadataVisible(); diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java index 13717e1..55e9c45 100644 --- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java @@ -26,7 +26,7 @@ import java.io.IOException; import java.util.Collections; import org.apache.commons.io.FileUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; +import org.apache.iotdb.db.conf.adapter.ActiveTimeseriesCounter; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -182,7 +182,7 @@ public class UnseqTsFileRecoverTest { public void test() throws StorageGroupProcessorException, IOException { TsFileRecoverPerformer performer = new TsFileRecoverPerformer(logNodePrefix, versionController, resource, false, false, Collections.emptyList()); - ActiveTimeSeriesCounter.getInstance() + ActiveTimeseriesCounter.getInstance() .init(resource.getTsFile().getParentFile().getParentFile().getName()); performer.recover().left.close(); diff --git a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java index 2e627ad..e637df4 100644 --- a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java +++ b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java @@ -24,7 +24,6 @@ import org.apache.iotdb.db.auth.authorizer.BasicAuthorizer; import org.apache.iotdb.db.auth.authorizer.IAuthorizer; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.cache.ChunkMetadataCache; @@ -129,7 +128,6 @@ public class EnvironmentUtils { config.setTsFileSizeThreshold(oldTsFileThreshold); config.setMemtableSizeThreshold(oldGroupSizeInByte); - IoTDBConfigDynamicAdapter.getInstance().reset(); } public static void cleanAllDir() throws IOException { @@ -168,10 +166,8 @@ public class EnvironmentUtils { * this function should be called before all code in the setup */ public static void envSetUp() throws StartupException, IOException { - IoTDBDescriptor.getInstance().getConfig().setEnableParameterAdapter(false); + IoTDBDescriptor.getInstance().getConfig().setEnableActiveTimeseriesCounter(false); IoTDB.metaManager.init(); - IoTDBConfigDynamicAdapter.getInstance().setInitialized(true); - createAllDir(); // disable the system monitor config.setEnableStatMonitor(false);
