This is an automated email from the ASF dual-hosted git repository. qiaojialin pushed a commit to branch make_dynamic_controlable in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit fa236efaf3b2cd0e2bde574281ae58765f262ed8 Author: qiaojialin <[email protected]> AuthorDate: Tue Apr 14 11:47:17 2020 +0800 make dynamic controlable --- .../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 9 +- .../iotdb/db/engine/flush/MemTableFlushTask.java | 5 +- .../db/engine/storagegroup/TsFileProcessor.java | 4 +- .../org/apache/iotdb/db/metadata/MManager.java | 98 +++++++++++++--------- 4 files changed, 71 insertions(+), 45 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java index 2069043..58f637a 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java @@ -127,6 +127,9 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter { @Override public synchronized boolean tryToAdaptParameters() { + if(!CONFIG.isEnableParameterAdapter()){ + return true; + } boolean canAdjust = true; double ratio = CompressionRatio.getInstance().getRatio(); long memtableSizeInByte = calcMemTableSize(ratio); @@ -216,12 +219,12 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter { @Override public void addOrDeleteStorageGroup(int diff) throws ConfigAdjusterException { - totalStorageGroup += diff; - maxMemTableNum += IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff; if(!CONFIG.isEnableParameterAdapter()){ - CONFIG.setMaxMemtableNumber(maxMemTableNum); return; } + totalStorageGroup += diff; + maxMemTableNum += IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff; + CONFIG.setMaxMemtableNumber(maxMemTableNum); if (!tryToAdaptParameters()) { totalStorageGroup -= diff; maxMemTableNum -= IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java index 1703844..78cdeda 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter; import org.apache.iotdb.db.engine.flush.pool.FlushSubTaskPoolManager; import org.apache.iotdb.db.engine.memtable.IMemTable; @@ -86,7 +87,9 @@ public class MemTableFlushTask { } encodingTaskQueue.add(new EndChunkGroupIoTask()); } - ActiveTimeSeriesCounter.getInstance().updateActiveRatio(storageGroup); + if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) { + ActiveTimeSeriesCounter.getInstance().updateActiveRatio(storageGroup); + } noMoreEncodingTask = true; logger.debug( "Storage group {} memtable {}, flushing into disk: data sort time cost {} ms.", diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 7e554cf..7d7dafe 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -261,6 +261,9 @@ public class TsFileProcessor { */ private long getMemtableSizeThresholdBasedOnSeriesNum() { IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + if(!config.isEnableParameterAdapter()){ + return config.getMemtableSizeThreshold(); + } long memTableSize = (long) (config.getMemtableSizeThreshold() * config.getMaxMemtableNumber() / IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * ActiveTimeSeriesCounter.getInstance() @@ -268,7 +271,6 @@ public class TsFileProcessor { return Math.max(memTableSize, config.getMemtableSizeThreshold()); } - public boolean shouldClose() { long fileSize = tsFileResource.getFileSize(); long fileSizeThreshold = IoTDBDescriptor.getInstance().getConfig() diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index 0ae8fbf..8f578c3 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -136,20 +136,22 @@ public class MManager { try { initFromLog(logFile); - // storage group name -> the series number - seriesNumberInStorageGroups = new HashMap<>(); - List<String> storageGroups = mtree.getAllStorageGroupNames(); - for (String sg : storageGroups) { - MNode node = mtree.getNodeByPath(sg); - seriesNumberInStorageGroups.put(sg, node.getLeafCount()); + if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) { + // storage group name -> the series number + seriesNumberInStorageGroups = new HashMap<>(); + List<String> storageGroups = mtree.getAllStorageGroupNames(); + for (String sg : storageGroups) { + MNode node = mtree.getNodeByPath(sg); + seriesNumberInStorageGroups.put(sg, node.getLeafCount()); + } + if (seriesNumberInStorageGroups.isEmpty()) { + maxSeriesNumberAmongStorageGroup = 0; + } else { + maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream() + .max(Integer::compareTo).get(); + } } - if (seriesNumberInStorageGroups.isEmpty()) { - maxSeriesNumberAmongStorageGroup = 0; - } else { - maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream() - .max(Integer::compareTo).get(); - } writeToLog = true; } catch (IOException | MetadataException e) { mtree = new MTree(); @@ -184,7 +186,9 @@ public class MManager { try { this.mtree = new MTree(); this.mNodeCache.clear(); - this.seriesNumberInStorageGroups.clear(); + if (seriesNumberInStorageGroups != null) { + this.seriesNumberInStorageGroups.clear(); + } this.maxSeriesNumberAmongStorageGroup = 0; if (logWriter != null) { logWriter.close(); @@ -294,10 +298,12 @@ public class MManager { createTimeseriesWithMemoryCheckAndLog(path, dataType, encoding, compressor, props); // update statistics - int size = seriesNumberInStorageGroups.get(storageGroupName); - seriesNumberInStorageGroups.put(storageGroupName, size + 1); - if (size + 1 > maxSeriesNumberAmongStorageGroup) { - maxSeriesNumberAmongStorageGroup = size + 1; + if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) { + int size = seriesNumberInStorageGroups.get(storageGroupName); + seriesNumberInStorageGroups.put(storageGroupName, size + 1); + if (size + 1 > maxSeriesNumberAmongStorageGroup) { + maxSeriesNumberAmongStorageGroup = size + 1; + } } } finally { lock.writeLock().unlock(); @@ -361,12 +367,16 @@ public class MManager { public Set<String> deleteTimeseries(String prefixPath) throws MetadataException { lock.writeLock().lock(); if (isStorageGroup(prefixPath)) { - int size = seriesNumberInStorageGroups.get(prefixPath); - seriesNumberInStorageGroups.put(prefixPath, 0); - if (size == maxSeriesNumberAmongStorageGroup) { - seriesNumberInStorageGroups.values().stream().max(Integer::compareTo) - .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val); + + if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) { + int size = seriesNumberInStorageGroups.get(prefixPath); + seriesNumberInStorageGroups.put(prefixPath, 0); + if (size == maxSeriesNumberAmongStorageGroup) { + seriesNumberInStorageGroups.values().stream().max(Integer::compareTo) + .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val); + } } + mNodeCache.clear(); } try { @@ -412,12 +422,15 @@ public class MManager { } catch (ConfigAdjusterException e) { throw new MetadataException(e); } - String storageGroup = getStorageGroupName(path); - int size = seriesNumberInStorageGroups.get(storageGroup); - seriesNumberInStorageGroups.put(storageGroup, size - 1); - if (size == maxSeriesNumberAmongStorageGroup) { - seriesNumberInStorageGroups.values().stream().max(Integer::compareTo) - .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val); + + if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) { + String storageGroup = getStorageGroupName(path); + int size = seriesNumberInStorageGroups.get(storageGroup); + seriesNumberInStorageGroups.put(storageGroup, size - 1); + if (size == maxSeriesNumberAmongStorageGroup) { + seriesNumberInStorageGroups.values().stream().max(Integer::compareTo) + .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val); + } } return storageGroupName; } finally { @@ -441,7 +454,9 @@ public class MManager { writer.flush(); } IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(1); - ActiveTimeSeriesCounter.getInstance().init(storageGroup); + if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) { + ActiveTimeSeriesCounter.getInstance().init(storageGroup); + } seriesNumberInStorageGroups.put(storageGroup, 0); } catch (IOException e) { throw new MetadataException(e.getMessage()); @@ -473,17 +488,20 @@ public class MManager { writer.flush(); } mNodeCache.clear(); - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(-1); - int size = seriesNumberInStorageGroups.get(storageGroup); - IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(size * -1); - ActiveTimeSeriesCounter.getInstance().delete(storageGroup); - seriesNumberInStorageGroups.remove(storageGroup); - if (size == maxSeriesNumberAmongStorageGroup) { - if (seriesNumberInStorageGroups.isEmpty()) { - maxSeriesNumberAmongStorageGroup = 0; - } else { - maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream() - .max(Integer::compareTo).get(); + + if (IoTDBDescriptor.getInstance().getConfig().isEnableParameterAdapter()) { + IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(-1); + int size = seriesNumberInStorageGroups.get(storageGroup); + IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(size * -1); + ActiveTimeSeriesCounter.getInstance().delete(storageGroup); + seriesNumberInStorageGroups.remove(storageGroup); + if (size == maxSeriesNumberAmongStorageGroup) { + if (seriesNumberInStorageGroups.isEmpty()) { + maxSeriesNumberAmongStorageGroup = 0; + } else { + maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream() + .max(Integer::compareTo).get(); + } } } }
