This is an automated email from the ASF dual-hosted git repository. yongzao pushed a commit to branch Move-ConfigNodeConfig-into-CommonConfig in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bd4d9621731bfd3c6b14ffeb55e6413b9f445037 Author: YongzaoDan <[email protected]> AuthorDate: Thu Jan 12 17:23:41 2023 +0800 stash --- .../iotdb/session/it/IoTDBSessionComplexIT.java | 6 +- .../iotdb/db/integration/IoTDBCompactionIT.java | 2 +- .../integration/IoTDBCompactionWithIDTableIT.java | 2 +- .../iotdb/db/integration/IoTDBMultiSeriesIT.java | 2 +- .../iotdb/db/integration/IoTDBTimePartitionIT.java | 2 +- .../db/integration/env/StandaloneEnvConfig.java | 2 +- .../IoTDBDeletionVersionAdaptionIT.java | 2 +- .../apache/iotdb/commons/conf/CommonConfig.java | 68 +++- .../iotdb/commons/conf/CommonDescriptor.java | 169 ++++++++ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 277 ++----------- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 443 +++++++-------------- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 2 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 2 +- .../config/metadata/GetTimeSlotListTask.java | 2 +- .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 2 +- .../java/org/apache/iotdb/db/service/DataNode.java | 1 + .../impl/DataNodeInternalRPCServiceImpl.java | 13 + .../apache/iotdb/db/utils/TimePartitionUtils.java | 2 +- .../SizeTieredCompactionSelectorTest.java | 4 +- .../db/engine/storagegroup/DataRegionTest.java | 18 +- .../iotdb/db/engine/storagegroup/TTLTest.java | 6 +- .../mpp/plan/analyze/QueryTimePartitionTest.java | 58 +-- .../plan/node/write/WritePlanNodeSplitTest.java | 6 +- .../iotdb/db/tools/TsFileAndModSettleToolTest.java | 6 +- 24 files changed, 496 insertions(+), 601 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java index 19e49c7904..2d2e8be5f3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionComplexIT.java @@ -268,7 +268,7 @@ public class IoTDBSessionComplexIT { private void insertRecords(ISession session, List<String> deviceIdList) throws IoTDBConnectionException, StatementExecutionException { - long timePartition = IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); + long timePartition = IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); List<String> measurements = new ArrayList<>(); measurements.add("s1"); @@ -317,7 +317,7 @@ public class IoTDBSessionComplexIT { private void insertMultiTablets(ISession session, List<String> deviceIdList) throws IoTDBConnectionException, StatementExecutionException { - long timePartition = IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); + long timePartition = IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); List<MeasurementSchema> schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); @@ -345,7 +345,7 @@ public class IoTDBSessionComplexIT { private void insertRecordsOfOneDevice(ISession session, String deviceId) throws IoTDBConnectionException, StatementExecutionException { - long timePartition = IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); + long timePartition = IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); List<String> measurements = new ArrayList<>(); measurements.add("s1"); diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCompactionIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCompactionIT.java index 7d9d5db749..d3f297d309 100644 --- a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCompactionIT.java +++ b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCompactionIT.java @@ -49,7 +49,7 @@ public class IoTDBCompactionIT { @Before public void setUp() throws Exception { - prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); + prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); ConfigFactory.getConfig().setPartitionInterval(1); EnvFactory.getEnv().initBeforeTest(); } diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCompactionWithIDTableIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCompactionWithIDTableIT.java index ba62d256c0..c6dd7ad71d 100644 --- a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCompactionWithIDTableIT.java +++ b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCompactionWithIDTableIT.java @@ -53,7 +53,7 @@ public class IoTDBCompactionWithIDTableIT { @Before public void setUp() throws Exception { - prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); + prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); ConfigFactory.getConfig().setPartitionInterval(1); isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable(); originalDeviceIDTransformationMethod = diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java index 68d76e6402..eae5f483e8 100644 --- a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java +++ b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java @@ -71,7 +71,7 @@ public class IoTDBMultiSeriesIT { ConfigFactory.getConfig().setPageSizeInByte(1024 * 150); ConfigFactory.getConfig().setGroupSizeInByte(1024 * 1000); ConfigFactory.getConfig().setMemtableSizeThreshold(1024 * 1000); - prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); + prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); ConfigFactory.getConfig().setPartitionInterval(100); ConfigFactory.getConfig().setCompressor("LZ4"); diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBTimePartitionIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBTimePartitionIT.java index 57a85927ba..5820226f29 100644 --- a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBTimePartitionIT.java +++ b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBTimePartitionIT.java @@ -43,7 +43,7 @@ public class IoTDBTimePartitionIT { @Before public void setUp() throws Exception { - prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); + prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); ConfigFactory.getConfig().setPartitionInterval(2592000); EnvFactory.getEnv().initBeforeTest(); } diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/env/StandaloneEnvConfig.java b/integration/src/test/java/org/apache/iotdb/db/integration/env/StandaloneEnvConfig.java index e8db7324a3..7bf97ccb59 100644 --- a/integration/src/test/java/org/apache/iotdb/db/integration/env/StandaloneEnvConfig.java +++ b/integration/src/test/java/org/apache/iotdb/db/integration/env/StandaloneEnvConfig.java @@ -51,7 +51,7 @@ public class StandaloneEnvConfig implements BaseConfig { } public BaseConfig setPartitionInterval(long partitionInterval) { - IoTDBDescriptor.getInstance().getConfig().setTimePartitionInterval(partitionInterval); + IoTDBDescriptor.getInstance().getConfig().setDnTimePartitionInterval(partitionInterval); return this; } diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/versionadaption/IoTDBDeletionVersionAdaptionIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/versionadaption/IoTDBDeletionVersionAdaptionIT.java index 69b027b20c..8123f8f8c2 100644 --- a/integration/src/test/java/org/apache/iotdb/db/integration/versionadaption/IoTDBDeletionVersionAdaptionIT.java +++ b/integration/src/test/java/org/apache/iotdb/db/integration/versionadaption/IoTDBDeletionVersionAdaptionIT.java @@ -65,7 +65,7 @@ public class IoTDBDeletionVersionAdaptionIT { @Before public void setUp() throws Exception { Locale.setDefault(Locale.ENGLISH); - prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); + prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); ConfigFactory.getConfig().setPartitionInterval(1000); EnvFactory.getEnv().initBeforeTest(); prepareSeries(); diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index f340c69b9d..d220a0d58e 100644 --- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.consensus.ConsensusProtocolClass; import org.apache.iotdb.commons.enums.HandleSystemErrorStrategy; import org.apache.iotdb.commons.loadbalance.LeaderDistributionPolicy; import org.apache.iotdb.commons.loadbalance.RegionGroupExtensionPolicy; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.datastructure.TVListSortAlgorithm; import org.apache.iotdb.commons.wal.WALMode; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -33,6 +34,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class CommonConfig { @@ -102,7 +105,7 @@ public class CommonConfig { /** Cluster management */ // Time partition interval in milliseconds - private long timePartitionInterval = 604_800_000; + private long timePartitionInterval = 604_800_000L; // The heartbeat interval in milliseconds private long heartbeatIntervalInMs = 1000; // Disk Monitor @@ -270,8 +273,6 @@ public class CommonConfig { // Maximum number of lines in a single TsBlock private int maxTsBlockLineNumber = 1000; - // TODO: - // Time cost(ms) threshold for slow query. Unit: millisecond private long slowQueryThreshold = 5000; // The max executing time of query in ms. Unit: millisecond @@ -904,6 +905,11 @@ public class CommonConfig { public void setUdfDir(String udfDir) { this.udfDir = udfDir; + updateUdfTemporaryLibDir(); + } + + public void updateUdfTemporaryLibDir() { + this.udfTemporaryLibDir = udfDir + File.separator + IoTDBConstant.TMP_FOLDER_NAME; } public String getUdfTemporaryLibDir() { @@ -920,6 +926,11 @@ public class CommonConfig { public void setTriggerDir(String triggerDir) { this.triggerDir = triggerDir; + updateTriggerTemporaryLibDir(); + } + + public void updateTriggerTemporaryLibDir() { + this.triggerTemporaryLibDir = triggerDir + File.separator + IoTDBConstant.TMP_FOLDER_NAME; } public String getTriggerTemporaryLibDir() { @@ -1537,6 +1548,7 @@ public class CommonConfig { public void setAllocateMemoryForStorageEngine(long allocateMemoryForStorageEngine) { this.allocateMemoryForStorageEngine = allocateMemoryForStorageEngine; + this.allocateMemoryForTimePartitionInfo = allocateMemoryForStorageEngine * 50 / 1001; } public long getAllocateMemoryForRead() { @@ -1545,6 +1557,21 @@ public class CommonConfig { public void setAllocateMemoryForRead(long allocateMemoryForRead) { this.allocateMemoryForRead = allocateMemoryForRead; + + this.allocateMemoryForBloomFilterCache = allocateMemoryForRead / 1001; + this.allocateMemoryForTimeSeriesMetaDataCache = allocateMemoryForRead * 200 / 1001; + this.allocateMemoryForChunkCache = allocateMemoryForRead * 100 / 1001; + this.allocateMemoryForCoordinator = allocateMemoryForRead * 50 / 1001; + this.allocateMemoryForOperators = allocateMemoryForRead * 200 / 1001; + this.allocateMemoryForDataExchange = allocateMemoryForRead * 200 / 1001; + this.allocateMemoryForTimeIndex = allocateMemoryForRead * 200 / 1001; + } + + public long getAllocateMemoryForFree() { + return Runtime.getRuntime().maxMemory() + - allocateMemoryForStorageEngine + - allocateMemoryForRead + - allocateMemoryForSchema; } public long getAllocateMemoryForSchema() { @@ -1553,6 +1580,10 @@ public class CommonConfig { public void setAllocateMemoryForSchema(long allocateMemoryForSchema) { this.allocateMemoryForSchema = allocateMemoryForSchema; + + this.allocateMemoryForSchemaRegion = allocateMemoryForSchema * 8 / 10; + this.allocateMemoryForSchemaCache = allocateMemoryForSchema / 10; + this.allocateMemoryForLastCache = allocateMemoryForSchema / 10; } public long getAllocateMemoryForConsensus() { @@ -1672,6 +1703,7 @@ public class CommonConfig { return maxBytesPerFragmentInstance; } + @TestOnly public void setMaxBytesPerFragmentInstance(long maxBytesPerFragmentInstance) { this.maxBytesPerFragmentInstance = maxBytesPerFragmentInstance; } @@ -2380,6 +2412,36 @@ public class CommonConfig { this.watermarkMethod = watermarkMethod; } + public String getWatermarkMethodName() { + return watermarkMethod.split("\\(")[0]; + } + + public int getWatermarkParamMarkRate() { + return Integer.parseInt(getWatermarkParamValue("embed_row_cycle", "5")); + } + + public int getWatermarkParamMaxRightBit() { + return Integer.parseInt(getWatermarkParamValue("embed_lsb_num", "5")); + } + + private String getWatermarkParamValue(String key, String defaultValue) { + String res = getWatermarkParamValue(key); + if (res != null) { + return res; + } + return defaultValue; + } + + private String getWatermarkParamValue(String key) { + String pattern = key + "=(\\w*)"; + Pattern r = Pattern.compile(pattern); + Matcher m = r.matcher(watermarkMethod); + if (m.find() && m.groupCount() > 0) { + return m.group(1); + } + return null; + } + public boolean isEnableInfluxDBRpcService() { return enableInfluxDBRpcService; } diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java index dc2d49ab5a..f71425a37c 100644 --- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java @@ -1618,6 +1618,175 @@ public class CommonDescriptor { } } + public void loadHotModifiedProps(Properties properties) { + + // update timed flush & close conf + loadTimedService(properties); + + // update tsfile-format config + loadTsFileProps(properties); + + // update max_deduplicated_path_num + CONF.setMaxDeduplicatedPathNum( + Integer.parseInt( + properties.getProperty( + "max_deduplicated_path_num", + Integer.toString(CONF.getMaxDeduplicatedPathNum())))); + // update slow_query_threshold + CONF.setSlowQueryThreshold( + Long.parseLong( + properties.getProperty( + "slow_query_threshold", Long.toString(CONF.getSlowQueryThreshold())))); + + // update insert-tablet-plan's row limit for select-into + CONF.setSelectIntoInsertTabletPlanRowLimit( + Integer.parseInt( + properties.getProperty( + "select_into_insert_tablet_plan_row_limit", + String.valueOf(CONF.getSelectIntoInsertTabletPlanRowLimit())))); + + // update sync config + CONF.setMaxNumberOfSyncFileRetry( + Integer.parseInt( + properties + .getProperty( + "max_number_of_sync_file_retry", + Integer.toString(CONF.getMaxNumberOfSyncFileRetry())) + .trim())); + + + } + + private void loadWALHotModifiedProps(Properties properties) { + long fsyncWalDelayInMs = + Long.parseLong( + properties.getProperty( + "fsync_wal_delay_in_ms", Long.toString(CONF.getFsyncWalDelayInMs()))); + if (fsyncWalDelayInMs > 0) { + CONF.setFsyncWalDelayInMs(fsyncWalDelayInMs); + } + + long walFileSizeThreshold = + Long.parseLong( + properties.getProperty( + "wal_file_size_threshold_in_byte", + Long.toString(CONF.getWalFileSizeThresholdInByte()))); + if (walFileSizeThreshold > 0) { + CONF.setWalFileSizeThresholdInByte(walFileSizeThreshold); + } + + double walMinEffectiveInfoRatio = + Double.parseDouble( + properties.getProperty( + "wal_min_effective_info_ratio", + Double.toString(CONF.getWalMinEffectiveInfoRatio()))); + if (walMinEffectiveInfoRatio > 0) { + CONF.setWalMinEffectiveInfoRatio(walMinEffectiveInfoRatio); + } + + long walMemTableSnapshotThreshold = + Long.parseLong( + properties.getProperty( + "wal_memtable_snapshot_threshold_in_byte", + Long.toString(CONF.getWalMemTableSnapshotThreshold()))); + if (walMemTableSnapshotThreshold > 0) { + CONF.setWalMemTableSnapshotThreshold(walMemTableSnapshotThreshold); + } + + int maxWalMemTableSnapshotNum = + Integer.parseInt( + properties.getProperty( + "max_wal_memtable_snapshot_num", + Integer.toString(CONF.getMaxWalMemTableSnapshotNum()))); + if (maxWalMemTableSnapshotNum > 0) { + CONF.setMaxWalMemTableSnapshotNum(maxWalMemTableSnapshotNum); + } + + long deleteWalFilesPeriod = + Long.parseLong( + properties.getProperty( + "delete_wal_files_period_in_ms", + Long.toString(CONF.getDeleteWalFilesPeriodInMs()))); + if (deleteWalFilesPeriod > 0) { + CONF.setDeleteWalFilesPeriodInMs(deleteWalFilesPeriod); + } + + long throttleDownThresholdInByte = + Long.parseLong( + properties.getProperty( + "iot_consensus_throttle_threshold_in_byte", + Long.toString(CONF.getIotConsensusThrottleThresholdInByte()))); + if (throttleDownThresholdInByte > 0) { + CONF.setIotConsensusThrottleThresholdInByte(throttleDownThresholdInByte); + } + + long cacheWindowInMs = + Long.parseLong( + properties.getProperty( + "iot_consensus_cache_window_time_in_ms", + Long.toString(CONF.getIotConsensusCacheWindowTimeInMs()))); + if (cacheWindowInMs > 0) { + CONF.setIotConsensusCacheWindowTimeInMs(cacheWindowInMs); + } + } + + /** Get default encode algorithm by data type */ + public TSEncoding getDefaultEncodingByType(TSDataType dataType) { + switch (dataType) { + case BOOLEAN: + return CONF.getDefaultBooleanEncoding(); + case INT32: + return CONF.getDefaultInt32Encoding(); + case INT64: + return CONF.getDefaultInt64Encoding(); + case FLOAT: + return CONF.getDefaultFloatEncoding(); + case DOUBLE: + return CONF.getDefaultDoubleEncoding(); + default: + return CONF.getDefaultTextEncoding(); + } + } + + public void reclaimConsensusMemory() { + CONF.setAllocateMemoryForStorageEngine( + CONF.getAllocateMemoryForStorageEngine() + CONF.getAllocateMemoryForConsensus()); + } + + public void initClusterSchemaMemoryAllocate() { + if (!CONF.isDefaultSchemaMemoryConfig()) { + // the config has already been updated as user config in properties file + return; + } + + // process the default schema memory allocate + + long schemaMemoryTotal = CONF.getAllocateMemoryForSchema(); + + int proportionSum = 10; + int schemaRegionProportion = 5; + int schemaCacheProportion = 3; + int partitionCacheProportion = 1; + int lastCacheProportion = 1; + + CONF.setAllocateMemoryForSchemaRegion( + schemaMemoryTotal * schemaRegionProportion / proportionSum); + LOGGER.info( + "Cluster allocateMemoryForSchemaRegion = {}", CONF.getAllocateMemoryForSchemaRegion()); + + CONF.setAllocateMemoryForSchemaCache(schemaMemoryTotal * schemaCacheProportion / proportionSum); + LOGGER.info( + "Cluster allocateMemoryForSchemaCache = {}", CONF.getAllocateMemoryForSchemaCache()); + + CONF.setAllocateMemoryForPartitionCache( + schemaMemoryTotal * partitionCacheProportion / proportionSum); + LOGGER.info( + "Cluster allocateMemoryForPartitionCache = {}", CONF.getAllocateMemoryForPartitionCache()); + + CONF.setAllocateMemoryForLastCache(schemaMemoryTotal * lastCacheProportion / proportionSum); + LOGGER.info("Cluster allocateMemoryForLastCache = {}", CONF.getAllocateMemoryForLastCache()); + } + public void loadGlobalConfig(TGlobalConfig globalConfig) { CONF.setDiskSpaceWarningThreshold(globalConfig.getDiskSpaceWarningThreshold()); } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index b8b25ce5d6..28ee504d3a 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -19,9 +19,7 @@ package org.apache.iotdb.db.conf; import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.commons.client.property.ClientPoolProperty.DefaultProperty; import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.audit.AuditLogOperation; import org.apache.iotdb.db.audit.AuditLogStorage; import org.apache.iotdb.db.conf.directories.DirectoryManager; @@ -190,10 +188,7 @@ public class IoTDBConfig { // Compact the unsequence files into the overlapped sequence files private boolean enableCrossSpaceCompaction = true; - /** - * The strategy of inner space compaction task. There are just one inner space compaction strategy - * SIZE_TIRED_COMPACTION: - */ + // The strategy of inner space compaction task. There are just one inner space compaction strategy SIZE_TIRED_COMPACTION: private InnerSequenceCompactionSelector innerSequenceCompactionSelector = InnerSequenceCompactionSelector.SIZE_TIERED; @@ -211,75 +206,69 @@ public class IoTDBConfig { private CrossCompactionSelector crossCompactionSelector = CrossCompactionSelector.REWRITE; private CrossCompactionPerformer crossCompactionPerformer = CrossCompactionPerformer.READ_POINT; - /** - * The priority of compaction task execution. There are three priority strategy INNER_CROSS: - * prioritize inner space compaction, reduce the number of files first CROSS INNER: prioritize - * cross space compaction, eliminate the unsequence files first BALANCE: alternate two compaction - * types - */ + // The priority of compaction task execution. There are three priority strategy INNER_CROSS: + // prioritize inner space compaction, reduce the number of files first CROSS INNER: prioritize + // cross space compaction, eliminate the unsequence files first BALANCE: alternate two compaction types private CompactionPriority compactionPriority = CompactionPriority.BALANCE; - /** The target tsfile size in compaction, 1 GB by default */ + // The target tsfile size in compaction, 1 GB by default private long targetCompactionFileSize = 1073741824L; - /** The target chunk size in compaction. */ + // The target chunk size in compaction private long targetChunkSize = 1048576L; - /** The target chunk point num in compaction. */ + // The target chunk point num in compaction private long targetChunkPointNum = 100000L; - /** - * If the chunk size is lower than this threshold, it will be deserialized into points, default is - * 10 KB - */ + // If the chunk size is lower than this threshold, it will be deserialized into points, default is 10 KB private long chunkSizeLowerBoundInCompaction = 10240L; - /** - * If the chunk point num is lower than this threshold, it will be deserialized into points, - * default is 1000 - */ + // If the chunk point num is lower than this threshold, it will be deserialized into points, default is 1000 private long chunkPointNumLowerBoundInCompaction = 1000; - /** - * If compaction thread cannot acquire the write lock within this timeout, the compaction task - * will be abort. - */ + // If compaction thread cannot acquire the write lock within this timeout, the compaction task will be abort. private long compactionAcquireWriteLockTimeout = 60_000L; - /** The max candidate file num in inner space compaction */ + // The max candidate file num in inner space compaction private int maxInnerCompactionCandidateFileNum = 30; - /** The max candidate file num in cross space compaction */ + // The max candidate file num in cross space compaction private int maxCrossCompactionCandidateFileNum = 1000; - /** The max total size of candidate files in cross space compaction */ + // The max total size of candidate files in cross space compaction private long maxCrossCompactionCandidateFileSize = 1024 * 1024 * 1024 * 5L; - /** The interval of compaction task schedulation in each virtual database. The unit is ms. */ + // The interval of compaction task schedulation in each virtual database. The unit is ms private long compactionScheduleIntervalInMs = 60_000L; - /** The interval of compaction task submission from queue in CompactionTaskMananger */ + // The interval of compaction task submission from queue in CompactionTaskMananger private long compactionSubmissionIntervalInMs = 60_000L; - /** - * The number of sub compaction threads to be set up to perform compaction. Currently only works - * for nonAligned data in cross space compaction and unseq inner space compaction. - */ + // The number of sub compaction threads to be set up to perform compaction. Currently only works + // for nonAligned data in cross space compaction and unseq inner space compaction. private int subCompactionTaskNum = 4; private boolean enableCompactionValidation = true; - - /** - * If one merge file selection runs for more than this time, it will be ended and its current - * selection will be used as final selection. When < 0, it means time is unbounded. Unit: - * millisecond - */ + // If one merge file selection runs for more than this time, it will be ended and its current + // selection will be used as final selection. When < 0, it means time is unbounded. Unit: millisecond private long crossCompactionFileSelectionTimeBudget = 30 * 1000L; - /** End Compaction Configurations */ + /** Retain Configuration */ + // Default TSfile storage is in local file system + private FSType tsFileStorageFs = FSType.LOCAL; + // Default core-site.xml file path is /etc/hadoop/conf/core-site.xml + private String coreSitePath = "/etc/hadoop/conf/core-site.xml"; + // Default hdfs-site.xml file path is /etc/hadoop/conf/hdfs-site.xml + private String hdfsSitePath = "/etc/hadoop/conf/hdfs-site.xml"; + + // Default HDFS ip is localhost + private String hdfsIp = "localhost"; + + // Default HDFS port is 9000 + private String hdfsPort = "9000"; /* Names of Watermark methods */ @@ -375,20 +364,8 @@ public class IoTDBConfig { */ private int compactionThreadCount = 10; - /** Default TSfile storage is in local file system */ - private FSType tsFileStorageFs = FSType.LOCAL; - - /** Default core-site.xml file path is /etc/hadoop/conf/core-site.xml */ - private String coreSitePath = "/etc/hadoop/conf/core-site.xml"; - - /** Default hdfs-site.xml file path is /etc/hadoop/conf/hdfs-site.xml */ - private String hdfsSitePath = "/etc/hadoop/conf/hdfs-site.xml"; - /** Default HDFS ip is localhost */ - private String hdfsIp = "localhost"; - /** Default HDFS port is 9000 */ - private String hdfsPort = "9000"; /** Default DFS NameServices is hdfsnamespace */ private String dfsNameServices = "hdfsnamespace"; @@ -415,11 +392,8 @@ public class IoTDBConfig { /** kerberos principal */ private String kerberosPrincipal = "your principal"; - /** the default fill interval in LinearFill and PreviousFill, -1 means infinite past time */ - private int defaultFillInterval = -1; - /** Time partition interval in milliseconds */ - private long timePartitionInterval = 604_800_000; + private long dnTimePartitionInterval = 604_800_000; /** * Level of TimeIndex, which records the start time and end time of TsFileResource. Currently, @@ -499,20 +473,12 @@ public class IoTDBConfig { this.groupByFillCacheSizeInMB = groupByFillCacheSizeInMB; } - public int getDefaultFillInterval() { - return defaultFillInterval; - } - - public void setDefaultFillInterval(int defaultFillInterval) { - this.defaultFillInterval = defaultFillInterval; - } - - public long getTimePartitionInterval() { - return timePartitionInterval; + public long getDnTimePartitionInterval() { + return dnTimePartitionInterval; } - public void setTimePartitionInterval(long timePartitionInterval) { - this.timePartitionInterval = timePartitionInterval; + public void setDnTimePartitionInterval(long dnTimePartitionInterval) { + this.dnTimePartitionInterval = dnTimePartitionInterval; } public TimeIndexLevel getTimeIndexLevel() { @@ -671,14 +637,6 @@ public class IoTDBConfig { this.dnRpcPort = dnRpcPort; } - public int getInfluxDBRpcPort() { - return influxDBRpcPort; - } - - public void setInfluxDBRpcPort(int influxDBRpcPort) { - this.influxDBRpcPort = influxDBRpcPort; - } - public String getDnSystemDir() { return dnSystemDir; } @@ -777,40 +735,6 @@ public class IoTDBConfig { this.dnExtDir = dnExtDir; } - public String getUdfDir() { - return udfDir; - } - - public void setUdfDir(String udfDir) { - this.udfDir = udfDir; - updateUdfTemporaryLibDir(); - } - - public String getUdfTemporaryLibDir() { - return udfTemporaryLibDir; - } - - public void updateUdfTemporaryLibDir() { - this.udfTemporaryLibDir = udfDir + File.separator + IoTDBConstant.TMP_FOLDER_NAME; - } - - public String getTriggerDir() { - return triggerDir; - } - - public void setTriggerDir(String triggerDir) { - this.triggerDir = triggerDir; - updateTriggerTemporaryLibDir(); - } - - public String getTriggerTemporaryLibDir() { - return triggerTemporaryLibDir; - } - - public void updateTriggerTemporaryLibDir() { - this.triggerTemporaryLibDir = triggerDir + File.separator + IoTDBConstant.TMP_FOLDER_NAME; - } - public String getMqttDir() { return mqttDir; } @@ -849,31 +773,6 @@ public class IoTDBConfig { this.maxMemtableNumber = maxMemtableNumber; } - public int getSubRawQueryThreadCount() { - return subRawQueryThreadCount; - } - - void setSubRawQueryThreadCount(int subRawQueryThreadCount) { - this.subRawQueryThreadCount = subRawQueryThreadCount; - } - - public long getMaxBytesPerFragmentInstance() { - return maxBytesPerFragmentInstance; - } - - @TestOnly - public void setMaxBytesPerFragmentInstance(long maxBytesPerFragmentInstance) { - this.maxBytesPerFragmentInstance = maxBytesPerFragmentInstance; - } - - public int getRawQueryBlockingQueueCapacity() { - return rawQueryBlockingQueueCapacity; - } - - public void setRawQueryBlockingQueueCapacity(int rawQueryBlockingQueueCapacity) { - this.rawQueryBlockingQueueCapacity = rawQueryBlockingQueueCapacity; - } - public int getWindowEvaluationThreadCount() { return windowEvaluationThreadCount; } @@ -930,22 +829,6 @@ public class IoTDBConfig { this.dnRpcMaxConcurrentClientNum = dnRpcMaxConcurrentClientNum; } - public int getmRemoteSchemaCacheSize() { - return mRemoteSchemaCacheSize; - } - - public void setmRemoteSchemaCacheSize(int mRemoteSchemaCacheSize) { - this.mRemoteSchemaCacheSize = mRemoteSchemaCacheSize; - } - - String getLanguageVersion() { - return languageVersion; - } - - void setLanguageVersion(String languageVersion) { - this.languageVersion = languageVersion; - } - public String getIoTDBVersion() { return IoTDBConstant.VERSION; } @@ -960,14 +843,6 @@ public class IoTDBConfig { : version.split("\\.")[0] + "." + version.split("\\.")[1]; } - public String getIpWhiteList() { - return ipWhiteList; - } - - public void setIpWhiteList(String ipWhiteList) { - this.ipWhiteList = ipWhiteList; - } - public long getCacheFileReaderClearPeriod() { return cacheFileReaderClearPeriod; } @@ -1012,55 +887,6 @@ public class IoTDBConfig { this.checkpointFileSizeThresholdInByte = checkpointFileSizeThresholdInByte; } - public boolean isChunkBufferPoolEnable() { - return chunkBufferPoolEnable; - } - - void setChunkBufferPoolEnable(boolean chunkBufferPoolEnable) { - this.chunkBufferPoolEnable = chunkBufferPoolEnable; - } - - public long getMergeIntervalSec() { - return mergeIntervalSec; - } - - void setMergeIntervalSec(long mergeIntervalSec) { - this.mergeIntervalSec = mergeIntervalSec; - } - - public void setAllocateMemoryForStorageEngine(long allocateMemoryForStorageEngine) { - this.allocateMemoryForStorageEngine = allocateMemoryForStorageEngine; - this.allocateMemoryForTimePartitionInfo = allocateMemoryForStorageEngine * 50 / 1001; - } - - - public void setAllocateMemoryForSchema(long allocateMemoryForSchema) { - this.allocateMemoryForSchema = allocateMemoryForSchema; - - this.allocateMemoryForSchemaRegion = allocateMemoryForSchema * 8 / 10; - this.allocateMemoryForSchemaCache = allocateMemoryForSchema / 10; - this.allocateMemoryForLastCache = allocateMemoryForSchema / 10; - } - - void setAllocateMemoryForRead(long allocateMemoryForRead) { - this.allocateMemoryForRead = allocateMemoryForRead; - - this.allocateMemoryForBloomFilterCache = allocateMemoryForRead / 1001; - this.allocateMemoryForTimeSeriesMetaDataCache = allocateMemoryForRead * 200 / 1001; - this.allocateMemoryForChunkCache = allocateMemoryForRead * 100 / 1001; - this.allocateMemoryForCoordinator = allocateMemoryForRead * 50 / 1001; - this.allocateMemoryForOperators = allocateMemoryForRead * 200 / 1001; - this.allocateMemoryForDataExchange = allocateMemoryForRead * 200 / 1001; - this.allocateMemoryForTimeIndex = allocateMemoryForRead * 200 / 1001; - } - - public long getAllocateMemoryForFree() { - return Runtime.getRuntime().maxMemory() - - allocateMemoryForStorageEngine - - allocateMemoryForRead - - allocateMemoryForSchema; - } - public int getCompactionThreadCount() { return compactionThreadCount; } @@ -1093,35 +919,6 @@ public class IoTDBConfig { this.dnRpcThriftCompressionEnable = dnRpcThriftCompressionEnable; } - public String getWatermarkMethodName() { - return watermarkMethod.split("\\(")[0]; - } - - public int getWatermarkParamMarkRate() { - return Integer.parseInt(getWatermarkParamValue("embed_row_cycle", "5")); - } - - public int getWatermarkParamMaxRightBit() { - return Integer.parseInt(getWatermarkParamValue("embed_lsb_num", "5")); - } - - private String getWatermarkParamValue(String key, String defaultValue) { - String res = getWatermarkParamValue(key); - if (res != null) { - return res; - } - return defaultValue; - } - - private String getWatermarkParamValue(String key) { - String pattern = key + "=(\\w*)"; - Pattern r = Pattern.compile(pattern); - Matcher m = r.matcher(watermarkMethod); - if (m.find() && m.groupCount() > 0) { - return m.group(1); - } - return null; - } FSType getTsFileStorageFs() { return tsFileStorageFs; diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 12af768942..d04a973537 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -19,13 +19,13 @@ package org.apache.iotdb.db.conf; import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.NodeUrlUtils; import org.apache.iotdb.confignode.rpc.thrift.TCQConfig; import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig; -import org.apache.iotdb.confignode.rpc.thrift.TRatisConfig; import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.compaction.execute.performer.constant.CrossCompactionPerformer; @@ -68,6 +68,7 @@ import java.util.ServiceLoader; public class IoTDBDescriptor { private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class); + private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); private final IoTDBConfig CONF = new IoTDBConfig(); @@ -217,193 +218,16 @@ public class IoTDBDescriptor { // TODO: Move to CommonDescriptor loadCompactionConfigurations(properties); - - CONF.setTsFileStorageFs( - properties.getProperty("tsfile_storage_fs", CONF.getTsFileStorageFs().toString())); - CONF.setCoreSitePath(properties.getProperty("core_site_path", CONF.getCoreSitePath())); - CONF.setHdfsSitePath(properties.getProperty("hdfs_site_path", CONF.getHdfsSitePath())); - CONF.setHdfsIp(properties.getProperty("hdfs_ip", CONF.getRawHDFSIp()).split(",")); - CONF.setHdfsPort(properties.getProperty("hdfs_port", CONF.getHdfsPort())); - CONF.setDfsNameServices(properties.getProperty("dfs_nameservices", CONF.getDfsNameServices())); - CONF.setDfsHaNamenodes( - properties.getProperty("dfs_ha_namenodes", CONF.getRawDfsHaNamenodes()).split(",")); - CONF.setDfsHaAutomaticFailoverEnabled( - Boolean.parseBoolean( - properties.getProperty( - "dfs_ha_automatic_failover_enabled", - String.valueOf(CONF.isDfsHaAutomaticFailoverEnabled())))); - CONF.setDfsClientFailoverProxyProvider( - properties.getProperty( - "dfs_client_failover_proxy_provider", CONF.getDfsClientFailoverProxyProvider())); - CONF.setUseKerberos( - Boolean.parseBoolean( - properties.getProperty("hdfs_use_kerberos", String.valueOf(CONF.isUseKerberos())))); - CONF.setKerberosKeytabFilePath( - properties.getProperty("kerberos_keytab_file_path", CONF.getKerberosKeytabFilePath())); - CONF.setKerberosPrincipal( - properties.getProperty("kerberos_principal", CONF.getKerberosPrincipal())); - - // the default fill interval in LinearFill and PreviousFill - CONF.setDefaultFillInterval( - Integer.parseInt( - properties.getProperty( - "default_fill_interval", String.valueOf(CONF.getDefaultFillInterval())))); - - if (CONF.getDnThriftMaxFrameSize() < IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2) { - CONF.setDnThriftMaxFrameSize(IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2); - } - - CONF.setFrequencyIntervalInMinute( - Integer.parseInt( - properties.getProperty( - "frequency_interval_in_minute", - String.valueOf(CONF.getFrequencyIntervalInMinute())))); - - CONF.setWindowEvaluationThreadCount( - Integer.parseInt( - properties.getProperty( - "window_evaluation_thread_count", - Integer.toString(CONF.getWindowEvaluationThreadCount())))); - if (CONF.getWindowEvaluationThreadCount() <= 0) { - CONF.setWindowEvaluationThreadCount(Runtime.getRuntime().availableProcessors()); - } - - CONF.setMaxPendingWindowEvaluationTasks( - Integer.parseInt( - properties.getProperty( - "max_pending_window_evaluation_tasks", - Integer.toString(CONF.getMaxPendingWindowEvaluationTasks())))); - if (CONF.getMaxPendingWindowEvaluationTasks() <= 0) { - CONF.setMaxPendingWindowEvaluationTasks(64); - } - - // id table related configuration - CONF.setDeviceIDTransformationMethod( - properties.getProperty( - "device_id_transformation_method", CONF.getDeviceIDTransformationMethod())); - - CONF.setEnableIDTable( - Boolean.parseBoolean( - properties.getProperty("enable_id_table", String.valueOf(CONF.isEnableIDTable())))); - - CONF.setEnableIDTableLogFile( - Boolean.parseBoolean( - properties.getProperty( - "enable_id_table_log_file", String.valueOf(CONF.isEnableIDTableLogFile())))); - - CONF.setSchemaEngineMode( - properties.getProperty("schema_engine_mode", String.valueOf(CONF.getSchemaEngineMode()))); - - CONF.setCachedMNodeSizeInSchemaFileMode( - Integer.parseInt( - properties.getProperty( - "cached_mnode_size_in_schema_file_mode", - String.valueOf(CONF.getCachedMNodeSizeInSchemaFileMode())))); - - CONF.setMinimumSegmentInSchemaFile( - Short.parseShort( - properties.getProperty( - "minimum_schema_file_segment_in_bytes", - String.valueOf(CONF.getMinimumSegmentInSchemaFile())))); - - CONF.setPageCacheSizeInSchemaFile( - Integer.parseInt( - properties.getProperty( - "page_cache_in_schema_file", String.valueOf(CONF.getPageCacheSizeInSchemaFile())))); - - CONF.setSchemaFileLogSize( - Integer.parseInt( - properties.getProperty( - "schema_file_log_size", String.valueOf(CONF.getSchemaFileLogSize())))); - - CONF.setExtPipeDir(properties.getProperty("ext_pipe_dir", CONF.getExtPipeDir()).trim()); - - // At the same time, set TSFileConfig - TSFileDescriptor.getInstance() - .getConfig() - .setTSFileStorageFs( - FSType.valueOf( - properties.getProperty("tsfile_storage_fs", CONF.getTsFileStorageFs().name()))); - TSFileDescriptor.getInstance() - .getConfig() - .setCoreSitePath(properties.getProperty("core_site_path", CONF.getCoreSitePath())); - TSFileDescriptor.getInstance() - .getConfig() - .setHdfsSitePath(properties.getProperty("hdfs_site_path", CONF.getHdfsSitePath())); - TSFileDescriptor.getInstance() - .getConfig() - .setHdfsIp(properties.getProperty("hdfs_ip", CONF.getRawHDFSIp()).split(",")); - TSFileDescriptor.getInstance() - .getConfig() - .setHdfsPort(properties.getProperty("hdfs_port", CONF.getHdfsPort())); - TSFileDescriptor.getInstance() - .getConfig() - .setDfsNameServices(properties.getProperty("dfs_nameservices", CONF.getDfsNameServices())); - TSFileDescriptor.getInstance() - .getConfig() - .setDfsHaNamenodes( - properties.getProperty("dfs_ha_namenodes", CONF.getRawDfsHaNamenodes()).split(",")); - TSFileDescriptor.getInstance() - .getConfig() - .setDfsHaAutomaticFailoverEnabled( - Boolean.parseBoolean( - properties.getProperty( - "dfs_ha_automatic_failover_enabled", - String.valueOf(CONF.isDfsHaAutomaticFailoverEnabled())))); - TSFileDescriptor.getInstance() - .getConfig() - .setDfsClientFailoverProxyProvider( - properties.getProperty( - "dfs_client_failover_proxy_provider", CONF.getDfsClientFailoverProxyProvider())); - TSFileDescriptor.getInstance() - .getConfig() - .setUseKerberos( - Boolean.parseBoolean( - properties.getProperty("hdfs_use_kerberos", String.valueOf(CONF.isUseKerberos())))); - TSFileDescriptor.getInstance() - .getConfig() - .setKerberosKeytabFilePath( - properties.getProperty("kerberos_keytab_file_path", CONF.getKerberosKeytabFilePath())); - TSFileDescriptor.getInstance() - .getConfig() - .setKerberosPrincipal( - properties.getProperty("kerberos_principal", CONF.getKerberosPrincipal())); - TSFileDescriptor.getInstance().getConfig().setBatchSize(CONF.getBatchSize()); - - // commons - commonDescriptor.loadCommonProps(properties); - commonDescriptor.initCommonConfigDir(CONF.getDnSystemDir()); - - // timed flush memtable - loadTimedService(properties); - - // set tsfile-format config - loadTsFileProps(properties); + /* Retain Configurations */ + loadRetainConfiguration(properties); // make RPCTransportFactory taking effect. RpcTransportFactory.reInit(); - // UDF - loadUDFProps(properties); - - // trigger - loadTriggerProps(properties); - - // CQ - loadCQProps(properties); - - // cluster - loadClusterProps(properties); - - // shuffle - loadShuffleProps(properties); - - // author cache - loadAuthorCache(properties); - - CONF.setTimePartitionInterval( - DateTimeUtils.convertMilliTimeWithPrecision( - CONF.getTimePartitionInterval(), CONF.getTimestampPrecision())); + // Set dn timePartitionInterval + CONF.setDnTimePartitionInterval( + DateTimeUtils.convertMilliTimeWithPrecision( + CONF.getDnTimePartitionInterval(), COMMON_CONFIG.getTimestampPrecision())); } private void loadDataNodeRPCConfiguration(Properties properties) { @@ -737,6 +561,145 @@ public class IoTDBDescriptor { Long.toString(CONF.getCrossCompactionFileSelectionTimeBudget())))); } + private void loadRetainConfiguration(Properties properties) { + CONF.setTsFileStorageFs( + properties.getProperty("tsfile_storage_fs", CONF.getTsFileStorageFs().toString())); + CONF.setCoreSitePath(properties.getProperty("core_site_path", CONF.getCoreSitePath())); + CONF.setHdfsSitePath(properties.getProperty("hdfs_site_path", CONF.getHdfsSitePath())); + CONF.setHdfsIp(properties.getProperty("hdfs_ip", CONF.getRawHDFSIp()).split(",")); + CONF.setHdfsPort(properties.getProperty("hdfs_port", CONF.getHdfsPort())); + CONF.setDfsNameServices(properties.getProperty("dfs_nameservices", CONF.getDfsNameServices())); + CONF.setDfsHaNamenodes( + properties.getProperty("dfs_ha_namenodes", CONF.getRawDfsHaNamenodes()).split(",")); + CONF.setDfsHaAutomaticFailoverEnabled( + Boolean.parseBoolean( + properties.getProperty( + "dfs_ha_automatic_failover_enabled", + String.valueOf(CONF.isDfsHaAutomaticFailoverEnabled())))); + CONF.setDfsClientFailoverProxyProvider( + properties.getProperty( + "dfs_client_failover_proxy_provider", CONF.getDfsClientFailoverProxyProvider())); + CONF.setUseKerberos( + Boolean.parseBoolean( + properties.getProperty("hdfs_use_kerberos", String.valueOf(CONF.isUseKerberos())))); + CONF.setKerberosKeytabFilePath( + properties.getProperty("kerberos_keytab_file_path", CONF.getKerberosKeytabFilePath())); + CONF.setKerberosPrincipal( + properties.getProperty("kerberos_principal", CONF.getKerberosPrincipal())); + + CONF.setWindowEvaluationThreadCount( + Integer.parseInt( + properties.getProperty( + "window_evaluation_thread_count", + Integer.toString(CONF.getWindowEvaluationThreadCount())))); + if (CONF.getWindowEvaluationThreadCount() <= 0) { + CONF.setWindowEvaluationThreadCount(Runtime.getRuntime().availableProcessors()); + } + + CONF.setMaxPendingWindowEvaluationTasks( + Integer.parseInt( + properties.getProperty( + "max_pending_window_evaluation_tasks", + Integer.toString(CONF.getMaxPendingWindowEvaluationTasks())))); + if (CONF.getMaxPendingWindowEvaluationTasks() <= 0) { + CONF.setMaxPendingWindowEvaluationTasks(64); + } + + // id table related configuration + CONF.setDeviceIDTransformationMethod( + properties.getProperty( + "device_id_transformation_method", CONF.getDeviceIDTransformationMethod())); + + CONF.setEnableIDTable( + Boolean.parseBoolean( + properties.getProperty("enable_id_table", String.valueOf(CONF.isEnableIDTable())))); + + CONF.setEnableIDTableLogFile( + Boolean.parseBoolean( + properties.getProperty( + "enable_id_table_log_file", String.valueOf(CONF.isEnableIDTableLogFile())))); + + CONF.setSchemaEngineMode( + properties.getProperty("schema_engine_mode", String.valueOf(CONF.getSchemaEngineMode()))); + + CONF.setCachedMNodeSizeInSchemaFileMode( + Integer.parseInt( + properties.getProperty( + "cached_mnode_size_in_schema_file_mode", + String.valueOf(CONF.getCachedMNodeSizeInSchemaFileMode())))); + + CONF.setMinimumSegmentInSchemaFile( + Short.parseShort( + properties.getProperty( + "minimum_schema_file_segment_in_bytes", + String.valueOf(CONF.getMinimumSegmentInSchemaFile())))); + + CONF.setPageCacheSizeInSchemaFile( + Integer.parseInt( + properties.getProperty( + "page_cache_in_schema_file", String.valueOf(CONF.getPageCacheSizeInSchemaFile())))); + + CONF.setSchemaFileLogSize( + Integer.parseInt( + properties.getProperty( + "schema_file_log_size", String.valueOf(CONF.getSchemaFileLogSize())))); + + CONF.setExtPipeDir(properties.getProperty("ext_pipe_dir", CONF.getExtPipeDir()).trim()); + + // At the same time, set TSFileConfig + TSFileDescriptor.getInstance() + .getConfig() + .setTSFileStorageFs( + FSType.valueOf( + properties.getProperty("tsfile_storage_fs", CONF.getTsFileStorageFs().name()))); + TSFileDescriptor.getInstance() + .getConfig() + .setCoreSitePath(properties.getProperty("core_site_path", CONF.getCoreSitePath())); + TSFileDescriptor.getInstance() + .getConfig() + .setHdfsSitePath(properties.getProperty("hdfs_site_path", CONF.getHdfsSitePath())); + TSFileDescriptor.getInstance() + .getConfig() + .setHdfsIp(properties.getProperty("hdfs_ip", CONF.getRawHDFSIp()).split(",")); + TSFileDescriptor.getInstance() + .getConfig() + .setHdfsPort(properties.getProperty("hdfs_port", CONF.getHdfsPort())); + TSFileDescriptor.getInstance() + .getConfig() + .setDfsNameServices(properties.getProperty("dfs_nameservices", CONF.getDfsNameServices())); + TSFileDescriptor.getInstance() + .getConfig() + .setDfsHaNamenodes( + properties.getProperty("dfs_ha_namenodes", CONF.getRawDfsHaNamenodes()).split(",")); + TSFileDescriptor.getInstance() + .getConfig() + .setDfsHaAutomaticFailoverEnabled( + Boolean.parseBoolean( + properties.getProperty( + "dfs_ha_automatic_failover_enabled", + String.valueOf(CONF.isDfsHaAutomaticFailoverEnabled())))); + TSFileDescriptor.getInstance() + .getConfig() + .setDfsClientFailoverProxyProvider( + properties.getProperty( + "dfs_client_failover_proxy_provider", CONF.getDfsClientFailoverProxyProvider())); + TSFileDescriptor.getInstance() + .getConfig() + .setUseKerberos( + Boolean.parseBoolean( + properties.getProperty("hdfs_use_kerberos", String.valueOf(CONF.isUseKerberos())))); + TSFileDescriptor.getInstance() + .getConfig() + .setKerberosKeytabFilePath( + properties.getProperty("kerberos_keytab_file_path", CONF.getKerberosKeytabFilePath())); + TSFileDescriptor.getInstance() + .getConfig() + .setKerberosPrincipal( + properties.getProperty("kerberos_principal", CONF.getKerberosPrincipal())); + + TSFileDescriptor.getInstance().getConfig().setBatchSize(COMMON_CONFIG.getBatchSize()); + } + public void loadHotModifiedProps(Properties properties) throws QueryProcessException { try { // update data dirs @@ -754,52 +717,14 @@ public class IoTDBDescriptor { DirectoryManager.getInstance().updateDirectoryStrategy(); } - // update timed flush & close conf - loadTimedService(properties); StorageEngine.getInstance().rebootTimedService(); - // update tsfile-format config - loadTsFileProps(properties); - - // update max_deduplicated_path_num - CONF.setMaxQueryDeduplicatedPathNum( - Integer.parseInt( - properties.getProperty( - "max_deduplicated_path_num", - Integer.toString(CONF.getMaxQueryDeduplicatedPathNum())))); - // update slow_query_threshold - CONF.setSlowQueryThreshold( - Long.parseLong( - properties.getProperty( - "slow_query_threshold", Long.toString(CONF.getSlowQueryThreshold())))); // update merge_write_throughput_mb_per_sec CONF.setCompactionWriteThroughputMbPerSec( Integer.parseInt( properties.getProperty( "merge_write_throughput_mb_per_sec", Integer.toString(CONF.getCompactionWriteThroughputMbPerSec())))); - // update insert-tablet-plan's row limit for select-into - CONF.setSelectIntoInsertTabletPlanRowLimit( - Integer.parseInt( - properties.getProperty( - "select_into_insert_tablet_plan_row_limit", - String.valueOf(CONF.getSelectIntoInsertTabletPlanRowLimit())))); - - // update sync config - CONF.setMaxNumberOfSyncFileRetry( - Integer.parseInt( - properties - .getProperty( - "max_number_of_sync_file_retry", - Integer.toString(CONF.getMaxNumberOfSyncFileRetry())) - .trim())); - - // update wal config - long prevDeleteWalFilesPeriodInMs = CONF.getDeleteWalFilesPeriodInMs(); - loadWALHotModifiedProps(properties); - if (prevDeleteWalFilesPeriodInMs != CONF.getDeleteWalFilesPeriodInMs()) { - WALManager.getInstance().rebootWALDeleteThread(); - } } catch (Exception e) { throw new QueryProcessException(String.format("Fail to reload configuration because %s", e)); } @@ -854,45 +779,11 @@ public class IoTDBDescriptor { } } - private void loadCQProps(Properties properties) { - CONF.setContinuousQueryThreadNum( - Integer.parseInt( - properties.getProperty( - "continuous_query_thread_num", - Integer.toString(CONF.getContinuousQueryThreadNum())))); - if (CONF.getContinuousQueryThreadNum() <= 0) { - CONF.setContinuousQueryThreadNum(Runtime.getRuntime().availableProcessors() / 2); - } - - CONF.setContinuousQueryMinimumEveryInterval( - DateTimeUtils.convertDurationStrToLong( - properties.getProperty("continuous_query_minimum_every_interval", "1s"), - CONF.getTimestampPrecision())); - } - - /** Get default encode algorithm by data type */ - public TSEncoding getDefaultEncodingByType(TSDataType dataType) { - switch (dataType) { - case BOOLEAN: - return CONF.getDefaultBooleanEncoding(); - case INT32: - return CONF.getDefaultInt32Encoding(); - case INT64: - return CONF.getDefaultInt64Encoding(); - case FLOAT: - return CONF.getDefaultFloatEncoding(); - case DOUBLE: - return CONF.getDefaultDoubleEncoding(); - default: - return CONF.getDefaultTextEncoding(); - } - } - // These configurations are received from config node when registering public void loadGlobalConfig(TGlobalConfig globalConfig) { CONF.setSeriesPartitionExecutorClass(globalConfig.getSeriesPartitionExecutorClass()); CONF.setSeriesPartitionSlotNum(globalConfig.getSeriesPartitionSlotNum()); - CONF.setTimePartitionInterval( + CONF.setDnTimePartitionInterval( DateTimeUtils.convertMilliTimeWithPrecision( globalConfig.timePartitionInterval, CONF.getTimestampPrecision())); CONF.setReadConsistencyLevel(globalConfig.getReadConsistencyLevel()); @@ -902,43 +793,5 @@ public class IoTDBDescriptor { CONF.setCqMinEveryIntervalInMs(cqConfig.getCqMinEveryIntervalInMs()); } - public void reclaimConsensusMemory() { - CONF.setAllocateMemoryForStorageEngine( - CONF.getAllocateMemoryForStorageEngine() + CONF.getAllocateMemoryForConsensus()); - SystemInfo.getInstance().allocateWriteMemory(); - } - - public void initClusterSchemaMemoryAllocate() { - if (!CONF.isDefaultSchemaMemoryConfig()) { - // the config has already been updated as user config in properties file - return; - } - - // process the default schema memory allocate - - long schemaMemoryTotal = CONF.getAllocateMemoryForSchema(); - int proportionSum = 10; - int schemaRegionProportion = 5; - int schemaCacheProportion = 3; - int partitionCacheProportion = 1; - int lastCacheProportion = 1; - - CONF.setAllocateMemoryForSchemaRegion( - schemaMemoryTotal * schemaRegionProportion / proportionSum); - logger.info( - "Cluster allocateMemoryForSchemaRegion = {}", CONF.getAllocateMemoryForSchemaRegion()); - - CONF.setAllocateMemoryForSchemaCache(schemaMemoryTotal * schemaCacheProportion / proportionSum); - logger.info( - "Cluster allocateMemoryForSchemaCache = {}", CONF.getAllocateMemoryForSchemaCache()); - - CONF.setAllocateMemoryForPartitionCache( - schemaMemoryTotal * partitionCacheProportion / proportionSum); - logger.info( - "Cluster allocateMemoryForPartitionCache = {}", CONF.getAllocateMemoryForPartitionCache()); - - CONF.setAllocateMemoryForLastCache(schemaMemoryTotal * lastCacheProportion / proportionSum); - logger.info("Cluster allocateMemoryForLastCache = {}", CONF.getAllocateMemoryForLastCache()); - } } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java index a6f8349313..be73577905 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java @@ -88,7 +88,7 @@ public class IoTDBStartCheck { static { constantParamValueTable.put(TIMESTAMP_PRECISION_STRING, config::getTimestampPrecision); constantParamValueTable.put( - PARTITION_INTERVAL_STRING, () -> String.valueOf(config.getTimePartitionInterval())); + PARTITION_INTERVAL_STRING, () -> String.valueOf(config.getDnTimePartitionInterval())); constantParamValueTable.put( TSFILE_FILE_SYSTEM_STRING, () -> config.getTsFileStorageFs().toString()); constantParamValueTable.put( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java index e165ba2d01..250f1b36b5 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java @@ -146,7 +146,7 @@ public class StorageEngine implements IService { } private static void initTimePartition() { - timePartitionInterval = IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); + timePartitionInterval = IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); } public static long getTimePartitionInterval() { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/GetTimeSlotListTask.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/GetTimeSlotListTask.java index 27d6381f4c..e7f5f63223 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/GetTimeSlotListTask.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/GetTimeSlotListTask.java @@ -61,7 +61,7 @@ public class GetTimeSlotListTask implements IConfigTask { .getColumnBuilder(0) .writeLong( timePartitionSlot.getStartTime() - / IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval()); + / IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval()); builder.declarePosition(); } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java index 919b943b8b..507476f8a4 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java @@ -3046,7 +3046,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { if (ctx.timeSlot != null) { getRegionIdStatement.setTimeSlotId( new TTimePartitionSlot( - Long.parseLong(ctx.timeSlot.getText()) * CONFIG.getTimePartitionInterval())); + Long.parseLong(ctx.timeSlot.getText()) * CONFIG.getDnTimePartitionInterval())); } else if (ctx.timeStamp != null) { getRegionIdStatement.setTimeStamp(Long.parseLong(ctx.timeStamp.getText())); } diff --git a/server/src/main/java/org/apache/iotdb/db/service/DataNode.java b/server/src/main/java/org/apache/iotdb/db/service/DataNode.java index 475a8ddc9a..770e5ac3b0 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/server/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -286,6 +286,7 @@ public class DataNode implements DataNodeMBean { if (!config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS)) { // In current implementation, only IoTConsensus need separated memory from Consensus IoTDBDescriptor.getInstance().reclaimConsensusMemory(); + SystemInfo.getInstance().allocateWriteMemory(); } } catch (Exception e) { throw new StartupException(e.getMessage()); diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java index 956b4be9f2..d9b1f291d3 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.service.thrift.impl; +import org.apache.iotdb.db.wal.WALManager; import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; @@ -1115,7 +1116,19 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface @Override public TSStatus loadConfiguration() throws TException { try { + long prevDeleteWalFilesPeriodInMs = CommonDescriptor.getInstance().getConfig().getDeleteWalFilesPeriodInMs(); + + CommonDescriptor.getInstance().loadHotModifiedProps(); IoTDBDescriptor.getInstance().loadHotModifiedProps(); + + try { + if (prevDeleteWalFilesPeriodInMs != CommonDescriptor.getInstance().getConfig().getDeleteWalFilesPeriodInMs()) { + WALManager.getInstance().rebootWALDeleteThread(); + } + } catch (Exception e) { + throw new QueryProcessException(String.format("Fail to reload configuration because %s", e)); + } + } catch (QueryProcessException e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java index 763b7e0d5a..737372f789 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java @@ -26,7 +26,7 @@ import org.apache.iotdb.db.conf.ServerConfigConsistent; public class TimePartitionUtils { @ServerConfigConsistent public static long timePartitionInterval = - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); public static TTimePartitionSlot getTimePartition(long time) { TTimePartitionSlot timePartitionSlot = new TTimePartitionSlot(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java index 470dd48a94..bd439ef804 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java @@ -35,8 +35,8 @@ public class SizeTieredCompactionSelectorTest { @Test public void testSubmitWhenNextTimePartitionExists() { long originPartitionInterval = - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); - IoTDBDescriptor.getInstance().getConfig().setTimePartitionInterval(1000000); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); + IoTDBDescriptor.getInstance().getConfig().setDnTimePartitionInterval(1000000); List<TsFileResource> resources = new ArrayList<>(); for (int i = 0; i < 100; ++i) { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java index 34b2655e6c..74a1e456ec 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java @@ -413,9 +413,9 @@ public class DataRegionTest { public void testEnableDiscardOutOfOrderDataForInsertTablet1() throws QueryProcessException, IllegalPathException, IOException, WriteProcessException { boolean defaultEnableDiscard = config.isEnableDiscardOutOfOrderData(); - long defaultTimePartition = config.getTimePartitionInterval(); + long defaultTimePartition = config.getDnTimePartitionInterval(); config.setEnableDiscardOutOfOrderData(true); - config.setTimePartitionInterval(100000); + config.setDnTimePartitionInterval(100000); String[] measurements = new String[2]; measurements[0] = "s0"; @@ -494,16 +494,16 @@ public class DataRegionTest { } config.setEnableDiscardOutOfOrderData(defaultEnableDiscard); - config.setTimePartitionInterval(defaultTimePartition); + config.setDnTimePartitionInterval(defaultTimePartition); } @Test public void testEnableDiscardOutOfOrderDataForInsertTablet2() throws QueryProcessException, IllegalPathException, IOException, WriteProcessException { boolean defaultEnableDiscard = config.isEnableDiscardOutOfOrderData(); - long defaultTimePartition = config.getTimePartitionInterval(); + long defaultTimePartition = config.getDnTimePartitionInterval(); config.setEnableDiscardOutOfOrderData(true); - config.setTimePartitionInterval(1200000); + config.setDnTimePartitionInterval(1200000); String[] measurements = new String[2]; measurements[0] = "s0"; @@ -582,16 +582,16 @@ public class DataRegionTest { } config.setEnableDiscardOutOfOrderData(defaultEnableDiscard); - config.setTimePartitionInterval(defaultTimePartition); + config.setDnTimePartitionInterval(defaultTimePartition); } @Test public void testEnableDiscardOutOfOrderDataForInsertTablet3() throws QueryProcessException, IllegalPathException, IOException, WriteProcessException { boolean defaultEnableDiscard = config.isEnableDiscardOutOfOrderData(); - long defaultTimePartition = config.getTimePartitionInterval(); + long defaultTimePartition = config.getDnTimePartitionInterval(); config.setEnableDiscardOutOfOrderData(true); - config.setTimePartitionInterval(1000000); + config.setDnTimePartitionInterval(1000000); String[] measurements = new String[2]; measurements[0] = "s0"; @@ -670,7 +670,7 @@ public class DataRegionTest { } config.setEnableDiscardOutOfOrderData(defaultEnableDiscard); - config.setTimePartitionInterval(defaultTimePartition); + config.setDnTimePartitionInterval(defaultTimePartition); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java index aa679b1c70..0c1472ae28 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java @@ -82,8 +82,8 @@ public class TTLTest { @Before public void setUp() throws MetadataException, DataRegionException { - prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); - IoTDBDescriptor.getInstance().getConfig().setTimePartitionInterval(86400000); + prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); + IoTDBDescriptor.getInstance().getConfig().setDnTimePartitionInterval(86400000); EnvironmentUtils.envSetUp(); dataRegion = new DataRegion( @@ -98,7 +98,7 @@ public class TTLTest { public void tearDown() throws IOException, StorageEngineException { dataRegion.syncCloseAllWorkingTsFileProcessors(); EnvironmentUtils.cleanEnv(); - IoTDBDescriptor.getInstance().getConfig().setTimePartitionInterval(prevPartitionInterval); + IoTDBDescriptor.getInstance().getConfig().setDnTimePartitionInterval(prevPartitionInterval); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java index e795634438..4a240f0f32 100644 --- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java @@ -397,16 +397,16 @@ public class QueryTimePartitionTest { new AndFilter( TimeFilter.gt(0), TimeFilter.ltEq( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 3 + 1))); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 3 + 1))); expected = Arrays.asList( new TTimePartitionSlot(0), new TTimePartitionSlot( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval()), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval()), new TTimePartitionSlot( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 2), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 2), new TTimePartitionSlot( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 3)); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 3)); assertEquals(expected.size(), res.left.size()); for (int i = 0; i < expected.size(); i++) { assertEquals(expected.get(i), res.left.get(i)); @@ -420,14 +420,14 @@ public class QueryTimePartitionTest { getTimePartitionSlotList( new AndFilter( TimeFilter.gtEq( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() - 1), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() - 1), TimeFilter.lt( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1))); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() + 1))); expected = Arrays.asList( new TTimePartitionSlot(0), new TTimePartitionSlot( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval())); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval())); assertEquals(expected.size(), res.left.size()); for (int i = 0; i < expected.size(); i++) { assertEquals(expected.get(i), res.left.get(i)); @@ -440,14 +440,14 @@ public class QueryTimePartitionTest { res = getTimePartitionSlotList( TimeFilter.between( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() - 1, - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() - 1, + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(), false)); expected = Arrays.asList( new TTimePartitionSlot(0), new TTimePartitionSlot( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval())); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval())); assertEquals(expected.size(), res.left.size()); for (int i = 0; i < expected.size(); i++) { assertEquals(expected.get(i), res.left.get(i)); @@ -461,13 +461,13 @@ public class QueryTimePartitionTest { getTimePartitionSlotList( new AndFilter( TimeFilter.gtEq( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval()), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval()), TimeFilter.ltEq( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1))); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() + 1))); expected = Collections.singletonList( new TTimePartitionSlot( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval())); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval())); assertEquals(expected.size(), res.left.size()); for (int i = 0; i < expected.size(); i++) { assertEquals(expected.get(i), res.left.get(i)); @@ -480,13 +480,13 @@ public class QueryTimePartitionTest { res = getTimePartitionSlotList( TimeFilter.between( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(), - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1, + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() + 1, false)); expected = Collections.singletonList( new TTimePartitionSlot( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval())); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval())); assertEquals(expected.size(), res.left.size()); for (int i = 0; i < expected.size(); i++) { assertEquals(expected.get(i), res.left.get(i)); @@ -509,38 +509,38 @@ public class QueryTimePartitionTest { new AndFilter( TimeFilter.gtEq(10), TimeFilter.lt( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval())), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval())), new AndFilter( - TimeFilter.gt(IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval()), + TimeFilter.gt(IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval()), TimeFilter.lt( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 2 + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 2 - 100))); OrFilter orFilter2 = new OrFilter( orFilter1, new AndFilter( TimeFilter.gt( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 2 - 50), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 2 - 50), TimeFilter.ltEq( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 2 + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 2 - 40))); OrFilter orFilter3 = new OrFilter( orFilter2, new AndFilter( TimeFilter.gt( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 2 - 20), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 2 - 20), TimeFilter.ltEq( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 3 + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 3 + 10))); OrFilter orFilter4 = new OrFilter( orFilter3, new AndFilter( TimeFilter.gt( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 5 + 1), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 5 + 1), TimeFilter.lt( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 5 + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 5 + 10))); res = getTimePartitionSlotList(orFilter4); @@ -548,13 +548,13 @@ public class QueryTimePartitionTest { Arrays.asList( new TTimePartitionSlot(0), new TTimePartitionSlot( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval()), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval()), new TTimePartitionSlot( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 2), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 2), new TTimePartitionSlot( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 3), + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 3), new TTimePartitionSlot( - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() * 5)); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval() * 5)); assertEquals(expected.size(), res.left.size()); for (int i = 0; i < expected.size(); i++) { assertEquals(expected.get(i), res.left.get(i)); diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/WritePlanNodeSplitTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/WritePlanNodeSplitTest.java index d7a6b90d87..2fba2f07fa 100644 --- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/WritePlanNodeSplitTest.java +++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/write/WritePlanNodeSplitTest.java @@ -69,8 +69,8 @@ public class WritePlanNodeSplitTest { @Before public void setUp() { prevTimePartitionInterval = - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(); - IoTDBDescriptor.getInstance().getConfig().setTimePartitionInterval(100); + IoTDBDescriptor.getInstance().getConfig().getDnTimePartitionInterval(); + IoTDBDescriptor.getInstance().getConfig().setDnTimePartitionInterval(100); TimePartitionUtils.setTimePartitionInterval(100); executorClassName = IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionExecutorClass(); @@ -305,6 +305,6 @@ public class WritePlanNodeSplitTest { @After public void tearDown() { TimePartitionUtils.setTimePartitionInterval(prevTimePartitionInterval); - IoTDBDescriptor.getInstance().getConfig().setTimePartitionInterval(prevTimePartitionInterval); + IoTDBDescriptor.getInstance().getConfig().setDnTimePartitionInterval(prevTimePartitionInterval); } } diff --git a/server/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java b/server/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java index e7b3ddb5e1..f397a0bc21 100644 --- a/server/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java +++ b/server/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java @@ -71,8 +71,8 @@ public class TsFileAndModSettleToolTest { @Before public void setUp() { config = IoTDBDescriptor.getInstance().getConfig(); - originPartitionInterval = config.getTimePartitionInterval(); - config.setTimePartitionInterval(newPartitionInterval); + originPartitionInterval = config.getDnTimePartitionInterval(); + config.setDnTimePartitionInterval(newPartitionInterval); EnvironmentUtils.envSetUp(); File f = new File(folder); @@ -100,7 +100,7 @@ public class TsFileAndModSettleToolTest { } catch (Exception e) { Assert.fail(e.getMessage()); } finally { - config.setTimePartitionInterval(originPartitionInterval); + config.setDnTimePartitionInterval(originPartitionInterval); } }
