This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ExternalLib013 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 727fd206b88eb9d73abee57f4da9f730304f7b98 Author: JackieTien97 <[email protected]> AuthorDate: Thu Aug 25 09:14:36 2022 +0800 Support loading external lib in 0.13 --- external-api/pom.xml | 60 + .../iotdb/external/api/IPropertiesLoader.java | 37 + .../iotdb/external/api/ISeriesNumerLimiter.java | 48 + pom.xml | 1 + server/pom.xml | 5 + .../resources/conf/iotdb-engine.properties | 25 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 29 + .../org/apache/iotdb/db/conf/IoTDBConstant.java | 4 + .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 1428 ++++++++++---------- .../metadata/SeriesNumberOverflowException.java | 28 + .../org/apache/iotdb/db/metadata/MManager.java | 191 ++- .../java/org/apache/iotdb/db/service/IoTDB.java | 5 + .../org/apache/iotdb/db/utils/JarLoaderUtil.java | 156 +++ .../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 + .../iotdb/tsfile/common/conf/TSFileDescriptor.java | 2 +- 15 files changed, 1268 insertions(+), 753 deletions(-) diff --git a/external-api/pom.xml b/external-api/pom.xml new file mode 100644 index 0000000000..dd28dcdc69 --- /dev/null +++ b/external-api/pom.xml @@ -0,0 +1,60 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>iotdb-parent</artifactId> + <groupId>org.apache.iotdb</groupId> + <version>0.13.2-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>external-api</artifactId> + <profiles> + <profile> + <id>get-jar-with-dependencies</id> + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <version>${maven.assembly.version}</version> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <!-- this is used for inheritance merges --> + <phase>package</phase> + <!-- bind to the packaging phase --> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> diff --git a/external-api/src/main/java/org/apche/iotdb/external/api/IPropertiesLoader.java b/external-api/src/main/java/org/apche/iotdb/external/api/IPropertiesLoader.java new file mode 100644 index 0000000000..10c5b21e7e --- /dev/null +++ b/external-api/src/main/java/org/apche/iotdb/external/api/IPropertiesLoader.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apche.iotdb.external.api; + +import java.nio.file.Path; +import java.util.Properties; + +/** + * An interface to load properties from external properties file to override the default + * configurations + */ +public interface IPropertiesLoader { + + /** + * Load Properties from specific file + * + * @param file The path of the properties file to open + * @return a property list with values in file. + */ + Properties loadProperties(Path file); +} diff --git a/external-api/src/main/java/org/apche/iotdb/external/api/ISeriesNumerLimiter.java b/external-api/src/main/java/org/apche/iotdb/external/api/ISeriesNumerLimiter.java new file mode 100644 index 0000000000..7b36b5d8b7 --- /dev/null +++ b/external-api/src/main/java/org/apche/iotdb/external/api/ISeriesNumerLimiter.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apche.iotdb.external.api; + +import java.util.Properties; + +/** An interface for series number limiting, users can implement their own limitation strategy */ +public interface ISeriesNumerLimiter { + + /** + * do the necessary initialization + * + * @param properties Properties containing all the parameters needed to init + */ + void init(Properties properties); + + /** + * add time series + * + * @param number time series number for current createTimeSeries operation + * @return true if totalTimeSeriesNumber doesn't exceed the limit and current createTimeSeries + * operation is allowed, otherwise false + */ + boolean addTimeSeries(int number); + + /** + * delete time series + * + * @param number time series number for current deleteTimeSeries operation + */ + void deleteTimeSeries(int number); +} diff --git a/pom.xml b/pom.xml index 31d9b94d56..0b742019b3 100644 --- a/pom.xml +++ b/pom.xml @@ -110,6 +110,7 @@ <module>metrics</module> <module>integration</module> <module>rewriteFileTool</module> + <module>external-api</module> <!-- <module>library-udf</module>--> </modules> <!-- Properties Management --> diff --git a/server/pom.xml b/server/pom.xml index cadf11281e..a8cdc0088a 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -67,6 +67,11 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>external-api</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>org.apache.iotdb</groupId> <artifactId>iotdb-thrift-sync</artifactId> diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 0be00a433d..ebd1a35242 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -935,4 +935,27 @@ timestamp_precision=ms ### Group By Fill Configuration #################### # Datatype: float -# group_by_fill_cache_size_in_mb=1.0 \ No newline at end of file +# group_by_fill_cache_size_in_mb=1.0 + + +#################### +### External Lib Configuration +#################### + +# external lib directory for properties loader +# For Window platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is +# absolute. Otherwise, it is relative. +# external_properties_loader_dir=ext\\loader +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# external_properties_loader_dir=ext/loader + +# external lib directory for limiter +# For Window platform +# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is +# absolute. Otherwise, it is relative. +# external_limiter_dir=ext\\limiter +# For Linux platform +# If its prefix is "/", then the path is absolute. Otherwise, it is relative. +# external_limiter_dir=ext/limiter \ No newline at end of file diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 997ea249d6..c3e0bdcde9 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -49,6 +49,7 @@ public class IoTDBConfig { /* Names of Watermark methods */ public static final String WATERMARK_GROUPED_LSB = "GroupBasedLSBMethod"; static final String CONFIG_NAME = "iotdb-engine.properties"; + public static final String EXTERNAL_CONFIG_NAME = "iotdb-engine-external.properties"; private static final Logger logger = LoggerFactory.getLogger(IoTDBConfig.class); private static final String MULTI_DIR_STRATEGY_PREFIX = "org.apache.iotdb.db.conf.directories.strategy."; @@ -245,6 +246,16 @@ public class IoTDBConfig { private String triggerDir = IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.TRIGGER_FOLDER_NAME; + /** External lib directory for properties loader, stores user-uploaded JAR files */ + private String externalPropertiesLoaderDir = + IoTDBConstant.EXT_FOLDER_NAME + + File.separator + + IoTDBConstant.EXT_PROPERTIES_LOADER_FOLDER_NAME; + + /** External lib directory for limiter, stores user uploaded JAR files */ + private String externalLimiterDir = + IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.EXT_LIMITER; + /** Data directory of data. It can be settled as dataDirs = {"data1", "data2", "data3"}; */ private String[] dataDirs = {"data" + File.separator + "data"}; @@ -988,6 +999,8 @@ public class IoTDBConfig { udfDir = addHomeDir(udfDir); triggerDir = addHomeDir(triggerDir); operationSyncLogDir = addHomeDir(operationSyncLogDir); + externalPropertiesLoaderDir = addHomeDir(externalPropertiesLoaderDir); + externalLimiterDir = addHomeDir(externalLimiterDir); if (TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs().equals(FSType.HDFS)) { String hdfsDir = getHdfsDir(); @@ -1212,6 +1225,22 @@ public class IoTDBConfig { this.triggerDir = triggerDir; } + public String getExternalPropertiesLoaderDir() { + return externalPropertiesLoaderDir; + } + + public void setExternalPropertiesLoaderDir(String externalPropertiesLoaderDir) { + this.externalPropertiesLoaderDir = externalPropertiesLoaderDir; + } + + public String getExternalLimiterDir() { + return externalLimiterDir; + } + + public void setExternalLimiterDir(String externalLimiterDir) { + this.externalLimiterDir = externalLimiterDir; + } + public String getMultiDirStrategyClassName() { return multiDirStrategyClassName; } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java index 1294002707..337a9e0a6e 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java @@ -191,6 +191,10 @@ public class IoTDBConstant { // compaction mods of previous version (<0.13) public static final String COMPACTION_MODIFICATION_FILE_NAME_FROM_OLD = "merge.mods"; + public static final String EXT_PROPERTIES_LOADER_FOLDER_NAME = "loader"; + + public static final String EXT_LIMITER = "limiter"; + // client version number public enum ClientVersion { V_0_12, diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 19d57c8699..cdb0057a2d 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -45,8 +45,11 @@ import java.io.IOException; import java.io.InputStream; import java.net.InetAddress; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.net.UnknownHostException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Properties; public class IoTDBDescriptor { @@ -114,6 +117,52 @@ public class IoTDBDescriptor { } } + /** + * get external props url location + * + * @return url object if location exit, otherwise null. + */ + public Path getExternalPropsPath() { + // Check if a config-directory was specified first. + String urlString = System.getProperty(IoTDBConstant.IOTDB_CONF, null); + // If it wasn't, check if a home directory was provided (This usually contains a config) + if (urlString == null) { + urlString = System.getProperty(IoTDBConstant.IOTDB_HOME, null); + if (urlString != null) { + urlString = + urlString + + File.separatorChar + + "conf" + + File.separatorChar + + IoTDBConfig.EXTERNAL_CONFIG_NAME; + } else { + // If this too wasn't provided, try to find a default config in the root of the classpath. + URL uri = IoTDBConfig.class.getResource("/" + IoTDBConfig.EXTERNAL_CONFIG_NAME); + if (uri != null) { + try { + return Paths.get(uri.toURI()); + } catch (URISyntaxException e) { + return null; + } + } + logger.warn( + "Cannot find IOTDB_HOME or IOTDB_EXTERNAL_CONF environment variable when loading " + + "config file {}, use default configuration", + IoTDBConfig.EXTERNAL_CONFIG_NAME); + // update all data seriesPath + conf.updatePath(); + return null; + } + } + // If a config location was provided, but it doesn't end with a properties file, + // append the default location. + else if (!urlString.endsWith(".properties")) { + urlString += (File.separatorChar + IoTDBConfig.EXTERNAL_CONFIG_NAME); + } + + return Paths.get(urlString); + } + /** load an property file and set TsfileDBConfig variables. */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning private void loadProps() { @@ -129,814 +178,817 @@ public class IoTDBDescriptor { Properties properties = new Properties(); properties.load(inputStream); - conf.setRpcAddress(properties.getProperty("rpc_address", conf.getRpcAddress())); - replaceHostnameWithIP(); + loadProperties(properties); - conf.setRpcThriftCompressionEnable( - Boolean.parseBoolean( - properties.getProperty( - "rpc_thrift_compression_enable", - Boolean.toString(conf.isRpcThriftCompressionEnable())))); + } catch (FileNotFoundException e) { + logger.warn("Fail to find config file {}", url, e); + } catch (IOException e) { + logger.warn("Cannot load config file, use default configuration", e); + } catch (Exception e) { + logger.warn("Incorrect format in config file, use default configuration", e); + } finally { + // update all data seriesPath + conf.updatePath(); + // update instance in metric + MetricConfigDescriptor.getInstance() + .getMetricConfig() + .updateRpcInstance(conf.getRpcAddress(), conf.getRpcPort()); + } + } - conf.setRpcAdvancedCompressionEnable( - Boolean.parseBoolean( - properties.getProperty( - "rpc_advanced_compression_enable", - Boolean.toString(conf.isRpcAdvancedCompressionEnable())))); + public void loadProperties(Properties properties) throws UnknownHostException { - conf.setRpcPort( - Integer.parseInt( - properties.getProperty("rpc_port", Integer.toString(conf.getRpcPort())))); + conf.setRpcAddress(properties.getProperty("rpc_address", conf.getRpcAddress())); + replaceHostnameWithIP(); - conf.setEnableInfluxDBRpcService( - Boolean.parseBoolean( - properties.getProperty( - "enable_influxdb_rpc_service", - Boolean.toString(conf.isEnableInfluxDBRpcService())))); + conf.setRpcThriftCompressionEnable( + Boolean.parseBoolean( + properties.getProperty( + "rpc_thrift_compression_enable", + Boolean.toString(conf.isRpcThriftCompressionEnable())))); - conf.setInfluxDBRpcPort( - Integer.parseInt( - properties.getProperty( - "influxdb_rpc_port", Integer.toString(conf.getInfluxDBRpcPort())))); + conf.setRpcAdvancedCompressionEnable( + Boolean.parseBoolean( + properties.getProperty( + "rpc_advanced_compression_enable", + Boolean.toString(conf.isRpcAdvancedCompressionEnable())))); - conf.setTimestampPrecision( - properties.getProperty("timestamp_precision", conf.getTimestampPrecision())); + conf.setRpcPort( + Integer.parseInt(properties.getProperty("rpc_port", Integer.toString(conf.getRpcPort())))); - conf.setBufferedArraysMemoryProportion( - Double.parseDouble( - properties.getProperty( - "buffered_arrays_memory_proportion", - Double.toString(conf.getBufferedArraysMemoryProportion())))); + conf.setEnableInfluxDBRpcService( + Boolean.parseBoolean( + properties.getProperty( + "enable_influxdb_rpc_service", + Boolean.toString(conf.isEnableInfluxDBRpcService())))); - conf.setTimeIndexMemoryProportion( - Double.parseDouble( - properties.getProperty( - "time_index_memory_proportion", - Double.toString(conf.getTimeIndexMemoryProportion())))); + conf.setInfluxDBRpcPort( + Integer.parseInt( + properties.getProperty( + "influxdb_rpc_port", Integer.toString(conf.getInfluxDBRpcPort())))); - conf.setFlushProportion( - Double.parseDouble( - properties.getProperty( - "flush_proportion", Double.toString(conf.getFlushProportion())))); + conf.setTimestampPrecision( + properties.getProperty("timestamp_precision", conf.getTimestampPrecision())); - conf.setRejectProportion( - Double.parseDouble( - properties.getProperty( - "reject_proportion", Double.toString(conf.getRejectProportion())))); + conf.setBufferedArraysMemoryProportion( + Double.parseDouble( + properties.getProperty( + "buffered_arrays_memory_proportion", + Double.toString(conf.getBufferedArraysMemoryProportion())))); - conf.setStorageGroupSizeReportThreshold( - Long.parseLong( - properties.getProperty( - "storage_group_report_threshold", - Long.toString(conf.getStorageGroupSizeReportThreshold())))); + conf.setTimeIndexMemoryProportion( + Double.parseDouble( + properties.getProperty( + "time_index_memory_proportion", + Double.toString(conf.getTimeIndexMemoryProportion())))); - conf.setMetaDataCacheEnable( - Boolean.parseBoolean( - properties.getProperty( - "meta_data_cache_enable", Boolean.toString(conf.isMetaDataCacheEnable())))); + conf.setFlushProportion( + Double.parseDouble( + properties.getProperty( + "flush_proportion", Double.toString(conf.getFlushProportion())))); - conf.setEnableLastCache( - Boolean.parseBoolean( - properties.getProperty( - "enable_last_cache", Boolean.toString(conf.isLastCacheEnabled())))); + conf.setRejectProportion( + Double.parseDouble( + properties.getProperty( + "reject_proportion", Double.toString(conf.getRejectProportion())))); + + conf.setStorageGroupSizeReportThreshold( + Long.parseLong( + properties.getProperty( + "storage_group_report_threshold", + Long.toString(conf.getStorageGroupSizeReportThreshold())))); - initMemoryAllocate(properties); + conf.setMetaDataCacheEnable( + Boolean.parseBoolean( + properties.getProperty( + "meta_data_cache_enable", Boolean.toString(conf.isMetaDataCacheEnable())))); - loadWALProps(properties); + conf.setEnableLastCache( + Boolean.parseBoolean( + properties.getProperty( + "enable_last_cache", Boolean.toString(conf.isLastCacheEnabled())))); - String systemDir = properties.getProperty("system_dir"); - if (systemDir == null) { - systemDir = properties.getProperty("base_dir"); - if (systemDir != null) { - systemDir = FilePathUtils.regularizePath(systemDir) + IoTDBConstant.SYSTEM_FOLDER_NAME; - } else { - systemDir = conf.getSystemDir(); - } + initMemoryAllocate(properties); + + loadWALProps(properties); + + String systemDir = properties.getProperty("system_dir"); + if (systemDir == null) { + systemDir = properties.getProperty("base_dir"); + if (systemDir != null) { + systemDir = FilePathUtils.regularizePath(systemDir) + IoTDBConstant.SYSTEM_FOLDER_NAME; + } else { + systemDir = conf.getSystemDir(); } - conf.setSystemDir(systemDir); + } + conf.setSystemDir(systemDir); - conf.setSchemaDir( - FilePathUtils.regularizePath(conf.getSystemDir()) + IoTDBConstant.SCHEMA_FOLDER_NAME); + conf.setSchemaDir( + FilePathUtils.regularizePath(conf.getSystemDir()) + IoTDBConstant.SCHEMA_FOLDER_NAME); - conf.setSyncDir( - FilePathUtils.regularizePath(conf.getSystemDir()) + IoTDBConstant.SYNC_FOLDER_NAME); + conf.setSyncDir( + FilePathUtils.regularizePath(conf.getSystemDir()) + IoTDBConstant.SYNC_FOLDER_NAME); - conf.setQueryDir( - FilePathUtils.regularizePath(conf.getSystemDir() + IoTDBConstant.QUERY_FOLDER_NAME)); + conf.setQueryDir( + FilePathUtils.regularizePath(conf.getSystemDir() + IoTDBConstant.QUERY_FOLDER_NAME)); - conf.setTracingDir(properties.getProperty("tracing_dir", conf.getTracingDir())); + conf.setTracingDir(properties.getProperty("tracing_dir", conf.getTracingDir())); - conf.setDataDirs(properties.getProperty("data_dirs", conf.getDataDirs()[0]).split(",")); + conf.setDataDirs(properties.getProperty("data_dirs", conf.getDataDirs()[0]).split(",")); - conf.setWalDir(properties.getProperty("wal_dir", conf.getWalDir())); + conf.setWalDir(properties.getProperty("wal_dir", conf.getWalDir())); - int mlogBufferSize = - Integer.parseInt( - properties.getProperty( - "mlog_buffer_size", Integer.toString(conf.getMlogBufferSize()))); - if (mlogBufferSize > 0) { - conf.setMlogBufferSize(mlogBufferSize); - } + int mlogBufferSize = + Integer.parseInt( + properties.getProperty("mlog_buffer_size", Integer.toString(conf.getMlogBufferSize()))); + if (mlogBufferSize > 0) { + conf.setMlogBufferSize(mlogBufferSize); + } - long forceMlogPeriodInMs = - Long.parseLong( - properties.getProperty( - "sync_mlog_period_in_ms", Long.toString(conf.getSyncMlogPeriodInMs()))); - if (forceMlogPeriodInMs > 0) { - conf.setSyncMlogPeriodInMs(forceMlogPeriodInMs); - } + long forceMlogPeriodInMs = + Long.parseLong( + properties.getProperty( + "sync_mlog_period_in_ms", Long.toString(conf.getSyncMlogPeriodInMs()))); + if (forceMlogPeriodInMs > 0) { + conf.setSyncMlogPeriodInMs(forceMlogPeriodInMs); + } - conf.setMultiDirStrategyClassName( - properties.getProperty("multi_dir_strategy", conf.getMultiDirStrategyClassName())); + conf.setMultiDirStrategyClassName( + properties.getProperty("multi_dir_strategy", conf.getMultiDirStrategyClassName())); - conf.setBatchSize( - Integer.parseInt( - properties.getProperty("batch_size", Integer.toString(conf.getBatchSize())))); + conf.setBatchSize( + Integer.parseInt( + properties.getProperty("batch_size", Integer.toString(conf.getBatchSize())))); - conf.setEnableMemControl( - (Boolean.parseBoolean( - properties.getProperty( - "enable_mem_control", Boolean.toString(conf.isEnableMemControl()))))); - logger.info("IoTDB enable memory control: {}", conf.isEnableMemControl()); + conf.setEnableMemControl( + (Boolean.parseBoolean( + properties.getProperty( + "enable_mem_control", Boolean.toString(conf.isEnableMemControl()))))); + logger.info("IoTDB enable memory control: {}", conf.isEnableMemControl()); - long seqTsFileSize = - Long.parseLong( - properties - .getProperty("seq_tsfile_size", Long.toString(conf.getSeqTsFileSize())) - .trim()); - if (seqTsFileSize >= 0) { - conf.setSeqTsFileSize(seqTsFileSize); - } + long seqTsFileSize = + Long.parseLong( + properties + .getProperty("seq_tsfile_size", Long.toString(conf.getSeqTsFileSize())) + .trim()); + if (seqTsFileSize >= 0) { + conf.setSeqTsFileSize(seqTsFileSize); + } - long unSeqTsFileSize = - Long.parseLong( - properties - .getProperty("unseq_tsfile_size", Long.toString(conf.getUnSeqTsFileSize())) - .trim()); - if (unSeqTsFileSize >= 0) { - conf.setUnSeqTsFileSize(unSeqTsFileSize); - } + long unSeqTsFileSize = + Long.parseLong( + properties + .getProperty("unseq_tsfile_size", Long.toString(conf.getUnSeqTsFileSize())) + .trim()); + if (unSeqTsFileSize >= 0) { + conf.setUnSeqTsFileSize(unSeqTsFileSize); + } - long memTableSizeThreshold = - Long.parseLong( - properties - .getProperty( - "memtable_size_threshold", Long.toString(conf.getMemtableSizeThreshold())) - .trim()); - if (memTableSizeThreshold > 0) { - conf.setMemtableSizeThreshold(memTableSizeThreshold); - } + long memTableSizeThreshold = + Long.parseLong( + properties + .getProperty( + "memtable_size_threshold", Long.toString(conf.getMemtableSizeThreshold())) + .trim()); + if (memTableSizeThreshold > 0) { + conf.setMemtableSizeThreshold(memTableSizeThreshold); + } - conf.setAvgSeriesPointNumberThreshold( - Integer.parseInt( - properties.getProperty( - "avg_series_point_number_threshold", - Integer.toString(conf.getAvgSeriesPointNumberThreshold())))); + conf.setAvgSeriesPointNumberThreshold( + Integer.parseInt( + properties.getProperty( + "avg_series_point_number_threshold", + Integer.toString(conf.getAvgSeriesPointNumberThreshold())))); - conf.setCheckPeriodWhenInsertBlocked( - Integer.parseInt( - properties.getProperty( - "check_period_when_insert_blocked", - Integer.toString(conf.getCheckPeriodWhenInsertBlocked())))); + conf.setCheckPeriodWhenInsertBlocked( + Integer.parseInt( + properties.getProperty( + "check_period_when_insert_blocked", + Integer.toString(conf.getCheckPeriodWhenInsertBlocked())))); - conf.setMaxWaitingTimeWhenInsertBlocked( - Integer.parseInt( - properties.getProperty( - "max_waiting_time_when_insert_blocked", - Integer.toString(conf.getMaxWaitingTimeWhenInsertBlocked())))); + conf.setMaxWaitingTimeWhenInsertBlocked( + Integer.parseInt( + properties.getProperty( + "max_waiting_time_when_insert_blocked", + Integer.toString(conf.getMaxWaitingTimeWhenInsertBlocked())))); - conf.setEstimatedSeriesSize( - Integer.parseInt( - properties.getProperty( - "estimated_series_size", Integer.toString(conf.getEstimatedSeriesSize())))); + conf.setEstimatedSeriesSize( + Integer.parseInt( + properties.getProperty( + "estimated_series_size", Integer.toString(conf.getEstimatedSeriesSize())))); - conf.setIoTaskQueueSizeForFlushing( - Integer.parseInt( - properties.getProperty( - "io_task_queue_size_for_flushing", - Integer.toString(conf.getIoTaskQueueSizeForFlushing())))); + conf.setIoTaskQueueSizeForFlushing( + Integer.parseInt( + properties.getProperty( + "io_task_queue_size_for_flushing", + Integer.toString(conf.getIoTaskQueueSizeForFlushing())))); - conf.setCompactionScheduleIntervalInMs( - Long.parseLong( - properties.getProperty( - "compaction_schedule_interval_in_ms", - Long.toString(conf.getCompactionScheduleIntervalInMs())))); + conf.setCompactionScheduleIntervalInMs( + Long.parseLong( + properties.getProperty( + "compaction_schedule_interval_in_ms", + Long.toString(conf.getCompactionScheduleIntervalInMs())))); - conf.setCompactionSubmissionIntervalInMs( - Long.parseLong( - properties.getProperty( - "compaction_submission_interval_in_ms", - Long.toString(conf.getCompactionSubmissionIntervalInMs())))); + conf.setCompactionSubmissionIntervalInMs( + Long.parseLong( + properties.getProperty( + "compaction_submission_interval_in_ms", + Long.toString(conf.getCompactionSubmissionIntervalInMs())))); - conf.setEnableCrossSpaceCompaction( - Boolean.parseBoolean( - properties.getProperty( - "enable_cross_space_compaction", - Boolean.toString(conf.isEnableCrossSpaceCompaction())))); + conf.setEnableCrossSpaceCompaction( + Boolean.parseBoolean( + properties.getProperty( + "enable_cross_space_compaction", + Boolean.toString(conf.isEnableCrossSpaceCompaction())))); - conf.setEnableSeqSpaceCompaction( - Boolean.parseBoolean( - properties.getProperty( - "enable_seq_space_compaction", - Boolean.toString(conf.isEnableSeqSpaceCompaction())))); + conf.setEnableSeqSpaceCompaction( + Boolean.parseBoolean( + properties.getProperty( + "enable_seq_space_compaction", + Boolean.toString(conf.isEnableSeqSpaceCompaction())))); - conf.setEnableUnseqSpaceCompaction( - Boolean.parseBoolean( - properties.getProperty( - "enable_unseq_space_compaction", - Boolean.toString(conf.isEnableUnseqSpaceCompaction())))); + conf.setEnableUnseqSpaceCompaction( + Boolean.parseBoolean( + properties.getProperty( + "enable_unseq_space_compaction", + Boolean.toString(conf.isEnableUnseqSpaceCompaction())))); - conf.setCrossCompactionStrategy( - CrossCompactionStrategy.getCrossCompactionStrategy( - properties.getProperty( - "cross_compaction_strategy", conf.getCrossCompactionStrategy().toString()))); + conf.setCrossCompactionStrategy( + CrossCompactionStrategy.getCrossCompactionStrategy( + properties.getProperty( + "cross_compaction_strategy", conf.getCrossCompactionStrategy().toString()))); - conf.setInnerCompactionStrategy( - InnerCompactionStrategy.getInnerCompactionStrategy( - properties.getProperty( - "inner_compaction_strategy", conf.getInnerCompactionStrategy().toString()))); + conf.setInnerCompactionStrategy( + InnerCompactionStrategy.getInnerCompactionStrategy( + properties.getProperty( + "inner_compaction_strategy", conf.getInnerCompactionStrategy().toString()))); - conf.setCompactionPriority( - CompactionPriority.valueOf( - properties.getProperty( - "compaction_priority", conf.getCompactionPriority().toString()))); + conf.setCompactionPriority( + CompactionPriority.valueOf( + properties.getProperty( + "compaction_priority", conf.getCompactionPriority().toString()))); - int subtaskNum = - Integer.parseInt( - properties.getProperty( - "sub_compaction_thread_num", Integer.toString(conf.getSubCompactionTaskNum()))); - subtaskNum = subtaskNum <= 0 ? 1 : subtaskNum; - conf.setSubCompactionTaskNum(subtaskNum); + int subtaskNum = + Integer.parseInt( + properties.getProperty( + "sub_compaction_thread_num", Integer.toString(conf.getSubCompactionTaskNum()))); + subtaskNum = subtaskNum <= 0 ? 1 : subtaskNum; + conf.setSubCompactionTaskNum(subtaskNum); - conf.setQueryTimeoutThreshold( - Integer.parseInt( - properties.getProperty( - "query_timeout_threshold", Integer.toString(conf.getQueryTimeoutThreshold())))); + conf.setQueryTimeoutThreshold( + Integer.parseInt( + properties.getProperty( + "query_timeout_threshold", Integer.toString(conf.getQueryTimeoutThreshold())))); - conf.setSessionTimeoutThreshold( - Integer.parseInt( - properties.getProperty( - "session_timeout_threshold", - Integer.toString(conf.getSessionTimeoutThreshold())))); + conf.setSessionTimeoutThreshold( + Integer.parseInt( + properties.getProperty( + "session_timeout_threshold", Integer.toString(conf.getSessionTimeoutThreshold())))); - conf.setSyncEnable( - Boolean.parseBoolean( - properties.getProperty("is_sync_enable", Boolean.toString(conf.isSyncEnable())))); + conf.setSyncEnable( + Boolean.parseBoolean( + properties.getProperty("is_sync_enable", Boolean.toString(conf.isSyncEnable())))); - conf.setSyncServerPort( - Integer.parseInt( - properties - .getProperty("sync_server_port", Integer.toString(conf.getSyncServerPort())) - .trim())); + conf.setSyncServerPort( + Integer.parseInt( + properties + .getProperty("sync_server_port", Integer.toString(conf.getSyncServerPort())) + .trim())); - conf.setIpWhiteList(properties.getProperty("ip_white_list", conf.getIpWhiteList())); + conf.setIpWhiteList(properties.getProperty("ip_white_list", conf.getIpWhiteList())); - conf.setConcurrentFlushThread( - Integer.parseInt( - properties.getProperty( - "concurrent_flush_thread", Integer.toString(conf.getConcurrentFlushThread())))); + conf.setConcurrentFlushThread( + Integer.parseInt( + properties.getProperty( + "concurrent_flush_thread", Integer.toString(conf.getConcurrentFlushThread())))); - if (conf.getConcurrentFlushThread() <= 0) { - conf.setConcurrentFlushThread(Runtime.getRuntime().availableProcessors()); - } + if (conf.getConcurrentFlushThread() <= 0) { + conf.setConcurrentFlushThread(Runtime.getRuntime().availableProcessors()); + } - // start: index parameter setting - conf.setIndexRootFolder(properties.getProperty("index_root_dir", conf.getIndexRootFolder())); + // start: index parameter setting + conf.setIndexRootFolder(properties.getProperty("index_root_dir", conf.getIndexRootFolder())); - conf.setEnableIndex( - Boolean.parseBoolean( - properties.getProperty("enable_index", Boolean.toString(conf.isEnableIndex())))); + conf.setEnableIndex( + Boolean.parseBoolean( + properties.getProperty("enable_index", Boolean.toString(conf.isEnableIndex())))); - conf.setConcurrentIndexBuildThread( - Integer.parseInt( - properties.getProperty( - "concurrent_index_build_thread", - Integer.toString(conf.getConcurrentIndexBuildThread())))); - if (conf.getConcurrentIndexBuildThread() <= 0) { - conf.setConcurrentIndexBuildThread(Runtime.getRuntime().availableProcessors()); - } + conf.setConcurrentIndexBuildThread( + Integer.parseInt( + properties.getProperty( + "concurrent_index_build_thread", + Integer.toString(conf.getConcurrentIndexBuildThread())))); + if (conf.getConcurrentIndexBuildThread() <= 0) { + conf.setConcurrentIndexBuildThread(Runtime.getRuntime().availableProcessors()); + } - conf.setDefaultIndexWindowRange( - Integer.parseInt( - properties.getProperty( - "default_index_window_range", - Integer.toString(conf.getDefaultIndexWindowRange())))); + conf.setDefaultIndexWindowRange( + Integer.parseInt( + properties.getProperty( + "default_index_window_range", + Integer.toString(conf.getDefaultIndexWindowRange())))); - conf.setIndexBufferSize( - Long.parseLong( - properties.getProperty( - "index_buffer_size", Long.toString(conf.getIndexBufferSize())))); - // end: index parameter setting + conf.setIndexBufferSize( + Long.parseLong( + properties.getProperty("index_buffer_size", Long.toString(conf.getIndexBufferSize())))); + // end: index parameter setting - conf.setConcurrentQueryThread( - Integer.parseInt( - properties.getProperty( - "concurrent_query_thread", Integer.toString(conf.getConcurrentQueryThread())))); + conf.setConcurrentQueryThread( + Integer.parseInt( + properties.getProperty( + "concurrent_query_thread", Integer.toString(conf.getConcurrentQueryThread())))); - if (conf.getConcurrentQueryThread() <= 0) { - conf.setConcurrentQueryThread(Runtime.getRuntime().availableProcessors()); - } + if (conf.getConcurrentQueryThread() <= 0) { + conf.setConcurrentQueryThread(Runtime.getRuntime().availableProcessors()); + } - conf.setConcurrentSubRawQueryThread( - Integer.parseInt( - properties.getProperty( - "concurrent_sub_rawQuery_thread", - Integer.toString(conf.getConcurrentSubRawQueryThread())))); + conf.setConcurrentSubRawQueryThread( + Integer.parseInt( + properties.getProperty( + "concurrent_sub_rawQuery_thread", + Integer.toString(conf.getConcurrentSubRawQueryThread())))); - if (conf.getConcurrentSubRawQueryThread() <= 0) { - conf.setConcurrentSubRawQueryThread(Runtime.getRuntime().availableProcessors()); - } + if (conf.getConcurrentSubRawQueryThread() <= 0) { + conf.setConcurrentSubRawQueryThread(Runtime.getRuntime().availableProcessors()); + } - conf.setRawQueryBlockingQueueCapacity( - Integer.parseInt( - properties.getProperty( - "raw_query_blocking_queue_capacity", - Integer.toString(conf.getRawQueryBlockingQueueCapacity())))); + conf.setRawQueryBlockingQueueCapacity( + Integer.parseInt( + properties.getProperty( + "raw_query_blocking_queue_capacity", + Integer.toString(conf.getRawQueryBlockingQueueCapacity())))); - conf.setmManagerCacheSize( - Integer.parseInt( - properties - .getProperty( - "metadata_node_cache_size", Integer.toString(conf.getmManagerCacheSize())) - .trim())); + conf.setmManagerCacheSize( + Integer.parseInt( + properties + .getProperty( + "metadata_node_cache_size", Integer.toString(conf.getmManagerCacheSize())) + .trim())); - conf.setmRemoteSchemaCacheSize( - Integer.parseInt( - properties - .getProperty( - "remote_schema_cache_size", - Integer.toString(conf.getmRemoteSchemaCacheSize())) - .trim())); + conf.setmRemoteSchemaCacheSize( + Integer.parseInt( + properties + .getProperty( + "remote_schema_cache_size", Integer.toString(conf.getmRemoteSchemaCacheSize())) + .trim())); - conf.setLanguageVersion( - properties.getProperty("language_version", conf.getLanguageVersion()).trim()); + conf.setLanguageVersion( + properties.getProperty("language_version", conf.getLanguageVersion()).trim()); - if (properties.containsKey("chunk_buffer_pool_enable")) { - conf.setChunkBufferPoolEnable( - Boolean.parseBoolean(properties.getProperty("chunk_buffer_pool_enable"))); - } + if (properties.containsKey("chunk_buffer_pool_enable")) { + conf.setChunkBufferPoolEnable( + Boolean.parseBoolean(properties.getProperty("chunk_buffer_pool_enable"))); + } - conf.setEnableExternalSort( - Boolean.parseBoolean( - properties.getProperty( - "enable_external_sort", Boolean.toString(conf.isEnableExternalSort())))); - conf.setExternalSortThreshold( - Integer.parseInt( - properties.getProperty( - "external_sort_threshold", Integer.toString(conf.getExternalSortThreshold())))); - conf.setUpgradeThreadNum( - Integer.parseInt( - properties.getProperty( - "upgrade_thread_num", Integer.toString(conf.getUpgradeThreadNum())))); - conf.setCrossCompactionMemoryBudget( - Long.parseLong( - properties.getProperty( - "cross_compaction_memory_budget", - Long.toString(conf.getCrossCompactionMemoryBudget())))); - conf.setCrossCompactionFileSelectionTimeBudget( - Long.parseLong( - properties.getProperty( - "cross_compaction_file_selection_time_budget", - Long.toString(conf.getCrossCompactionFileSelectionTimeBudget())))); - conf.setMergeIntervalSec( - Long.parseLong( - properties.getProperty( - "merge_interval_sec", Long.toString(conf.getMergeIntervalSec())))); - conf.setConcurrentCompactionThread( - Integer.parseInt( - properties.getProperty( - "concurrent_compaction_thread", - Integer.toString(conf.getConcurrentCompactionThread())))); - conf.setTargetCompactionFileSize( - Long.parseLong( - properties.getProperty( - "target_compaction_file_size", - Long.toString(conf.getTargetCompactionFileSize())))); - conf.setTargetChunkSize( - Long.parseLong( - properties.getProperty( - "target_chunk_size", Long.toString(conf.getTargetChunkSize())))); - conf.setTargetChunkPointNum( - Long.parseLong( - properties.getProperty( - "target_chunk_point_num", Long.toString(conf.getTargetChunkPointNum())))); - conf.setChunkPointNumLowerBoundInCompaction( - Long.parseLong( - properties.getProperty( - "chunk_size_lower_bound_in_compaction", - Long.toString(conf.getChunkPointNumLowerBoundInCompaction())))); - conf.setChunkSizeLowerBoundInCompaction( - Long.parseLong( - properties.getProperty( - "chunk_size_lower_bound_in_compaction", - Long.toString(conf.getChunkSizeLowerBoundInCompaction())))); - conf.setMaxInnerCompactionCandidateFileNum( - Integer.parseInt( - properties.getProperty( - "max_inner_compaction_candidate_file_num", - Integer.toString(conf.getMaxInnerCompactionCandidateFileNum())))); - conf.setMaxCrossCompactionCandidateFileNum( - Integer.parseInt( - properties.getProperty( - "max_cross_compaction_candidate_file_num", - Integer.toString(conf.getMaxCrossCompactionCandidateFileNum())))); + conf.setEnableExternalSort( + Boolean.parseBoolean( + properties.getProperty( + "enable_external_sort", Boolean.toString(conf.isEnableExternalSort())))); + conf.setExternalSortThreshold( + Integer.parseInt( + properties.getProperty( + "external_sort_threshold", Integer.toString(conf.getExternalSortThreshold())))); + conf.setUpgradeThreadNum( + Integer.parseInt( + properties.getProperty( + "upgrade_thread_num", Integer.toString(conf.getUpgradeThreadNum())))); + conf.setCrossCompactionMemoryBudget( + Long.parseLong( + properties.getProperty( + "cross_compaction_memory_budget", + Long.toString(conf.getCrossCompactionMemoryBudget())))); + conf.setCrossCompactionFileSelectionTimeBudget( + Long.parseLong( + properties.getProperty( + "cross_compaction_file_selection_time_budget", + Long.toString(conf.getCrossCompactionFileSelectionTimeBudget())))); + conf.setMergeIntervalSec( + Long.parseLong( + properties.getProperty( + "merge_interval_sec", Long.toString(conf.getMergeIntervalSec())))); + conf.setConcurrentCompactionThread( + Integer.parseInt( + properties.getProperty( + "concurrent_compaction_thread", + Integer.toString(conf.getConcurrentCompactionThread())))); + conf.setTargetCompactionFileSize( + Long.parseLong( + properties.getProperty( + "target_compaction_file_size", Long.toString(conf.getTargetCompactionFileSize())))); + conf.setTargetChunkSize( + Long.parseLong( + properties.getProperty("target_chunk_size", Long.toString(conf.getTargetChunkSize())))); + conf.setTargetChunkPointNum( + Long.parseLong( + properties.getProperty( + "target_chunk_point_num", Long.toString(conf.getTargetChunkPointNum())))); + conf.setChunkPointNumLowerBoundInCompaction( + Long.parseLong( + properties.getProperty( + "chunk_size_lower_bound_in_compaction", + Long.toString(conf.getChunkPointNumLowerBoundInCompaction())))); + conf.setChunkSizeLowerBoundInCompaction( + Long.parseLong( + properties.getProperty( + "chunk_size_lower_bound_in_compaction", + Long.toString(conf.getChunkSizeLowerBoundInCompaction())))); + conf.setMaxInnerCompactionCandidateFileNum( + Integer.parseInt( + properties.getProperty( + "max_inner_compaction_candidate_file_num", + Integer.toString(conf.getMaxInnerCompactionCandidateFileNum())))); + conf.setMaxCrossCompactionCandidateFileNum( + Integer.parseInt( + properties.getProperty( + "max_cross_compaction_candidate_file_num", + Integer.toString(conf.getMaxCrossCompactionCandidateFileNum())))); - conf.setCompactionWriteThroughputMbPerSec( - Integer.parseInt( - properties.getProperty( - "compaction_write_throughput_mb_per_sec", - Integer.toString(conf.getCompactionWriteThroughputMbPerSec())))); + conf.setCompactionWriteThroughputMbPerSec( + Integer.parseInt( + properties.getProperty( + "compaction_write_throughput_mb_per_sec", + Integer.toString(conf.getCompactionWriteThroughputMbPerSec())))); - conf.setEnablePartialInsert( - Boolean.parseBoolean( - properties.getProperty( - "enable_partial_insert", String.valueOf(conf.isEnablePartialInsert())))); + conf.setEnablePartialInsert( + Boolean.parseBoolean( + properties.getProperty( + "enable_partial_insert", String.valueOf(conf.isEnablePartialInsert())))); - conf.setEnableMTreeSnapshot( - Boolean.parseBoolean( - properties.getProperty( - "enable_mtree_snapshot", Boolean.toString(conf.isEnableMTreeSnapshot())))); - conf.setMtreeSnapshotInterval( - Integer.parseInt( - properties.getProperty( - "mtree_snapshot_interval", Integer.toString(conf.getMtreeSnapshotInterval())))); - conf.setMtreeSnapshotThresholdTime( - Integer.parseInt( - properties.getProperty( - "mtree_snapshot_threshold_time", - Integer.toString(conf.getMtreeSnapshotThresholdTime())))); + conf.setEnableMTreeSnapshot( + Boolean.parseBoolean( + properties.getProperty( + "enable_mtree_snapshot", Boolean.toString(conf.isEnableMTreeSnapshot())))); + conf.setMtreeSnapshotInterval( + Integer.parseInt( + properties.getProperty( + "mtree_snapshot_interval", Integer.toString(conf.getMtreeSnapshotInterval())))); + conf.setMtreeSnapshotThresholdTime( + Integer.parseInt( + properties.getProperty( + "mtree_snapshot_threshold_time", + Integer.toString(conf.getMtreeSnapshotThresholdTime())))); - int maxConcurrentClientNum = - Integer.parseInt( - properties.getProperty( - "rpc_max_concurrent_client_num", - Integer.toString(conf.getRpcMaxConcurrentClientNum()).trim())); - if (maxConcurrentClientNum <= 0) { - maxConcurrentClientNum = 65535; - } + int maxConcurrentClientNum = + Integer.parseInt( + properties.getProperty( + "rpc_max_concurrent_client_num", + Integer.toString(conf.getRpcMaxConcurrentClientNum()).trim())); + if (maxConcurrentClientNum <= 0) { + maxConcurrentClientNum = 65535; + } - conf.setEnableWatermark( - Boolean.parseBoolean( - properties.getProperty( - "watermark_module_opened", Boolean.toString(conf.isEnableWatermark()).trim()))); - conf.setWatermarkSecretKey( - properties.getProperty("watermark_secret_key", conf.getWatermarkSecretKey())); - conf.setWatermarkBitString( - properties.getProperty("watermark_bit_string", conf.getWatermarkBitString())); - conf.setWatermarkMethod( - properties.getProperty("watermark_method", conf.getWatermarkMethod())); + conf.setEnableWatermark( + Boolean.parseBoolean( + properties.getProperty( + "watermark_module_opened", Boolean.toString(conf.isEnableWatermark()).trim()))); + conf.setWatermarkSecretKey( + properties.getProperty("watermark_secret_key", conf.getWatermarkSecretKey())); + conf.setWatermarkBitString( + properties.getProperty("watermark_bit_string", conf.getWatermarkBitString())); + conf.setWatermarkMethod(properties.getProperty("watermark_method", conf.getWatermarkMethod())); + + loadAutoCreateSchemaProps(properties); + + conf.setRpcMaxConcurrentClientNum(maxConcurrentClientNum); + + conf.setTsFileStorageFs( + properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().toString())); + conf.setCoreSitePath(properties.getProperty("core_site_path", conf.getCoreSitePath())); + conf.setHdfsSitePath(properties.getProperty("hdfs_site_path", conf.getHdfsSitePath())); + conf.setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(",")); + conf.setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort())); + conf.setDfsNameServices(properties.getProperty("dfs_nameservices", conf.getDfsNameServices())); + conf.setDfsHaNamenodes( + properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(",")); + conf.setDfsHaAutomaticFailoverEnabled( + Boolean.parseBoolean( + properties.getProperty( + "dfs_ha_automatic_failover_enabled", + String.valueOf(conf.isDfsHaAutomaticFailoverEnabled())))); + conf.setDfsClientFailoverProxyProvider( + properties.getProperty( + "dfs_client_failover_proxy_provider", conf.getDfsClientFailoverProxyProvider())); + conf.setUseKerberos( + Boolean.parseBoolean( + properties.getProperty("hdfs_use_kerberos", String.valueOf(conf.isUseKerberos())))); + conf.setKerberosKeytabFilePath( + properties.getProperty("kerberos_keytab_file_path", conf.getKerberosKeytabFilePath())); + conf.setKerberosPrincipal( + properties.getProperty("kerberos_principal", conf.getKerberosPrincipal())); - loadAutoCreateSchemaProps(properties); + conf.setDefaultTTL( + Long.parseLong( + properties.getProperty("default_ttl", String.valueOf(conf.getDefaultTTL())))); - conf.setRpcMaxConcurrentClientNum(maxConcurrentClientNum); - - conf.setTsFileStorageFs( - properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().toString())); - conf.setCoreSitePath(properties.getProperty("core_site_path", conf.getCoreSitePath())); - conf.setHdfsSitePath(properties.getProperty("hdfs_site_path", conf.getHdfsSitePath())); - conf.setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(",")); - conf.setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort())); - conf.setDfsNameServices( - properties.getProperty("dfs_nameservices", conf.getDfsNameServices())); - conf.setDfsHaNamenodes( - properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(",")); - conf.setDfsHaAutomaticFailoverEnabled( - Boolean.parseBoolean( - properties.getProperty( - "dfs_ha_automatic_failover_enabled", - String.valueOf(conf.isDfsHaAutomaticFailoverEnabled())))); - conf.setDfsClientFailoverProxyProvider( - properties.getProperty( - "dfs_client_failover_proxy_provider", conf.getDfsClientFailoverProxyProvider())); - conf.setUseKerberos( - Boolean.parseBoolean( - properties.getProperty("hdfs_use_kerberos", String.valueOf(conf.isUseKerberos())))); - conf.setKerberosKeytabFilePath( - properties.getProperty("kerberos_keytab_file_path", conf.getKerberosKeytabFilePath())); - conf.setKerberosPrincipal( - properties.getProperty("kerberos_principal", conf.getKerberosPrincipal())); - - conf.setDefaultTTL( - Long.parseLong( - properties.getProperty("default_ttl", String.valueOf(conf.getDefaultTTL())))); + conf.setAllowReadOnlyWhenErrorsOccur( + Boolean.parseBoolean( + properties.getProperty( + "allow_read_only_when_errors_occur", + String.valueOf(conf.isAllowReadOnlyWhenErrorsOccur())))); - conf.setAllowReadOnlyWhenErrorsOccur( - Boolean.parseBoolean( - properties.getProperty( - "allow_read_only_when_errors_occur", - String.valueOf(conf.isAllowReadOnlyWhenErrorsOccur())))); + // the num of memtables in each storage group + conf.setConcurrentWritingTimePartition( + Integer.parseInt( + properties.getProperty( + "concurrent_writing_time_partition", + String.valueOf(conf.getConcurrentWritingTimePartition())))); - // the num of memtables in each storage group - conf.setConcurrentWritingTimePartition( - Integer.parseInt( - properties.getProperty( - "concurrent_writing_time_partition", - String.valueOf(conf.getConcurrentWritingTimePartition())))); + // the default fill interval in LinearFill and PreviousFill + conf.setDefaultFillInterval( + Integer.parseInt( + properties.getProperty( + "default_fill_interval", String.valueOf(conf.getDefaultFillInterval())))); - // the default fill interval in LinearFill and PreviousFill - conf.setDefaultFillInterval( - Integer.parseInt( - properties.getProperty( - "default_fill_interval", String.valueOf(conf.getDefaultFillInterval())))); + conf.setTagAttributeTotalSize( + Integer.parseInt( + properties.getProperty( + "tag_attribute_total_size", String.valueOf(conf.getTagAttributeTotalSize())))); - conf.setTagAttributeTotalSize( - Integer.parseInt( - properties.getProperty( - "tag_attribute_total_size", String.valueOf(conf.getTagAttributeTotalSize())))); + conf.setTagAttributeFlushInterval( + Integer.parseInt( + properties.getProperty( + "tag_attribute_flush_interval", + String.valueOf(conf.getTagAttributeFlushInterval())))); - conf.setTagAttributeFlushInterval( - Integer.parseInt( - properties.getProperty( - "tag_attribute_flush_interval", - String.valueOf(conf.getTagAttributeFlushInterval())))); + conf.setPrimitiveArraySize( + (Integer.parseInt( + properties.getProperty( + "primitive_array_size", String.valueOf(conf.getPrimitiveArraySize()))))); - conf.setPrimitiveArraySize( - (Integer.parseInt( - properties.getProperty( - "primitive_array_size", String.valueOf(conf.getPrimitiveArraySize()))))); + conf.setThriftMaxFrameSize( + Integer.parseInt( + properties.getProperty( + "thrift_max_frame_size", String.valueOf(conf.getThriftMaxFrameSize())))); - conf.setThriftMaxFrameSize( - Integer.parseInt( - properties.getProperty( - "thrift_max_frame_size", String.valueOf(conf.getThriftMaxFrameSize())))); + if (conf.getThriftMaxFrameSize() < IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2) { + conf.setThriftMaxFrameSize(IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2); + } - if (conf.getThriftMaxFrameSize() < IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2) { - conf.setThriftMaxFrameSize(IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2); - } + conf.setThriftDefaultBufferSize( + Integer.parseInt( + properties.getProperty( + "thrift_init_buffer_size", String.valueOf(conf.getThriftDefaultBufferSize())))); - conf.setThriftDefaultBufferSize( - Integer.parseInt( - properties.getProperty( - "thrift_init_buffer_size", String.valueOf(conf.getThriftDefaultBufferSize())))); + conf.setFrequencyIntervalInMinute( + Integer.parseInt( + properties.getProperty( + "frequency_interval_in_minute", + String.valueOf(conf.getFrequencyIntervalInMinute())))); - conf.setFrequencyIntervalInMinute( - Integer.parseInt( - properties.getProperty( - "frequency_interval_in_minute", - String.valueOf(conf.getFrequencyIntervalInMinute())))); + conf.setSlowQueryThreshold( + Long.parseLong( + properties.getProperty( + "slow_query_threshold", String.valueOf(conf.getSlowQueryThreshold())))); - conf.setSlowQueryThreshold( - Long.parseLong( - properties.getProperty( - "slow_query_threshold", String.valueOf(conf.getSlowQueryThreshold())))); + conf.setVirtualStorageGroupNum( + Integer.parseInt( + properties.getProperty( + "virtual_storage_group_num", String.valueOf(conf.getVirtualStorageGroupNum())))); - conf.setVirtualStorageGroupNum( - Integer.parseInt( - properties.getProperty( - "virtual_storage_group_num", String.valueOf(conf.getVirtualStorageGroupNum())))); + conf.setRecoveryLogIntervalInMs( + Long.parseLong( + properties.getProperty( + "recovery_log_interval_in_ms", String.valueOf(conf.getRecoveryLogIntervalInMs())))); - conf.setRecoveryLogIntervalInMs( - Long.parseLong( - properties.getProperty( - "recovery_log_interval_in_ms", - String.valueOf(conf.getRecoveryLogIntervalInMs())))); + conf.setConcurrentWindowEvaluationThread( + Integer.parseInt( + properties.getProperty( + "concurrent_window_evaluation_thread", + Integer.toString(conf.getConcurrentWindowEvaluationThread())))); + if (conf.getConcurrentWindowEvaluationThread() <= 0) { + conf.setConcurrentWindowEvaluationThread(Runtime.getRuntime().availableProcessors()); + } - conf.setConcurrentWindowEvaluationThread( - Integer.parseInt( - properties.getProperty( - "concurrent_window_evaluation_thread", - Integer.toString(conf.getConcurrentWindowEvaluationThread())))); - if (conf.getConcurrentWindowEvaluationThread() <= 0) { - conf.setConcurrentWindowEvaluationThread(Runtime.getRuntime().availableProcessors()); - } + conf.setMaxPendingWindowEvaluationTasks( + Integer.parseInt( + properties.getProperty( + "max_pending_window_evaluation_tasks", + Integer.toString(conf.getMaxPendingWindowEvaluationTasks())))); + if (conf.getMaxPendingWindowEvaluationTasks() <= 0) { + conf.setMaxPendingWindowEvaluationTasks(64); + } - conf.setMaxPendingWindowEvaluationTasks( - Integer.parseInt( - properties.getProperty( - "max_pending_window_evaluation_tasks", - Integer.toString(conf.getMaxPendingWindowEvaluationTasks())))); - if (conf.getMaxPendingWindowEvaluationTasks() <= 0) { - conf.setMaxPendingWindowEvaluationTasks(64); - } + // id table related configuration + // conf.setDeviceIDTransformationMethod( + // properties.getProperty( + // "device_id_transformation_method", conf.getDeviceIDTransformationMethod())); + + // conf.setEnableIDTable( + // Boolean.parseBoolean( + // properties.getProperty("enable_id_table", + // String.valueOf(conf.isEnableIDTable())))); + + // conf.setEnableIDTableLogFile( + // Boolean.parseBoolean( + // properties.getProperty( + // "enable_id_table_log_file", + // String.valueOf(conf.isEnableIDTableLogFile())))); + + // mqtt + if (properties.getProperty(IoTDBConstant.MQTT_HOST_NAME) != null) { + conf.setMqttHost(properties.getProperty(IoTDBConstant.MQTT_HOST_NAME)); + } + if (properties.getProperty(IoTDBConstant.MQTT_PORT_NAME) != null) { + conf.setMqttPort(Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_PORT_NAME))); + } + if (properties.getProperty(IoTDBConstant.MQTT_HANDLER_POOL_SIZE_NAME) != null) { + conf.setMqttHandlerPoolSize( + Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_HANDLER_POOL_SIZE_NAME))); + } + if (properties.getProperty(IoTDBConstant.MQTT_PAYLOAD_FORMATTER_NAME) != null) { + conf.setMqttPayloadFormatter( + properties.getProperty(IoTDBConstant.MQTT_PAYLOAD_FORMATTER_NAME)); + } + if (properties.getProperty(IoTDBConstant.ENABLE_MQTT) != null) { + conf.setEnableMQTTService( + Boolean.parseBoolean(properties.getProperty(IoTDBConstant.ENABLE_MQTT))); + } + if (properties.getProperty(IoTDBConstant.MQTT_MAX_MESSAGE_SIZE) != null) { + conf.setMqttMaxMessageSize( + Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_MAX_MESSAGE_SIZE))); + } - // id table related configuration - // conf.setDeviceIDTransformationMethod( - // properties.getProperty( - // "device_id_transformation_method", conf.getDeviceIDTransformationMethod())); - - // conf.setEnableIDTable( - // Boolean.parseBoolean( - // properties.getProperty("enable_id_table", - // String.valueOf(conf.isEnableIDTable())))); - - // conf.setEnableIDTableLogFile( - // Boolean.parseBoolean( - // properties.getProperty( - // "enable_id_table_log_file", - // String.valueOf(conf.isEnableIDTableLogFile())))); - - // mqtt - if (properties.getProperty(IoTDBConstant.MQTT_HOST_NAME) != null) { - conf.setMqttHost(properties.getProperty(IoTDBConstant.MQTT_HOST_NAME)); - } - if (properties.getProperty(IoTDBConstant.MQTT_PORT_NAME) != null) { - conf.setMqttPort(Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_PORT_NAME))); - } - if (properties.getProperty(IoTDBConstant.MQTT_HANDLER_POOL_SIZE_NAME) != null) { - conf.setMqttHandlerPoolSize( - Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_HANDLER_POOL_SIZE_NAME))); - } - if (properties.getProperty(IoTDBConstant.MQTT_PAYLOAD_FORMATTER_NAME) != null) { - conf.setMqttPayloadFormatter( - properties.getProperty(IoTDBConstant.MQTT_PAYLOAD_FORMATTER_NAME)); - } - if (properties.getProperty(IoTDBConstant.ENABLE_MQTT) != null) { - conf.setEnableMQTTService( - Boolean.parseBoolean(properties.getProperty(IoTDBConstant.ENABLE_MQTT))); - } - if (properties.getProperty(IoTDBConstant.MQTT_MAX_MESSAGE_SIZE) != null) { - conf.setMqttMaxMessageSize( - Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_MAX_MESSAGE_SIZE))); - } + conf.setAuthorizerProvider( + properties.getProperty("authorizer_provider_class", conf.getAuthorizerProvider())); + // if using org.apache.iotdb.db.auth.authorizer.OpenIdAuthorizer, openID_url is needed. + conf.setOpenIdProviderUrl(properties.getProperty("openID_url", conf.getOpenIdProviderUrl())); - conf.setAuthorizerProvider( - properties.getProperty("authorizer_provider_class", conf.getAuthorizerProvider())); - // if using org.apache.iotdb.db.auth.authorizer.OpenIdAuthorizer, openID_url is needed. - conf.setOpenIdProviderUrl(properties.getProperty("openID_url", conf.getOpenIdProviderUrl())); + conf.setEnablePartition( + Boolean.parseBoolean( + properties.getProperty("enable_partition", String.valueOf(conf.isEnablePartition())))); - conf.setEnablePartition( - Boolean.parseBoolean( - properties.getProperty( - "enable_partition", String.valueOf(conf.isEnablePartition())))); + conf.setPartitionInterval( + Long.parseLong( + properties.getProperty( + "partition_interval", String.valueOf(conf.getPartitionInterval())))); - conf.setPartitionInterval( - Long.parseLong( - properties.getProperty( - "partition_interval", String.valueOf(conf.getPartitionInterval())))); + conf.setAdminName(properties.getProperty("admin_name", conf.getAdminName())); - conf.setAdminName(properties.getProperty("admin_name", conf.getAdminName())); + conf.setAdminPassword(properties.getProperty("admin_password", conf.getAdminPassword())); - conf.setAdminPassword(properties.getProperty("admin_password", conf.getAdminPassword())); + conf.setSelectIntoInsertTabletPlanRowLimit( + Integer.parseInt( + properties.getProperty( + "select_into_insert_tablet_plan_row_limit", + String.valueOf(conf.getSelectIntoInsertTabletPlanRowLimit())))); - conf.setSelectIntoInsertTabletPlanRowLimit( - Integer.parseInt( - properties.getProperty( - "select_into_insert_tablet_plan_row_limit", - String.valueOf(conf.getSelectIntoInsertTabletPlanRowLimit())))); + conf.setInsertMultiTabletEnableMultithreadingColumnThreshold( + Integer.parseInt( + properties.getProperty( + "insert_multi_tablet_enable_multithreading_column_threshold", + String.valueOf(conf.getInsertMultiTabletEnableMultithreadingColumnThreshold())))); - conf.setInsertMultiTabletEnableMultithreadingColumnThreshold( - Integer.parseInt( - properties.getProperty( - "insert_multi_tablet_enable_multithreading_column_threshold", - String.valueOf(conf.getInsertMultiTabletEnableMultithreadingColumnThreshold())))); + conf.setEncryptDecryptProvider( + properties.getProperty( + "iotdb_server_encrypt_decrypt_provider", conf.getEncryptDecryptProvider())); - conf.setEncryptDecryptProvider( - properties.getProperty( - "iotdb_server_encrypt_decrypt_provider", conf.getEncryptDecryptProvider())); + conf.setEncryptDecryptProviderParameter( + properties.getProperty( + "iotdb_server_encrypt_decrypt_provider_parameter", + conf.getEncryptDecryptProviderParameter())); - conf.setEncryptDecryptProviderParameter( - properties.getProperty( - "iotdb_server_encrypt_decrypt_provider_parameter", - conf.getEncryptDecryptProviderParameter())); + // set OperationSync config + conf.setEnableOperationSync( + Boolean.parseBoolean( + properties.getProperty( + "enable_operation_sync", String.valueOf(conf.isEnableOperationSync())))); - // set OperationSync config - conf.setEnableOperationSync( - Boolean.parseBoolean( - properties.getProperty( - "enable_operation_sync", String.valueOf(conf.isEnableOperationSync())))); + conf.setSecondaryAddress( + properties.getProperty("secondary_address", conf.getSecondaryAddress())); - conf.setSecondaryAddress( - properties.getProperty("secondary_address", conf.getSecondaryAddress())); + conf.setSecondaryPort( + Integer.parseInt( + properties.getProperty("secondary_port", String.valueOf(conf.getSecondaryPort())))); - conf.setSecondaryPort( - Integer.parseInt( - properties.getProperty("secondary_port", String.valueOf(conf.getSecondaryPort())))); + conf.setSecondaryUser(properties.getProperty("secondary_user", conf.getSecondaryUser())); - conf.setSecondaryUser(properties.getProperty("secondary_user", conf.getSecondaryUser())); + conf.setSecondaryPassword( + properties.getProperty("secondary_password", conf.getSecondaryPassword())); - conf.setSecondaryPassword( - properties.getProperty("secondary_password", conf.getSecondaryPassword())); + conf.setOperationSyncSessionConcurrencySize( + Integer.parseInt( + properties.getProperty( + "operation_sync_session_concurrency_size", + String.valueOf(conf.getOperationSyncSessionConcurrencySize())))); - conf.setOperationSyncSessionConcurrencySize( - Integer.parseInt( - properties.getProperty( - "operation_sync_session_concurrency_size", - String.valueOf(conf.getOperationSyncSessionConcurrencySize())))); + conf.setOperationSyncLogDir( + properties.getProperty("operation_sync_log_dir", conf.getOperationSyncLogDir())); - conf.setOperationSyncLogDir( - properties.getProperty("operation_sync_log_dir", conf.getOperationSyncLogDir())); + conf.setOperationSyncLogValidity( + Integer.parseInt( + properties.getProperty( + "operation_sync_log_file_validity", + String.valueOf(conf.getOperationSyncLogValidity())))); - conf.setOperationSyncLogValidity( - Integer.parseInt( - properties.getProperty( - "operation_sync_log_file_validity", - String.valueOf(conf.getOperationSyncLogValidity())))); + conf.setOperationSyncLogNum( + Integer.parseInt( + properties.getProperty( + "operation_sync_log_file_num", String.valueOf(conf.getOperationSyncLogNum())))); - conf.setOperationSyncLogNum( - Integer.parseInt( - properties.getProperty( - "operation_sync_log_file_num", String.valueOf(conf.getOperationSyncLogNum())))); + conf.setOperationSyncMaxLogSize( + Long.parseLong( + properties.getProperty( + "operation_sync_max_log_size", String.valueOf(conf.getOperationSyncMaxLogSize())))); - conf.setOperationSyncMaxLogSize( - Long.parseLong( - properties.getProperty( - "operation_sync_max_log_size", - String.valueOf(conf.getOperationSyncMaxLogSize())))); + conf.setOperationSyncProducerCacheSize( + Integer.parseInt( + properties.getProperty( + "operation_sync_producer_cache_size", + String.valueOf(conf.getOperationSyncProducerCacheSize())))); - conf.setOperationSyncProducerCacheSize( - Integer.parseInt( - properties.getProperty( - "operation_sync_producer_cache_size", - String.valueOf(conf.getOperationSyncProducerCacheSize())))); + conf.setOperationSyncProducerCacheNum( + Integer.parseInt( + properties.getProperty( + "operation_sync_producer_cache_num", + String.valueOf(conf.getOperationSyncProducerCacheNum())))); - conf.setOperationSyncProducerCacheNum( - Integer.parseInt( - properties.getProperty( - "operation_sync_producer_cache_num", - String.valueOf(conf.getOperationSyncProducerCacheNum())))); + conf.setSchemaQueryFetchSize( + Integer.parseInt( + properties.getProperty( + "schema_query_fetch_size", String.valueOf(conf.getSchemaQueryFetchSize())))); - conf.setSchemaQueryFetchSize( - Integer.parseInt( - properties.getProperty( - "schema_query_fetch_size", String.valueOf(conf.getSchemaQueryFetchSize())))); + // At the same time, set TSFileConfig + TSFileDescriptor.getInstance() + .getConfig() + .setTSFileStorageFs( + FSType.valueOf( + properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().name()))); + TSFileDescriptor.getInstance() + .getConfig() + .setCoreSitePath(properties.getProperty("core_site_path", conf.getCoreSitePath())); + TSFileDescriptor.getInstance() + .getConfig() + .setHdfsSitePath(properties.getProperty("hdfs_site_path", conf.getHdfsSitePath())); + TSFileDescriptor.getInstance() + .getConfig() + .setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(",")); + TSFileDescriptor.getInstance() + .getConfig() + .setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort())); + TSFileDescriptor.getInstance() + .getConfig() + .setDfsNameServices(properties.getProperty("dfs_nameservices", conf.getDfsNameServices())); + TSFileDescriptor.getInstance() + .getConfig() + .setDfsHaNamenodes( + properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(",")); + TSFileDescriptor.getInstance() + .getConfig() + .setDfsHaAutomaticFailoverEnabled( + Boolean.parseBoolean( + properties.getProperty( + "dfs_ha_automatic_failover_enabled", + String.valueOf(conf.isDfsHaAutomaticFailoverEnabled())))); + TSFileDescriptor.getInstance() + .getConfig() + .setDfsClientFailoverProxyProvider( + properties.getProperty( + "dfs_client_failover_proxy_provider", conf.getDfsClientFailoverProxyProvider())); + TSFileDescriptor.getInstance() + .getConfig() + .setUseKerberos( + Boolean.parseBoolean( + properties.getProperty("hdfs_use_kerberos", String.valueOf(conf.isUseKerberos())))); + TSFileDescriptor.getInstance() + .getConfig() + .setKerberosKeytabFilePath( + properties.getProperty("kerberos_keytab_file_path", conf.getKerberosKeytabFilePath())); + TSFileDescriptor.getInstance() + .getConfig() + .setKerberosPrincipal( + properties.getProperty("kerberos_principal", conf.getKerberosPrincipal())); + TSFileDescriptor.getInstance().getConfig().setBatchSize(conf.getBatchSize()); - // At the same time, set TSFileConfig - TSFileDescriptor.getInstance() - .getConfig() - .setTSFileStorageFs( - FSType.valueOf( - properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().name()))); - TSFileDescriptor.getInstance() - .getConfig() - .setCoreSitePath(properties.getProperty("core_site_path", conf.getCoreSitePath())); - TSFileDescriptor.getInstance() - .getConfig() - .setHdfsSitePath(properties.getProperty("hdfs_site_path", conf.getHdfsSitePath())); - TSFileDescriptor.getInstance() - .getConfig() - .setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(",")); - TSFileDescriptor.getInstance() - .getConfig() - .setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort())); - TSFileDescriptor.getInstance() - .getConfig() - .setDfsNameServices( - properties.getProperty("dfs_nameservices", conf.getDfsNameServices())); - TSFileDescriptor.getInstance() - .getConfig() - .setDfsHaNamenodes( - properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(",")); - TSFileDescriptor.getInstance() - .getConfig() - .setDfsHaAutomaticFailoverEnabled( - Boolean.parseBoolean( - properties.getProperty( - "dfs_ha_automatic_failover_enabled", - String.valueOf(conf.isDfsHaAutomaticFailoverEnabled())))); - TSFileDescriptor.getInstance() - .getConfig() - .setDfsClientFailoverProxyProvider( - properties.getProperty( - "dfs_client_failover_proxy_provider", conf.getDfsClientFailoverProxyProvider())); - TSFileDescriptor.getInstance() - .getConfig() - .setUseKerberos( - Boolean.parseBoolean( - properties.getProperty( - "hdfs_use_kerberos", String.valueOf(conf.isUseKerberos())))); - TSFileDescriptor.getInstance() - .getConfig() - .setKerberosKeytabFilePath( - properties.getProperty( - "kerberos_keytab_file_path", conf.getKerberosKeytabFilePath())); - TSFileDescriptor.getInstance() - .getConfig() - .setKerberosPrincipal( - properties.getProperty("kerberos_principal", conf.getKerberosPrincipal())); - TSFileDescriptor.getInstance().getConfig().setBatchSize(conf.getBatchSize()); + // timed flush memtable, timed close tsfile + loadTimedService(properties); - // timed flush memtable, timed close tsfile - loadTimedService(properties); + // set tsfile-format config + loadTsFileProps(properties); - // set tsfile-format config - loadTsFileProps(properties); + // make RPCTransportFactory taking effect. + RpcTransportFactory.reInit(); + + // UDF + loadUDFProps(properties); - // make RPCTransportFactory taking effect. - RpcTransportFactory.reInit(); + // trigger + loadTriggerProps(properties); - // UDF - loadUDFProps(properties); + // CQ + loadCQProps(properties); - // trigger - loadTriggerProps(properties); + // external lib props + loadExternalLibProps(properties); + } - // CQ - loadCQProps(properties); + private void loadExternalLibProps(Properties properties) { - } catch (FileNotFoundException e) { - logger.warn("Fail to find config file {}", url, e); - } catch (IOException e) { - logger.warn("Cannot load config file, use default configuration", e); - } catch (Exception e) { - logger.warn("Incorrect format in config file, use default configuration", e); - } finally { - // update all data seriesPath - conf.updatePath(); - // update instance in metric - MetricConfigDescriptor.getInstance() - .getMetricConfig() - .updateRpcInstance(conf.getRpcAddress(), conf.getRpcPort()); - } + conf.setExternalPropertiesLoaderDir( + properties.getProperty( + "external_properties_loader_dir", conf.getExternalPropertiesLoaderDir())); + + conf.setExternalLimiterDir( + properties.getProperty("external_limiter_dir", conf.getExternalLimiterDir())); } // to keep consistent with the cluster module. diff --git a/server/src/main/java/org/apache/iotdb/db/exception/metadata/SeriesNumberOverflowException.java b/server/src/main/java/org/apache/iotdb/db/exception/metadata/SeriesNumberOverflowException.java new file mode 100644 index 0000000000..482495eaa2 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/exception/metadata/SeriesNumberOverflowException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.exception.metadata; + +import org.apache.iotdb.rpc.TSStatusCode; + +public class SeriesNumberOverflowException extends MetadataException { + + public SeriesNumberOverflowException() { + super("exceed max allowed series number.", TSStatusCode.SERIES_OVERFLOW.getStatusCode()); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index b4fcd40d24..5689d599a3 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -35,6 +35,7 @@ import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.exception.metadata.NoTemplateOnMNodeException; import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.exception.metadata.SeriesNumberOverflowException; import org.apache.iotdb.db.exception.metadata.StorageGroupAlreadySetException; import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException; import org.apache.iotdb.db.exception.metadata.TemplateIsInUseException; @@ -104,6 +105,7 @@ import org.apache.iotdb.tsfile.write.schema.TimeseriesSchema; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; +import org.apche.iotdb.external.api.ISeriesNumerLimiter; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -120,6 +122,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -186,8 +189,8 @@ public class MManager { private boolean initialized; private boolean allowToCreateNewSeries = true; - private AtomicLong totalNormalSeriesNumber = new AtomicLong(); - private AtomicLong totalTemplateSeriesNumber = new AtomicLong(); + private final AtomicLong totalNormalSeriesNumber = new AtomicLong(); + private final AtomicLong totalTemplateSeriesNumber = new AtomicLong(); private final int mtreeSnapshotInterval; private final long mtreeSnapshotThresholdTime; @@ -205,6 +208,23 @@ public class MManager { private TagManager tagManager = TagManager.getInstance(); private TemplateManager templateManager = TemplateManager.getInstance(); + private ISeriesNumerLimiter seriesNumerLimiter = + new ISeriesNumerLimiter() { + @Override + public void init(Properties properties) {} + + @Override + public boolean addTimeSeries(int number) { + // always return true, don't limit the number of series + return true; + } + + @Override + public void deleteTimeSeries(int number) { + // do nothing + } + }; + // region MManager Singleton private static class MManagerHolder { @@ -215,6 +235,10 @@ public class MManager { private static final MManager INSTANCE = new MManager(); } + public void setSeriesNumerLimiter(ISeriesNumerLimiter seriesNumerLimiter) { + this.seriesNumerLimiter = seriesNumerLimiter; + } + /** we should not use this function in other place, but only in IoTDB class */ public static MManager getInstance() { return MManagerHolder.INSTANCE; @@ -580,35 +604,47 @@ public class MManager { "IoTDB system load is too large to create timeseries, " + "please increase MAX_HEAP_SIZE in iotdb-env.sh/bat and restart"); } - try { - PartialPath path = plan.getPath(); - SchemaUtils.checkDataTypeWithEncoding(plan.getDataType(), plan.getEncoding()); - ensureStorageGroup(path); - - TSDataType type = plan.getDataType(); - // create time series in MTree - IMeasurementMNode leafMNode = - mtree.createTimeseries( - path, - type, - plan.getEncoding(), - plan.getCompressor(), - plan.getProps(), - plan.getAlias()); - - // the cached mNode may be replaced by new entityMNode in mtree - mNodeCache.invalidate(path.getDevicePath()); - - // update tag index - - if (offset != -1 && isRecovering) { - // the timeseries has already been created and now system is recovering, using the tag info - // in tagFile to recover index directly - tagManager.recoverIndex(offset, leafMNode); - } else if (plan.getTags() != null) { - // tag key, tag value - tagManager.addIndex(plan.getTags(), leafMNode); + if (!seriesNumerLimiter.addTimeSeries(1)) { + throw new SeriesNumberOverflowException(); + } + try { + IMeasurementMNode leafMNode; + try { + PartialPath path = plan.getPath(); + SchemaUtils.checkDataTypeWithEncoding(plan.getDataType(), plan.getEncoding()); + + ensureStorageGroup(path); + + TSDataType type = plan.getDataType(); + // create time series in MTree + leafMNode = + mtree.createTimeseries( + path, + type, + plan.getEncoding(), + plan.getCompressor(), + plan.getProps(), + plan.getAlias()); + + // the cached mNode may be replaced by new entityMNode in mtree + mNodeCache.invalidate(path.getDevicePath()); + + // update tag index + + if (offset != -1 && isRecovering) { + // the timeseries has already been created and now system is recovering, using the tag + // info + // in tagFile to recover index directly + tagManager.recoverIndex(offset, leafMNode); + } else if (plan.getTags() != null) { + // tag key, tag value + tagManager.addIndex(plan.getTags(), leafMNode); + } + } catch (Throwable t) { + // roll back + seriesNumerLimiter.deleteTimeSeries(1); + throw t; } // update statistics and schemaDataTypeNumMap @@ -692,31 +728,43 @@ public class MManager { "IoTDB system load is too large to create timeseries, " + "please increase MAX_HEAP_SIZE in iotdb-env.sh/bat and restart"); } + int seriesCount = plan.getMeasurements().size(); + + if (!seriesNumerLimiter.addTimeSeries(seriesCount)) { + throw new SeriesNumberOverflowException(); + } + try { PartialPath prefixPath = plan.getPrefixPath(); List<String> measurements = plan.getMeasurements(); List<TSDataType> dataTypes = plan.getDataTypes(); List<TSEncoding> encodings = plan.getEncodings(); - for (int i = 0; i < measurements.size(); i++) { - SchemaUtils.checkDataTypeWithEncoding(dataTypes.get(i), encodings.get(i)); - } + try { + for (int i = 0; i < measurements.size(); i++) { + SchemaUtils.checkDataTypeWithEncoding(dataTypes.get(i), encodings.get(i)); + } - ensureStorageGroup(prefixPath); + ensureStorageGroup(prefixPath); - // create time series in MTree - mtree.createAlignedTimeseries( - prefixPath, - measurements, - plan.getDataTypes(), - plan.getEncodings(), - plan.getCompressors()); + // create time series in MTree + mtree.createAlignedTimeseries( + prefixPath, + measurements, + plan.getDataTypes(), + plan.getEncodings(), + plan.getCompressors()); - // the cached mNode may be replaced by new entityMNode in mtree - mNodeCache.invalidate(prefixPath); + // the cached mNode may be replaced by new entityMNode in mtree + mNodeCache.invalidate(prefixPath); + } catch (Throwable t) { + // roll back + seriesNumerLimiter.deleteTimeSeries(seriesCount); + throw t; + } // update statistics and schemaDataTypeNumMap - totalNormalSeriesNumber.addAndGet(measurements.size()); + totalNormalSeriesNumber.addAndGet(seriesCount); if (totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE >= MTREE_SIZE_THRESHOLD) { logger.warn("Current series number {} is too large...", totalNormalSeriesNumber); allowToCreateNewSeries = false; @@ -844,6 +892,7 @@ public class MManager { node = node.getParent(); } totalNormalSeriesNumber.addAndGet(-1); + seriesNumerLimiter.deleteTimeSeries(1); if (!allowToCreateNewSeries && totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) { logger.info("Current series number {} come back to normal level", totalNormalSeriesNumber); @@ -883,9 +932,11 @@ public class MManager { public void deleteStorageGroups(List<PartialPath> storageGroups) throws MetadataException { try { for (PartialPath storageGroup : storageGroups) { - totalNormalSeriesNumber.addAndGet( - -mtree.getAllTimeseriesCount( - storageGroup.concatNode(MULTI_LEVEL_PATH_WILDCARD), false, false)); + int timeSeriesCount = + mtree.getAllTimeseriesCount( + storageGroup.concatNode(MULTI_LEVEL_PATH_WILDCARD), false, false); + totalNormalSeriesNumber.addAndGet(-timeSeriesCount); + seriesNumerLimiter.deleteTimeSeries(timeSeriesCount); // clear cached MNode if (!allowToCreateNewSeries && totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) { @@ -2476,7 +2527,9 @@ public class MManager { } node.setUseTemplate(false); - totalTemplateSeriesNumber.addAndGet(-node.getUpperTemplate().getMeasurementsCount()); + int seriesCount = -node.getUpperTemplate().getMeasurementsCount(); + totalTemplateSeriesNumber.addAndGet(-seriesCount); + seriesNumerLimiter.deleteTimeSeries(seriesCount); // clear caches within MManger mNodeCache.invalidate(node); @@ -2502,24 +2555,36 @@ public class MManager { String.format("Path [%s] has not been set any template.", node.getFullPath())); } - // this operation may change mtree structure and node type - // invoke mnode.setUseTemplate is invalid + if (!seriesNumerLimiter.addTimeSeries(template.getMeasurementsCount())) { + throw new SeriesNumberOverflowException(); + } + + IMNode mountedMNode; + try { + // this operation may change mtree structure and node type + // invoke mnode.setUseTemplate is invalid - // check alignment of template and mounted node - // if direct measurement exists, node will be replaced - IMNode mountedMNode = mtree.checkTemplateAlignmentWithMountedNode(node, template); + // check alignment of template and mounted node + // if direct measurement exists, node will be replaced + mountedMNode = mtree.checkTemplateAlignmentWithMountedNode(node, template); - // if has direct measurement (be a EntityNode), to ensure alignment adapt with former node or - // template - if (mountedMNode.isEntity()) { - mountedMNode - .getAsEntityMNode() - .setAligned( - node.isEntity() - ? node.getAsEntityMNode().isAligned() - : node.getUpperTemplate().isDirectAligned()); + // if has direct measurement (be a EntityNode), to ensure alignment adapt with former node or + // template + if (mountedMNode.isEntity()) { + mountedMNode + .getAsEntityMNode() + .setAligned( + node.isEntity() + ? node.getAsEntityMNode().isAligned() + : node.getUpperTemplate().isDirectAligned()); + } + mountedMNode.setUseTemplate(true); + } catch (Throwable t) { + // roll back + seriesNumerLimiter.deleteTimeSeries(template.getMeasurementsCount()); + throw t; } - mountedMNode.setUseTemplate(true); + totalTemplateSeriesNumber.addAndGet(template.getMeasurementsCount()); if (node != mountedMNode) { diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java index fcd152f612..66ffc23ac4 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java +++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java @@ -53,6 +53,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import static org.apache.iotdb.db.utils.JarLoaderUtil.loadExternLib; + public class IoTDB implements IoTDBMBean { private static final Logger logger = LoggerFactory.getLogger(IoTDB.class); @@ -78,6 +80,9 @@ public class IoTDB implements IoTDBMBean { System.exit(1); } IoTDB daemon = IoTDB.getInstance(); + + loadExternLib(config); + daemon.active(); } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java b/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java new file mode 100644 index 0000000000..a2bfd94cc9 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.utils; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; +import org.apache.iotdb.db.metadata.MManager; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; + +import org.apache.commons.io.FileUtils; +import org.apche.iotdb.external.api.IPropertiesLoader; +import org.apche.iotdb.external.api.ISeriesNumerLimiter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.net.UnknownHostException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.ServiceLoader; + +public class JarLoaderUtil { + + private static final Logger logger = LoggerFactory.getLogger(JarLoaderUtil.class); + + public static URL[] getExternalJarURLs(String jarDir) throws IOException { + HashSet<File> fileSet = + new HashSet<>( + FileUtils.listFiles( + SystemFileFactory.INSTANCE.getFile(jarDir), new String[] {"jar"}, true)); + return FileUtils.toURLs(fileSet.toArray(new File[0])); + } + + public static void loadExternLib(IoTDBConfig config) { + // load external properties + String loaderDir = config.getExternalPropertiesLoaderDir(); + + if (!(new File(loaderDir).exists())) { + return; + } + + Path externalPropertiesFile = IoTDBDescriptor.getInstance().getExternalPropsPath(); + URL[] loaderJarURLs; + List<Properties> externalPropertiesList = new ArrayList<>(); + try { + loaderJarURLs = getExternalJarURLs(loaderDir); + + if (loaderJarURLs == null || loaderJarURLs.length == 0) { + return; + } + + ClassLoader classLoader = new URLClassLoader(loaderJarURLs); + + // Use SPI to get all plugins' class + ServiceLoader<IPropertiesLoader> loaders = + ServiceLoader.load(IPropertiesLoader.class, classLoader); + + for (IPropertiesLoader loader : loaders) { + if (loader == null) { + logger.error("IPropertiesLoader(), loader is null."); + continue; + } + Properties properties = loader.loadProperties(externalPropertiesFile.toAbsolutePath()); + if (properties != null) { + externalPropertiesList.add(properties); + } + } + } catch (Throwable t) { + logger.error("error happened while loading external loader. ", t); + // ignore + } + + if (externalPropertiesList.size() != 1) { + return; + } + + // overwrite the default properties; + for (Properties properties : externalPropertiesList) { + try { + IoTDBDescriptor.getInstance().loadProperties(properties); + } catch (UnknownHostException e) { + // ignore + logger.error("error happened while loading external properties. ", e); + } + TSFileDescriptor.getInstance() + .overwriteConfigByCustomSettings(TSFileDescriptor.getInstance().getConfig(), properties); + } + + String limiterDir = config.getExternalLimiterDir(); + + if (!(new File(loaderDir).exists())) { + return; + } + + URL[] limiterJarURLs; + + List<ISeriesNumerLimiter> limiterList = new ArrayList<>(); + + try { + limiterJarURLs = getExternalJarURLs(limiterDir); + + if (limiterJarURLs == null || limiterJarURLs.length == 0) { + return; + } + + ClassLoader classLoader = new URLClassLoader(limiterJarURLs); + + // Use SPI to get all plugins' class + ServiceLoader<ISeriesNumerLimiter> limiters = + ServiceLoader.load(ISeriesNumerLimiter.class, classLoader); + + for (ISeriesNumerLimiter limiter : limiters) { + if (limiter == null) { + logger.error("ISeriesNumerLimiter(), limiter is null."); + continue; + } + for (Properties properties : externalPropertiesList) { + limiter.init(properties); + } + limiterList.add(limiter); + } + } catch (Throwable t) { + // ignore + logger.error("error happened while loading external limiter. ", t); + } + + if (limiterList.size() != 1) { + return; + } + + MManager.getInstance().setSeriesNumerLimiter(limiterList.get(0)); + } +} diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index 0de8ca7779..e330d04731 100644 --- a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -59,6 +59,8 @@ public enum TSStatusCode { TEMPLATE_IS_IN_USE(326), TEMPLATE_IMCOMPATIBLE(327), + SERIES_OVERFLOW(337), + EXECUTE_STATEMENT_ERROR(400), SQL_PARSE_ERROR(401), GENERATE_TIME_ZONE_ERROR(402), diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java index 840e64d756..711c5a4fd2 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java @@ -61,7 +61,7 @@ public class TSFileDescriptor { } } - private void overwriteConfigByCustomSettings(TSFileConfig conf, Properties properties) { + public void overwriteConfigByCustomSettings(TSFileConfig conf, Properties properties) { PropertiesOverWriter writer = new PropertiesOverWriter(properties); writer.setInt(conf::setGroupSizeInByte, "group_size_in_byte");
