This is an automated email from the ASF dual-hosted git repository. hxd pushed a commit to branch re-ext-lib-0.13 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ae156817544481cab463ecc9cf6c4570c2e9e193 Author: xiangdong huang <[email protected]> AuthorDate: Wed Aug 31 01:07:17 2022 +0800 refactor the mechanism of using external-api --- .../iotdb/external/api/IPropertiesLoader.java | 11 +- ...sNumerLimiter.java => ISeriesNumerMonitor.java} | 7 +- .../resources/conf/iotdb-engine.properties | 25 +--- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 41 ++---- .../org/apache/iotdb/db/conf/IoTDBConstant.java | 4 - .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 79 +++-------- .../org/apache/iotdb/db/metadata/MManager.java | 69 +++++---- .../java/org/apache/iotdb/db/service/IoTDB.java | 4 - .../org/apache/iotdb/db/utils/JarLoaderUtil.java | 156 --------------------- .../iotdb/tsfile/common/conf/TSFileConfig.java | 11 ++ .../iotdb/tsfile/common/conf/TSFileDescriptor.java | 4 +- 11 files changed, 94 insertions(+), 317 deletions(-) diff --git a/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java b/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java index 4e62929c15..5cf90fc950 100644 --- a/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java +++ b/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.external.api; -import java.nio.file.Path; import java.util.Properties; /** @@ -30,8 +29,14 @@ public interface IPropertiesLoader { /** * Load Properties from specific file * - * @param file The path of the properties file to open * @return a property list with values in file. */ - Properties loadProperties(Path file); + Properties loadProperties(); + + /** + * some properties that iotdb does not use by default. + * + * @return properties. + */ + Properties getCustomizedProperties(); } diff --git a/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java b/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java similarity index 82% rename from external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java rename to external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java index 2a740ff1ae..acbc2bbfa2 100644 --- a/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java +++ b/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java @@ -20,8 +20,8 @@ package org.apache.iotdb.external.api; import java.util.Properties; -/** An interface for series number limiting, users can implement their own limitation strategy */ -public interface ISeriesNumerLimiter { +/** An interface for series number monitoring, users can implement their own limitation strategy */ +public interface ISeriesNumerMonitor { /** * do the necessary initialization @@ -34,8 +34,7 @@ public interface ISeriesNumerLimiter { * add time series * * @param number time series number for current createTimeSeries operation - * @return true if totalTimeSeriesNumber doesn't exceed the limit and current createTimeSeries - * operation is allowed, otherwise false + * @return true if we want to allow the operation, otherwise false */ boolean addTimeSeries(int number); diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index ebd1a35242..0be00a433d 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -935,27 +935,4 @@ timestamp_precision=ms ### Group By Fill Configuration #################### # Datatype: float -# group_by_fill_cache_size_in_mb=1.0 - - -#################### -### External Lib Configuration -#################### - -# external lib directory for properties loader -# For Window platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is -# absolute. Otherwise, it is relative. -# external_properties_loader_dir=ext\\loader -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# external_properties_loader_dir=ext/loader - -# external lib directory for limiter -# For Window platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is -# absolute. Otherwise, it is relative. -# external_limiter_dir=ext\\limiter -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# external_limiter_dir=ext/limiter \ No newline at end of file +# group_by_fill_cache_size_in_mb=1.0 \ No newline at end of file diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index c3e0bdcde9..74cb6fa4aa 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -39,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -49,7 +50,6 @@ public class IoTDBConfig { /* Names of Watermark methods */ public static final String WATERMARK_GROUPED_LSB = "GroupBasedLSBMethod"; static final String CONFIG_NAME = "iotdb-engine.properties"; - public static final String EXTERNAL_CONFIG_NAME = "iotdb-engine-external.properties"; private static final Logger logger = LoggerFactory.getLogger(IoTDBConfig.class); private static final String MULTI_DIR_STRATEGY_PREFIX = "org.apache.iotdb.db.conf.directories.strategy."; @@ -246,16 +246,6 @@ public class IoTDBConfig { private String triggerDir = IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.TRIGGER_FOLDER_NAME; - /** External lib directory for properties loader, stores user-uploaded JAR files */ - private String externalPropertiesLoaderDir = - IoTDBConstant.EXT_FOLDER_NAME - + File.separator - + IoTDBConstant.EXT_PROPERTIES_LOADER_FOLDER_NAME; - - /** External lib directory for limiter, stores user uploaded JAR files */ - private String externalLimiterDir = - IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.EXT_LIMITER; - /** Data directory of data. It can be settled as dataDirs = {"data1", "data2", "data3"}; */ private String[] dataDirs = {"data" + File.separator + "data"}; @@ -865,6 +855,9 @@ public class IoTDBConfig { // The max record num returned in one schema query. private int schemaQueryFetchSize = 10000000; + // customizedProperties, this should be empty by default. + private Properties customizedProperties = new Properties(); + public IoTDBConfig() { // empty constructor } @@ -999,8 +992,6 @@ public class IoTDBConfig { udfDir = addHomeDir(udfDir); triggerDir = addHomeDir(triggerDir); operationSyncLogDir = addHomeDir(operationSyncLogDir); - externalPropertiesLoaderDir = addHomeDir(externalPropertiesLoaderDir); - externalLimiterDir = addHomeDir(externalLimiterDir); if (TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs().equals(FSType.HDFS)) { String hdfsDir = getHdfsDir(); @@ -1225,22 +1216,6 @@ public class IoTDBConfig { this.triggerDir = triggerDir; } - public String getExternalPropertiesLoaderDir() { - return externalPropertiesLoaderDir; - } - - public void setExternalPropertiesLoaderDir(String externalPropertiesLoaderDir) { - this.externalPropertiesLoaderDir = externalPropertiesLoaderDir; - } - - public String getExternalLimiterDir() { - return externalLimiterDir; - } - - public void setExternalLimiterDir(String externalLimiterDir) { - this.externalLimiterDir = externalLimiterDir; - } - public String getMultiDirStrategyClassName() { return multiDirStrategyClassName; } @@ -2765,4 +2740,12 @@ public class IoTDBConfig { public void setSchemaQueryFetchSize(int schemaQueryFetchSize) { this.schemaQueryFetchSize = schemaQueryFetchSize; } + + public Properties getCustomizedProperties() { + return customizedProperties; + } + + public void setCustomizedProperties(Properties customizedProperties) { + this.customizedProperties = customizedProperties; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java index 337a9e0a6e..1294002707 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java @@ -191,10 +191,6 @@ public class IoTDBConstant { // compaction mods of previous version (<0.13) public static final String COMPACTION_MODIFICATION_FILE_NAME_FROM_OLD = "merge.mods"; - public static final String EXT_PROPERTIES_LOADER_FOLDER_NAME = "loader"; - - public static final String EXT_LIMITER = "limiter"; - // client version number public enum ClientVersion { V_0_12, diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index cdb0057a2d..05e400ac87 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.engine.compaction.inner.InnerCompactionStrategy; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.qp.utils.DatetimeUtils; import org.apache.iotdb.db.service.metrics.MetricService; +import org.apache.iotdb.external.api.IPropertiesLoader; import org.apache.iotdb.metrics.config.MetricConfigDescriptor; import org.apache.iotdb.metrics.config.ReloadLevel; import org.apache.iotdb.rpc.RpcTransportFactory; @@ -45,12 +46,10 @@ import java.io.IOException; import java.io.InputStream; import java.net.InetAddress; import java.net.MalformedURLException; -import java.net.URISyntaxException; import java.net.URL; import java.net.UnknownHostException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Properties; +import java.util.ServiceLoader; public class IoTDBDescriptor { @@ -60,6 +59,21 @@ public class IoTDBDescriptor { protected IoTDBDescriptor() { loadProps(); + ServiceLoader<IPropertiesLoader> propertiesLoaderServiceLoader = + ServiceLoader.load(IPropertiesLoader.class); + for (IPropertiesLoader loader : propertiesLoaderServiceLoader) { + try { + Properties properties = loader.loadProperties(); + loadProperties(properties); + conf.setCustomizedProperties(loader.getCustomizedProperties()); + TSFileDescriptor.getInstance().overwriteConfigByCustomSettings(properties); + TSFileDescriptor.getInstance() + .getConfig() + .setCustomizedProperties(loader.getCustomizedProperties()); + } catch (UnknownHostException e) { + logger.warn("load property from {} failed.", loader.getClass().getName(), e); + } + } } public static IoTDBDescriptor getInstance() { @@ -117,52 +131,6 @@ public class IoTDBDescriptor { } } - /** - * get external props url location - * - * @return url object if location exit, otherwise null. - */ - public Path getExternalPropsPath() { - // Check if a config-directory was specified first. - String urlString = System.getProperty(IoTDBConstant.IOTDB_CONF, null); - // If it wasn't, check if a home directory was provided (This usually contains a config) - if (urlString == null) { - urlString = System.getProperty(IoTDBConstant.IOTDB_HOME, null); - if (urlString != null) { - urlString = - urlString - + File.separatorChar - + "conf" - + File.separatorChar - + IoTDBConfig.EXTERNAL_CONFIG_NAME; - } else { - // If this too wasn't provided, try to find a default config in the root of the classpath. - URL uri = IoTDBConfig.class.getResource("/" + IoTDBConfig.EXTERNAL_CONFIG_NAME); - if (uri != null) { - try { - return Paths.get(uri.toURI()); - } catch (URISyntaxException e) { - return null; - } - } - logger.warn( - "Cannot find IOTDB_HOME or IOTDB_EXTERNAL_CONF environment variable when loading " - + "config file {}, use default configuration", - IoTDBConfig.EXTERNAL_CONFIG_NAME); - // update all data seriesPath - conf.updatePath(); - return null; - } - } - // If a config location was provided, but it doesn't end with a properties file, - // append the default location. - else if (!urlString.endsWith(".properties")) { - urlString += (File.separatorChar + IoTDBConfig.EXTERNAL_CONFIG_NAME); - } - - return Paths.get(urlString); - } - /** load an property file and set TsfileDBConfig variables. */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning private void loadProps() { @@ -976,19 +944,6 @@ public class IoTDBDescriptor { // CQ loadCQProps(properties); - - // external lib props - loadExternalLibProps(properties); - } - - private void loadExternalLibProps(Properties properties) { - - conf.setExternalPropertiesLoaderDir( - properties.getProperty( - "external_properties_loader_dir", conf.getExternalPropertiesLoaderDir())); - - conf.setExternalLimiterDir( - properties.getProperty("external_limiter_dir", conf.getExternalLimiterDir())); } // to keep consistent with the cluster module. diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java index 0f16a467c4..8f429f50ad 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -91,7 +91,7 @@ import org.apache.iotdb.db.service.metrics.enums.Tag; import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.db.utils.TypeInferenceUtils; -import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.external.api.ISeriesNumerMonitor; import org.apache.iotdb.metrics.config.MetricConfigDescriptor; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; @@ -122,7 +122,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Properties; +import java.util.ServiceLoader; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -208,22 +208,8 @@ public class MManager { private TagManager tagManager = TagManager.getInstance(); private TemplateManager templateManager = TemplateManager.getInstance(); - private ISeriesNumerLimiter seriesNumerLimiter = - new ISeriesNumerLimiter() { - @Override - public void init(Properties properties) {} - - @Override - public boolean addTimeSeries(int number) { - // always return true, don't limit the number of series - return true; - } - - @Override - public void deleteTimeSeries(int number) { - // do nothing - } - }; + // seriesNumerLimiter may be null, so we must check it before use it. + private ISeriesNumerMonitor seriesNumerMonitor = null; // region MManager Singleton private static class MManagerHolder { @@ -235,8 +221,8 @@ public class MManager { private static final MManager INSTANCE = new MManager(); } - public void setSeriesNumerLimiter(ISeriesNumerLimiter seriesNumerLimiter) { - this.seriesNumerLimiter = seriesNumerLimiter; + public void setSeriesNumerMonitor(ISeriesNumerMonitor seriesNumerMonitor) { + this.seriesNumerMonitor = seriesNumerMonitor; } /** we should not use this function in other place, but only in IoTDB class */ @@ -247,6 +233,18 @@ public class MManager { // region Interfaces and Implementation of MManager initialization、snapshot、recover and clear protected MManager() { + // init seriesNumerLimiter if there is. + // each mmanager instance will generate an ISeriesNumerLimiter instance + // So, if you want to share the ISeriesNumerLimiter instance, pls change this part of code. + ServiceLoader<ISeriesNumerMonitor> limiterServiceLoader = + ServiceLoader.load(ISeriesNumerMonitor.class); + for (ISeriesNumerMonitor loader : limiterServiceLoader) { + if (this.seriesNumerMonitor != null) { + // it means there is more than one ISeriesNumerLimiter implementation. + logger.warn("There are more than one ISeriesNumerLimiter implementation. pls check."); + } + this.seriesNumerMonitor = loader; + } mtreeSnapshotInterval = config.getMtreeSnapshotInterval(); mtreeSnapshotThresholdTime = config.getMtreeSnapshotThresholdTime() * 1000L; String schemaDir = config.getSchemaDir(); @@ -605,7 +603,7 @@ public class MManager { + "please increase MAX_HEAP_SIZE in iotdb-env.sh/bat and restart"); } - if (!seriesNumerLimiter.addTimeSeries(1)) { + if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(1)) { throw new SeriesNumberOverflowException(); } try { @@ -643,7 +641,9 @@ public class MManager { } } catch (Throwable t) { // roll back - seriesNumerLimiter.deleteTimeSeries(1); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(1); + } throw t; } @@ -730,7 +730,7 @@ public class MManager { } int seriesCount = plan.getMeasurements().size(); - if (!seriesNumerLimiter.addTimeSeries(seriesCount)) { + if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(seriesCount)) { throw new SeriesNumberOverflowException(); } @@ -759,7 +759,9 @@ public class MManager { mNodeCache.invalidate(prefixPath); } catch (Throwable t) { // roll back - seriesNumerLimiter.deleteTimeSeries(seriesCount); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(seriesCount); + } throw t; } @@ -892,7 +894,9 @@ public class MManager { node = node.getParent(); } totalNormalSeriesNumber.addAndGet(-1); - seriesNumerLimiter.deleteTimeSeries(1); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(1); + } if (!allowToCreateNewSeries && totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) { logger.info("Current series number {} come back to normal level", totalNormalSeriesNumber); @@ -936,7 +940,9 @@ public class MManager { mtree.getAllTimeseriesCount( storageGroup.concatNode(MULTI_LEVEL_PATH_WILDCARD), false, false); totalNormalSeriesNumber.addAndGet(-timeSeriesCount); - seriesNumerLimiter.deleteTimeSeries(timeSeriesCount); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(timeSeriesCount); + } // clear cached MNode if (!allowToCreateNewSeries && totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) { @@ -2529,7 +2535,9 @@ public class MManager { node.setUseTemplate(false); int seriesCount = node.getUpperTemplate().getMeasurementsCount(); totalTemplateSeriesNumber.addAndGet(-seriesCount); - seriesNumerLimiter.deleteTimeSeries(seriesCount); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(seriesCount); + } // clear caches within MManger mNodeCache.invalidate(node); @@ -2555,7 +2563,8 @@ public class MManager { String.format("Path [%s] has not been set any template.", node.getFullPath())); } - if (!seriesNumerLimiter.addTimeSeries(template.getMeasurementsCount())) { + if (seriesNumerMonitor != null + && !seriesNumerMonitor.addTimeSeries(template.getMeasurementsCount())) { throw new SeriesNumberOverflowException(); } @@ -2581,7 +2590,9 @@ public class MManager { mountedMNode.setUseTemplate(true); } catch (Throwable t) { // roll back - seriesNumerLimiter.deleteTimeSeries(template.getMeasurementsCount()); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(template.getMeasurementsCount()); + } throw t; } diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java index 66ffc23ac4..0c9c9c7dd2 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java +++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java @@ -53,8 +53,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import static org.apache.iotdb.db.utils.JarLoaderUtil.loadExternLib; - public class IoTDB implements IoTDBMBean { private static final Logger logger = LoggerFactory.getLogger(IoTDB.class); @@ -81,8 +79,6 @@ public class IoTDB implements IoTDBMBean { } IoTDB daemon = IoTDB.getInstance(); - loadExternLib(config); - daemon.active(); } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java b/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java deleted file mode 100644 index 30d0c28f47..0000000000 --- a/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.utils; - -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; -import org.apache.iotdb.db.metadata.MManager; -import org.apache.iotdb.external.api.IPropertiesLoader; -import org.apache.iotdb.external.api.ISeriesNumerLimiter; -import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; - -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.net.URLClassLoader; -import java.net.UnknownHostException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Properties; -import java.util.ServiceLoader; - -public class JarLoaderUtil { - - private static final Logger logger = LoggerFactory.getLogger(JarLoaderUtil.class); - - public static URL[] getExternalJarURLs(String jarDir) throws IOException { - HashSet<File> fileSet = - new HashSet<>( - FileUtils.listFiles( - SystemFileFactory.INSTANCE.getFile(jarDir), new String[] {"jar"}, true)); - return FileUtils.toURLs(fileSet.toArray(new File[0])); - } - - public static void loadExternLib(IoTDBConfig config) { - // load external properties - String loaderDir = config.getExternalPropertiesLoaderDir(); - - if (!(new File(loaderDir).exists())) { - return; - } - - Path externalPropertiesFile = IoTDBDescriptor.getInstance().getExternalPropsPath(); - URL[] loaderJarURLs; - List<Properties> externalPropertiesList = new ArrayList<>(); - try { - loaderJarURLs = getExternalJarURLs(loaderDir); - - if (loaderJarURLs == null || loaderJarURLs.length == 0) { - return; - } - - ClassLoader classLoader = new URLClassLoader(loaderJarURLs); - - // Use SPI to get all plugins' class - ServiceLoader<IPropertiesLoader> loaders = - ServiceLoader.load(IPropertiesLoader.class, classLoader); - - for (IPropertiesLoader loader : loaders) { - if (loader == null) { - logger.error("IPropertiesLoader(), loader is null."); - continue; - } - Properties properties = loader.loadProperties(externalPropertiesFile.toAbsolutePath()); - if (properties != null) { - externalPropertiesList.add(properties); - } - } - } catch (Throwable t) { - logger.error("error happened while loading external loader. ", t); - // ignore - } - - if (externalPropertiesList.size() != 1) { - return; - } - - // overwrite the default properties; - for (Properties properties : externalPropertiesList) { - try { - IoTDBDescriptor.getInstance().loadProperties(properties); - } catch (UnknownHostException e) { - // ignore - logger.error("error happened while loading external properties. ", e); - } - TSFileDescriptor.getInstance() - .overwriteConfigByCustomSettings(TSFileDescriptor.getInstance().getConfig(), properties); - } - - String limiterDir = config.getExternalLimiterDir(); - - if (!(new File(loaderDir).exists())) { - return; - } - - URL[] limiterJarURLs; - - List<ISeriesNumerLimiter> limiterList = new ArrayList<>(); - - try { - limiterJarURLs = getExternalJarURLs(limiterDir); - - if (limiterJarURLs == null || limiterJarURLs.length == 0) { - return; - } - - ClassLoader classLoader = new URLClassLoader(limiterJarURLs); - - // Use SPI to get all plugins' class - ServiceLoader<ISeriesNumerLimiter> limiters = - ServiceLoader.load(ISeriesNumerLimiter.class, classLoader); - - for (ISeriesNumerLimiter limiter : limiters) { - if (limiter == null) { - logger.error("ISeriesNumerLimiter(), limiter is null."); - continue; - } - for (Properties properties : externalPropertiesList) { - limiter.init(properties); - } - limiterList.add(limiter); - } - } catch (Throwable t) { - // ignore - logger.error("error happened while loading external limiter. ", t); - } - - if (limiterList.size() != 1) { - return; - } - - MManager.getInstance().setSeriesNumerLimiter(limiterList.get(0)); - } -} diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java index 060aba9189..df5df28446 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java @@ -24,6 +24,7 @@ import org.apache.iotdb.tsfile.fileSystem.FSType; import java.io.Serializable; import java.nio.charset.Charset; +import java.util.Properties; /** TSFileConfig is a configuration class. Every variable is public and has default value. */ public class TSFileConfig implements Serializable { @@ -145,6 +146,8 @@ public class TSFileConfig implements Serializable { private double bloomFilterErrorRate = 0.05; /** The amount of data iterate each time */ private int batchSize = 1000; + /** customizedProperties, this should be empty by default. */ + private Properties customizedProperties = new Properties(); public TSFileConfig() {} @@ -411,4 +414,12 @@ public class TSFileConfig implements Serializable { public void setBatchSize(int batchSize) { this.batchSize = batchSize; } + + public Properties getCustomizedProperties() { + return customizedProperties; + } + + public void setCustomizedProperties(Properties customizedProperties) { + this.customizedProperties = customizedProperties; + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java index 711c5a4fd2..765bea8684 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java @@ -57,11 +57,11 @@ public class TSFileDescriptor { private void init() { Properties properties = loadProperties(); if (properties != null) { - overwriteConfigByCustomSettings(this.conf, properties); + overwriteConfigByCustomSettings(properties); } } - public void overwriteConfigByCustomSettings(TSFileConfig conf, Properties properties) { + public void overwriteConfigByCustomSettings(Properties properties) { PropertiesOverWriter writer = new PropertiesOverWriter(properties); writer.setInt(conf::setGroupSizeInByte, "group_size_in_byte");
