This is an automated email from the ASF dual-hosted git repository. hxd pushed a commit to branch re-ext-lib in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 76d5a898dfcb9b6914ba496c8c8b4a8ada927843 Author: xiangdong huang <[email protected]> AuthorDate: Wed Oct 19 18:15:06 2022 +0800 refactor seriesNumberMonitor and additional properties, cherry pick from rel/0.13 --- .../iotdb/external/api/IPropertiesLoader.java | 12 +- ...sNumerLimiter.java => ISeriesNumerMonitor.java} | 8 +- .../iotdb/flink/tsfile/util/TSFileConfigUtil.java | 1 + .../util/TSFileConfigUtilCompletenessTest.java | 12 +- metrics/dropwizard-metrics/pom.xml | 6 - .../apache/iotdb/commons/conf/IoTDBConstant.java | 4 - .../resources/conf/iotdb-datanode.properties | 21 --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 42 ++---- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 73 ++-------- .../db/metadata/schemaregion/SchemaEngine.java | 46 +++---- .../schemaregion/SchemaRegionMemoryImpl.java | 38 ++++-- .../schemaregion/SchemaRegionSchemaFileImpl.java | 34 +++-- .../db/protocol/mqtt/PayloadFormatManager.java | 5 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 4 - .../java/org/apache/iotdb/db/service/NewIoTDB.java | 3 - .../java/org/apache/iotdb/db/utils/FilesUtils.java | 43 ++++++ .../org/apache/iotdb/db/utils/JarLoaderUtil.java | 150 --------------------- .../iotdb/tsfile/common/conf/TSFileConfig.java | 12 ++ .../iotdb/tsfile/common/conf/TSFileDescriptor.java | 4 +- 19 files changed, 175 insertions(+), 343 deletions(-) diff --git a/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java b/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java index 4e62929c15..a19a10ec2d 100644 --- a/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java +++ b/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.external.api; -import java.nio.file.Path; import java.util.Properties; /** @@ -26,12 +25,17 @@ import java.util.Properties; * configurations */ public interface IPropertiesLoader { - /** * Load Properties from specific file * - * @param file The path of the properties file to open * @return a property list with values in file. */ - Properties loadProperties(Path file); + Properties loadProperties(); + + /** + * some properties that iotdb does not use by default. + * + * @return properties. + */ + Properties getCustomizedProperties(); } diff --git a/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java b/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java similarity index 82% rename from external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java rename to external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java index 2a740ff1ae..de1a60e3e3 100644 --- a/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java +++ b/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java @@ -20,8 +20,8 @@ package org.apache.iotdb.external.api; import java.util.Properties; -/** An interface for series number limiting, users can implement their own limitation strategy */ -public interface ISeriesNumerLimiter { +/** An interface for series number monitoring, users can implement their own limitation strategy */ +public interface ISeriesNumerMonitor { /** * do the necessary initialization @@ -29,13 +29,11 @@ public interface ISeriesNumerLimiter { * @param properties Properties containing all the parameters needed to init */ void init(Properties properties); - /** * add time series * * @param number time series number for current createTimeSeries operation - * @return true if totalTimeSeriesNumber doesn't exceed the limit and current createTimeSeries - * operation is allowed, otherwise false + * @return true if we want to allow the operation, otherwise false */ boolean addTimeSeries(int number); diff --git a/flink-tsfile-connector/src/main/java/org/apache/iotdb/flink/tsfile/util/TSFileConfigUtil.java b/flink-tsfile-connector/src/main/java/org/apache/iotdb/flink/tsfile/util/TSFileConfigUtil.java index 44862a3bbb..c3be71c07b 100644 --- a/flink-tsfile-connector/src/main/java/org/apache/iotdb/flink/tsfile/util/TSFileConfigUtil.java +++ b/flink-tsfile-connector/src/main/java/org/apache/iotdb/flink/tsfile/util/TSFileConfigUtil.java @@ -60,5 +60,6 @@ public class TSFileConfigUtil { globalConfig.setTSFileStorageFs(config.getTSFileStorageFs()); globalConfig.setUseKerberos(config.isUseKerberos()); globalConfig.setValueEncoder(config.getValueEncoder()); + globalConfig.setCustomizedProperties(config.getCustomizedProperties()); } } diff --git a/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java b/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java index a3c40fbfc9..c46c700f2c 100644 --- a/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java +++ b/flink-tsfile-connector/src/test/java/org/apache/iotdb/flink/util/TSFileConfigUtilCompletenessTest.java @@ -25,6 +25,8 @@ import org.junit.Test; import java.lang.reflect.Method; import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -38,7 +40,7 @@ public class TSFileConfigUtilCompletenessTest { @Test public void testTSFileConfigUtilCompleteness() { - String[] addedSetters = { + String[] setters = { "setBatchSize", "setBloomFilterErrorRate", "setCompressor", @@ -75,13 +77,17 @@ public class TSFileConfigUtilCompletenessTest { "setFreqEncodingBlockSize", "setMaxTsBlockLineNumber", "setMaxTsBlockSizeInBytes", - "setPatternMatchingThreshold" + "setPatternMatchingThreshold", + "setCustomizedProperties" }; + Set<String> addedSetters = new HashSet<>(); + Collections.addAll(addedSetters, setters); + Set<String> newSetters = Arrays.stream(TSFileConfig.class.getMethods()) .map(Method::getName) .filter(s -> s.startsWith("set")) - .filter(s -> !Arrays.asList(addedSetters).contains(s)) + .filter(s -> !addedSetters.contains(s)) .collect(Collectors.toSet()); assertTrue( String.format( diff --git a/metrics/dropwizard-metrics/pom.xml b/metrics/dropwizard-metrics/pom.xml index 51b7b65af7..62e88b3a22 100644 --- a/metrics/dropwizard-metrics/pom.xml +++ b/metrics/dropwizard-metrics/pom.xml @@ -43,12 +43,6 @@ <artifactId>metrics-jmx</artifactId> <version>${dropwizard.metrics.version}</version> </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-core</artifactId> - <version>${jersey-core.version}</version> - <scope>compile</scope> - </dependency> <dependency> <groupId>io.projectreactor.netty</groupId> <artifactId>reactor-netty-http</artifactId> diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index d5f647a82a..a135d84907 100644 --- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@ -228,10 +228,6 @@ public class IoTDBConstant { public static final String WAL_FOLDER_NAME = "wal"; public static final String EXT_PIPE_FOLDER_NAME = "extPipe"; - public static final String EXT_PROPERTIES_LOADER_FOLDER_NAME = "loader"; - - public static final String EXT_LIMITER = "limiter"; - // mqtt public static final String ENABLE_MQTT = "enable_mqtt_service"; public static final String MQTT_HOST_NAME = "mqtt_host"; diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties index 5e413f3e4c..b43262ad3f 100644 --- a/server/src/assembly/resources/conf/iotdb-datanode.properties +++ b/server/src/assembly/resources/conf/iotdb-datanode.properties @@ -1166,25 +1166,4 @@ trigger_forward_mqtt_pool_size=4 # Datatype: int # data_region_num=1 -#################### -### External Lib Configuration -#################### - -# external lib directory for properties loader -# For Window platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is -# absolute. Otherwise, it is relative. -# external_properties_loader_dir=ext\\loader -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# external_properties_loader_dir=ext/loader - -# external lib directory for limiter -# For Window platform -# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is -# absolute. Otherwise, it is relative. -# external_limiter_dir=ext\\limiter -# For Linux platform -# If its prefix is "/", then the path is absolute. Otherwise, it is relative. -# external_limiter_dir=ext/limiter diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 5df7d0ef19..6db2094591 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -53,6 +53,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -64,7 +65,6 @@ public class IoTDBConfig { /* Names of Watermark methods */ public static final String WATERMARK_GROUPED_LSB = "GroupBasedLSBMethod"; public static final String CONFIG_NAME = "iotdb-datanode.properties"; - public static final String EXTERNAL_CONFIG_NAME = "iotdb-datanode-external.properties"; private static final Logger logger = LoggerFactory.getLogger(IoTDBConfig.class); private static final String MULTI_DIR_STRATEGY_PREFIX = "org.apache.iotdb.db.conf.directories.strategy."; @@ -285,16 +285,6 @@ public class IoTDBConfig { private String mqttDir = IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.MQTT_FOLDER_NAME; - /** External lib directory for properties loader, stores user-uploaded JAR files */ - private String externalPropertiesLoaderDir = - IoTDBConstant.EXT_FOLDER_NAME - + File.separator - + IoTDBConstant.EXT_PROPERTIES_LOADER_FOLDER_NAME; - - /** External lib directory for limiter, stores user uploaded JAR files */ - private String externalLimiterDir = - IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.EXT_LIMITER; - /** Data directories. It can be settled as dataDirs = {"data1", "data2", "data3"}; */ private String[] dataDirs = { IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.DATA_FOLDER_NAME @@ -1051,6 +1041,9 @@ public class IoTDBConfig { private long dataRatisConsensusMaxSleepTimeMs = 10000L; private long schemaRatisConsensusMaxSleepTimeMs = 10000L; + // customizedProperties, this should be empty by default. + private Properties customizedProperties = new Properties(); + IoTDBConfig() {} public float getUdfMemoryBudgetInMB() { @@ -1171,8 +1164,7 @@ public class IoTDBConfig { triggerDir = addHomeDir(triggerDir); triggerTemporaryLibDir = addHomeDir(triggerTemporaryLibDir); mqttDir = addHomeDir(mqttDir); - externalPropertiesLoaderDir = addHomeDir(externalPropertiesLoaderDir); - externalLimiterDir = addHomeDir(externalLimiterDir); + extPipeDir = addHomeDir(extPipeDir); if (TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs().equals(FSType.HDFS)) { @@ -1423,22 +1415,6 @@ public class IoTDBConfig { this.mqttDir = mqttDir; } - public String getExternalPropertiesLoaderDir() { - return externalPropertiesLoaderDir; - } - - public void setExternalPropertiesLoaderDir(String externalPropertiesLoaderDir) { - this.externalPropertiesLoaderDir = externalPropertiesLoaderDir; - } - - public String getExternalLimiterDir() { - return externalLimiterDir; - } - - public void setExternalLimiterDir(String externalLimiterDir) { - this.externalLimiterDir = externalLimiterDir; - } - public String getMultiDirStrategyClassName() { return multiDirStrategyClassName; } @@ -3525,4 +3501,12 @@ public class IoTDBConfig { public void setSchemaRatisConsensusMaxSleepTimeMs(long schemaRatisConsensusMaxSleepTimeMs) { this.schemaRatisConsensusMaxSleepTimeMs = schemaRatisConsensusMaxSleepTimeMs; } + + public Properties getCustomizedProperties() { + return customizedProperties; + } + + public void setCustomizedProperties(Properties customizedProperties) { + this.customizedProperties = customizedProperties; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 2fc236ebfb..c0ba2105cd 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -42,6 +42,7 @@ import org.apache.iotdb.db.service.metrics.MetricService; import org.apache.iotdb.db.utils.datastructure.TVListSortAlgorithm; import org.apache.iotdb.db.wal.WALManager; import org.apache.iotdb.db.wal.utils.WALMode; +import org.apache.iotdb.external.api.IPropertiesLoader; import org.apache.iotdb.metrics.config.MetricConfigDescriptor; import org.apache.iotdb.metrics.config.ReloadLevel; import org.apache.iotdb.rpc.RpcTransportFactory; @@ -61,12 +62,10 @@ import java.io.IOException; import java.io.InputStream; import java.net.InetAddress; import java.net.MalformedURLException; -import java.net.URISyntaxException; import java.net.URL; import java.net.UnknownHostException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Properties; +import java.util.ServiceLoader; public class IoTDBDescriptor { @@ -78,6 +77,18 @@ public class IoTDBDescriptor { protected IoTDBDescriptor() { loadProps(); + ServiceLoader<IPropertiesLoader> propertiesLoaderServiceLoader = + ServiceLoader.load(IPropertiesLoader.class); + for (IPropertiesLoader loader : propertiesLoaderServiceLoader) { + logger.info("Will reload properties from {} ", loader.getClass().getName()); + Properties properties = loader.loadProperties(); + loadProperties(properties); + conf.setCustomizedProperties(loader.getCustomizedProperties()); + TSFileDescriptor.getInstance().overwriteConfigByCustomSettings(properties); + TSFileDescriptor.getInstance() + .getConfig() + .setCustomizedProperties(loader.getCustomizedProperties()); + } } public static IoTDBDescriptor getInstance() { @@ -135,52 +146,6 @@ public class IoTDBDescriptor { } } - /** - * get props url location - * - * @return url object if location exit, otherwise null. - */ - public Path getExternalPropsPath() { - // Check if a config-directory was specified first. - String urlString = System.getProperty(IoTDBConstant.IOTDB_CONF, null); - // If it wasn't, check if a home directory was provided (This usually contains a config) - if (urlString == null) { - urlString = System.getProperty(IoTDBConstant.IOTDB_HOME, null); - if (urlString != null) { - urlString = - urlString - + File.separatorChar - + "conf" - + File.separatorChar - + IoTDBConfig.EXTERNAL_CONFIG_NAME; - } else { - // If this too wasn't provided, try to find a default config in the root of the classpath. - URL uri = IoTDBConfig.class.getResource("/" + IoTDBConfig.EXTERNAL_CONFIG_NAME); - if (uri != null) { - try { - return Paths.get(uri.toURI()); - } catch (URISyntaxException e) { - return null; - } - } - logger.warn( - "Cannot find IOTDB_HOME or IOTDB_EXTERNAL_CONF environment variable when loading " - + "config file {}, use default configuration", - IoTDBConfig.EXTERNAL_CONFIG_NAME); - // update all data seriesPath - conf.updatePath(); - return null; - } - } - // If a config location was provided, but it doesn't end with a properties file, - // append the default location. - else if (!urlString.endsWith(".properties")) { - urlString += (File.separatorChar + IoTDBConfig.EXTERNAL_CONFIG_NAME); - } - - return Paths.get(urlString); - } - /** load an property file and set TsfileDBConfig variables. */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning private void loadProps() { @@ -1361,16 +1326,6 @@ public class IoTDBDescriptor { } } - private void loadExternalLibProps(Properties properties) { - - conf.setExternalPropertiesLoaderDir( - properties.getProperty( - "external_properties_loader_dir", conf.getExternalPropertiesLoaderDir())); - - conf.setExternalLimiterDir( - properties.getProperty("external_limiter_dir", conf.getExternalLimiterDir())); - } - // timed flush memtable private void loadTimedService(Properties properties) { conf.setEnableTimedFlushSeqMemtable( diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java index 358d37f7b8..c044886aad 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java @@ -37,7 +37,7 @@ import org.apache.iotdb.db.metadata.mtree.ConfigMTree; import org.apache.iotdb.db.metadata.rescon.SchemaResourceManager; import org.apache.iotdb.db.metadata.visitor.SchemaExecutionVisitor; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.external.api.ISeriesNumerMonitor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +48,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Properties; +import java.util.ServiceLoader; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; @@ -71,22 +71,8 @@ public class SchemaEngine { private ScheduledExecutorService timedForceMLogThread; - private ISeriesNumerLimiter seriesNumerLimiter = - new ISeriesNumerLimiter() { - @Override - public void init(Properties properties) {} - - @Override - public boolean addTimeSeries(int number) { - // always return true, don't limit the number of series - return true; - } - - @Override - public void deleteTimeSeries(int number) { - // do nothing - } - }; + // seriesNumberMonitor may be null + private ISeriesNumerMonitor seriesNumerMonitor = null; public TSStatus write(SchemaRegionId schemaRegionId, PlanNode planNode) { return planNode.accept(new SchemaExecutionVisitor(), schemaRegionMap.get(schemaRegionId)); @@ -99,7 +85,21 @@ public class SchemaEngine { private SchemaEngineManagerHolder() {} } - private SchemaEngine() {} + private SchemaEngine() { + // init ISeriesNumerMonitor if there is. + // each mmanager instance will generate an ISeriesNumerMonitor instance + // So, if you want to share the ISeriesNumerMonitor instance, pls change this part of code. + ServiceLoader<ISeriesNumerMonitor> monitorServiceLoader = + ServiceLoader.load(ISeriesNumerMonitor.class); + for (ISeriesNumerMonitor loader : monitorServiceLoader) { + if (this.seriesNumerMonitor != null) { + // it means there is more than one ISeriesNumerMonitor implementation. + logger.warn("There are more than one ISeriesNumerMonitor implementation. pls check."); + } + logger.info("Will set seriesNumerMonitor from {} ", loader.getClass().getName()); + this.seriesNumerMonitor = loader; + } + } public static SchemaEngine getInstance() { return SchemaEngineManagerHolder.INSTANCE; @@ -311,12 +311,12 @@ public class SchemaEngine { case Memory: schemaRegion = new SchemaRegionMemoryImpl( - storageGroup, schemaRegionId, storageGroupMNode, seriesNumerLimiter); + storageGroup, schemaRegionId, storageGroupMNode, seriesNumerMonitor); break; case Schema_File: schemaRegion = new SchemaRegionSchemaFileImpl( - storageGroup, schemaRegionId, storageGroupMNode, seriesNumerLimiter); + storageGroup, schemaRegionId, storageGroupMNode, seriesNumerMonitor); break; case Rocksdb_based: schemaRegion = @@ -386,7 +386,7 @@ public class SchemaEngine { } } - public void setSeriesNumerLimiter(ISeriesNumerLimiter seriesNumerLimiter) { - this.seriesNumerLimiter = seriesNumerLimiter; + public void setSeriesNumerMonitor(ISeriesNumerMonitor seriesNumerMonitor) { + this.seriesNumerMonitor = seriesNumerMonitor; } } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java index b9f11f3271..d787ec70f1 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java @@ -89,7 +89,7 @@ import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.dataset.ShowDevicesResult; import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; import org.apache.iotdb.db.utils.SchemaUtils; -import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.external.api.ISeriesNumerMonitor; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -187,14 +187,15 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { private LoadingCache<PartialPath, IMNode> mNodeCache; private TagManager tagManager; - private final ISeriesNumerLimiter seriesNumerLimiter; + // seriesNumberMonitor may be null + private final ISeriesNumerMonitor seriesNumerMonitor; // region Interfaces and Implementation of initialization、snapshot、recover and clear public SchemaRegionMemoryImpl( PartialPath storageGroup, SchemaRegionId schemaRegionId, IStorageGroupMNode storageGroupMNode, - ISeriesNumerLimiter seriesNumerLimiter) + ISeriesNumerMonitor seriesNumerMonitor) throws MetadataException { storageGroupFullPath = storageGroup.getFullPath(); @@ -228,7 +229,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { } } - this.seriesNumerLimiter = seriesNumerLimiter; + this.seriesNumerMonitor = seriesNumerMonitor; init(); } @@ -445,8 +446,9 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { int seriesCount = leafMNodes.size(); schemaStatisticsManager.deleteTimeseries(seriesCount); - seriesNumerLimiter.deleteTimeSeries(seriesCount); - + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(seriesCount); + } // drop triggers with no exceptions TriggerEngine.drop(leafMNodes); @@ -575,14 +577,14 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { throw new SeriesOverflowException(); } - if (!seriesNumerLimiter.addTimeSeries(1)) { + if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(1)) { throw new SeriesNumberOverflowException(); } try { IMeasurementMNode leafMNode; - // using try-catch to restore seriesNumerLimiter's state while create failed + // using try-catch to restore seriesNumberMonitor's state while create failed try { PartialPath path = plan.getPath(); SchemaUtils.checkDataTypeWithEncoding(plan.getDataType(), plan.getEncoding()); @@ -601,7 +603,9 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { // the cached mNode may be replaced by new entityMNode in mtree mNodeCache.invalidate(path.getDevicePath()); } catch (Throwable t) { - seriesNumerLimiter.deleteTimeSeries(1); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(1); + } throw t; } @@ -656,7 +660,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { throw new SeriesOverflowException(); } - if (!seriesNumerLimiter.addTimeSeries(seriesCount)) { + if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(seriesCount)) { throw new SeriesNumberOverflowException(); } @@ -669,7 +673,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { List<Map<String, String>> attributesList = plan.getAttributesList(); List<IMeasurementMNode> measurementMNodeList; - // using try-catch to restore seriesNumerLimiter's state while create failed + // using try-catch to restore seriesNumberMonitor's state while create failed try { for (int i = 0; i < measurements.size(); i++) { SchemaUtils.checkDataTypeWithEncoding(dataTypes.get(i), encodings.get(i)); @@ -688,7 +692,9 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { // the cached mNode may be replaced by new entityMNode in mtree mNodeCache.invalidate(prefixPath); } catch (Throwable t) { - seriesNumerLimiter.deleteTimeSeries(seriesCount); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(seriesCount); + } throw t; } @@ -871,7 +877,9 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { mNodeCache.invalidate(node.getPartialPath()); schemaStatisticsManager.deleteTimeseries(1); - seriesNumerLimiter.deleteTimeSeries(1); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(1); + } } private void recoverRollbackPreDeleteTimeseries(PartialPath path) throws MetadataException { @@ -931,7 +939,9 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { mNodeCache.invalidate(node.getPartialPath()); schemaStatisticsManager.deleteTimeseries(1); - seriesNumerLimiter.deleteTimeSeries(1); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(1); + } return storageGroupPath; } // endregion diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java index d8404e50a9..13932d2449 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java @@ -84,7 +84,7 @@ import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.dataset.ShowDevicesResult; import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; import org.apache.iotdb.db.utils.SchemaUtils; -import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.external.api.ISeriesNumerMonitor; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -179,14 +179,15 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion { private LoadingCache<PartialPath, IMNode> mNodeCache; private TagManager tagManager; - private final ISeriesNumerLimiter seriesNumerLimiter; + // seriesNumberMonitor may be null + private final ISeriesNumerMonitor seriesNumerMonitor; // region Interfaces and Implementation of initialization、snapshot、recover and clear public SchemaRegionSchemaFileImpl( PartialPath storageGroup, SchemaRegionId schemaRegionId, IStorageGroupMNode storageGroupMNode, - ISeriesNumerLimiter seriesNumerLimiter) + ISeriesNumerMonitor seriesNumerMonitor) throws MetadataException { storageGroupFullPath = storageGroup.getFullPath(); @@ -212,7 +213,7 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion { } }); this.storageGroupMNode = storageGroupMNode; - this.seriesNumerLimiter = seriesNumerLimiter; + this.seriesNumerMonitor = seriesNumerMonitor; init(); } @@ -406,8 +407,9 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion { int seriesCount = leafMNodes.size(); schemaStatisticsManager.deleteTimeseries(seriesCount); - seriesNumerLimiter.deleteTimeSeries(seriesCount); - + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(seriesCount); + } // drop triggers with no exceptions TriggerEngine.drop(leafMNodes); @@ -469,14 +471,14 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion { throw new SeriesOverflowException(); } - if (!seriesNumerLimiter.addTimeSeries(1)) { + if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(1)) { throw new SeriesNumberOverflowException(); } try { PartialPath path = plan.getPath(); IMeasurementMNode leafMNode; - // using try-catch to restore seriesNumerLimiter's state while create failed + // using try-catch to restore seriesNumberMonitor's state while create failed try { SchemaUtils.checkDataTypeWithEncoding(plan.getDataType(), plan.getEncoding()); @@ -491,7 +493,9 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion { plan.getProps(), plan.getAlias()); } catch (Throwable t) { - seriesNumerLimiter.deleteTimeSeries(1); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(1); + } throw t; } @@ -615,7 +619,7 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion { throw new SeriesOverflowException(); } - if (!seriesNumerLimiter.addTimeSeries(seriesCount)) { + if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(seriesCount)) { throw new SeriesNumberOverflowException(); } @@ -627,7 +631,7 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion { List<Map<String, String>> tagsList = plan.getTagsList(); List<Map<String, String>> attributesList = plan.getAttributesList(); List<IMeasurementMNode> measurementMNodeList; - // using try-catch to restore seriesNumerLimiter's state while create failed + // using try-catch to restore seriesNumberMonitor's state while create failed try { for (int i = 0; i < measurements.size(); i++) { SchemaUtils.checkDataTypeWithEncoding(dataTypes.get(i), encodings.get(i)); @@ -643,7 +647,9 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion { plan.getCompressors(), plan.getAliasList()); } catch (Throwable t) { - seriesNumerLimiter.deleteTimeSeries(seriesCount); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(seriesCount); + } throw t; } @@ -822,7 +828,9 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion { mNodeCache.invalidate(node.getPartialPath()); schemaStatisticsManager.deleteTimeseries(1); - seriesNumerLimiter.deleteTimeSeries(1); + if (seriesNumerMonitor != null) { + seriesNumerMonitor.deleteTimeSeries(1); + } return storageGroupPath; } // endregion diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/mqtt/PayloadFormatManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/mqtt/PayloadFormatManager.java index b666c8be2e..40a7631474 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/mqtt/PayloadFormatManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/mqtt/PayloadFormatManager.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.protocol.mqtt; import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.utils.FilesUtils; import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; @@ -33,8 +34,6 @@ import java.util.HashMap; import java.util.Map; import java.util.ServiceLoader; -import static org.apache.iotdb.db.utils.JarLoaderUtil.getExternalJarURLs; - /** PayloadFormatManager loads payload formatter from SPI services. */ public class PayloadFormatManager { private static final Logger logger = LoggerFactory.getLogger(PayloadFormatManager.class); @@ -87,7 +86,7 @@ public class PayloadFormatManager { logger.info("PayloadFormatManager(), find MQTT Payload Plugin {}.", pluginName); } - URL[] jarURLs = getExternalJarURLs(mqttDir); + URL[] jarURLs = FilesUtils.getPluginJarURLs(mqttDir); logger.debug("MQTT Plugin jarURLs: {}", jarURLs); for (URL jarUrl : jarURLs) { diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java index cca8a7dc2b..4bfc5be5ef 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java +++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java @@ -57,8 +57,6 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import static org.apache.iotdb.db.utils.JarLoaderUtil.loadExternLib; - public class IoTDB implements IoTDBMBean { private static final Logger logger = LoggerFactory.getLogger(IoTDB.class); @@ -84,8 +82,6 @@ public class IoTDB implements IoTDBMBean { } IoTDB daemon = IoTDB.getInstance(); - loadExternLib(config); - daemon.active(); } diff --git a/server/src/main/java/org/apache/iotdb/db/service/NewIoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/NewIoTDB.java index b8fa07d27b..f24e0b3466 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/NewIoTDB.java +++ b/server/src/main/java/org/apache/iotdb/db/service/NewIoTDB.java @@ -57,8 +57,6 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import static org.apache.iotdb.db.utils.JarLoaderUtil.loadExternLib; - public class NewIoTDB implements NewIoTDBMBean { private static final Logger logger = LoggerFactory.getLogger(NewIoTDB.class); @@ -85,7 +83,6 @@ public class NewIoTDB implements NewIoTDBMBean { config.setMppMode(true); // In standalone mode, Consensus memory should be reclaimed IoTDBDescriptor.getInstance().reclaimConsensusMemory(); - loadExternLib(config); daemon.active(); } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FilesUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FilesUtils.java new file mode 100644 index 0000000000..8a77f35db7 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/utils/FilesUtils.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.db.utils; + +import org.apache.iotdb.commons.file.SystemFileFactory; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.HashSet; + +public class FilesUtils { + + /** + * get all jar files in the given folder + * + * @param folderPath + * @return all jar files' URL + * @throws IOException + */ + public static URL[] getPluginJarURLs(String folderPath) throws IOException { + HashSet<File> fileSet = + new HashSet<>( + org.apache.commons.io.FileUtils.listFiles( + SystemFileFactory.INSTANCE.getFile(folderPath), new String[] {"jar"}, true)); + return org.apache.commons.io.FileUtils.toURLs(fileSet.toArray(new File[0])); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java b/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java deleted file mode 100644 index f7fc3a059c..0000000000 --- a/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.utils; - -import org.apache.iotdb.commons.file.SystemFileFactory; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.metadata.schemaregion.SchemaEngine; -import org.apache.iotdb.external.api.IPropertiesLoader; -import org.apache.iotdb.external.api.ISeriesNumerLimiter; -import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; - -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Properties; -import java.util.ServiceLoader; - -public class JarLoaderUtil { - - private static final Logger logger = LoggerFactory.getLogger(JarLoaderUtil.class); - - public static URL[] getExternalJarURLs(String jarDir) throws IOException { - HashSet<File> fileSet = - new HashSet<>( - FileUtils.listFiles( - SystemFileFactory.INSTANCE.getFile(jarDir), new String[] {"jar"}, true)); - return FileUtils.toURLs(fileSet.toArray(new File[0])); - } - - public static void loadExternLib(IoTDBConfig config) { - // load external properties - String loaderDir = config.getExternalPropertiesLoaderDir(); - - if (!(new File(loaderDir).exists())) { - return; - } - - Path externalPropertiesFile = IoTDBDescriptor.getInstance().getExternalPropsPath(); - URL[] loaderJarURLs; - List<Properties> externalPropertiesList = new ArrayList<>(); - try { - loaderJarURLs = getExternalJarURLs(loaderDir); - - if (loaderJarURLs == null || loaderJarURLs.length == 0) { - return; - } - - ClassLoader classLoader = new URLClassLoader(loaderJarURLs); - - // Use SPI to get all plugins' class - ServiceLoader<IPropertiesLoader> loaders = - ServiceLoader.load(IPropertiesLoader.class, classLoader); - - for (IPropertiesLoader loader : loaders) { - if (loader == null) { - logger.error("IPropertiesLoader(), loader is null."); - continue; - } - Properties properties = loader.loadProperties(externalPropertiesFile.toAbsolutePath()); - if (properties != null) { - externalPropertiesList.add(properties); - } - } - } catch (Throwable t) { - logger.error("error happened while loading external loader. ", t); - // ignore - } - - if (externalPropertiesList.size() != 1) { - return; - } - - // overwrite the default properties; - for (Properties properties : externalPropertiesList) { - IoTDBDescriptor.getInstance().loadProperties(properties); - TSFileDescriptor.getInstance() - .overwriteConfigByCustomSettings(TSFileDescriptor.getInstance().getConfig(), properties); - } - - String limiterDir = config.getExternalLimiterDir(); - - if (!(new File(loaderDir).exists())) { - return; - } - - URL[] limiterJarURLs; - - List<ISeriesNumerLimiter> limiterList = new ArrayList<>(); - - try { - limiterJarURLs = getExternalJarURLs(limiterDir); - - if (limiterJarURLs == null || limiterJarURLs.length == 0) { - return; - } - - ClassLoader classLoader = new URLClassLoader(limiterJarURLs); - - // Use SPI to get all plugins' class - ServiceLoader<ISeriesNumerLimiter> limiters = - ServiceLoader.load(ISeriesNumerLimiter.class, classLoader); - - for (ISeriesNumerLimiter limiter : limiters) { - if (limiter == null) { - logger.error("ISeriesNumerLimiter(), limiter is null."); - continue; - } - for (Properties properties : externalPropertiesList) { - limiter.init(properties); - } - limiterList.add(limiter); - } - } catch (Throwable t) { - // ignore - logger.error("error happened while loading external limiter. ", t); - } - - if (limiterList.size() != 1) { - return; - } - - SchemaEngine.getInstance().setSeriesNumerLimiter(limiterList.get(0)); - } -} diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java index c79c55d229..837f9e395a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java @@ -24,6 +24,7 @@ import org.apache.iotdb.tsfile.fileSystem.FSType; import java.io.Serializable; import java.nio.charset.Charset; +import java.util.Properties; /** TSFileConfig is a configuration class. Every variable is public and has default value. */ public class TSFileConfig implements Serializable { @@ -158,6 +159,9 @@ public class TSFileConfig implements Serializable { private int patternMatchingThreshold = 1000000; + /** customizedProperties, this should be empty by default. */ + private Properties customizedProperties = new Properties(); + public TSFileConfig() {} public int getGroupSizeInByte() { @@ -463,4 +467,12 @@ public class TSFileConfig implements Serializable { public void setPatternMatchingThreshold(int patternMatchingThreshold) { this.patternMatchingThreshold = patternMatchingThreshold; } + + public Properties getCustomizedProperties() { + return customizedProperties; + } + + public void setCustomizedProperties(Properties customizedProperties) { + this.customizedProperties = customizedProperties; + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java index 0ee217c32e..db26a79483 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java @@ -57,11 +57,11 @@ public class TSFileDescriptor { private void init() { Properties properties = loadProperties(); if (properties != null) { - overwriteConfigByCustomSettings(this.conf, properties); + overwriteConfigByCustomSettings(properties); } } - public void overwriteConfigByCustomSettings(TSFileConfig conf, Properties properties) { + public void overwriteConfigByCustomSettings(Properties properties) { PropertiesOverWriter writer = new PropertiesOverWriter(properties); writer.setInt(conf::setGroupSizeInByte, "group_size_in_byte");
