This is an automated email from the ASF dual-hosted git repository. hxd pushed a commit to branch change_rpc_port in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fbe6e3debe5dca9386948463c92da55617aeeb7b Merge: caf26ad 70c8fa59 Author: xiangdong huang <[email protected]> AuthorDate: Wed Dec 30 12:10:56 2020 +0800 merge with master .github/pull_request_template.md | 93 +++ .github/workflows/sonar_and_coverall.yml | 2 +- .mvn/wrapper/maven-wrapper.properties | 2 +- .travis.yml | 10 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 52 +- .../java/org/apache/iotdb/cluster/ClusterMain.java | 1 + .../cluster/client/async/AsyncClientPool.java | 1 - .../apache/iotdb/cluster/config/ClusterConfig.java | 14 +- .../iotdb/cluster/config/ClusterConstant.java | 6 +- .../iotdb/cluster/coordinator/Coordinator.java | 598 ++++++++++++++++++ .../apache/iotdb/cluster/metadata/CMManager.java | 8 +- .../cluster/query/ClusterConcatPathOptimizer.java | 4 +- .../iotdb/cluster/query/ClusterPlanExecutor.java | 10 +- .../iotdb/cluster/query/ClusterPlanRouter.java | 9 - .../apache/iotdb/cluster/server/ClientServer.java | 26 +- .../iotdb/cluster/server/MetaClusterServer.java | 5 + .../org/apache/iotdb/cluster/server/Timer.java | 13 +- .../cluster/server/member/DataGroupMember.java | 2 +- .../cluster/server/member/MetaGroupMember.java | 576 ++--------------- .../iotdb/cluster/server/member/RaftMember.java | 21 +- .../cluster/server/service/BaseSyncService.java | 7 +- .../apache/iotdb/cluster/utils/ClusterUtils.java | 4 +- .../org/apache/iotdb/cluster/common/IoTDBTest.java | 2 +- .../cluster/log/applier/DataLogApplierTest.java | 6 +- .../manage/MetaSingleSnapshotLogManagerTest.java | 3 + .../cluster/log/snapshot/DataSnapshotTest.java | 5 + .../log/snapshot/MetaSimpleSnapshotTest.java | 2 + .../cluster/partition/SlotPartitionTableTest.java | 14 - .../query/ClusterPhysicalGeneratorTest.java | 10 +- .../cluster/query/ClusterPlanExecutorTest.java | 4 +- .../iotdb/cluster/server/member/MemberTest.java | 10 +- .../cluster/server/member/MetaGroupMemberTest.java | 24 +- docs/UserGuide/Client/Status Codes.md | 8 + docs/UserGuide/Concept/SDT.md | 63 ++ .../Ecosystem Integration/Zeppelin-IoTDB.md | 9 +- .../Operation Manual/UDF User Defined Function.md | 476 ++++++++++++++ docs/zh/UserGuide/Client/Status Codes.md | 9 +- docs/zh/UserGuide/Concept/SDT.md | 61 ++ .../Ecosystem Integration/Zeppelin-IoTDB.md | 9 +- .../Operation Manual/UDF User Defined Function.md | 476 ++++++++++++++ example/pom.xml | 1 + example/udf/pom.xml | 82 +++ .../java/org/apache/iotdb/udf/UDTFExample.java | 56 ++ hadoop/pom.xml | 2 +- hive-connector/pom.xml | 2 +- .../main/java/org/apache/iotdb/jdbc/Config.java | 2 +- .../apache/iotdb/jdbc/IoTDBConnectionParams.java | 2 +- .../resources/conf/iotdb-engine.properties | 24 +- server/src/assembly/resources/conf/logback.xml | 24 +- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 20 +- .../apache/iotdb/db/auth/entity/PrivilegeType.java | 2 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 107 +++- .../org/apache/iotdb/db/conf/IoTDBConstant.java | 5 + .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 43 ++ .../compaction/CompactionMergeTaskPoolManager.java | 32 +- .../db/engine/compaction/TsFileManagement.java | 22 +- .../level/LevelCompactionTsFileManagement.java | 190 +++--- .../no/NoCompactionTsFileManagement.java | 5 - .../engine/compaction/utils/CompactionUtils.java | 12 +- .../iotdb/db/engine/merge/task/MergeFileTask.java | 2 +- .../io/LocalTextModificationAccessor.java | 61 +- .../engine/storagegroup/StorageGroupProcessor.java | 90 +-- .../UDFRegistrationException.java} | 72 +-- .../iotdb/db/metadata/logfile/MLogWriter.java | 22 +- .../main/java/org/apache/iotdb/db/qp/Planner.java | 9 +- .../apache/iotdb/db/qp/constant/DatetimeUtils.java | 40 ++ .../apache/iotdb/db/qp/constant/SQLConstant.java | 14 +- .../apache/iotdb/db/qp/executor/IPlanExecutor.java | 11 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 70 ++- .../org/apache/iotdb/db/qp/logical/Operator.java | 3 +- .../iotdb/db/qp/logical/crud/SFWOperator.java | 16 +- .../iotdb/db/qp/logical/crud/SelectOperator.java | 47 +- .../CreateFunctionOperator.java} | 99 +-- .../DropFunctionOperator.java} | 81 ++- .../ShowFunctionsOperator.java} | 78 ++- .../iotdb/db/qp/physical/crud/QueryPlan.java | 2 +- .../apache/iotdb/db/qp/physical/crud/UDFPlan.java | 52 ++ .../apache/iotdb/db/qp/physical/crud/UDTFPlan.java | 133 ++++ .../iotdb/db/qp/physical/crud/UpdatePlan.java | 135 ---- .../sys/{ShowPlan.java => CreateFunctionPlan.java} | 114 ++-- .../iotdb/db/qp/physical/sys/DropFunctionPlan.java | 81 +-- .../sys/ShowFunctionsPlan.java} | 75 +-- .../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 6 +- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 152 +++-- .../iotdb/db/qp/strategy/LogicalGenerator.java | 7 +- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 130 ++-- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 192 ++++-- .../db/query/aggregation/AggregateResult.java | 2 +- .../db/query/aggregation/AggregationType.java | 1 - .../db/query/control/QueryResourceManager.java | 8 +- .../db/query/dataset/AlignByDeviceDataSet.java | 4 +- .../dataset/DirectAlignByTimeDataSet.java} | 74 +-- .../dataset/DirectNonAlignDataSet.java} | 72 +-- .../apache/iotdb/db/query/dataset/ListDataSet.java | 4 +- .../db/query/dataset/NonAlignEngineDataSet.java | 28 +- .../dataset/RawQueryDataSetWithValueFilter.java | 4 +- .../dataset/RawQueryDataSetWithoutValueFilter.java | 24 +- .../db/query/dataset/ShowTimeseriesDataSet.java | 4 +- .../db/query/dataset/UDTFAlignByTimeDataSet.java | 277 ++++++++ .../apache/iotdb/db/query/dataset/UDTFDataSet.java | 152 +++++ .../db/query/dataset/UDTFNonAlignDataSet.java | 191 ++++++ .../dataset/groupby/GroupByEngineDataSet.java | 4 +- .../query/dataset/groupby/GroupByFillDataSet.java | 4 +- .../query/dataset/groupby/GroupByTimeDataSet.java | 4 +- .../groupby/GroupByWithValueFilterDataSet.java | 2 +- .../groupby/GroupByWithoutValueFilterDataSet.java | 2 +- .../iotdb/db/query/executor/IQueryRouter.java | 8 +- .../iotdb/db/query/executor/QueryRouter.java | 51 +- .../db/query/executor/RawDataQueryExecutor.java | 22 +- .../iotdb/db/query/executor/UDTFQueryExecutor.java | 84 +++ .../UpdateOperator.java => query/udf/api/UDF.java} | 70 +-- .../org/apache/iotdb/db/query/udf/api/UDTF.java | 113 ++++ .../apache/iotdb/db/query/udf/api/access/Row.java | 119 ++++ .../udf/api/access/RowIterator.java} | 90 +-- .../iotdb/db/query/udf/api/access/RowWindow.java | 61 ++ .../db/query/udf/api/collector/PointCollector.java | 137 ++++ .../api/customizer/config/UDFConfigurations.java} | 79 ++- .../api/customizer/config/UDTFConfigurations.java | 105 ++++ .../api/customizer/parameter/UDFParameters.java | 119 ++++ .../api/customizer/strategy/AccessStrategy.java | 64 ++ .../strategy/RowByRowAccessStrategy.java | 61 ++ .../strategy/SlidingSizeWindowAccessStrategy.java | 123 ++++ .../strategy/SlidingTimeWindowAccessStrategy.java | 249 ++++++++ .../iotdb/db/query/udf/core/access/RowImpl.java | 93 +++ .../db/query/udf/core/access/RowIteratorImpl.java | 58 ++ .../db/query/udf/core/access/RowWindowImpl.java | 75 +++ .../db/query/udf/core/context/UDFContext.java | 134 ++++ .../db/query/udf/core/executor/UDTFExecutor.java | 105 ++++ .../iotdb/db/query/udf/core/input/InputLayer.java | 563 +++++++++++++++++ .../iotdb/db/query/udf/core/input/SafetyLine.java | 60 ++ .../udf/core/reader/LayerPointReader.java} | 89 +-- .../udf/core/reader/LayerRowReader.java} | 79 ++- .../udf/core/reader/LayerRowWindowReader.java} | 77 ++- .../core/transformer/RawQueryPointTransformer.java | 72 +++ .../db/query/udf/core/transformer/Transformer.java | 93 +++ .../core/transformer/UDFQueryRowTransformer.java | 84 +-- .../transformer/UDFQueryRowWindowTransformer.java | 46 ++ .../udf/core/transformer/UDFQueryTransformer.java | 101 +++ .../query/udf/datastructure/SerializableList.java | 165 +++++ .../primitive/ElasticSerializableIntList.java | 109 ++++ .../udf/datastructure/primitive/IntList.java} | 74 +-- .../primitive/SerializableIntList.java | 131 ++++ .../datastructure/primitive/WrappedIntArray.java | 90 +-- .../row/ElasticSerializableRowRecordList.java | 234 +++++++ .../row/SerializableRowRecordList.java | 245 ++++++++ .../tv/ElasticSerializableBinaryTVList.java | 121 ++++ .../tv/ElasticSerializableTVList.java | 312 +++++++++ .../datastructure/tv/SerializableBinaryTVList.java | 79 +++ .../tv/SerializableBooleanTVList.java | 69 ++ .../datastructure/tv/SerializableDoubleTVList.java | 69 ++ .../datastructure/tv/SerializableFloatTVList.java | 69 ++ .../datastructure/tv/SerializableIntTVList.java | 69 ++ .../datastructure/tv/SerializableLongTVList.java | 69 ++ .../udf/datastructure/tv/SerializableTVList.java | 99 +++ .../udf/service/TemporaryQueryDataFileService.java | 136 ++++ .../iotdb/db/query/udf/service/UDFClassLoader.java | 83 +++ .../query/udf/service/UDFClassLoaderManager.java | 122 ++++ .../iotdb/db/query/udf/service/UDFLogWriter.java | 69 ++ .../udf/service/UDFRegistrationInformation.java | 57 ++ .../query/udf/service/UDFRegistrationService.java | 339 ++++++++++ .../java/org/apache/iotdb/db/service/IoTDB.java | 10 +- .../org/apache/iotdb/db/service/ServiceType.java | 9 +- .../org/apache/iotdb/db/service/TSServiceImpl.java | 113 ++-- .../db/sync/receiver/load/FileLoaderManager.java | 4 +- .../db/sync/receiver/transfer/SyncServiceImpl.java | 25 +- .../iotdb/db/sync/sender/transfer/SyncClient.java | 48 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 25 +- .../iotdb/db/writelog/recover/LogReplayer.java | 9 - .../apache/iotdb/db/auth/AuthorityCheckerTest.java | 3 - .../compaction/LevelCompactionMergeTest.java | 4 +- .../LevelCompactionTsFileManagementTest.java | 1 - .../NoCompactionTsFileManagementTest.java | 1 - .../iotdb/db/engine/storagegroup/TTLTest.java | 2 +- .../iotdb/db/integration/IoTDBDeletionIT.java | 35 ++ .../db/integration/IoTDBLevelCompactionIT.java | 3 + .../iotdb/db/integration/IoTDBRestartIT.java | 40 +- .../db/integration/IoTDBRpcCompressionIT.java | 1 - .../iotdb/db/integration/IoTDBSimpleQueryIT.java | 6 +- .../iotdb/db/integration/IoTDBUDFManagementIT.java | 308 +++++++++ .../db/integration/IoTDBUDFWindowQueryIT.java | 513 +++++++++++++++ .../integration/IoTDBUDTFAlignByTimeQueryIT.java | 694 +++++++++++++++++++++ .../db/integration/IoTDBUDTFHybridQueryIT.java | 166 +++++ .../db/integration/IoTDBUDTFNonAlignQueryIT.java | 337 ++++++++++ .../apache/iotdb/db/metadata/PartialPathTest.java | 34 +- .../iotdb/db/qp/plan/ConcatOptimizerTest.java | 7 +- .../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 213 ++++++- .../query/dataset/UDTFAlignByTimeDataSetTest.java | 578 +++++++++++++++++ .../ElasticSerializableRowRecordListTest.java | 224 +++++++ .../ElasticSerializableTVListTest.java | 251 ++++++++ .../SerializableBinaryTVListTest.java | 80 +++ .../SerializableBooleanTVListTest.java | 80 +++ .../SerializableDoubleTVListTest.java | 79 +++ .../datastructure/SerializableFloatTVListTest.java | 78 +++ .../datastructure/SerializableIntTVListTest.java | 78 +++ .../udf/datastructure/SerializableListTest.java} | 81 ++- .../datastructure/SerializableLongTVListTest.java | 78 +++ .../SerializableRowRecordListTest.java | 143 +++++ .../udf/datastructure/SerializableTVListTest.java} | 79 ++- .../iotdb/db/query/udf/example/Accumulator.java | 82 +++ .../apache/iotdb/db/query/udf/example/Adder.java | 78 +++ .../apache/iotdb/db/query/udf/example/Counter.java | 76 +++ .../org/apache/iotdb/db/query/udf/example/Max.java | 64 ++ .../iotdb/db/query/udf/example/Multiplier.java | 54 ++ .../SlidingSizeWindowConstructorTester0.java | 53 ++ .../SlidingSizeWindowConstructorTester1.java | 54 ++ .../SlidingTimeWindowConstructionTester.java | 61 ++ .../db/query/udf/example/TerminateTester.java | 64 ++ .../apache/iotdb/db/utils/EnvironmentUtils.java | 8 + .../apache/iotdb/db/writelog/PerformanceTest.java | 15 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 - .../java/org/apache/iotdb/session/SessionUT.java | 2 +- site/src/main/.vuepress/config.js | 26 +- spark-iotdb-connector/pom.xml | 2 +- .../tsfile/encoding/bitpacking/IntPacker.java | 4 +- .../tsfile/encoding/bitpacking/LongPacker.java | 4 +- .../org/apache/iotdb/tsfile/read/common/Field.java | 4 - .../apache/iotdb/tsfile/read/common/RowRecord.java | 2 +- .../query/dataset/DataSetWithTimeGenerator.java | 4 +- .../query/dataset/DataSetWithoutTimeGenerator.java | 4 +- .../tsfile/read/query/dataset/EmptyDataSet.java | 4 +- .../tsfile/read/query/dataset/QueryDataSet.java | 4 +- .../org/apache/iotdb/tsfile/utils/BytesUtils.java | 4 +- .../iotdb/tsfile/utils/ReadWriteIOUtils.java | 11 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 2 +- 224 files changed, 14868 insertions(+), 2296 deletions(-) diff --cc cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java index bf6b504,149d1aa..f616728 --- a/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java @@@ -76,35 -64,21 +76,36 @@@ public class ClusterMain public static void main(String[] args) { if (args.length < 1) { - logger.error("Usage: <-s|-a|-r> [-internal_meta_port <internal meta port>] " - + "[-internal_data_port <internal data port>] " - + "[-cluster_rpc_port <cluster rpc port>] " - + "[-seed_nodes <node1:meta_port:data_port:cluster_rpc_port," - + "node2:meta_port:data_port:cluster_rpc_port," - + "...,noden:meta_port:data_port:cluster_rpc_port>] " - + "[-sc] " - + "[-rpc_port <rpc port>]"); + logger.error("Usage: <-s|-a|-r> " + + "[-{} <internal meta port>] " + + "[-{} <internal data port>] " + + "[-{} <cluster rpc port>] " + + "[-{} <cluster RPC address>]\n" + + "[-{} <node1:meta_port:data_port:cluster_rpc_port," + + "node2:meta_port:data_port:cluster_rpc_port," + + "...,noden:meta_port:data_port:cluster_rpc_port,>] " + + "[-{} <debug rpc port>]" + + "-s: start the node as a seed\n" + + "-a: start the node as a new node\n" + + "-r: remove the node out of the cluster\n" + + "or: set CLUSTER_CONF and IOTDB_CONF environment variable", + OPTION_INTERVAL_META_PORT, + OPTION_INTERVAL_DATA_PORT, + OPTION_CLUSTER_RPC_PORT, + OPTION_CLUSTER_RPC_IP, + OPTION_SEED_NODES, + //debug rpc is for starting another rpc port for the singleton IoTDB server. + OPTION_DEBUG_RPC_PORT + ); ++ return; } - String mode = args[0]; - if (args.length > 1) { - String[] params = Arrays.copyOfRange(args, 1, args.length); - replaceDefaultPrams(params); - } + + // init server's configuration first, because the cluster configuration may read settings from + // the server's configuration. + IoTDBDescriptor.getInstance().getConfig().setEnableRPCService(false); + IoTDBDescriptor.getInstance().getConfig().setSyncEnable(false); + IoTDBDescriptor.getInstance().getConfig().setAutoCreateSchemaEnabled(false); // params check if (!checkConfig()) { diff --cc server/src/main/java/org/apache/iotdb/db/service/IoTDB.java index c07180e,8b03141..8c925e4 --- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java +++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java @@@ -101,14 -107,15 +104,19 @@@ public class IoTDB implements IoTDBMBea registerManager.register(Measurement.INSTANCE); registerManager.register(TVListAllocator.getInstance()); registerManager.register(CacheHitRatioMonitor.getInstance()); + registerManager.register(MergeManager.getINSTANCE()); + registerManager.register(CompactionMergeTaskPoolManager.getInstance()); JMXService.registerMBean(getInstance(), mbeanName); registerManager.register(StorageEngine.getInstance()); + registerManager.register(TemporaryQueryDataFileService.getInstance()); + registerManager.register(UDFClassLoaderManager.getInstance()); + registerManager.register(UDFRegistrationService.getInstance()); - registerManager.register(RPCService.getInstance()); + //in cluster mode, RPC service is not enabled. + if (IoTDBDescriptor.getInstance().getConfig().isEnableRPCService()) { + registerManager.register(RPCService.getInstance()); + } + if (IoTDBDescriptor.getInstance().getConfig().isEnableMetricService()) { registerManager.register(MetricsService.getInstance()); }
