This is an automated email from the ASF dual-hosted git repository. xiangweiwei pushed a commit to branch addAggregationIT in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3e8b7b6f3e1ae7ecc08112a7362ea7efc3f27fc3 Merge: 989551f800 1fb7229c2d Author: Alima777 <[email protected]> AuthorDate: Mon Jun 20 11:14:40 2022 +0800 Merge branch 'master' into addAggregationIT .github/workflows/cluster-it.yml | 17 +- .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 2 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 8 + client-py/README.md | 95 +++++ client-py/iotdb/dbapi/Cursor.py | 10 +- client-py/iotdb/sqlalchemy/IoTDBDialect.py | 136 +++++++ .../sqlalchemy/IoTDBIdentifierPreparer.py} | 14 +- client-py/iotdb/sqlalchemy/IoTDBSQLCompiler.py | 243 +++++++++++++ .../sqlalchemy/IoTDBTypeCompiler.py} | 32 +- .../sqlalchemy/__init__.py} | 6 - .../sqlalchemy/tests/__init__.py} | 6 - client-py/iotdb/sqlalchemy/tests/test_dialect.py | 92 +++++ client-py/requirements.txt | 5 +- client-py/setup.py | 7 + .../src/assembly/resources/conf/confignode-env.bat | 8 +- .../resources/conf/iotdb-confignode.properties | 2 +- confignode/src/assembly/resources/conf/logback.xml | 13 +- .../client/ConfigNodeClientPoolFactory.java | 4 +- .../{ConfigNodeConf.java => ConfigNodeConfig.java} | 14 +- .../confignode/conf/ConfigNodeDescriptor.java | 8 +- .../confignode/conf/ConfigNodeStartupCheck.java | 2 +- .../consensus/request/ConfigRequest.java | 4 + .../consensus/request/ConfigRequestType.java | 3 +- .../request/read/GetRegionLocationsReq.java | 62 ++++ .../consensus/response/RegionLocationsResp.java | 32 +- .../iotdb/confignode/manager/ConfigManager.java | 18 +- .../iotdb/confignode/manager/ConsensusManager.java | 4 +- .../apache/iotdb/confignode/manager/Manager.java | 4 + .../iotdb/confignode/manager/PartitionManager.java | 9 +- .../iotdb/confignode/manager/ProcedureManager.java | 11 +- .../iotdb/confignode/persistence/UDFInfo.java | 4 +- .../executor/ConfigRequestExecutor.java | 3 + .../persistence/partition/PartitionInfo.java | 25 ++ .../partition/StorageGroupPartitionTable.java | 39 ++ .../iotdb/confignode/service/ConfigNode.java | 10 +- .../service/thrift/ConfigNodeRPCService.java | 4 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 15 + .../consensus/request/ConfigRequestSerDeTest.java | 11 + .../confignode/persistence/PartitionInfoTest.java | 68 +++- .../thrift/ConfigNodeRPCServiceProcessorTest.java | 10 +- .../iotdb/consensus/ratis/RatisConsensus.java | 23 +- .../consensus/standalone/StandAloneServerImpl.java | 2 + .../iotdb/consensus/ratis/RatisConsensusTest.java | 120 +++---- docs/UserGuide/API/Programming-MQTT.md | 7 +- .../UserGuide/API/Programming-Python-Native-API.md | 100 +++++- .../Maintenance-Tools/Maintenance-Command.md | 67 +++- docs/zh/UserGuide/API/Programming-MQTT.md | 7 +- .../UserGuide/API/Programming-Python-Native-API.md | 97 ++++- .../Maintenance-Tools/Maintenance-Command.md | 63 ++++ .../java/org/apache/iotdb/mqtt/MQTTClient.java | 7 + .../java/org/apache/iotdb/udf/UDTFExample.java | 22 +- .../iotdb/db/query/udf/example/Accumulator.java | 28 +- .../apache/iotdb/db/query/udf/example/Adder.java | 26 +- .../apache/iotdb/db/query/udf/example/Counter.java | 22 +- .../org/apache/iotdb/db/query/udf/example/Max.java | 22 +- .../iotdb/db/query/udf/example/Multiplier.java | 22 +- .../SlidingSizeWindowConstructorTester0.java | 16 +- .../SlidingSizeWindowConstructorTester1.java | 22 +- .../SlidingTimeWindowConstructionTester.java | 22 +- .../db/query/udf/example/TerminateTester.java | 18 +- .../iotdb/db/query/udf/example/ValidateTester.java | 20 +- .../iotdb/db/query/udf/example/WindowStartEnd.java | 18 +- .../java/org/apache/iotdb/it/env/Cluster1Env.java | 4 +- .../org/apache/iotdb/it/env/ClusterEnvBase.java | 165 ++++++--- .../org/apache/iotdb/it/env/ClusterNodeBase.java | 188 ++++++---- .../java/org/apache/iotdb/it/env/ConfigNode.java | 149 ++++---- .../java/org/apache/iotdb/it/env/DataNode.java | 128 +++---- .../iotdb/itbase/runtime/ClusterTestStatement.java | 42 ++- .../itbase/runtime/ParallelRequestDelegate.java | 2 +- ...gregationWithoutValueFilterWithDeletion2IT.java | 3 + ...ggregationWithoutValueFilterWithDeletionIT.java | 3 + ...upByQueryWithoutValueFilterWithDeletion2IT.java | 3 + ...oupByQueryWithoutValueFilterWithDeletionIT.java | 3 + .../db/it/aligned/IoTDBInsertAlignedValuesIT.java | 5 +- .../db/it/udf/BuiltinAggregationFunctionEnum.java | 58 +++ .../BuiltinTimeSeriesGeneratingFunctionEnum.java | 87 +++++ .../iotdb/db/it/udf/IoTDBUDFManagementIT.java | 339 ++++++++++++++++++ .../it/{ => udf}/IoTDBUDTFBuiltinFunctionIT.java | 20 +- integration/checkstyle.xml | 2 +- .../iotdb/db/query/udf/example/Accumulator.java | 28 +- .../apache/iotdb/db/query/udf/example/Adder.java | 26 +- .../apache/iotdb/db/query/udf/example/Counter.java | 22 +- .../org/apache/iotdb/db/query/udf/example/Max.java | 22 +- .../iotdb/db/query/udf/example/Multiplier.java | 22 +- .../SlidingSizeWindowConstructorTester0.java | 16 +- .../SlidingSizeWindowConstructorTester1.java | 22 +- .../SlidingTimeWindowConstructionTester.java | 22 +- .../db/query/udf/example/TerminateTester.java | 18 +- .../iotdb/db/query/udf/example/ValidateTester.java | 20 +- .../iotdb/db/query/udf/example/WindowStartEnd.java | 18 +- .../db/integration/IoTDBLoadExternalTsfileIT.java | 8 +- .../IoTDBLoadExternalTsfileWithVirtualSGIT.java | 138 +++++++ .../IoTDBSyntaxConventionIdentifierIT.java | 5 + .../iotdb/db/integration/IoTDBUDFManagementIT.java | 398 --------------------- .../org/apache/iotdb/library/anomaly/UDTFIQR.java | 23 +- .../apache/iotdb/library/anomaly/UDTFKSigma.java | 21 +- .../org/apache/iotdb/library/anomaly/UDTFLOF.java | 23 +- .../iotdb/library/anomaly/UDTFMissDetect.java | 21 +- .../apache/iotdb/library/anomaly/UDTFRange.java | 21 +- .../iotdb/library/anomaly/UDTFTwoSidedFilter.java | 21 +- .../iotdb/library/anomaly/util/MissDetector.java | 4 +- .../iotdb/library/anomaly/util/WindowDetect.java | 4 +- .../org/apache/iotdb/library/dmatch/UDAFCov.java | 26 +- .../org/apache/iotdb/library/dmatch/UDAFDtw.java | 26 +- .../apache/iotdb/library/dmatch/UDAFPearson.java | 26 +- .../apache/iotdb/library/dmatch/UDTFPtnSym.java | 23 +- .../org/apache/iotdb/library/dmatch/UDTFXCorr.java | 24 +- .../iotdb/library/dprofile/UDAFIntegral.java | 20 +- .../iotdb/library/dprofile/UDAFIntegralAvg.java | 23 +- .../org/apache/iotdb/library/dprofile/UDAFMad.java | 28 +- .../apache/iotdb/library/dprofile/UDAFMedian.java | 28 +- .../apache/iotdb/library/dprofile/UDAFMode.java | 17 +- .../iotdb/library/dprofile/UDAFPercentile.java | 25 +- .../apache/iotdb/library/dprofile/UDAFPeriod.java | 25 +- .../apache/iotdb/library/dprofile/UDAFSkew.java | 23 +- .../apache/iotdb/library/dprofile/UDAFSpread.java | 25 +- .../apache/iotdb/library/dprofile/UDAFStddev.java | 23 +- .../org/apache/iotdb/library/dprofile/UDTFACF.java | 23 +- .../iotdb/library/dprofile/UDTFDistinct.java | 26 +- .../iotdb/library/dprofile/UDTFHistogram.java | 23 +- .../apache/iotdb/library/dprofile/UDTFMinMax.java | 23 +- .../apache/iotdb/library/dprofile/UDTFMvAvg.java | 25 +- .../apache/iotdb/library/dprofile/UDTFPACF.java | 23 +- .../org/apache/iotdb/library/dprofile/UDTFQLB.java | 23 +- .../iotdb/library/dprofile/UDTFResample.java | 23 +- .../apache/iotdb/library/dprofile/UDTFSample.java | 23 +- .../apache/iotdb/library/dprofile/UDTFSegment.java | 23 +- .../apache/iotdb/library/dprofile/UDTFSpline.java | 23 +- .../apache/iotdb/library/dprofile/UDTFZScore.java | 23 +- .../dprofile/util/ExactOrderStatistics.java | 41 ++- .../iotdb/library/dquality/UDTFCompleteness.java | 18 +- .../iotdb/library/dquality/UDTFConsistency.java | 18 +- .../iotdb/library/dquality/UDTFTimeliness.java | 18 +- .../iotdb/library/dquality/UDTFValidity.java | 18 +- .../library/dquality/util/TimeSeriesQuality.java | 4 +- .../iotdb/library/drepair/UDTFTimestampRepair.java | 19 +- .../iotdb/library/drepair/UDTFValueFill.java | 19 +- .../iotdb/library/drepair/UDTFValueRepair.java | 19 +- .../apache/iotdb/library/drepair/util/ARFill.java | 4 +- .../iotdb/library/drepair/util/LikelihoodFill.java | 2 +- .../iotdb/library/drepair/util/LinearFill.java | 2 +- .../iotdb/library/drepair/util/LsGreedy.java | 2 +- .../apache/iotdb/library/drepair/util/MAFill.java | 2 +- .../iotdb/library/drepair/util/MeanFill.java | 2 +- .../iotdb/library/drepair/util/PreviousFill.java | 2 +- .../apache/iotdb/library/drepair/util/Screen.java | 2 +- .../iotdb/library/drepair/util/ScreenFill.java | 4 +- .../library/drepair/util/TimestampRepair.java | 4 +- .../iotdb/library/drepair/util/ValueFill.java | 6 +- .../iotdb/library/drepair/util/ValueRepair.java | 4 +- .../apache/iotdb/library/frequency/UDTFConv.java | 26 +- .../apache/iotdb/library/frequency/UDTFDWT.java | 23 +- .../apache/iotdb/library/frequency/UDTFDeconv.java | 26 +- .../apache/iotdb/library/frequency/UDTFFFT.java | 23 +- .../iotdb/library/frequency/UDTFHighPass.java | 23 +- .../apache/iotdb/library/frequency/UDTFIDWT.java | 23 +- .../apache/iotdb/library/frequency/UDTFIFFT.java | 23 +- .../iotdb/library/frequency/UDTFLowPass.java | 23 +- .../iotdb/library/frequency/util/FFTUtil.java | 2 +- .../library/series/UDTFConsecutiveSequences.java | 20 +- .../library/series/UDTFConsecutiveWindows.java | 20 +- .../iotdb/library/series/util/ConsecutiveUtil.java | 4 +- .../iotdb/library/string/UDTFRegexMatch.java | 22 +- .../iotdb/library/string/UDTFRegexReplace.java | 22 +- .../iotdb/library/string/UDTFRegexSplit.java | 22 +- .../iotdb/library/string/UDTFStrReplace.java | 22 +- .../iotdb/library/util/LinearRegression.java | 2 +- .../java/org/apache/iotdb/library/util/Util.java | 4 +- node-commons/pom.xml | 5 + .../RegionStatus.java} | 17 +- .../commons/udf/builtin/String/UDTFConcat.java | 22 +- .../commons/udf/builtin/String/UDTFEndsWith.java | 22 +- .../commons/udf/builtin/String/UDTFLower.java | 22 +- .../commons/udf/builtin/String/UDTFStartsWith.java | 22 +- .../commons/udf/builtin/String/UDTFStrCompare.java | 24 +- .../commons/udf/builtin/String/UDTFStrLength.java | 22 +- .../commons/udf/builtin/String/UDTFStrLocate.java | 22 +- .../commons/udf/builtin/String/UDTFSubstr.java | 22 +- .../iotdb/commons/udf/builtin/String/UDTFTrim.java | 22 +- .../commons/udf/builtin/String/UDTFUpper.java | 22 +- .../apache/iotdb/commons/udf/builtin/UDTFAbs.java | 28 +- .../iotdb/commons/udf/builtin/UDTFBottomK.java | 17 +- .../apache/iotdb/commons/udf/builtin/UDTFCast.java | 28 +- .../commons/udf/builtin/UDTFCommonDerivative.java | 16 +- .../udf/builtin/UDTFCommonValueDifference.java | 16 +- .../iotdb/commons/udf/builtin/UDTFConst.java | 24 +- .../iotdb/commons/udf/builtin/UDTFConstE.java | 18 +- .../iotdb/commons/udf/builtin/UDTFConstPi.java | 18 +- .../iotdb/commons/udf/builtin/UDTFContains.java | 24 +- .../udf/builtin/UDTFContinuouslySatisfy.java | 47 +-- .../iotdb/commons/udf/builtin/UDTFDerivative.java | 21 +- .../udf/builtin/UDTFEqualSizeBucketAggSample.java | 29 +- .../udf/builtin/UDTFEqualSizeBucketM4Sample.java | 28 +- .../builtin/UDTFEqualSizeBucketOutlierSample.java | 32 +- .../builtin/UDTFEqualSizeBucketRandomSample.java | 26 +- .../udf/builtin/UDTFEqualSizeBucketSample.java | 14 +- .../iotdb/commons/udf/builtin/UDTFInRange.java | 36 +- .../apache/iotdb/commons/udf/builtin/UDTFJexl.java | 64 ++-- .../iotdb/commons/udf/builtin/UDTFMatches.java | 24 +- .../apache/iotdb/commons/udf/builtin/UDTFMath.java | 36 +- .../udf/builtin/UDTFNonNegativeDerivative.java | 16 +- .../builtin/UDTFNonNegativeValueDifference.java | 16 +- .../iotdb/commons/udf/builtin/UDTFOnOff.java | 35 +- .../iotdb/commons/udf/builtin/UDTFSelectK.java | 58 ++- .../commons/udf/builtin/UDTFTimeDifference.java | 18 +- .../apache/iotdb/commons/udf/builtin/UDTFTopK.java | 17 +- .../commons/udf/builtin/UDTFValueDifference.java | 19 +- .../iotdb/commons/udf/builtin/UDTFValueTrend.java | 24 +- .../udf/service/UDFRegistrationInformation.java | 2 +- .../udf/service/UDFRegistrationService.java | 4 +- .../UDFBinaryTransformer.java} | 28 +- .../commons/udf/utils/UDFDataTypeTransformer.java | 77 ++++ pom.xml | 1 + .../resources/conf/iotdb-engine.properties | 2 +- server/src/assembly/resources/conf/iotdb-env.bat | 8 +- server/src/assembly/resources/conf/logback.xml | 13 +- .../apache/iotdb/db/client/ConfigNodeClient.java | 28 ++ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 + .../org/apache/iotdb/db/engine/StorageEngine.java | 75 +++- .../db/engine/compaction/CompactionScheduler.java | 6 +- .../DefaultCompactionTaskComparatorImpl.java | 17 +- .../compaction/cross/CrossSpaceCompactionTask.java | 6 +- .../compaction/inner/InnerSpaceCompactionTask.java | 6 +- .../compaction/task/AbstractCompactionTask.java | 9 +- .../iotdb/db/engine/snapshot/SnapshotTaker.java | 56 +-- .../iotdb/db/engine/storagegroup/DataRegion.java | 13 +- .../db/engine/storagegroup/TsFileManager.java | 6 + .../dataregion/StorageGroupManager.java | 19 +- .../db/metadata/schemaregion/SchemaEngine.java | 4 + .../iotdb/db/mpp/common/header/HeaderConstant.java | 24 ++ .../execution/datatransfer/DataBlockManager.java | 14 +- .../execution/datatransfer/LocalSinkHandle.java | 72 ++-- .../execution/datatransfer/LocalSourceHandle.java | 43 ++- .../execution/datatransfer/SharedTsBlockQueue.java | 23 +- .../execution/fragment/FragmentInstanceState.java | 4 +- .../org/apache/iotdb/db/mpp/plan/Coordinator.java | 3 + .../db/mpp/plan/analyze/ClusterSchemaFetcher.java | 55 +-- .../mpp/plan/analyze/FakePartitionFetcherImpl.java | 53 ++- .../plan/execution/config/ConfigTaskVisitor.java | 6 + .../mpp/plan/execution/config/ShowRegionTask.java | 86 +++++ .../config/executor/ClusterConfigTaskExecutor.java | 35 +- .../config/executor/IConfigTaskExecutor.java | 3 + .../executor/StandaloneConfigTaskExecutor.java | 14 +- .../plan/expression/multi/FunctionExpression.java | 5 +- .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 18 + .../iotdb/db/mpp/plan/planner/LogicalPlanner.java | 72 +++- .../plan/planner/distribution/SourceRewriter.java | 7 +- .../scheduler/AbstractFragInsStateTracker.java | 1 - .../db/mpp/plan/scheduler/ClusterScheduler.java | 20 +- .../scheduler/FixedRateFragInsStateTracker.java | 5 +- .../mpp/plan/scheduler/SimpleQueryTerminator.java | 1 - .../db/mpp/plan/statement/StatementVisitor.java | 5 + .../statement/metadata/ShowRegionStatement.java | 54 +++ .../db/mpp/transformation/api/LayerRowReader.java | 2 +- .../transformation/api/LayerRowWindowReader.java | 2 +- ...ializableRowRecordListBackedMultiColumnRow.java | 16 +- ...izableRowRecordListBackedMultiColumnWindow.java | 12 +- ...wRecordListBackedMultiColumnWindowIterator.java | 4 +- ...ticSerializableTVListBackedSingleColumnRow.java | 14 +- ...SerializableTVListBackedSingleColumnWindow.java | 13 +- ...ableTVListBackedSingleColumnWindowIterator.java | 4 +- .../LayerPointReaderBackedSingleColumnRow.java | 14 +- .../intermediate/ConstantIntermediateLayer.java | 4 +- .../dag/intermediate/IntermediateLayer.java | 6 +- .../MultiInputColumnIntermediateLayer.java | 8 +- ...InputColumnMultiReferenceIntermediateLayer.java | 8 +- ...nputColumnSingleReferenceIntermediateLayer.java | 8 +- .../dag/transformer/multi/UDFQueryTransformer.java | 5 +- .../mpp/transformation/dag/udf/UDTFExecutor.java | 24 +- .../transformation/dag/udf/UDTFTypeInferrer.java | 17 +- .../transformation/dag/util/LayerCacheUtils.java | 5 +- .../tv/ElasticSerializableBinaryTVList.java | 15 +- .../tv/ElasticSerializableTVList.java | 27 +- .../db/protocol/mqtt/JSONPayloadFormatter.java | 15 +- .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 163 +++++++++ .../apache/iotdb/db/qp/executor/PlanExecutor.java | 2 +- .../java/org/apache/iotdb/db/service/DataNode.java | 16 +- .../org/apache/iotdb/db/service/MQTTService.java | 12 +- .../service/thrift/impl/InternalServiceImpl.java | 57 +-- .../iotdb/db/wal/buffer/AbstractWALBuffer.java | 8 +- .../org/apache/iotdb/db/wal/buffer/IWALBuffer.java | 3 + .../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 30 +- .../org/apache/iotdb/db/wal/io/ILogWriter.java | 3 +- .../java/org/apache/iotdb/db/wal/io/LogWriter.java | 3 +- .../java/org/apache/iotdb/db/wal/node/WALNode.java | 19 +- .../compaction/CompactionTaskComparatorTest.java | 74 +++- .../compaction/CompactionTaskManagerTest.java | 27 +- .../compaction/cross/CrossSpaceCompactionTest.java | 9 +- .../cross/RewriteCrossSpaceCompactionTest.java | 12 +- .../inner/InnerCompactionEmptyTsFileTest.java | 3 +- .../db/engine/storagegroup/DataRegionTest.java | 3 +- .../storagegroup/StorageGroupProcessorTest.java | 3 +- .../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 16 +- .../db/protocol/mqtt/JSONPayloadFormatterTest.java | 58 +++ .../iotdb/db/protocol/mqtt/PublishHandlerTest.java | 7 +- .../ElasticSerializableTVListTest.java | 6 +- .../apache/iotdb/db/query/udf/example/Adder.java | 26 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 2 +- thrift-commons/src/main/thrift/common.thrift | 10 + .../src/main/thrift/confignode.thrift | 14 + .../tsfile/file/metadata/enums/TSDataType.java | 6 +- udf-api/pom.xml | 31 ++ .../main/java/org/apache/iotdb}/udf/api/UDF.java | 8 +- .../main/java/org/apache/iotdb}/udf/api/UDTF.java | 20 +- .../java/org/apache/iotdb}/udf/api/access/Row.java | 8 +- .../apache/iotdb}/udf/api/access/RowIterator.java | 2 +- .../apache/iotdb}/udf/api/access/RowWindow.java | 10 +- .../iotdb}/udf/api/collector/PointCollector.java | 56 +-- .../api/customizer/config/UDFConfigurations.java | 8 +- .../api/customizer/config/UDTFConfigurations.java | 24 +- .../parameter/UDFParameterValidator.java | 47 +-- .../api/customizer/parameter/UDFParameters.java | 29 +- .../api/customizer/strategy/AccessStrategy.java | 8 +- .../strategy/RowByRowAccessStrategy.java | 12 +- .../strategy/SlidingSizeWindowAccessStrategy.java | 12 +- .../strategy/SlidingTimeWindowAccessStrategy.java | 12 +- .../UDFAttributeNotProvidedException.java | 2 +- .../iotdb}/udf/api/exception/UDFException.java | 2 +- .../UDFInputSeriesDataTypeNotValidException.java | 10 +- .../UDFInputSeriesIndexNotValidException.java | 2 +- .../UDFInputSeriesNumberNotValidException.java | 2 +- .../UDFOutputSeriesDataTypeNotValidException.java | 2 +- .../exception/UDFParameterNotValidException.java | 2 +- .../api/exception/UDFRegistrationException.java | 2 +- .../java/org/apache/iotdb/udf/api/type/Binary.java | 150 ++++++++ .../java/org/apache/iotdb/udf/api/type/Type.java | 35 +- 327 files changed, 5566 insertions(+), 2870 deletions(-)
