This is an automated email from the ASF dual-hosted git repository. yongzao pushed a commit to branch optimize-getOrCreatePartition in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3ce69c53089f408adb886dcde226a9a0d9bc1eac Merge: 4eea85a2173 b833e3b66d1 Author: YongzaoDan <[email protected]> AuthorDate: Mon May 22 10:39:49 2023 +0800 Merge branch 'master' into optimize-getOrCreatePartition .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 65 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 + client-py/tests/test_dataframe.py | 2 + .../confignode/client/DataNodeRequestType.java | 7 +- .../client/async/AsyncDataNodeClientPool.java | 15 +- .../client/async/handlers/AsyncClientHandler.java | 1 + .../heartbeat/DataNodeHeartbeatHandler.java | 31 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 49 ++ .../confignode/conf/ConfigNodeDescriptor.java | 16 + .../consensus/request/ConfigPhysicalPlan.java | 8 + .../consensus/request/ConfigPhysicalPlanType.java | 4 + .../request/read/pipe/task/ShowPipePlanV2.java | 23 +- .../coordinator/PipeHandleLeaderChangePlan.java | 74 +++ .../response/pipe/task/PipeTableResp.java | 123 ++++ .../statemachine/ConfigRegionStateMachine.java | 5 +- .../iotdb/confignode/manager/ConfigManager.java | 36 +- .../apache/iotdb/confignode/manager/IManager.java | 9 +- .../iotdb/confignode/manager/ProcedureManager.java | 40 ++ .../iotdb/confignode/manager/load/LoadManager.java | 118 ++-- .../manager/load/balancer/RegionBalancer.java | 2 +- .../manager/load/balancer/RouteBalancer.java | 268 ++------- .../load/balancer/router/RegionRouteMap.java | 176 ------ .../confignode/manager/load/cache/LoadCache.java | 157 ++++- .../manager/load/cache/route/RegionRouteCache.java | 142 +++++ .../manager/load/service/HeartbeatService.java | 5 +- .../manager/load/service/StatisticsService.java | 73 ++- .../manager/load/subscriber/RouteChangeEvent.java | 35 +- .../iotdb/confignode/manager/node/NodeManager.java | 5 +- .../manager/partition/PartitionManager.java | 23 +- .../manager/partition/PartitionMetrics.java | 2 +- .../iotdb/confignode/manager/pipe/PipeManager.java | 10 + .../pipe/{ => plugin}/PipePluginCoordinator.java | 2 +- .../manager/pipe/runtime/PipeMetaSyncer.java | 80 +++ .../pipe/runtime/PipeRuntimeCoordinator.java | 98 +++ .../pipe/{ => task}/PipeTaskCoordinator.java | 37 +- .../manager/{ => schema}/ClusterSchemaManager.java | 74 ++- .../schema/ClusterSchemaQuotaStatistics.java | 28 +- .../persistence/executor/ConfigPlanExecutor.java | 14 +- .../partition/DatabasePartitionTable.java | 6 +- .../persistence/partition/PartitionInfo.java | 18 + .../persistence/pipe/PipePluginInfo.java | 57 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 145 ++++- .../persistence/pipe/PipeTaskOperation.java | 2 + .../procedure/env/ConfigNodeProcedureEnv.java | 66 +-- .../procedure/env/DataNodeRemoveHandler.java | 4 +- .../pipe/plugin/CreatePipePluginProcedure.java | 2 +- .../impl/pipe/plugin/DropPipePluginProcedure.java | 2 +- .../runtime/PipeHandleLeaderChangeProcedure.java | 195 ++++++ .../impl/pipe/runtime/PipeMetaSyncProcedure.java | 126 ++++ .../pipe/task/AbstractOperatePipeProcedureV2.java | 87 ++- .../impl/pipe/task/CreatePipeProcedureV2.java | 77 +-- .../impl/pipe/task/DropPipeProcedureV2.java | 28 +- .../impl/pipe/task/StartPipeProcedureV2.java | 40 +- .../impl/pipe/task/StopPipeProcedureV2.java | 40 +- .../impl/schema/DataNodeRegionTaskExecutor.java | 4 +- .../impl/schema/DeleteDatabaseProcedure.java | 10 +- .../statemachine/CreateRegionGroupsProcedure.java | 1 - .../procedure/store/ProcedureFactory.java | 20 + .../confignode/procedure/store/ProcedureType.java | 12 +- .../iotdb/confignode/service/ConfigNode.java | 6 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 7 - .../request/ConfigPhysicalPlanSerDeTest.java | 18 + .../manager/ClusterSchemaManagerTest.java | 2 + .../load/balancer/router/RegionRouteMapTest.java | 82 --- .../iotdb/confignode/persistence/PipeInfoTest.java | 1 + .../PipeHandleLeaderChangeProcedureTest.java | 63 ++ .../pipe/runtime/PipeMetaSyncProcedureTest.java | 55 ++ consensus/pom.xml | 2 +- .../iotdb/consensus/config/IoTConsensusConfig.java | 36 +- .../consensus/iot/IoTConsensusServerImpl.java | 2 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 10 +- .../logdispatcher/LogDispatcherThreadMetrics.java | 22 + .../consensus/iot/logdispatcher/SyncStatus.java | 10 +- .../consensus/iot/wal/ConsensusReqReader.java | 3 + .../ratis/ApplicationStateMachineProxy.java | 10 +- .../apache/iotdb/consensus/ratis/RatisClient.java | 18 +- .../iotdb/consensus/ratis/RatisConsensus.java | 3 +- .../apache/iotdb/consensus/ratis/utils/Utils.java | 17 + .../iot/logdispatcher/SyncStatusTest.java | 10 +- .../consensus/iot/util/FakeConsensusReqReader.java | 5 + distribution/pom.xml | 14 + distribution/src/assembly/spark-connector.xml | 48 ++ docs/Community/Materials.md | 103 ++-- docs/UserGuide/API/Programming-Java-Native-API.md | 98 --- .../UserGuide/Ecosystem-Integration/Spark-IoTDB.md | 273 +++++---- docs/UserGuide/Monitor-Alert/Metric-Tool.md | 8 +- docs/UserGuide/Operate-Metadata/Node.md | 4 +- docs/UserGuide/Operate-Metadata/Timeseries.md | 89 ++- docs/UserGuide/QuickStart/Files.md | 2 +- docs/UserGuide/QuickStart/QuickStart.md | 2 +- docs/UserGuide/QuickStart/WayToGetIoTDB.md | 2 +- docs/UserGuide/Reference/Common-Config-Manual.md | 97 ++- docs/UserGuide/Reference/DataNode-Config-Manual.md | 18 +- docs/UserGuide/Reference/Keywords.md | 1 + docs/UserGuide/Reference/Status-Codes.md | 2 + .../Syntax-Conventions/Session-And-TsFile-API.md | 4 +- .../UserGuide/API/Programming-Java-Native-API.md | 93 --- .../UserGuide/Ecosystem-Integration/Spark-IoTDB.md | 261 ++++---- .../UserGuide/Ecosystem-Integration/Workbench.md | 79 +-- docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 24 +- docs/zh/UserGuide/Operate-Metadata/Node.md | 4 +- docs/zh/UserGuide/Operate-Metadata/Timeseries.md | 84 ++- docs/zh/UserGuide/QuickStart/Files.md | 2 +- docs/zh/UserGuide/QuickStart/QuickStart.md | 3 +- .../zh/UserGuide/Reference/Common-Config-Manual.md | 95 ++- .../UserGuide/Reference/DataNode-Config-Manual.md | 27 +- docs/zh/UserGuide/Reference/Keywords.md | 1 + docs/zh/UserGuide/Reference/Status-Codes.md | 258 ++++---- .../Syntax-Conventions/Session-And-TsFile-API.md | 4 +- integration-test/pom.xml | 18 + .../iotdb/it/env/cluster/MppCommonConfig.java | 24 + .../it/env/cluster/MppSharedCommonConfig.java | 28 + .../iotdb/it/env/remote/RemoteCommonConfig.java | 20 + .../org/apache/iotdb/it/utils/TsFileGenerator.java | 42 ++ .../org/apache/iotdb/itbase/env/CommonConfig.java | 8 + .../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 18 + .../db/it/aligned/IoTDBAlignedDataDeletionIT.java | 17 + .../db/it/last/IoTDBLastQueryLastCacheIT.java | 17 +- .../iotdb/db/it/orderBy/IoTDBOrderBy2IT.java | 48 ++ .../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java | 32 +- .../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 2 + .../db/it/schema/IoTDBAutoCreateSchemaIT.java | 17 +- .../db/it/schema/IoTDBClusterDeviceQuotaIT.java | 44 ++ .../it/schema/IoTDBClusterMeasurementQuotaIT.java | 126 ++++ .../it/schema/IoTDBCreateAlignedTimeseriesIT.java | 17 +- .../db/it/schema/IoTDBCreateStorageGroupIT.java | 18 +- .../db/it/schema/IoTDBCreateTimeseriesIT.java | 17 +- .../db/it/schema/IoTDBDeactivateTemplateIT.java | 48 +- .../it/schema/IoTDBDeleteAlignedTimeseriesIT.java | 18 +- .../db/it/schema/IoTDBDeleteStorageGroupIT.java | 17 +- .../db/it/schema/IoTDBDeleteTimeseriesIT.java | 18 +- .../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 58 +- .../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 166 ++++-- .../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 94 ++- .../db/it/schema/IoTDBSortedShowTimeseriesIT.java | 23 +- .../apache/iotdb/db/it/schema/IoTDBTagAlterIT.java | 27 +- .../org/apache/iotdb/db/it/schema/IoTDBTagIT.java | 37 +- .../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 30 + .../session/it/IoTDBSessionSchemaTemplateIT.java | 107 ++++ .../org/apache/iotdb/spark/it/AbstractTest.java | 55 ++ .../java/org/apache/iotdb/spark/it/ReadTest.java | 145 +++++ .../test/java/org/apache/iotdb/spark/it/Utils.java | 96 +++ .../java/org/apache/iotdb/spark/it/WriteTest.java | 116 ++++ .../org/apache/iotdb/util/AbstractSchemaIT.java | 48 +- .../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 14 +- .../metrics/metricsets/jvm/JvmMemoryMetrics.java | 6 +- .../metrics/metricsets/jvm/JvmThreadMetrics.java | 31 +- .../metricsets/net/LinuxNetMetricManager.java | 27 +- .../iotdb/metrics/metricsets/net/NetMetrics.java | 2 +- .../resources/conf/iotdb-common.properties | 72 ++- .../iotdb/commons/concurrent/ThreadName.java | 1 + .../apache/iotdb/commons/conf/CommonConfig.java | 6 +- .../iotdb/commons/conf/CommonDescriptor.java | 8 - .../org/apache/iotdb/commons/path/AlignedPath.java | 15 +- .../apache/iotdb/commons/path/MeasurementPath.java | 31 +- .../org/apache/iotdb/commons/path/PartialPath.java | 51 +- .../apache/iotdb/commons/path/PathPatternNode.java | 2 +- .../apache/iotdb/commons/path/PathPatternTree.java | 34 +- .../apache/iotdb/commons/path/PathPatternUtil.java | 52 ++ .../iotdb/commons/path/fa/nfa/SimpleNFA.java | 3 +- .../pipe/plugin/builtin/BuiltinPipePlugin.java | 10 +- .../plugin/builtin/collector/IoTDBCollector.java | 60 ++ .../builtin/connector/IoTDBThriftConnector.java | 78 +++ .../iotdb/commons/pipe/task/meta/PipeMeta.java | 19 +- .../commons/pipe/task/meta/PipeMetaKeeper.java | 8 + .../commons/pipe/task/meta/PipeRuntimeMeta.java | 45 +- .../commons/pipe/task/meta/PipeStaticMeta.java | 145 +++-- .../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 102 +++- .../commons/schema/ClusterSchemaQuotaLevel.java | 19 +- .../iotdb/commons/schema/filter/SchemaFilter.java | 75 +++ .../commons/schema/filter/SchemaFilterType.java | 42 +- .../commons/schema/filter/SchemaFilterVisitor.java | 59 ++ .../commons/schema/filter/impl/DataTypeFilter.java | 65 ++ .../schema/filter/impl/PathContainsFilter.java | 64 ++ .../commons/schema/filter/impl/TagFilter.java | 83 +++ .../node/common/AbstractMeasurementMNode.java | 4 + .../schema/node/role/IMeasurementMNode.java | 2 + .../commons/schema/node/utils/IMNodeFactory.java | 4 + .../commons/schema/tree/AbstractTreeVisitor.java | 122 +++- .../commons/schema/view/LogicalViewSchema.java | 224 +++++++ .../schema/view/viewExpression/ViewExpression.java | 318 ++++++++++ .../view/viewExpression/ViewExpressionType.java | 76 +++ .../binary/BinaryViewExpression.java | 119 ++++ .../binary/arithmetic/AdditionViewExpression.java | 61 ++ .../arithmetic/ArithmeticBinaryViewExpression.java | 52 ++ .../binary/arithmetic/DivisionViewExpression.java | 61 ++ .../binary/arithmetic/ModuloViewExpression.java | 61 ++ .../arithmetic/MultiplicationViewExpression.java | 63 ++ .../arithmetic/SubtractionViewExpression.java | 61 ++ .../compare/CompareBinaryViewExpression.java | 52 ++ .../binary/compare/EqualToViewExpression.java | 57 ++ .../binary/compare/GreaterEqualViewExpression.java | 57 ++ .../binary/compare/GreaterThanViewExpression.java | 57 ++ .../binary/compare/LessEqualViewExpression.java | 57 ++ .../binary/compare/LessThanViewExpression.java | 57 ++ .../binary/compare/NonEqualViewExpression.java | 57 ++ .../binary/logic/LogicAndViewExpression.java | 57 ++ .../binary/logic/LogicBinaryViewExpression.java | 52 ++ .../binary/logic/LogicOrViewExpression.java | 57 ++ .../viewExpression/leaf/ConstantViewOperand.java | 98 +++ .../view/viewExpression/leaf/LeafViewOperand.java | 20 +- .../view/viewExpression/leaf/NullViewOperand.java | 61 ++ .../viewExpression/leaf/TimeSeriesViewOperand.java | 87 +++ .../viewExpression/leaf/TimestampViewOperand.java | 72 +++ .../multi/FunctionViewExpression.java | 198 +++++++ .../ternary/BetweenViewExpression.java | 111 ++++ .../ternary/TernaryViewExpression.java | 105 ++++ .../viewExpression/unary/InViewExpression.java | 106 ++++ .../viewExpression/unary/IsNullViewExpression.java | 89 +++ .../viewExpression/unary/LikeViewExpression.java | 163 +++++ .../unary/LogicNotViewExpression.java | 61 ++ .../unary/NegationViewExpression.java | 61 ++ .../unary/RegularViewExpression.java | 105 ++++ .../viewExpression/unary/UnaryViewExpression.java | 73 +++ .../visitor/ViewExpressionVisitor.java | 219 +++++++ .../apache/iotdb/commons/service/ServiceType.java | 1 + .../iotdb/commons/path/PathPatternTreeTest.java | 61 +- .../commons/pipe/task/meta/PipeMetaDeSerTest.java | 77 +++ .../org/apache/iotdb/pipe/api/PipeCollector.java | 1 + .../org/apache/iotdb/pipe/api/PipeConnector.java | 4 + .../iotdb/pipe/api/customizer/PipeParameters.java | 22 + .../api/exception/PipeConnectionException.java | 15 +- .../exception/PipeRuntimeCriticalException.java | 19 +- .../pipe/api/exception/PipeRuntimeException.java | 19 +- .../exception/PipeRuntimeNonCriticalException.java | 19 +- pom.xml | 39 +- .../schemaregion/rocksdb/RSchemaRegion.java | 22 +- .../rocksdb/mnode/RMeasurementMNode.java | 5 + .../metadata/tagSchemaRegion/TagSchemaRegion.java | 22 +- .../resources/conf/iotdb-datanode.properties | 15 +- .../apache/iotdb/db/client/ConfigNodeClient.java | 17 - .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 181 +++++- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 110 +++- .../org/apache/iotdb/db/conf/OperationType.java | 3 +- .../db/consensus/DataRegionConsensusImpl.java | 9 + .../statemachine/DataRegionStateMachine.java | 3 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 41 ++ .../iotdb/db/engine/TsFileMetricManager.java | 63 +- .../iotdb/db/engine/cache/BloomFilterCache.java | 2 +- .../db/engine/cache/CacheHitRatioMonitor.java | 2 +- .../db/engine/cache/TimeSeriesMetadataCache.java | 13 +- .../compaction/constant/CompactionTaskStatus.java} | 9 +- .../compaction/constant/CompactionTaskType.java} | 10 +- .../exception/CompactionExceptionHandler.java | 2 +- .../CompactionMemoryNotEnoughException.java} | 9 +- .../performer/impl/FastCompactionPerformer.java | 16 +- .../impl/ReadChunkCompactionPerformer.java | 11 +- .../impl/ReadPointCompactionPerformer.java | 21 +- .../execute/task/AbstractCompactionTask.java | 5 +- .../execute/task/CompactionTaskSummary.java | 18 + .../execute/task/CrossSpaceCompactionTask.java | 37 +- .../execute/task/InnerSpaceCompactionTask.java | 6 +- .../compaction/execute/utils/CompactionUtils.java | 22 +- .../execute/utils/MultiTsFileDeviceIterator.java | 3 +- .../readchunk/AlignedSeriesCompactionExecutor.java | 6 - .../compaction/schedule/CompactionTaskManager.java | 70 ++- .../compaction/schedule/CompactionWorker.java | 3 - .../estimator/AbstractCompactionEstimator.java | 3 +- .../ReadPointCrossCompactionEstimator.java | 6 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 26 +- .../impl/SizeTieredCompactionSelector.java | 2 +- .../utils/CrossSpaceCompactionCandidate.java | 6 +- .../memtable/AlignedWritableMemChunkGroup.java | 10 +- .../db/engine/memtable/WritableMemChunkGroup.java | 10 +- .../db/engine/settle/SettleRequestHandler.java | 2 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 317 +++------- .../db/engine/storagegroup/TsFileResource.java | 15 +- .../engine/storagegroup/TsFileResourceStatus.java | 3 +- .../SchemaQuotaExceededException.java} | 23 +- .../db/exception/query/OutOfTTLException.java | 4 +- .../apache/iotdb/db/metadata/MetadataConstant.java | 6 + .../db/metadata/cache/DataNodeSchemaCache.java | 18 +- .../metadata/cache/DataNodeSchemaCacheMetrics.java | 31 +- .../cache/DeviceUsingTemplateSchemaCache.java | 50 +- .../iotdb/db/metadata/cache/SchemaCacheEntry.java | 36 +- .../db/metadata/cache/TimeSeriesSchemaCache.java | 9 +- .../dualkeycache/impl/DualKeyCacheBuilder.java | 18 +- .../dualkeycache/impl/DualKeyCachePolicy.java | 3 +- .../dualkeycache/impl/FIFOCacheEntryManager.java | 189 ++++++ .../mnode/config/factory/ConfigMNodeFactory.java | 7 + .../mnode/mem/factory/MemMNodeFactory.java | 14 + ...MeasurementMNode.java => LogicalViewMNode.java} | 31 +- .../metadata/mnode/mem/impl/MeasurementMNode.java | 5 + .../metadata/mnode/mem/info/LogicalViewInfo.java | 163 +++++ .../schemafile/factory/CacheMNodeFactory.java | 7 + .../schemafile/impl/CachedMeasurementMNode.java | 5 + .../iotdb/db/metadata/mtree/ConfigMTree.java | 8 +- .../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 43 +- .../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 103 +++- .../mtree/snapshot/MemMTreeSnapshotUtil.java | 48 +- .../db/metadata/mtree/store/CachedMTreeStore.java | 3 +- .../db/metadata/mtree/store/MemMTreeStore.java | 4 + .../db/metadata/mtree/traverser/Traverser.java | 68 +++ .../traverser/TraverserWithLimitOffsetWrapper.java | 5 + .../mtree/traverser/basic/DatabaseTraverser.java | 5 + .../mtree/traverser/basic/EntityTraverser.java | 37 +- .../mtree/traverser/basic/MNodeTraverser.java | 5 + .../traverser/basic/MeasurementTraverser.java | 39 +- .../plan/schemaregion/SchemaRegionPlanType.java | 2 + .../plan/schemaregion/SchemaRegionPlanVisitor.java | 5 + .../impl/SchemaRegionPlanDeserializer.java | 24 + .../impl/SchemaRegionPlanSerializer.java | 27 + .../impl/SchemaRegionPlanTxtSerializer.java | 21 + .../impl/read/SchemaRegionReadPlanFactory.java | 37 +- .../impl/read/ShowDevicesPlanImpl.java | 15 +- .../impl/read/ShowTimeSeriesPlanImpl.java | 37 +- .../impl/write/CreateLogicalViewPlanImpl.java | 75 +++ .../plan/schemaregion/read/IShowDevicesPlan.java | 4 + .../schemaregion/read/IShowTimeSeriesPlan.java | 7 +- .../schemaregion/result/ShowTimeSeriesResult.java | 13 +- .../schemaregion/write/ICreateLogicalViewPlan.java | 66 +++ .../metadata/query/info/ITimeSeriesSchemaInfo.java | 6 +- .../rescon/DataNodeSchemaQuotaManager.java | 92 +++ .../metadata/rescon/MemSchemaRegionStatistics.java | 8 +- .../db/metadata/schemaregion/ISchemaRegion.java | 21 +- .../db/metadata/schemaregion/SchemaEngine.java | 92 ++- .../schemaregion/SchemaRegionMemoryImpl.java | 126 ++-- .../schemaregion/SchemaRegionSchemaFileImpl.java | 83 +-- .../apache/iotdb/db/metadata/tag/TagManager.java | 16 +- .../metadata/template/ClusterTemplateManager.java | 40 +- .../template/alter/TemplateExtendInfo.java | 78 +++ .../apache/iotdb/db/metadata/utils/MetaUtils.java | 3 +- .../visitor/GetSourcePathsVisitor.java | 96 +++ .../visitor/TransformToExpressionVisitor.java | 321 ++++++++++ .../metadata/visitor/SchemaExecutionVisitor.java | 22 + .../db/mpp/common/header/ColumnHeaderConstant.java | 6 +- .../mpp/common/schematree/ClusterSchemaTree.java | 6 +- .../db/mpp/common/schematree/DeviceSchemaInfo.java | 18 +- .../common/schematree/IMeasurementSchemaInfo.java | 8 +- .../common/schematree/MeasurementSchemaInfo.java | 22 +- .../schematree/node/SchemaMeasurementNode.java | 42 +- .../visitor/SchemaTreeDeviceVisitor.java | 5 + .../visitor/SchemaTreeMeasurementVisitor.java | 5 + .../SchemaTreeVisitorWithLimitOffsetWrapper.java | 6 + .../iotdb/db/mpp/execution/driver/Driver.java | 36 +- .../db/mpp/execution/driver/DriverContext.java | 9 + .../mpp/execution/exchange/SharedTsBlockQueue.java | 12 +- .../execution/executor/RegionWriteExecutor.java | 173 ++++-- .../fragment/FragmentInstanceContext.java | 10 + .../fragment/FragmentInstanceExecution.java | 17 + .../operator/process/DeviceViewIntoOperator.java | 6 +- .../operator/process/MergeSortOperator.java | 9 +- .../execution/operator/process/SortOperator.java | 258 ++++++-- .../process/join/HorizontallyConcatOperator.java | 8 +- .../process/join/merge/MergeSortComparator.java | 45 +- ...rtKeyComparator.java => SortKeyComparator.java} | 11 +- .../operator/schema/source/DeviceSchemaSource.java | 13 +- .../schema/source/SchemaSourceFactory.java | 27 +- .../schema/source/TimeSeriesSchemaSource.java | 31 +- .../mpp/metric/TimeSeriesMetadataCacheMetrics.java | 4 +- .../org/apache/iotdb/db/mpp/plan/Coordinator.java | 2 +- .../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 16 +- .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 150 ++++- .../analyze/schema/AutoCreateSchemaExecutor.java | 10 +- .../analyze/schema/ClusterSchemaFetchExecutor.java | 7 +- .../plan/analyze/schema/ClusterSchemaFetcher.java | 50 +- .../mpp/plan/analyze/schema/SchemaValidator.java | 27 +- .../db/mpp/plan/execution/IQueryExecution.java | 1 + .../db/mpp/plan/execution/QueryExecution.java | 13 +- .../config/executor/ClusterConfigTaskExecutor.java | 26 +- .../visitor/TransformToViewExpressionVisitor.java | 391 ++++++++++++ .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 207 +++++-- .../db/mpp/plan/planner/LogicalPlanBuilder.java | 53 +- .../db/mpp/plan/planner/LogicalPlanVisitor.java | 106 ++-- .../db/mpp/plan/planner/OperatorTreeGenerator.java | 49 +- .../plan/planner/distribution/SourceRewriter.java | 4 +- .../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +- .../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 + .../plan/node/load/LoadSingleTsFileNode.java | 4 + .../node/metedata/read/DevicesSchemaScanNode.java | 23 +- .../metedata/read/LevelTimeSeriesCountNode.java | 47 +- .../node/metedata/read/TimeSeriesCountNode.java | 43 +- .../metedata/read/TimeSeriesSchemaScanNode.java | 95 +-- .../node/metedata/write/CreateLogicalViewNode.java | 250 ++++++++ .../write/InternalCreateMultiTimeSeriesNode.java | 1 + .../plan/node/write/InsertMultiTabletsNode.java | 22 +- .../plan/planner/plan/node/write/InsertNode.java | 133 +---- .../planner/plan/node/write/InsertRowNode.java | 171 +----- .../planner/plan/node/write/InsertRowsNode.java | 33 +- .../plan/node/write/InsertRowsOfOneDeviceNode.java | 32 +- .../planner/plan/node/write/InsertTabletNode.java | 145 +---- .../db/mpp/plan/scheduler/AsyncPlanNodeSender.java | 74 ++- .../plan/scheduler/AsyncSendPlanNodeHandler.java | 27 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 25 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 8 +- .../iotdb/db/mpp/plan/statement/StatementType.java | 2 + .../db/mpp/plan/statement/StatementVisitor.java | 7 + .../plan/statement/component/OrderByComponent.java | 6 +- .../component/{SortKey.java => OrderByKey.java} | 2 +- .../plan/statement/crud/InsertBaseStatement.java | 157 ++++- .../crud/InsertMultiTabletsStatement.java | 30 + .../plan/statement/crud/InsertRowStatement.java | 183 +++++- .../crud/InsertRowsOfOneDeviceStatement.java | 42 ++ .../plan/statement/crud/InsertRowsStatement.java | 41 ++ .../plan/statement/crud/InsertTabletStatement.java | 155 ++++- .../db/mpp/plan/statement/crud/QueryStatement.java | 57 +- .../metadata/CountLevelTimeSeriesStatement.java | 30 +- .../metadata/CountTimeSeriesStatement.java | 30 +- .../metadata/CreateLogicalViewStatement.java | 246 ++++++++ .../statement/metadata/ShowDevicesStatement.java | 10 + .../metadata/ShowTimeSeriesStatement.java | 30 +- .../plan/statement/sys/ShowQueriesStatement.java | 4 +- .../org/apache/iotdb/db/pipe/agent/PipeAgent.java | 14 +- .../db/pipe/agent/plugin/PipePluginAgent.java | 26 +- .../IoTDBThriftReceiver.java} | 24 +- .../db/pipe/agent/receiver/PipeReceiverAgent.java | 90 +++ .../iotdb/db/pipe/agent/runtime/PipeLauncher.java | 170 ++++++ .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 47 +- .../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 511 +++++++++++++++- .../PipeCollectorConstant.java} | 19 +- .../apache/iotdb/db/pipe/config/PipeConfig.java | 22 + .../PipeConnectorConstant.java} | 19 +- .../config/PipeProcessorConstant.java} | 11 +- .../core/collector/IoTDBDataRegionCollector.java | 122 ++++ .../PipeHistoricalDataRegionTsFileCollector.java | 112 ++++ .../realtime/PipeRealtimeDataRegionCollector.java | 23 +- ... => PipeRealtimeDataRegionHybridCollector.java} | 62 +- .../realtime/assigner/PipeDataRegionAssigner.java | 19 +- .../listener/PipeInsertionDataNodeListener.java | 4 +- .../matcher/CachedSchemaPatternMatcher.java | 6 +- .../realtime/matcher/PipeDataRegionMatcher.java | 9 +- .../impl/iotdb/IoTDBThriftConnectorClient.java | 70 +++ .../impl/iotdb/IoTDBThriftConnectorVersion.java} | 20 +- .../impl/iotdb/v1/IoTDBThriftConnectorV1.java | 231 ++++++++ .../impl/iotdb/v1/IoTDBThriftReceiverV1.java | 275 +++++++++ .../connector/impl/iotdb/v1/PipeRequestType.java} | 38 +- .../v1/reponse/PipeTransferFilePieceResp.java | 80 +++ .../iotdb/v1/request/PipeTransferFilePieceReq.java | 88 +++ .../iotdb/v1/request/PipeTransferFileSealReq.java | 79 +++ .../iotdb/v1/request/PipeTransferHandshakeReq.java | 71 +++ .../v1/request/PipeTransferInsertNodeReq.java | 102 ++++ .../PipeConnectorSubtaskLifeCycle.java | 33 +- .../{ => manager}/PipeConnectorSubtaskManager.java | 56 +- .../iotdb/db/pipe/core/event/EnrichedEvent.java | 54 ++ .../core/event/impl/PipeTabletInsertionEvent.java | 31 +- .../core/event/impl/PipeTsFileInsertionEvent.java | 84 ++- .../event/realtime/PipeRealtimeCollectEvent.java | 20 +- .../realtime/PipeRealtimeCollectEventFactory.java | 8 +- .../event/view/collector/PipeEventCollector.java | 64 +- .../execution/executor/PipeSubtaskExecutor.java | 17 +- ...anager.java => PipeSubtaskExecutorManager.java} | 12 +- .../execution/scheduler/PipeTaskScheduler.java | 18 +- .../db/pipe/resource/PipeFileResourceManager.java | 186 ++++++ .../iotdb/db/pipe/resource/PipeRaftlogHolder.java | 22 - .../db/pipe/resource/PipeResourceManager.java | 16 +- .../iotdb/db/pipe/resource/PipeTsFileHolder.java | 22 - .../iotdb/db/pipe/resource/PipeWALHolder.java | 22 - .../org/apache/iotdb/db/pipe/task/PipeBuilder.java | 66 +++ .../org/apache/iotdb/db/pipe/task/PipeTask.java | 9 +- .../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 60 +- .../apache/iotdb/db/pipe/task/PipeTaskManager.java | 96 +++ .../task/queue/EventSupplier.java} | 20 +- .../ListenableBlockingPendingQueue.java} | 18 +- .../db/pipe/task/queue/ListenablePendingQueue.java | 159 +++++ .../ListenableUnblockingPendingQueue.java} | 18 +- .../queue/PendingQueueEmptyToNotEmptyListener.java | 12 +- .../queue/PendingQueueFullToNotFullListener.java} | 8 +- .../queue/PendingQueueNotEmptyToEmptyListener.java | 12 +- .../queue/PendingQueueNotFullToFullListener.java} | 8 +- .../db/pipe/task/stage/PipeTaskCollectorStage.java | 109 +++- .../db/pipe/task/stage/PipeTaskConnectorStage.java | 105 +--- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 138 ++++- .../iotdb/db/pipe/task/stage/PipeTaskStage.java | 95 ++- .../db/pipe/task/subtask/PipeConnectorSubtask.java | 124 +++- .../db/pipe/task/subtask/PipeProcessorSubtask.java | 49 +- .../iotdb/db/pipe/task/subtask/PipeSubtask.java | 34 +- .../org/apache/iotdb/db/rescon/SystemInfo.java | 20 +- .../java/org/apache/iotdb/db/service/DataNode.java | 102 +--- .../iotdb/db/service/metrics/FileMetrics.java | 26 +- .../metrics/recorder/CompactionMetricsManager.java | 81 +-- .../db/service/thrift/ProcessorWithMetrics.java | 6 +- .../service/thrift/impl/ClientRPCServiceImpl.java | 209 ++++--- .../impl/DataNodeInternalRPCServiceImpl.java | 87 +-- .../db/sync/common/ClusterSyncInfoFetcher.java | 11 +- .../org/apache/iotdb/db/tools/DiskSpiller.java | 166 ++++++ .../apache/iotdb/db/tools/FileSpillerReader.java | 143 +++++ .../apache/iotdb/db/tools/IoTDBDataDirViewer.java | 4 +- .../MemoryReader.java} | 35 +- .../apache/iotdb/db/tools/SortBufferManager.java | 73 +++ .../SortReader.java} | 25 +- .../iotdb/db/tools/TsFileResourcePrinter.java | 4 +- .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 2 +- .../org/apache/iotdb/db/tools/TsFileSplitTool.java | 2 +- .../db/tools/settle/TsFileAndModSettleTool.java | 6 +- .../db/tools/validate/TsFileValidationTool.java | 16 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 2 +- .../org/apache/iotdb/db/utils/UpgradeUtils.java | 2 +- .../db/utils/datastructure/AlignedTVList.java | 6 +- .../datastructure/FixedPriorityBlockingQueue.java | 10 + .../db/utils/datastructure/MergeSortHeap.java | 4 +- .../iotdb/db/utils/datastructure/MergeSortKey.java | 25 +- .../{MergeSortKey.java => SortKey.java} | 14 +- .../iotdb/db/wal/buffer/AbstractWALBuffer.java | 26 +- .../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 41 +- .../org/apache/iotdb/db/wal/buffer/WALEntry.java | 4 +- .../iotdb/db/wal/checkpoint/CheckpointManager.java | 118 +++- .../iotdb/db/wal/checkpoint/MemTableInfo.java | 28 +- .../exception/MemTablePinException.java} | 20 +- .../exception/WALPipeException.java} | 20 +- .../apache/iotdb/db/wal/io/WALByteBufReader.java | 6 +- .../org/apache/iotdb/db/wal/node/WALFakeNode.java | 24 +- .../java/org/apache/iotdb/db/wal/node/WALNode.java | 57 +- .../db/wal/recover/file/TsFilePlanRedoer.java | 4 - .../iotdb/db/wal/utils/WALEntryPosition.java | 157 +++++ .../apache/iotdb/db/wal/utils/WALFileUtils.java | 11 + .../iotdb/db/wal/utils/WALInsertNodeCache.java | 170 ++++++ .../apache/iotdb/db/wal/utils/WALPipeHandler.java | 132 +++++ .../db/wal/utils/listener/WALFlushListener.java | 13 +- .../iotdb/db/engine/cache/ChunkCacheTest.java | 4 +- .../engine/compaction/AbstractCompactionTest.java | 9 +- .../FastCrossCompactionPerformerTest.java | 5 + .../db/engine/compaction/MemoryControlTest.java | 75 +++ .../cross/CrossSpaceCompactionExceptionTest.java | 2 +- .../cross/CrossSpaceCompactionSelectorTest.java | 171 ++++++ .../CrossSpaceCompactionWithFastPerformerTest.java | 6 +- ...eCompactionWithFastPerformerValidationTest.java | 20 +- ...sSpaceCompactionWithReadPointPerformerTest.java | 6 +- .../db/engine/compaction/cross/MergeTest.java | 6 +- .../cross/RewriteCompactionFileSelectorTest.java | 46 +- .../inner/AbstractInnerSpaceCompactionTest.java | 4 +- .../inner/InnerCompactionMoreDataTest.java | 2 +- .../compaction/inner/InnerCompactionTest.java | 4 +- .../ReadChunkCompactionPerformerAlignedTest.java | 76 +++ .../inner/sizetiered/SizeTieredCompactionTest.java | 10 +- .../compaction/utils/CompactionCheckerUtils.java | 4 + .../utils/MultiTsFileDeviceIteratorTest.java | 12 +- .../db/engine/snapshot/IoTDBSnapshotTest.java | 2 +- .../db/engine/storagegroup/DataRegionTest.java | 16 +- .../engine/storagegroup/FakedTsFileResource.java | 2 +- .../db/engine/storagegroup/TsFileResourceTest.java | 2 +- .../cache/dualkeycache/DualKeyCacheTest.java | 17 + .../schemaRegion/SchemaRegionBasicTest.java | 270 ++++++++- .../schemaRegion/SchemaRegionTestUtil.java | 4 +- .../metadata/view/ViewExpressionToStringTest.java | 183 ++++++ .../execution/operator/AlignedSeriesTestUtil.java | 6 +- .../execution/operator/MergeSortOperatorTest.java | 36 +- .../mpp/execution/operator/OperatorMemoryTest.java | 48 +- .../mpp/execution/operator/SortOperatorTest.java | 273 +++++++++ .../schema/SchemaFetchScanOperatorTest.java | 3 +- .../schema/SchemaQueryScanOperatorTest.java | 4 +- .../db/mpp/plan/analyze/FakeSchemaFetcherImpl.java | 7 +- .../db/mpp/plan/optimization/TestPlanBuilder.java | 6 +- .../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 120 +++- .../db/mpp/plan/plan/QueryLogicalPlanUtil.java | 16 +- .../distribution/DistributionPlannerBasicTest.java | 10 +- .../read/DeviceSchemaScanNodeSerdeTest.java | 3 +- .../metadata/read/SchemaCountNodeSerdeTest.java | 8 +- .../read/TimeSeriesSchemaScanNodeSerdeTest.java | 3 +- .../plan/node/process/DeviceViewNodeSerdeTest.java | 6 +- .../plan/plan/node/process/SortNodeSerdeTest.java | 4 +- .../collector/CachedSchemaPatternMatcherTest.java | 54 +- .../core/collector/PipeRealtimeCollectTest.java | 104 +++- .../pipe/core/connector/PipeThriftRequestTest.java | 118 ++++ .../executor/PipeAssignerSubtaskExecutorTest.java | 40 -- .../executor/PipeConnectorSubtaskExecutorTest.java | 8 +- .../executor/PipeProcessorSubtaskExecutorTest.java | 10 +- .../pipe/resource/PipeFileResourceManagerTest.java | 218 +++++++ .../AlignedSeriesScanLimitOffsetPushDownTest.java | 10 +- .../query/reader/series/SeriesReaderTestUtil.java | 6 +- .../series/SeriesScanLimitOffsetPushDownTest.java | 8 +- .../iotdb/db/rescon/ResourceManagerTest.java | 12 +- .../DataNodeInternalRPCServiceImplTest.java | 31 +- .../sync/pipedata/BufferedPipeDataQueueTest.java | 658 --------------------- .../db/utils/datastructure/VectorTVListTest.java | 11 + .../org/apache/iotdb/db/wal/io/WALFileTest.java | 25 +- .../iotdb/db/wal/node/ConsensusReqReaderTest.java | 27 +- .../org/apache/iotdb/db/wal/node/WALNodeTest.java | 33 +- .../iotdb/db/wal/node/WALPipeHandlerTest.java | 256 ++++++++ .../db/wal/recover/file/TsFilePlanRedoerTest.java | 32 +- .../file/UnsealedTsFileRecoverPerformerTest.java | 5 +- .../iotdb/db/wal/utils/WALInsertNodeCacheTest.java | 167 ++++++ .../java/org/apache/iotdb/rpc/TSStatusCode.java | 11 +- .../java/org/apache/iotdb/session/Session.java | 4 + .../org/apache/iotdb/session/pool/SessionPool.java | 319 +++++++++- spark-iotdb-connector/pom.xml | 84 +-- spark-iotdb-connector/{ => scala_2.11}/pom.xml | 108 ++-- spark-iotdb-connector/{ => scala_2.12}/pom.xml | 103 ++-- .../src/test/resources/iotdb-datanode.properties | 23 - .../apache/iotdb/spark/db/EnvironmentUtils.java | 263 -------- .../org/apache/iotdb/spark/db/IoTDBTest.scala | 150 ----- .../org/apache/iotdb/spark/db/IoTDBWriteTest.scala | 118 ---- .../iotdb/spark/db/unit/DataFrameToolsTest.scala | 96 --- spark-tsfile/pom.xml | 50 +- .../iotdb/spark/tsfile/qp/common/SQLConstant.java | 118 ++-- .../src/main/thrift/confignode.thrift | 14 +- thrift/src/main/thrift/client.thrift | 13 + thrift/src/main/thrift/datanode.thrift | 53 +- .../java/org/apache/iotdb/tsfile/utils/Binary.java | 20 +- .../tsfile/write/schema/IMeasurementSchema.java | 4 + .../tsfile/write/schema/MeasurementSchema.java | 10 + .../tsfile/write/schema/MeasurementSchemaType.java | 23 +- .../write/schema/VectorMeasurementSchema.java | 10 + 593 files changed, 22876 insertions(+), 7123 deletions(-)
