This is an automated email from the ASF dual-hosted git repository. marklau99 pushed a commit to branch refactor-compaction-metrics in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cedb22f27dd4e1ba24800b4102413bb5e8a51e9f Merge: 1f9c235f058 8f637838819 Author: Liu Xuxin <[email protected]> AuthorDate: Tue Jun 20 20:03:02 2023 +0800 Merge remote-tracking branch 'upstream/master' into refactor-compaction-metrics .github/workflows/sonar-coveralls.yml | 39 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 142 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 49 + .../resources/conf/iotdb-confignode.properties | 34 +- .../confignode/client/DataNodeRequestType.java | 2 + .../client/async/AsyncDataNodeClientPool.java | 29 +- .../client/async/handlers/AsyncClientHandler.java | 5 +- ...RPCHandler.java => SchemaUpdateRPCHandler.java} | 6 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 11 - .../confignode/conf/ConfigNodeDescriptor.java | 7 - .../confignode/conf/ConfigNodeStartupCheck.java | 16 + .../confignode/conf/SystemPropertiesUtils.java | 58 +- .../request/write/sync/PreCreatePipePlanV1.java | 2 +- .../request/write/sync/RecordPipeMessagePlan.java | 2 +- .../request/write/sync/SetPipeStatusPlanV1.java | 2 +- .../response/pipe/task/PipeTableResp.java | 2 +- .../iotdb/confignode/manager/ConfigManager.java | 31 +- .../apache/iotdb/confignode/manager/IManager.java | 3 + .../iotdb/confignode/manager/ProcedureManager.java | 100 +- .../partition/GreedyPartitionAllocator.java | 4 +- .../manager/load/service/StatisticsService.java | 3 +- .../iotdb/confignode/manager/node/NodeManager.java | 5 +- .../manager/partition/PartitionManager.java | 158 +- .../manager/pipe/runtime/PipeHeartbeatParser.java | 238 +++ ...ordinator.java => PipeLeaderChangeHandler.java} | 81 +- .../manager/pipe/runtime/PipeMetaSyncer.java | 4 +- .../pipe/runtime/PipeRuntimeCoordinator.java | 100 +- .../manager/pipe/task/PipeTaskCoordinator.java | 13 +- .../manager/schema/ClusterSchemaManager.java | 27 +- .../persistence/executor/ConfigPlanExecutor.java | 39 +- .../persistence/partition/PartitionInfo.java | 52 +- .../persistence/pipe/PipePluginInfo.java | 24 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 25 +- .../{task => }/AbstractOperatePipeProcedureV2.java | 17 +- .../impl}/pipe/PipeTaskOperation.java | 2 +- .../pipe/plugin/CreatePipePluginProcedure.java | 8 +- .../impl/pipe/plugin/DropPipePluginProcedure.java | 6 +- .../runtime/PipeHandleLeaderChangeProcedure.java | 10 +- .../runtime/PipeHandleMetaChangeProcedure.java | 182 +-- .../impl/pipe/runtime/PipeMetaSyncProcedure.java | 4 +- .../impl/pipe/task/CreatePipeProcedureV2.java | 55 +- .../impl/pipe/task/DropPipeProcedureV2.java | 25 +- .../impl/pipe/task/StartPipeProcedureV2.java | 38 +- .../impl/pipe/task/StopPipeProcedureV2.java | 37 +- ...ocedure.java => AlterLogicalViewProcedure.java} | 288 ++-- .../impl/schema/DeleteLogicalViewProcedure.java | 3 +- .../procedure/impl/sync/CreatePipeProcedure.java | 3 +- .../procedure/impl/sync/DropPipeProcedure.java | 1 + .../procedure/impl/sync/StartPipeProcedure.java | 3 +- .../procedure/impl/sync/StopPipeProcedure.java | 3 +- .../state/schema/AlterLogicalViewState.java | 8 +- .../procedure/store/ProcedureFactory.java | 6 + .../confignode/procedure/store/ProcedureType.java | 5 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 27 +- .../request/ConfigPhysicalPlanSerDeTest.java | 8 +- .../runtime/PipeHandleMetaChangeProcedureTest.java | 7 +- .../consensus/iot/IoTConsensusServerImpl.java | 7 +- .../logdispatcher/LogDispatcherThreadMetrics.java | 38 +- .../iotdb/consensus/ratis/RatisConsensusTest.java | 153 +- .../apache/iotdb/consensus/ratis/TestUtils.java | 160 ++ docs/Download/README.md | 22 +- docs/UserGuide/API/InfluxDB-Protocol.md | 344 ---- docs/UserGuide/Data-Concept/Encoding.md | 22 +- .../Data-Modeling/SchemaRegion-rocksdb.md | 2 +- .../Edge-Cloud-Collaboration/Sync-Tool.md | 375 ----- .../Maintenance-Tools/SchemaFileSketch-Tool.md | 10 +- docs/UserGuide/Maintenance-Tools/Watermark-Tool.md | 196 --- docs/UserGuide/Query-Data/Group-By.md | 1204 +++++++------- docs/UserGuide/Query-Data/Overview.md | 2 +- docs/UserGuide/Reference/Common-Config-Manual.md | 124 +- .../Reference/ConfigNode-Config-Manual.md | 8 +- docs/UserGuide/Reference/DataNode-Config-Manual.md | 16 +- docs/UserGuide/Reference/Status-Codes.md | 260 +-- docs/zh/Download/README.md | 22 +- docs/zh/UserGuide/API/InfluxDB-Protocol.md | 347 ---- docs/zh/UserGuide/Data-Concept/Encoding.md | 22 +- .../Data-Modeling/SchemaRegion-rocksdb.md | 2 +- .../Edge-Cloud-Collaboration/Sync-Tool.md | 362 ----- .../Maintenance-Tools/SchemaFileSketch-Tool.md | 10 +- .../UserGuide/Maintenance-Tools/Watermark-Tool.md | 196 --- docs/zh/UserGuide/Query-Data/Group-By.md | 880 +++++----- .../zh/UserGuide/Reference/Common-Config-Manual.md | 99 +- .../Reference/ConfigNode-Config-Manual.md | 8 +- .../UserGuide/Reference/DataNode-Config-Manual.md | 26 +- docs/zh/UserGuide/Reference/Status-Codes.md | 260 +-- example/ext-pipe-plugin-example/README.md | 60 - example/ext-pipe-plugin-example/pom.xml | 65 - .../iotdb/extpipe/ExtPipeSinkWriterFactory.java | 139 -- .../iotdb/extpipe/ExtPipeSinkWriterImpl.java | 164 -- ...ipe.external.api.IExternalPipeSinkWriterFactory | 1 - example/pom.xml | 1 - integration-test/import-control.xml | 1 - .../iotdb/it/env/cluster/MppCommonConfig.java | 26 +- .../it/env/cluster/MppSharedCommonConfig.java | 28 - .../iotdb/it/env/remote/RemoteCommonConfig.java | 20 - .../org/apache/iotdb/itbase/env/CommonConfig.java | 8 - .../it/partition/IoTDBPartitionGetterIT.java | 2 +- .../confignode/it/utils/ConfigNodeTestUtils.java | 2 + .../org/apache/iotdb/db/it/IoTDBEncodingIT.java | 59 - .../db/it/aligned/IoTDBInsertAlignedValuesIT.java | 60 +- .../db/it/groupby/IoTDBGroupByConditionIT.java | 44 + .../iotdb/db/it/groupby/IoTDBGroupByCountIT.java | 44 + .../db/it/groupby/IoTDBGroupByVariationIT.java | 23 +- .../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java | 74 +- .../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 32 +- .../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 56 +- .../db/it/schema/IoTDBSortedShowTimeseriesIT.java | 2 +- .../org/apache/iotdb/db/it/schema/IoTDBTagIT.java | 2 +- .../db/it/schema/view/IoTDBAliasSeriesIT.java | 102 ++ .../iotdb/db/it/schema/view/IoTDBAlterViewIT.java | 117 ++ .../it/schema/view/IoTDBCreateAndShowViewIT.java | 429 +++++ .../view/IoTDBShowDevicesContainedViewIT.java | 226 +++ .../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 38 +- .../iotdb/db/it/view/IoTDBCreateAndShowViewIT.java | 294 ---- .../iotdb/db/it/watermark/IoTDBWatermarkIT.java | 229 --- .../iotdb/session/it/IoTDBSessionComplexIT.java | 8 +- .../org/apache/iotdb/util/AbstractSchemaIT.java | 17 +- .../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 12 +- iotdb-api/external-pipe-api/pom.xml | 35 - .../apache/iotdb/pipe/external/api/DataType.java | 51 - .../external/api/ExternalPipeSinkWriterStatus.java | 77 - .../pipe/external/api/IExternalPipeSinkWriter.java | 175 -- .../api/IExternalPipeSinkWriterFactory.java | 63 - .../org/apache/iotdb/pipe/api/PipeCollector.java | 7 +- .../org/apache/iotdb/pipe/api/PipeConnector.java | 6 +- .../org/apache/iotdb/pipe/api/PipeProcessor.java | 6 +- .../apache/iotdb/pipe/api/access/RowIterator.java | 75 - .../api/customizer/PipeRuntimeConfiguration.java | 28 - .../iotdb/pipe/api/customizer/PipeStrategy.java | 28 - .../PipeCollectorRuntimeConfiguration.java | 30 - .../PipeCollectorRuntimeConfiguration.java | 7 +- .../PipeConnectorRuntimeConfiguration.java | 7 +- .../PipeProcessorRuntimeConfiguration.java | 7 +- .../configuration/PipeRuntimeConfiguration.java | 6 +- .../configuration/PipeRuntimeEnvironment.java | 8 +- .../PipeConnectorRuntimeConfiguration.java | 84 - .../connector/parallel/ParallelStrategy.java | 24 - .../retry/EqualRetryIntervalStrategy.java | 63 - .../retry/ExponentialRetryIntervalStrategy.java | 70 - .../customizer/connector/retry/RetryStrategy.java | 34 - .../customizer/connector/reuse/ReuseStrategy.java | 24 - .../{ => parameter}/PipeParameterValidator.java | 2 +- .../customizer/{ => parameter}/PipeParameters.java | 6 +- .../PipeProcessorRuntimeConfiguration.java | 35 - .../event/dml/insertion/TabletInsertionEvent.java | 12 +- .../event/dml/insertion/TsFileInsertionEvent.java | 8 - .../api/exception/PipeManagementException.java | 32 - iotdb-client/client-cpp/src/main/Session.h | 10 +- iotdb-client/client-py/iotdb/Session.py | 20 +- .../client-py/iotdb/sqlalchemy/IoTDBSQLCompiler.py | 56 +- .../client-py/iotdb/utils/IoTDBConstants.py | 1 - iotdb-client/client-py/tests/test_dataframe.py | 2 +- .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 14 - .../iotdb/jdbc/IoTDBDatabaseMetadataTest.java | 4 - .../java/org/apache/iotdb/rpc/TSStatusCode.java | 5 +- .../iotdb/flink/tsfile/util/TSFileConfigUtil.java | 1 - .../util/TSFileConfigUtilCompletenessTest.java | 2 - .../src/main/thrift/confignode.thrift | 12 + .../thrift/src/main/thrift/client.thrift | 12 +- .../thrift/src/main/thrift/datanode.thrift | 7 + .../metrics/metricsets/cpu/CpuUsageMetrics.java | 41 +- .../iotdb/metrics/utils/IoTDBMetricsUtils.java | 13 +- .../resources/conf/iotdb-common.properties | 78 +- .../iotdb/commons/auth/entity/PrivilegeType.java | 4 + .../iotdb/commons/client/ClientPoolFactory.java | 24 + .../async/AsyncPipeDataTransferServiceClient.java | 167 ++ .../iotdb/commons/concurrent/ThreadName.java | 19 +- .../apache/iotdb/commons/conf/CommonConfig.java | 56 +- .../iotdb/commons/conf/CommonDescriptor.java | 31 +- .../consensus/index/impl/RecoverProgressIndex.java | 6 + .../exception/sync/PipeAlreadyExistException.java | 31 - .../exception/sync/PipeDataLoadException.java | 25 - .../exception/sync/PipeNotExistException.java | 25 - .../sync/PipeSinkAlreadyExistException.java | 25 - .../exception/sync/PipeSinkBeingUsedException.java | 27 - .../commons/exception/sync/PipeSinkException.java | 42 - .../exception/sync/PipeSinkNotExistException.java | 25 - .../exception/sync/SyncConnectionException.java | 39 - .../apache/iotdb/commons/path/MeasurementPath.java | 11 +- .../org/apache/iotdb/commons/path/PartialPath.java | 10 + .../apache/iotdb/commons/path/PathPatternUtil.java | 4 + .../iotdb/commons/pipe/config/PipeConfig.java | 4 - .../pipe/plugin/builtin/BuiltinPipePlugin.java | 8 +- .../plugin/builtin/collector/IoTDBCollector.java | 6 +- .../builtin/connector/DoNothingConnector.java | 6 +- ...hriftConnector.java => IoTDBSyncConnector.java} | 16 +- .../builtin/connector/IoTDBThriftConnector.java | 22 +- .../builtin/connector/IoTDBThriftConnectorV1.java | 5 +- .../builtin/connector/IoTDBThriftConnectorV2.java | 8 +- .../builtin/processor/DoNothingProcessor.java | 6 +- .../service/PipePluginExecutableManager.java | 6 +- .../commons/pipe/task/meta/PipeStaticMeta.java | 2 +- .../iotdb/commons/schema/filter/SchemaFilter.java | 12 +- ...FilterVisitor.java => SchemaFilterFactory.java} | 45 +- .../commons/schema/filter/SchemaFilterType.java | 9 +- .../commons/schema/filter/SchemaFilterVisitor.java | 28 +- .../impl/{DataTypeFilter.java => AndFilter.java} | 41 +- .../commons/schema/filter/impl/DataTypeFilter.java | 6 +- .../schema/filter/impl/PathContainsFilter.java | 8 +- .../commons/schema/filter/impl/TagFilter.java | 6 +- .../{DataTypeFilter.java => ViewTypeFilter.java} | 32 +- .../node/common/AbstractDatabaseDeviceMNode.java | 11 +- .../schema/node/common/AbstractDeviceMNode.java | 11 +- .../node/common/AbstractMeasurementMNode.java | 2 +- .../commons/schema/node/info/IDeviceInfo.java | 4 +- .../commons/schema/node/role/IDeviceMNode.java | 4 +- .../PipeStatus.java => schema/view/ViewType.java} | 51 +- .../iotdb/commons/sync/{pipe => }/PipeInfo.java | 46 +- .../iotdb/commons/sync/{pipe => }/PipeMessage.java | 5 +- .../iotdb/commons/sync/{pipe => }/PipeStatus.java | 3 +- .../commons/sync/{pipe => }/TsFilePipeInfo.java | 36 +- .../iotdb/commons/sync/metadata/SyncMetadata.java | 241 --- .../commons/sync/persistence/SyncLogReader.java | 111 -- .../commons/sync/persistence/SyncLogWriter.java | 94 -- .../iotdb/commons/sync/pipe/SyncOperation.java | 29 - .../iotdb/commons/sync/pipesink/IoTDBPipeSink.java | 166 -- .../iotdb/commons/sync/pipesink/PipeSink.java | 88 - .../commons/sync/transport/SyncIdentityInfo.java | 57 - .../iotdb/commons/sync/utils/SyncConstant.java | 87 - .../iotdb/commons/sync/utils/SyncPathUtil.java | 156 -- .../org/apache/iotdb/commons/utils/AuthUtils.java | 8 + .../CommonDateTimeUtils.java} | 28 +- .../pipe/{task/meta => }/PipeMetaDeSerTest.java | 6 +- .../commons/sync/metedata/SyncMetadataTest.java | 189 --- pom.xml | 22 +- server/pom.xml | 5 - .../resources/conf/iotdb-datanode.properties | 40 +- ...rint-schema-file.bat => print-pb-tree-file.bat} | 2 +- ...{print-schema-file.sh => print-pb-tree-file.sh} | 0 .../org/apache/iotdb/db/audit/AuditLogger.java | 6 +- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 8 + .../apache/iotdb/db/client/ConfigNodeClient.java | 22 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 330 +--- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 252 ++- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 69 +- .../iotdb/db/conf/directories/TierManager.java | 7 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 4 +- .../constant/CompactionValidationLevel.java} | 9 +- .../CompactionFileCountExceededException.java | 8 +- .../execute/task/AbstractCompactionTask.java | 6 +- .../execute/task/CrossSpaceCompactionTask.java | 56 +- .../execute/task/InnerSpaceCompactionTask.java | 30 +- .../compaction/execute/utils/CompactionUtils.java | 116 ++ .../utils/validator/CompactionValidator.java | 50 + .../utils/validator/NoneCompactionValidator.java | 33 +- .../ResourceAndTsfileCompactionValidator.java | 52 + .../validator/ResourceOnlyCompactionValidator.java | 51 + .../estimator/AbstractCompactionEstimator.java | 6 +- .../ReadPointCrossCompactionEstimator.java | 42 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 22 +- .../impl/SizeTieredCompactionSelector.java | 2 +- .../utils/CrossSpaceCompactionCandidate.java | 63 +- .../db/engine/settle/SettleRequestHandler.java | 2 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 99 +- .../db/engine/storagegroup/TsFileManager.java | 74 - .../db/engine/storagegroup/TsFileProcessor.java | 34 +- .../db/engine/storagegroup/TsFileResource.java | 13 +- ...xception.java => DuplicateInsertException.java} | 15 +- .../metadata/SeriesOverflowException.java | 8 +- .../SchemaFileLogCorruptedException.java | 4 +- .../metadata/schemafile/SchemaFileNotExists.java | 4 +- .../schemafile/SchemaPageOverflowException.java | 2 +- .../UnsupportedViewException.java} | 22 +- .../view/ViewContainsAggregationException.java | 17 +- .../metadata/view/ViewNotExistException.java | 25 - .../apache/iotdb/db/metadata/MetadataConstant.java | 8 +- .../metadata/cache/DataNodeLastCacheManager.java | 4 +- .../db/metadata/cache/TimeSeriesSchemaCache.java | 8 +- .../idtable/AppendOnlyDiskSchemaManager.java | 2 +- .../metadata/metric/SchemaEngineCachedMetric.java | 1 + .../db/metadata/metric/SchemaEngineMemMetric.java | 3 +- .../db/metadata/metric/SchemaMetricManager.java | 4 +- .../db/metadata/mnode/mem/info/DeviceInfo.java | 6 +- .../metadata/mnode/mem/info/LogicalViewInfo.java | 10 +- .../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 6 +- .../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 36 +- .../mtree/snapshot/MemMTreeSnapshotUtil.java | 4 +- .../db/metadata/mtree/store/CachedMTreeStore.java | 12 +- .../mtree/store/disk/cache/CacheMemoryManager.java | 2 +- .../ReleaseFlushStrategyNumBasedImpl.java | 2 +- .../mtree/store/disk/schemafile/ISchemaFile.java | 4 +- .../store/disk/schemafile/MockSchemaFile.java | 4 +- .../mtree/store/disk/schemafile/SchemaFile.java | 34 +- .../store/disk/schemafile/SchemaFileConfig.java | 14 +- .../disk/schemafile/pagemgr/BTreePageManager.java | 2 +- .../store/disk/schemafile/pagemgr/PageManager.java | 2 +- .../traverser/collector/MeasurementCollector.java | 2 +- .../plan/schemaregion/SchemaRegionPlanType.java | 1 + .../plan/schemaregion/SchemaRegionPlanVisitor.java | 5 + .../impl/SchemaRegionPlanDeserializer.java | 9 + .../impl/SchemaRegionPlanSerializer.java | 13 + .../impl/SchemaRegionPlanTxtSerializer.java | 11 + .../impl/read/SchemaRegionReadPlanFactory.java | 9 +- .../impl/write/AlterLogicalViewPlanImpl.java | 56 + .../impl/write/SchemaRegionWritePlanFactory.java | 8 + .../schemaregion/result/ShowDevicesResult.java | 6 +- .../write/view/IAlterLogicalViewPlan.java | 46 + .../db/metadata/query/info/IDeviceSchemaInfo.java | 2 +- .../rescon/CachedSchemaEngineStatistics.java | 2 +- .../rescon/CachedSchemaRegionStatistics.java | 4 +- .../metadata/rescon/ISchemaRegionStatistics.java | 4 + .../metadata/rescon/MemSchemaRegionStatistics.java | 10 + .../db/metadata/rescon/SchemaResourceManager.java | 10 +- .../db/metadata/schemaregion/ISchemaRegion.java | 3 + .../db/metadata/schemaregion/SchemaEngine.java | 11 +- .../db/metadata/schemaregion/SchemaEngineMode.java | 2 +- .../schemaregion/SchemaRegionMemoryImpl.java | 35 +- ...maFileImpl.java => SchemaRegionPBTreeImpl.java} | 74 +- .../apache/iotdb/db/metadata/tag/TagLogFile.java | 3 +- .../apache/iotdb/db/metadata/tag/TagManager.java | 21 +- .../iotdb/db/metadata/view/ViewPathType.java | 8 +- .../apache/iotdb/db/metadata/view/ViewPaths.java | 26 + .../db/metadata/visitor/DeviceFilterVisitor.java | 6 +- .../metadata/visitor/SchemaExecutionVisitor.java | 20 + .../metadata/visitor/TimeseriesFilterVisitor.java | 15 +- .../mpp/common/schematree/ClusterSchemaTree.java | 10 +- .../execution/executor/RegionWriteExecutor.java | 28 + .../operator/process/last/LastQueryUtil.java | 4 +- .../schema/source/LogicalViewSchemaSource.java | 77 +- .../schema/source/TimeSeriesSchemaSource.java | 7 +- .../operator/source/ShowQueriesOperator.java | 6 +- .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 188 ++- .../iotdb/db/mpp/plan/analyze/SelectIntoUtils.java | 34 +- .../mpp/plan/analyze/schema/ISchemaValidation.java | 6 + .../plan/analyze/schema/NormalSchemaFetcher.java | 66 +- .../plan/execution/config/ConfigTaskVisitor.java | 30 +- .../config/executor/ClusterConfigTaskExecutor.java | 161 +- .../config/executor/IConfigTaskExecutor.java | 13 +- .../config/metadata/GetTimeSlotListTask.java | 4 +- .../execution/config/sys/pipe/ShowPipeTask.java | 10 +- .../config/sys/sync/CreatePipeSinkTask.java | 42 - .../config/sys/sync/DropPipeSinkTask.java | 42 - .../config/sys/sync/ShowPipeSinkTask.java | 96 -- .../memory/StatementMemorySourceVisitor.java | 22 - .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 230 ++- .../db/mpp/plan/planner/LogicalPlanVisitor.java | 25 +- .../plan/planner/distribution/SourceRewriter.java | 122 +- .../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +- .../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 + .../node/metedata/write/AlterTimeSeriesNode.java | 33 +- .../metedata/write/view/AlterLogicalViewNode.java | 186 +++ .../db/mpp/plan/statement/StatementVisitor.java | 26 +- .../plan/statement/crud/InsertBaseStatement.java | 27 + .../crud/InsertMultiTabletsStatement.java | 31 + .../plan/statement/crud/InsertRowStatement.java | 11 +- .../crud/InsertRowsOfOneDeviceStatement.java | 1 + .../plan/statement/crud/InsertRowsStatement.java | 31 + .../plan/statement/crud/InsertTabletStatement.java | 11 +- .../db/mpp/plan/statement/crud/QueryStatement.java | 24 +- .../metadata/AlterTimeSeriesStatement.java | 10 + .../metadata/view/CreateLogicalViewStatement.java | 55 +- ...wPipeStatement.java => ShowPipesStatement.java} | 6 +- .../sys/sync/CreatePipeSinkStatement.java | 120 -- .../statement/sys/sync/DropPipeSinkStatement.java | 62 - .../statement/sys/sync/ShowPipeSinkStatement.java | 48 - .../db/pipe/agent/plugin/PipePluginAgent.java | 27 +- .../pipe/agent/receiver/IoTDBThriftReceiver.java | 6 +- .../db/pipe/agent/receiver/PipeReceiverAgent.java | 12 +- .../db/pipe/agent/runtime/PipeAgentLauncher.java | 4 +- .../db/pipe/agent/runtime/PipeRuntimeAgent.java | 12 +- .../SimpleConsensusProgressIndexAssigner.java | 5 +- .../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 96 +- .../collector/IoTDBDataRegionCollector.java | 89 +- .../PipeHistoricalDataRegionCollector.java | 2 +- .../PipeHistoricalDataRegionTsFileCollector.java | 88 +- .../realtime/PipeRealtimeDataRegionCollector.java | 32 +- .../PipeRealtimeDataRegionFakeCollector.java | 16 +- .../PipeRealtimeDataRegionHybridCollector.java | 16 +- .../PipeRealtimeDataRegionLogCollector.java | 15 +- .../PipeRealtimeDataRegionTsFileCollector.java | 15 +- .../realtime/assigner/DisruptorQueue.java | 2 +- .../realtime/assigner/PipeDataRegionAssigner.java | 10 +- .../realtime/epoch}/TsFileEpoch.java | 4 +- .../realtime/epoch}/TsFileEpochManager.java | 7 +- .../realtime/epoch}/TsFileEpochStateMigrator.java | 2 +- .../listener/PipeInsertionDataNodeListener.java | 8 +- .../matcher/CachedSchemaPatternMatcher.java | 6 +- .../realtime/matcher/PipeDataRegionMatcher.java | 6 +- .../{ => constant}/PipeCollectorConstant.java | 4 +- .../{ => constant}/PipeConnectorConstant.java | 12 +- .../{ => constant}/PipeProcessorConstant.java | 2 +- .../configuraion/PipeTaskRuntimeConfiguration.java | 42 + .../env/PipeTaskCollectorRuntimeEnvironment.java | 32 +- .../plugin/env/PipeTaskRuntimeEnvironment.java} | 24 +- .../IoTDBThriftConnectorRequestVersion.java} | 9 +- .../pipe/connector/legacy/IoTDBSyncConnector.java | 287 ++++ .../connector/legacy/IoTDBSyncReceiver.java} | 491 +++--- .../connector/legacy/loader}/DeletionLoader.java | 19 +- .../connector/legacy/loader}/ILoader.java | 5 +- .../connector/legacy/loader}/TsFileLoader.java | 16 +- .../legacy}/pipedata/DeletionPipeData.java | 20 +- .../connector/legacy}/pipedata/PipeData.java | 13 +- .../connector/legacy}/pipedata/TsFilePipeData.java | 89 +- .../v1}/IoTDBThriftConnectorClient.java | 2 +- .../v1/IoTDBThriftConnectorV1.java | 81 +- .../v1/IoTDBThriftReceiverV1.java | 31 +- .../iotdb => connector}/v1/PipeRequestType.java | 2 +- .../v1/reponse/PipeTransferFilePieceResp.java | 2 +- .../v1/request/PipeTransferFilePieceReq.java | 8 +- .../v1/request/PipeTransferFileSealReq.java | 8 +- .../v1/request/PipeTransferHandshakeReq.java | 8 +- .../v1/request/PipeTransferInsertNodeReq.java | 8 +- .../v1/request/PipeTransferTabletReq.java | 56 +- .../pipe/connector/v2/IoTDBThriftConnectorV2.java | 325 ++++ ...nsferInsertNodeTabletInsertionEventHandler.java | 51 + ...ipeTransferRawTabletInsertionEventHandler.java} | 29 +- .../PipeTransferTabletInsertionEventHandler.java | 131 ++ .../PipeTransferTsFileInsertionEventHandler.java | 222 +++ .../PipeHistoricalDataRegionFakeCollector.java | 56 - .../event/impl/PipeEmptyTabletInsertionEvent.java | 39 - .../TsFileInsertionDataContainer.java | 136 -- .../TsFileInsertionDataTabletIterator.java | 287 ---- .../db/pipe/{core => }/event/EnrichedEvent.java | 12 +- .../event/common/row}/PipeBinaryTransformer.java | 7 +- .../event/common/row}/PipeDataTypeTransformer.java | 40 +- .../view/access => event/common/row}/PipeRow.java | 52 +- .../common/row}/PipeRowCollector.java | 38 +- .../PipeInsertNodeTabletInsertionEvent.java | 50 +- .../tablet/PipeRawTabletInsertionEvent.java} | 39 +- .../tablet}/TabletInsertionDataContainer.java | 202 ++- .../common/tsfile}/PipeTsFileInsertionEvent.java | 44 +- .../tsfile/TsFileInsertionDataContainer.java | 209 +++ .../tsfile/TsFileInsertionDataTabletIterator.java | 147 ++ .../event/realtime/PipeRealtimeCollectEvent.java | 13 +- .../realtime/PipeRealtimeCollectEventFactory.java | 10 +- .../executor/PipeConnectorSubtaskExecutor.java | 2 +- .../executor/PipeProcessorSubtaskExecutor.java | 2 +- .../processor/PipeDoNothingProcessor.java | 37 +- .../pipe/resource/wal/PipeWALResourceManager.java | 7 +- .../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 17 +- .../BlockingPendingQueue.java | 2 +- .../BoundedBlockingPendingQueue.java | 2 +- .../task/{queue => connection}/EventSupplier.java | 2 +- .../connection}/PipeEventCollector.java | 5 +- .../UnboundedBlockingPendingQueue.java | 2 +- .../db/pipe/task/stage/PipeTaskCollectorStage.java | 94 +- .../db/pipe/task/stage/PipeTaskConnectorStage.java | 14 +- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 77 +- .../db/pipe/task/subtask/PipeConnectorSubtask.java | 4 +- .../subtask}/PipeConnectorSubtaskLifeCycle.java | 5 +- .../subtask}/PipeConnectorSubtaskManager.java | 66 +- .../db/pipe/task/subtask/PipeProcessorSubtask.java | 2 +- .../iotdb/db/pipe/task/subtask/PipeSubtask.java | 2 +- .../influxdb/handler/QueryHandlerFactory.java | 3 +- .../influxdb/meta/InfluxDBMetaManagerFactory.java | 3 +- .../protocol/influxdb/util/QueryResultUtils.java | 2 +- .../rest/v1/impl/GrafanaApiServiceImpl.java | 3 +- .../rest/v2/impl/GrafanaApiServiceImpl.java | 3 +- .../org/apache/iotdb/db/rescon/SystemInfo.java | 54 + .../iotdb/db/rescon/TsFileResourceManager.java | 2 +- .../java/org/apache/iotdb/db/service/DataNode.java | 5 - .../iotdb/db/service/InfluxDBRPCService.java | 109 -- .../apache/iotdb/db/service/IoTDBShutdownHook.java | 2 +- .../db/service/metrics/DataNodeMetricsHelper.java | 1 + .../iotdb/db/service/metrics/WritingMetrics.java | 20 + .../service/thrift/impl/ClientRPCServiceImpl.java | 23 +- .../impl/DataNodeInternalRPCServiceImpl.java | 44 +- .../java/org/apache/iotdb/db/sync/SyncService.java | 561 ------- .../db/sync/common/ClusterSyncInfoFetcher.java | 133 -- .../iotdb/db/sync/common/ISyncInfoFetcher.java | 58 - .../apache/iotdb/db/sync/common/LocalSyncInfo.java | 142 -- .../iotdb/db/sync/common/LocalSyncInfoFetcher.java | 168 -- .../iotdb/db/sync/datasource/AbstractOpBlock.java | 119 -- .../iotdb/db/sync/datasource/DeletionGroup.java | 242 --- .../iotdb/db/sync/datasource/DeletionOpBlock.java | 106 -- .../iotdb/db/sync/datasource/PipeOpManager.java | 345 ---- .../iotdb/db/sync/datasource/PipeOpSgManager.java | 197 --- .../db/sync/datasource/PipeStorageGroupInfo.java | 50 - .../iotdb/db/sync/datasource/TsFileOpBlock.java | 1208 -------------- .../iotdb/db/sync/externalpipe/ExtPipePlugin.java | 676 -------- .../externalpipe/ExtPipePluginConfiguration.java | 140 -- .../db/sync/externalpipe/ExtPipePluginManager.java | 338 ---- .../sync/externalpipe/ExtPipePluginRegister.java | 139 -- .../db/sync/externalpipe/ExternalPipeStatus.java | 58 - .../externalpipe/operation/DeleteOperation.java | 78 - .../externalpipe/operation/InsertOperation.java | 61 - .../db/sync/externalpipe/operation/Operation.java | 80 - .../sync/pipedata/queue/BufferedPipeDataQueue.java | 426 ----- .../db/sync/pipedata/queue/PipeDataQueue.java | 41 - .../iotdb/db/sync/sender/manager/ISyncManager.java | 44 - .../db/sync/sender/manager/LocalSyncManager.java | 91 -- .../db/sync/sender/pipe/ExternalPipeSink.java | 130 -- .../org/apache/iotdb/db/sync/sender/pipe/Pipe.java | 141 -- .../iotdb/db/sync/sender/pipe/PipeSinkFactory.java | 41 - .../iotdb/db/sync/sender/pipe/TsFilePipe.java | 464 ------ .../db/sync/sender/recovery/TsFilePipeLogger.java | 155 -- .../db/sync/transport/client/ISyncClient.java | 44 - .../db/sync/transport/client/IoTDBSyncClient.java | 278 ---- .../db/sync/transport/client/SenderManager.java | 283 ---- .../sync/transport/client/SyncClientFactory.java | 59 - ...leSketchTool.java => PBTreeFileSketchTool.java} | 10 +- .../watermark/GroupedLSBWatermarkEncoder.java | 150 -- .../db/tools/watermark/WatermarkDetector.java | 220 --- .../iotdb/db/tools/watermark/WatermarkEncoder.java | 36 - .../org/apache/iotdb/db/utils/DateTimeUtils.java | 24 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 2 - .../apache/iotdb/db/utils/TimePartitionUtils.java | 4 +- .../iotdb/db/utils/TimestampPrecisionUtils.java | 57 + .../apache/iotdb/db/utils/sync/SyncPipeUtil.java | 84 - .../java/org/apache/iotdb/db/wal/WALManager.java | 15 +- .../java/org/apache/iotdb/db/wal/node/WALNode.java | 69 +- .../file/UnsealedTsFileRecoverPerformer.java | 2 +- .../apache/iotdb/db/wal/utils/WALEntryHandler.java | 9 +- .../iotdb/db/wal/utils/WALEntryPosition.java | 15 +- .../iotdb/db/wal/utils/WALInsertNodeCache.java | 6 +- .../engine/compaction/AbstractCompactionTest.java | 4 +- .../engine/compaction/CompactionSchedulerTest.java | 101 +- .../CompactionSchedulerWithFastPerformerTest.java | 96 +- .../compaction/CompactionTaskManagerTest.java | 2 +- .../compaction/CompactionValidationTest.java | 224 +++ .../FastNonAlignedCrossCompactionTest.java | 145 ++ .../db/engine/compaction/MemoryControlTest.java | 137 +- .../cross/CrossSpaceCompactionSelectorTest.java | 1694 ++++++++++++++++++++ ...eCompactionWithFastPerformerValidationTest.java | 22 +- ...actionWithReadPointPerformerValidationTest.java | 18 +- .../db/engine/compaction/cross/MergeTest.java | 1 + .../cross/RewriteCompactionFileSelectorTest.java | 62 +- .../inner/InnerCompactionSchedulerTest.java | 10 +- .../inner/InnerSpaceCompactionSelectorTest.java | 626 ++++++++ .../SizeTieredCompactionSelectorTest.java | 14 +- .../compaction/utils/CompactionConfigRestorer.java | 4 +- .../db/engine/settle/SettleRequestHandlerTest.java | 7 +- .../db/engine/storagegroup/DataRegionTest.java | 57 +- .../iotdb/db/engine/storagegroup/TTLTest.java | 7 +- .../mtree/schemafile/AliasIndexPageTest.java | 8 +- .../mtree/schemafile/InternalPageTest.java | 8 +- .../mtree/schemafile/SchemaFileLogTest.java | 21 +- .../metadata/mtree/schemafile/SchemaFileTest.java | 8 +- .../mtree/schemafile/WrappedSegmentTest.java | 8 +- .../schemaRegion/AbstractSchemaRegionTest.java | 21 +- .../schemaRegion/SchemaRegionBasicTest.java | 29 +- .../schemaRegion/SchemaStatisticsTest.java | 2 +- .../mpp/plan/analyze/QueryTimePartitionTest.java | 62 +- .../plan/node/write/WritePlanNodeSplitTest.java | 7 +- .../collector/CachedSchemaPatternMatcherTest.java | 41 +- .../collector/PipeRealtimeCollectTest.java | 46 +- .../connector/PipeThriftRequestTest.java | 12 +- .../PipeTabletInsertionEventTest.java} | 121 +- .../event/TsFileInsertionDataContainerTest.java | 679 ++++++++ .../PipeConnectorSubtaskExecutorTest.java | 5 +- .../PipeProcessorSubtaskExecutorTest.java | 5 +- .../{executor => }/PipeSubtaskExecutorTest.java | 3 +- .../org/apache/iotdb/db/qp/sql/ASTVisitorTest.java | 12 +- .../org/apache/iotdb/db/sync/SyncTestUtils.java | 41 - .../db/sync/datasource/DeletionGroupTest.java | 231 --- .../db/sync/datasource/DeletionOpBlockTest.java | 62 - .../db/sync/datasource/PipeOpManagerTest.java | 595 ------- .../db/sync/datasource/TsFileOpBlockTest.java | 1123 ------------- .../db/sync/persistence/LocalSyncInfoTest.java | 104 -- .../iotdb/db/sync/persistence/SyncLogTest.java | 121 -- .../iotdb/db/sync/pipedata/PipeDataTest.java | 65 - ...leSketchTest.java => PBTreeFileSketchTest.java} | 16 +- .../iotdb/db/tools/TsFileAndModSettleToolTest.java | 12 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 4 - .../iotdb/db/wal/buffer/WALBufferCommonTest.java | 1 + .../db/wal/checkpoint/CheckpointManagerTest.java | 1 + .../iotdb/db/wal/node/ConsensusReqReaderTest.java | 2 + .../iotdb/db/wal/node/WALEntryHandlerTest.java | 156 +- .../org/apache/iotdb/db/wal/node/WALNodeTest.java | 2 + .../db/wal/recover/WALRecoverManagerTest.java | 2 + .../src/test/resources/iotdb-datanode.properties | 4 +- site/iotdb-doap.rdf | 8 + site/src/main/.vuepress/components/IoTDB.vue | 4 +- site/src/main/.vuepress/components/IoTDBZH.vue | 4 +- .../docsearch/client/components/Docsearch.ts | 7 +- site/src/main/.vuepress/config.ts | 8 + site/src/main/.vuepress/navbar/en.ts | 4 +- site/src/main/.vuepress/navbar/zh.ts | 4 +- site/src/main/.vuepress/sidebar/V1.0.x/en.ts | 1 - site/src/main/.vuepress/sidebar/V1.0.x/zh.ts | 1 - site/src/main/.vuepress/sidebar/V1.1.x/en.ts | 2 +- site/src/main/.vuepress/sidebar/V1.1.x/zh.ts | 2 +- site/src/main/.vuepress/sidebar/V1.2.x/en.ts | 12 +- site/src/main/.vuepress/sidebar/V1.2.x/zh.ts | 12 +- site/src/main/.vuepress/sidebar/en.ts | 562 ++++--- site/src/main/.vuepress/sidebar/zh.ts | 568 ++++--- site/src/main/.vuepress/utils/index.ts | 3 + site/src/main/package-lock.json | 6 +- site/src/main/package.json | 1 + tsfile/pom.xml | 4 - .../iotdb/tsfile/common/conf/TSFileConfig.java | 20 - .../iotdb/tsfile/common/conf/TSFileDescriptor.java | 3 - .../iotdb/tsfile/encoding/decoder/Decoder.java | 2 - .../iotdb/tsfile/encoding/decoder/FreqDecoder.java | 144 -- .../iotdb/tsfile/encoding/encoder/FreqEncoder.java | 317 ---- .../tsfile/encoding/encoder/TSEncodingBuilder.java | 64 - .../tsfile/file/metadata/TimeseriesMetadata.java | 2 - .../tsfile/file/metadata/enums/TSEncoding.java | 3 +- .../iotdb/tsfile/fileSystem/FSFactoryProducer.java | 8 + .../tsfile/fileSystem/fsFactory/HDFSFactory.java | 8 - .../tsfile/fileSystem/fsFactory/OSFSFactory.java | 13 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 18 + .../apache/iotdb/tsfile/utils/BitConstructor.java | 94 -- .../org/apache/iotdb/tsfile/utils/BitReader.java | 70 - .../iotdb/tsfile/utils/ReadWriteIOUtils.java | 19 +- .../tsfile/encoding/decoder/FreqDecoderTest.java | 161 -- 596 files changed, 14489 insertions(+), 24173 deletions(-)
