This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a change to branch beyyes/sort_transform_elimate_topk
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 4982cef78b4 add SortElimination rule
add 0526599c961 Fixed the bug that add column procedure is not idempotent
& Added tests for table plan / procedures
add 8db2c3dbc9e Delete SchemaQueryMergeNode in LogicalPlanner
add ce6c81af584 Merge master0710
add 5997327f41f Fixed the bug that "show tables" fails when the database
is first created and has no tables under it
new 22e003e9c31 Merge branch 'ty/TableModelGrammar' into
beyyes/sort_transform_elimate_topk
add 3673568a4ed Fixed the bug that the configMTree can not be recovered
when there are tables
add 3145d189530 Introduce IterativeOptimizer and adapt
ProjectOffPushDownRules, InlineProjectionsRules
new 173d6eeb3e1 merge
new 4188b0edba9 add streamsort and sort elimination
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/daily-it.yml | 50 +
.github/workflows/multi-language-client.yml | 15 +-
distribution/src/assembly/all.xml | 9 -
distribution/src/assembly/confignode.xml | 9 -
distribution/src/assembly/datanode.xml | 9 -
docker/src/main/Dockerfile-1.0.0-datanode | 4 +-
.../apache/iotdb/SubscriptionSessionExample.java | 40 +-
integration-test/pom.xml | 14 +
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../iotdb/it/env/cluster/env/AbstractEnv.java | 13 +-
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../iotdb/it/env/remote/env/RemoteServerEnv.java | 6 +
.../org/apache/iotdb/itbase/category/DailyIT.java | 14 +-
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 3 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../apache/iotdb/cli/it/StartClientScriptIT.java | 65 +
.../it/cluster/IoTDBClusterNodeGetterIT.java | 1 -
.../it/partition/IoTDBTimePartitionIT.java | 150 ++
.../IoTDBRegionMigrateReliabilityITFramework.java | 55 +-
.../pass/IoTDBRegionMigrateClusterCrashIT.java | 3 +
.../pass/IoTDBRegionMigrateConfigNodeCrashIT.java | 3 +
.../pass/IoTDBRegionMigrateNormalIT.java | 3 +
...rateCoordinatorCrashWhenRemoveRemotePeerIT.java | 3 +
.../IoTDBRegionMigrateDataNodeCrashIT.java | 6 +
...nMigrateOriginalCrashWhenDeleteLocalPeerIT.java | 3 +
...MigrateOriginalCrashWhenRemoveRemotePeerIT.java | 3 +
.../confignode/it/utils/ConfigNodeTestUtils.java | 1 -
.../apache/iotdb/db/it/IoTDBJDBCMetadataIT.java | 135 ++
.../org/apache/iotdb/db/it/IoTDBRestServiceIT.java | 17 +-
.../iotdb/db/it/IoTDBSetConfigurationIT.java | 90 +-
...oTDBAlignByDeviceWithTemplateAggregationIT.java | 29 +
.../iotdb/db/it/auth/IoTDBSeriesPermissionIT.java | 2 +-
.../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 2 +-
.../pipe/it/autocreate/IoTDBPipeAutoDropIT.java | 6 +
.../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 100 +-
.../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 2 +-
.../pipe/it/manual/IoTDBPipeMetaHistoricalIT.java | 6 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 2 +-
.../iotdb/relational/it/schema/IoTDBTableIT.java | 61 +-
.../it/session/IoTDBTableModelSessionIT.java | 25 +-
.../session/pool/IoTDBTableModelSessionPoolIT.java | 47 +-
.../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 318 ++--
.../it/dual/IoTDBSubscriptionTopicIT.java | 300 ++--
.../it/local/IoTDBSubscriptionBasicIT.java | 302 +---
.../it/local/IoTDBSubscriptionDataTypeIT.java | 368 ++++
.../java/org/apache/iotdb/cli/AbstractCli.java | 27 +-
.../java/org/apache/iotdb/tool/ImportData.java | 2 +
.../java/org/apache/iotdb/tool/ImportSchema.java | 2 +
.../java/org/apache/iotdb/tool/ImportTsFile.java | 31 +-
.../client-py/iotdb/utils/IoTDBRpcDataSet.py | 2 +-
iotdb-client/client-py/requirements.txt | 4 +-
iotdb-client/client-py/requirements_dev.txt | 8 +-
iotdb-client/client-py/resources/setup.py | 6 +-
iotdb-client/client-py/tests/test_todf.py | 6 +-
.../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 23 +-
.../org/apache/iotdb/jdbc/IoTDBResultMetadata.java | 44 +-
.../apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java | 19 +-
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 2 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 +-
.../iotdb/rpc/subscription/config/TopicConfig.java | 23 +-
.../rpc/subscription/config/TopicConstant.java | 8 +-
.../payload/poll/SubscriptionCommitContext.java | 2 +
.../subscription/payload/poll/TabletsPayload.java | 10 +-
.../response/PipeSubscribeSubscribeResp.java | 32 +-
.../response/PipeSubscribeUnsubscribeResp.java | 32 +-
.../java/org/apache/iotdb/session/Session.java | 29 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 28 +
.../consumer/SubscriptionConsumer.java | 118 +-
.../consumer/SubscriptionProvider.java | 10 +-
.../consumer/SubscriptionPullConsumer.java | 25 +
.../consumer/SubscriptionPushConsumer.java | 29 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +-
.../src/assembly/resources/conf/confignode-env.sh | 6 +-
.../assembly/resources/sbin/start-confignode.sh | 1 +
.../CnToCnInternalServiceAsyncRequestManager.java | 10 +
.../CnToDnInternalServiceAsyncRequestManager.java | 10 +
.../handlers/ConfigNodeAsyncRequestContext.java | 6 +
.../iotdb/confignode/conf/ConfigNodeConfig.java | 12 -
.../confignode/conf/ConfigNodeDescriptor.java | 21 +-
.../confignode/conf/ConfigNodeStartupCheck.java | 8 +-
.../conf/ConfigNodeSystemPropertiesHandler.java | 18 +-
.../confignode/conf/SystemPropertiesUtils.java | 97 +-
.../request/write/database/SetTTLPlan.java | 6 +
.../response/pipe/task/PipeTableResp.java | 67 +-
.../iotdb/confignode/manager/ClusterManager.java | 12 +-
.../iotdb/confignode/manager/ConfigManager.java | 39 +-
.../apache/iotdb/confignode/manager/IManager.java | 11 +
.../iotdb/confignode/manager/TTLManager.java | 16 +-
.../partition/DataPartitionPolicyTable.java | 2 +-
.../manager/node/ClusterNodeStartUtils.java | 12 +-
.../iotdb/confignode/manager/node/NodeManager.java | 15 +-
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 37 +-
.../protocol/IoTDBConfigRegionConnector.java | 13 +-
.../runtime/heartbeat/PipeHeartbeat.java | 4 +
.../runtime/heartbeat/PipeHeartbeatParser.java | 39 +-
.../pipe/event/PipeConfigRegionSnapshotEvent.java | 9 +-
.../pipe/execution/PipeConfigNodeSubtask.java | 2 +
.../resource/PipeConfigNodeResourceManager.java | 51 +
.../PipeConfigNodeSnapshotResourceManager.java | 11 +-
.../manager/schema/ClusterSchemaManager.java | 64 +-
.../iotdb/confignode/persistence/TTLInfo.java | 24 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 51 +-
.../persistence/schema/ClusterSchemaInfo.java | 33 +-
.../confignode/persistence/schema/ConfigMTree.java | 83 +-
.../persistence/schema/TemplateTable.java | 10 +-
.../procedure/env/RegionMaintainHandler.java | 17 +-
.../impl/pipe/task/AlterPipeProcedureV2.java | 44 +-
.../impl/schema/AlterLogicalViewProcedure.java | 2 +-
.../impl/schema/DataNodeRegionTaskExecutor.java | 16 +-
.../impl/schema/DeactivateTemplateProcedure.java | 12 +-
.../impl/schema/DeleteLogicalViewProcedure.java | 8 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 18 +-
.../procedure/impl/schema/SchemaUtils.java | 17 +-
.../impl/schema/SetTemplateProcedure.java | 3 +-
.../impl/schema/table/AddTableColumnProcedure.java | 201 ++-
.../impl/schema/table/CreateTableProcedure.java | 134 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 33 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 76 +
.../iotdb/confignode/persistence/TTLInfoTest.java | 46 +-
.../persistence/schema/ConfigMTreeTest.java | 81 +
.../schema/table/AddTableColumnProcedureTest.java | 62 +
.../schema/table/CreateTableProcedureTest.java | 73 +
.../org/apache/iotdb/consensus/IConsensus.java | 8 +
.../apache/iotdb/consensus/iot/IoTConsensus.java | 6 +
.../iot/logdispatcher/IndexController.java | 4 +-
.../apache/iotdb/consensus/pipe/PipeConsensus.java | 6 +
.../ratis/ApplicationStateMachineProxy.java | 9 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 12 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 14 +-
.../iotdb/consensus/simple/SimpleConsensus.java | 5 +
.../apache/iotdb/consensus/iot/StabilityTest.java | 4 +
.../iotdb/consensus/ratis/RatisConsensusTest.java | 7 +
.../consensus/simple/SimpleConsensusTest.java | 6 +-
iotdb-core/datanode/pom.xml | 4 +-
.../src/assembly/resources/conf/datanode-env.sh | 6 +-
.../db/conf/DataNodeSystemPropertiesHandler.java | 16 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 18 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 303 ++--
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 17 +-
.../dataregion/DataRegionStateMachine.java | 2 +-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 95 +-
.../IoTDBDataNodeCacheLeaderClientManager.java | 8 +-
.../evolvable/batch/PipeTabletEventBatch.java | 133 ++
.../PipeTabletEventPlainBatch.java} | 121 +-
.../batch/PipeTabletEventTsFileBatch.java | 425 +++++
.../PipeTransferBatchReqBuilder.java | 96 +-
.../request/PipeTransferTabletRawReq.java | 133 +-
.../airgap/IoTDBDataRegionAirGapConnector.java | 3 +-
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 14 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 12 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 2 +
.../PipeConsensusTsFileInsertionEventHandler.java | 2 +
.../PipeConsensusTransferBatchReqBuilder.java | 12 +-
.../request/PipeConsensusTsFileSealReq.java | 9 +-
.../request/PipeConsensusTsFileSealWithModReq.java | 2 +
.../async/IoTDBDataRegionAsyncConnector.java | 238 ++-
.../PipeTransferTabletBatchEventHandler.java | 31 +-
...Handler.java => PipeTransferTsFileHandler.java} | 134 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 129 +-
.../thrift/sync/IoTDBSchemaRegionConnector.java | 13 +-
.../protocol/writeback/WriteBackConnector.java | 3 +-
.../pipe/connector/util/PipeTabletEventSorter.java | 202 +++
.../consensus/PipeConsensusConnectorMetrics.java | 19 +-
.../consensus/ProgressIndexDataNodeManager.java | 20 +-
.../schema/PipeSchemaRegionSnapshotEvent.java | 4 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 24 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 4 +-
.../tablet/TabletInsertionDataContainer.java | 6 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 50 +-
.../common/tsfile/TsFileInsertionPointCounter.java | 24 +-
.../container/TsFileInsertionDataContainer.java | 78 +
.../TsFileInsertionDataContainerProvider.java | 122 ++
.../query/TsFileInsertionQueryDataContainer.java} | 121 +-
.../TsFileInsertionQueryDataTabletIterator.java} | 22 +-
.../scan/AlignedSinglePageWholeChunkReader.java | 170 ++
.../container/scan/SinglePageWholeChunkReader.java | 116 ++
.../scan/TsFileInsertionScanDataContainer.java | 386 ++++
.../PipeHistoricalDataRegionTsFileExtractor.java | 71 +-
.../realtime/PipeRealtimeDataRegionExtractor.java | 43 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 6 +-
.../realtime/assigner/DisruptorQueue.java | 12 +-
.../PipeDataNodeRemainingEventAndTimeMetrics.java | 20 +
.../PipeDataNodeRemainingEventAndTimeOperator.java | 29 +-
.../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 37 +-
.../iotdb/db/pipe/metric/PipeResourceMetrics.java | 14 +-
.../downsampling/PartialPathLastObjectCache.java | 4 +-
.../pipeconsensus/PipeConsensusReceiver.java | 154 +-
.../pipeconsensus/PipeConsensusReceiverAgent.java | 53 +-
...nager.java => PipeDataNodeResourceManager.java} | 8 +-
.../db/pipe/resource/memory/PipeMemoryBlock.java | 18 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 63 +-
...oryWeighUtil.java => PipeMemoryWeightUtil.java} | 63 +-
.../pipe/resource/tsfile/PipeTsFileResource.java | 102 +-
.../resource/tsfile/PipeTsFileResourceManager.java | 10 +-
.../pipe/resource/wal/PipeWALResourceManager.java | 34 +-
.../hardlink/PipeWALHardlinkResourceManager.java | 23 +-
.../selfhost/PipeWALSelfHostResourceManager.java | 8 +-
.../pipe/task/builder/PipeDataNodeTaskBuilder.java | 23 +-
.../pipe/task/connection/PipeEventCollector.java | 29 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 20 +-
.../subtask/connector/PipeConnectorSubtask.java | 62 +-
.../PipeRealtimePriorityBlockingQueue.java | 11 +-
.../subtask/processor/PipeProcessorSubtask.java | 28 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 58 +-
.../iotdb/db/protocol/client/ConfigNodeInfo.java | 2 +-
.../DnToCnInternalServiceAsyncRequestManager.java | 10 +
.../impl/DataNodeInternalRPCServiceImpl.java | 49 +-
.../db/queryengine/common/MPPQueryContext.java | 51 +-
.../common/header/ColumnHeaderConstant.java | 10 +-
.../fragment/FragmentInstanceContext.java | 19 +
.../fragment/FragmentInstanceExecution.java | 2 +
.../execution/load/LoadTsFileManager.java | 72 +-
.../operator/schema/source/DeviceSchemaSource.java | 2 +-
.../AbstractSeriesAggregationScanOperator.java | 2 +-
.../operator/source/AlignedSeriesScanOperator.java | 4 +-
.../operator/source/SeriesScanOperator.java | 3 +-
.../execution/operator/source/SeriesScanUtil.java | 3 +-
.../execution/schedule/AbstractDriverThread.java | 12 +-
.../schedule/DriverTaskAbortedException.java | 3 +
.../metric/load/LoadTsFileCostMetricsSet.java | 10 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 2 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 4 +-
.../plan/analyze/LoadTsfileAnalyzer.java | 4 +-
.../analyze/cache/partition/PartitionCache.java | 10 +-
.../analyze/cache/schema/DataNodeTTLCache.java | 43 +-
.../cache/schema/TimeSeriesSchemaCache.java | 2 +-
.../cache/schema/dualkeycache/impl/CacheStats.java | 2 -
.../schema/dualkeycache/impl/DualKeyCacheImpl.java | 2 +-
.../schema/lastcache/DataNodeLastCacheManager.java | 3 -
.../execution/config/TableConfigTaskVisitor.java | 30 +-
.../config/executor/ClusterConfigTaskExecutor.java | 82 +-
.../config/metadata/GetTimeSlotListTask.java | 6 +-
.../config/metadata/ShowVariablesTask.java | 10 +-
.../config/metadata/relational/ShowTablesTask.java | 37 +-
.../plan/optimization/AggregationPushDown.java | 28 +-
.../plan/planner/LocalExecutionPlanner.java | 13 +-
.../planner/memory/MemoryReservationManager.java | 46 +
.../NotThreadSafeMemoryReservationManager.java | 94 +
.../ThreadSafeMemoryReservationManager.java} | 33 +-
.../plan/planner/plan/node/PlanGraphPrinter.java | 5 +
.../plan/planner/plan/node/PlanNode.java | 14 +-
.../plan/planner/plan/node/PlanVisitor.java | 5 +
.../plan/node/pipe/PipeEnrichedInsertNode.java | 5 -
.../planner/plan/node/write/DeleteDataNode.java | 18 +-
.../plan/planner/plan/node/write/InsertNode.java | 44 +-
.../planner/plan/node/write/InsertRowsNode.java | 11 +
.../plan/planner/plan/node/write/SearchNode.java | 51 +
.../querystats/PlanOptimizersStatsCollector.java | 75 +
.../querystats/QueryPlanOptimizerStatistics.java | 57 +
.../querystats/QueryPlanOptimizerStats.java | 84 +
.../plan/relational/planner/Assignments.java | 20 +
.../relational/planner/ExpressionExtractor.java | 117 ++
.../planner/ExpressionSymbolInliner.java | 65 +
.../plan/relational/planner/LogicalPlanner.java | 88 +-
.../plan/relational/planner/PlanBuilder.java | 4 +-
.../plan/relational/planner/PlanNodeSearcher.java | 226 +++
.../plan/relational/planner/QueryPlanner.java | 4 +-
.../plan/relational/planner/SymbolsExtractor.java | 200 +++
.../plan/relational/planner/ir/IrUtils.java | 5 +
.../GroupReference.java} | 72 +-
.../planner/iterative/IterativeOptimizer.java | 365 ++++
.../plan/relational/planner/iterative/Lookup.java | 65 +
.../plan/relational/planner/iterative/Memo.java | 225 +++
.../plan/relational/planner/iterative/Plans.java | 54 +
.../plan/relational/planner/iterative/Rule.java | 87 +
.../relational/planner/iterative/RuleIndex.java | 76 +
.../relational/planner/iterative/RuleStats.java | 53 +
.../planner/iterative/RuleStatsRecorder.java | 45 +
.../planner/iterative/rule/InlineProjections.java | 220 +++
.../iterative/rule/ProjectOffPushDownRule.java | 67 +
.../planner/iterative/rule/PruneFilterColumns.java | 46 +
.../planner/iterative/rule/PruneLimitColumns.java | 36 +
.../planner/iterative/rule/PruneOffsetColumns.java | 36 +
.../iterative/rule/PruneOutputSourceColumns.java | 43 +
.../iterative/rule/PruneProjectColumns.java | 39 +
.../planner/iterative/rule/PruneSortColumns.java | 44 +
.../iterative/rule/PruneTableScanColumns.java | 102 ++
.../rule/RemoveRedundantIdentityProjections.java | 48 +
.../relational/planner/iterative/rule/Util.java | 137 ++
.../relational/planner/node/ChildReplacer.java | 32 +
.../plan/relational/planner/node/CollectNode.java | 12 +
.../plan/relational/planner/node/FilterNode.java | 6 +
.../plan/relational/planner/node/LimitNode.java | 11 +
.../relational/planner/node/MergeSortNode.java | 18 +
.../plan/relational/planner/node/OffsetNode.java | 6 +
.../plan/relational/planner/node/OutputNode.java | 6 +
.../plan/relational/planner/node/Patterns.java | 429 +++++
.../plan/relational/planner/node/ProjectNode.java | 10 +
.../plan/relational/planner/node/SortNode.java | 6 +
.../relational/planner/node/StreamSortNode.java | 12 +
.../relational/planner/node/TableScanNode.java | 12 +
.../plan/relational/planner/node/TopKNode.java | 22 +
.../optimizations/AdaptivePlanOptimizer.java | 56 +
.../planner/optimizations/PlanOptimizer.java | 77 +
.../planner/optimizations/PruneUnUsedColumns.java | 151 --
.../RemoveRedundantIdentityProjections.java | 97 -
.../plan/relational/utils/MoreLists.java | 53 +
.../plan/relational/utils/matching/Capture.java} | 31 +-
.../plan/relational/utils/matching/Captures.java | 86 +
.../relational/utils/matching/DefaultPrinter.java | 77 +
.../plan/relational/utils/matching/Match.java | 66 +
.../plan/relational/utils/matching/Pattern.java | 120 ++
.../relational/utils/matching/PatternVisitor.java | 46 +
.../plan/relational/utils/matching/Property.java | 89 +
.../relational/utils/matching/PropertyPattern.java | 51 +
.../utils/matching/pattern/CapturePattern.java | 53 +
.../utils/matching/pattern/EqualsPattern.java | 53 +
.../utils/matching/pattern/FilterPattern.java | 55 +
.../utils/matching/pattern/TypeOfPattern.java | 59 +
.../utils/matching/pattern/WithPattern.java | 65 +
.../plan/scheduler/load/LoadTsFileScheduler.java | 28 +-
.../statement/metadata/ShowDatabaseStatement.java | 24 +-
.../mtree/impl/pbtree/schemafile/SchemaFile.java | 10 +-
.../db/schemaengine/table/DataNodeTableCache.java | 140 +-
.../iotdb/db/schemaengine/table/ITableCache.java | 22 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 3 +-
.../iotdb/db/service/metrics/WritingMetrics.java | 186 +-
.../iotdb/db/storageengine/StorageEngine.java | 6 -
.../db/storageengine/dataregion/DataRegion.java | 230 ++-
.../fast/reader/CompactionChunkReader.java | 17 +-
.../readchunk/loader/InstantChunkLoader.java | 12 +-
.../readchunk/loader/InstantPageLoader.java | 7 +-
.../compaction/repair/RepairDataFileScanUtil.java | 23 +-
.../compaction/schedule/CompactionScheduler.java | 5 +-
.../compaction/schedule/CompactionTaskManager.java | 8 +
.../estimator/AbstractCompactionEstimator.java | 6 +-
.../FastCompactionInnerCompactionEstimator.java | 15 +-
.../FastCrossSpaceCompactionEstimator.java | 15 +-
.../ReadChunkInnerCompactionEstimator.java | 19 +-
.../RepairUnsortedFileCompactionEstimator.java | 11 +-
.../dataregion/flush/CompressionRatio.java | 7 +
.../dataregion/memtable/AbstractMemTable.java | 32 +-
.../dataregion/memtable/TsFileProcessor.java | 15 +-
.../read/reader/common/PriorityMergeReader.java | 34 +-
.../storageengine/dataregion/wal/WALManager.java | 32 +-
.../dataregion/wal/buffer/WALBuffer.java | 53 +-
.../dataregion/wal/io/CheckpointWriter.java | 2 +-
.../storageengine/dataregion/wal/io/LogWriter.java | 53 +-
.../dataregion/wal/io/WALFileVersion.java | 61 +
.../dataregion/wal/io/WALInputStream.java | 166 +-
.../dataregion/wal/io/WALMetaData.java | 21 +-
.../storageengine/dataregion/wal/io/WALWriter.java | 36 +-
.../storageengine/dataregion/wal/node/WALNode.java | 18 +-
.../dataregion/wal/recover/WALNodeRecoverTask.java | 76 +-
...{WALRecoverWriter.java => WALRepairWriter.java} | 39 +-
.../file/UnsealedTsFileRecoverPerformer.java | 19 +
.../dataregion/wal/utils/WALEntryPosition.java | 1 -
.../dataregion/wal/utils/WALInsertNodeCache.java | 4 +-
.../agent/SubscriptionBrokerAgent.java | 3 +-
.../agent/SubscriptionConsumerAgent.java | 2 +-
.../subscription/agent/SubscriptionTopicAgent.java | 17 +
.../db/subscription/broker/SubscriptionBroker.java | 27 +-
.../broker/SubscriptionPrefetchingQueue.java | 243 ++-
.../broker/SubscriptionPrefetchingTabletQueue.java | 192 ++
.../SubscriptionPrefetchingTabletsQueue.java | 265 ---
.../broker/SubscriptionPrefetchingTsFileQueue.java | 388 ++--
.../db/subscription/event/SubscriptionEvent.java | 295 +++-
.../event/SubscriptionEventBinaryCache.java | 59 +-
.../event/SubscriptionTsFileEvent.java | 247 ---
.../batch/SubscriptionPipeTabletEventBatch.java | 149 ++
.../batch/SubscriptionPipeTsFileEventBatch.java | 61 +
.../event/pipe/SubscriptionPipeEmptyEvent.java} | 25 +-
.../event/pipe/SubscriptionPipeEvents.java} | 20 +-
.../pipe/SubscriptionPipeTabletBatchEvents.java} | 35 +-
.../pipe/SubscriptionPipeTsFileBatchEvents.java | 82 +
.../pipe/SubscriptionPipeTsFilePlainEvent.java | 56 +
.../receiver/SubscriptionReceiverV1.java | 39 +-
.../java/org/apache/iotdb/db/utils/MmapUtil.java | 8 +
.../pipe/connector/PipeTabletEventSorterTest.java | 218 +++
.../event/TsFileInsertionDataContainerTest.java | 232 ++-
.../PipeDataNodeSnapshotResourceManagerTest.java | 20 +-
.../execution/operator/OperatorMemoryTest.java | 19 +-
.../plan/planner/node/PlanGraphPrinterTest.java | 2 +-
.../plan/relational/analyzer/AnalyzerTest.java | 82 +-
.../analyzer/LimitOffsetPushDownTest.java | 45 +-
.../plan/relational/analyzer/SortTest.java | 1333 ++++----------
.../compaction/AbstractCompactionTest.java | 3 +
...eCompactionWithFastPerformerValidationTest.java | 4 +
...actionWithReadPointPerformerValidationTest.java | 4 +
...eCrossSpaceCompactionWithFastPerformerTest.java | 5 +
.../storageengine/dataregion/wal/WALTestUtils.java | 4 +-
.../wal/compression/WALCompressionTest.java | 39 +-
.../dataregion/wal/node/WALNodeTest.java | 2 +-
...verWriterTest.java => WALRepairWriterTest.java} | 31 +-
.../wal/utils/WALInsertNodeCacheTest.java | 2 +-
.../iotdb/metrics/core/type/IoTDBCounter.java | 4 +-
.../apache/iotdb/metrics/config/MetricConfig.java | 8 +-
iotdb-core/node-commons/pom.xml | 69 +-
.../resources/conf/generate_properties.bat | 65 -
.../assembly/resources/conf/generate_properties.sh | 59 -
.../resources/conf/iotdb-system.properties | 1855 +-------------------
...properties => iotdb-system.properties.template} | 1112 ++++++------
.../AsyncConfigNodeInternalServiceClient.java | 28 +
.../async/AsyncDataNodeInternalServiceClient.java | 28 +
.../client/request/AsyncRequestManager.java | 9 +-
.../{Utils.java => TestConnectionUtils.java} | 37 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 79 +-
.../iotdb/commons/conf/CommonDescriptor.java | 57 +-
.../iotdb/commons/conf/ConfigurationFileUtils.java | 124 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 -
.../consensus/index/impl/RecoverProgressIndex.java | 5 +-
.../commons/file/SystemPropertiesHandler.java | 62 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 5 +
.../config/constant/PipeConnectorConstant.java | 12 +-
.../config/constant/PipeExtractorConstant.java | 6 +-
.../request/PipeConsensusTransferFileSealReq.java | 11 +
.../PipeConsensusTransferFileSealWithModReq.java | 19 +
.../pipe/connector/protocol/IoTDBConnector.java | 40 +-
.../connector/protocol/IoTDBSslSyncConnector.java | 16 +-
.../iotdb/commons/pipe/event/EnrichedEvent.java | 2 +-
.../commons/pipe/pattern/IoTDBPipePattern.java | 4 +
.../commons}/pipe/resource/log/PipeLogManager.java | 7 +-
.../commons}/pipe/resource/log/PipeLogStatus.java | 6 +-
.../pipe/task/connection/BlockingPendingQueue.java | 22 +-
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 19 +-
.../task/subtask/PipeAbstractConnectorSubtask.java | 56 +-
.../apache/iotdb/commons/schema/table/TsTable.java | 18 +-
.../schema/table/TsTableInternalRPCType.java | 30 +-
.../schema/table/TsTableInternalRPCUtil.java | 12 +-
.../table/column/TsTableColumnSchemaUtil.java | 4 +-
.../apache/iotdb/commons/schema/ttl/TTLCache.java | 41 +-
.../iotdb/commons/service/metric/enums/Metric.java | 11 +-
.../subscription/config/SubscriptionConfig.java | 31 +-
.../iotdb/commons/utils/CommonDateTimeUtils.java | 10 +
.../iotdb/commons/utils/TimePartitionUtils.java | 35 +-
.../commons/utils/TimePartitionUtilsTest.java | 92 +
iotdb-doap.rdf | 29 +-
.../src/main/thrift/confignode.thrift | 48 +-
pom.xml | 14 +-
430 files changed, 15691 insertions(+), 8671 deletions(-)
create mode 100644 .github/workflows/daily-it.yml
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java
=>
integration-test/src/main/java/org/apache/iotdb/itbase/category/DailyIT.java
(66%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBTimePartitionIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBJDBCMetadataIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionDataTypeIT.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/resource/PipeConfigNodeResourceManager.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedureTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedureTest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/{builder/PipeEventBatch.java
=> batch/PipeTabletEventPlainBatch.java} (60%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/{builder
=> batch}/PipeTransferBatchReqBuilder.java (55%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/{PipeTransferTsFileInsertionEventHandler.java
=> PipeTransferTsFileHandler.java} (66%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/PipeTabletEventSorter.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainer.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainerProvider.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/{TsFileInsertionDataContainer.java
=> container/query/TsFileInsertionQueryDataContainer.java} (77%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/{TsFileInsertionDataTabletIterator.java
=> container/query/TsFileInsertionQueryDataTabletIterator.java} (89%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/AlignedSinglePageWholeChunkReader.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/SinglePageWholeChunkReader.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/{PipeResourceManager.java
=> PipeDataNodeResourceManager.java} (93%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/{PipeMemoryWeighUtil.java
=> PipeMemoryWeightUtil.java} (56%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/MemoryReservationManager.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/memory/NotThreadSafeMemoryReservationManager.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java
=> queryengine/plan/planner/memory/ThreadSafeMemoryReservationManager.java}
(50%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/SearchNode.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/execution/querystats/PlanOptimizersStatsCollector.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/execution/querystats/QueryPlanOptimizerStatistics.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/execution/querystats/QueryPlanOptimizerStats.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionExtractor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ExpressionSymbolInliner.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanNodeSearcher.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SymbolsExtractor.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/{node/OffsetNode.java
=> iterative/GroupReference.java} (51%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/IterativeOptimizer.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/Lookup.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/Memo.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/Plans.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/Rule.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/RuleIndex.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/RuleStats.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/RuleStatsRecorder.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/InlineProjections.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/ProjectOffPushDownRule.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneFilterColumns.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneLimitColumns.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneOffsetColumns.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneOutputSourceColumns.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneProjectColumns.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneSortColumns.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveRedundantIdentityProjections.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/Util.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/ChildReplacer.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/AdaptivePlanOptimizer.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PlanOptimizer.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PruneUnUsedColumns.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/RemoveRedundantIdentityProjections.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/MoreLists.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{pipe/resource/log/PipeLogManager.java
=> queryengine/plan/relational/utils/matching/Capture.java} (56%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/Captures.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/DefaultPrinter.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/Match.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/Pattern.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/PatternVisitor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/Property.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/PropertyPattern.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/pattern/CapturePattern.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/pattern/EqualsPattern.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/pattern/FilterPattern.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/pattern/TypeOfPattern.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/matching/pattern/WithPattern.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileVersion.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/{WALRecoverWriter.java
=> WALRepairWriter.java} (60%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletQueue.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletsQueue.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionTsFileEvent.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{storageengine/dataregion/wal/io/CheckpointWriter.java
=> subscription/event/pipe/SubscriptionPipeEmptyEvent.java} (68%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{storageengine/dataregion/wal/io/CheckpointWriter.java
=> subscription/event/pipe/SubscriptionPipeEvents.java} (68%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java
=> subscription/event/pipe/SubscriptionPipeTabletBatchEvents.java} (55%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFileBatchEvents.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFilePlainEvent.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeTabletEventSorterTest.java
rename
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/{WALRecoverWriterTest.java
=> WALRepairWriterTest.java} (93%)
delete mode 100644
iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat
delete mode 100644
iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh
copy
iotdb-core/node-commons/src/assembly/resources/conf/{iotdb-system.properties =>
iotdb-system.properties.template} (81%)
rename
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/{Utils.java
=> TestConnectionUtils.java} (71%)
rename iotdb-core/{datanode/src/main/java/org/apache/iotdb/db =>
node-commons/src/main/java/org/apache/iotdb/commons}/pipe/resource/log/PipeLogManager.java
(88%)
rename iotdb-core/{datanode/src/main/java/org/apache/iotdb/db =>
node-commons/src/main/java/org/apache/iotdb/commons}/pipe/resource/log/PipeLogStatus.java
(87%)
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/TimePartitionUtilsTest.java
