This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch ty/TableModelGrammar
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from c00011d0285 Add format
add b93cd0446bc Implement new plan optimizer: PredicatePushDown
add 458efdbaa89 Optimize ConfigNode ConsensusManager init logic (#12098)
add 9f809556e7e Do not wait to retry when configLeader exists (#12075)
add 00b309bb55c Faster repair data (#12039)
new b5b8c42268b fix conflicts
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 2 +-
.../db/it/aligned/IoTDBPredicatePushDown2IT.java | 55 ++
.../db/it/aligned/IoTDBPredicatePushDown3IT.java | 55 ++
.../db/it/aligned/IoTDBPredicatePushDown4IT.java | 55 ++
.../db/it/aligned/IoTDBPredicatePushDownIT.java | 601 +++++++++++++++++++++
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 13 +-
.../confignode/client/DataNodeRequestType.java | 3 +-
.../client/async/AsyncDataNodeClientPool.java | 9 +-
.../client/async/handlers/AsyncClientHandler.java | 3 +-
.../statemachine/ConfigRegionStateMachine.java | 6 +-
.../iotdb/confignode/manager/ClusterManager.java | 10 -
.../iotdb/confignode/manager/ConfigManager.java | 13 +-
.../apache/iotdb/confignode/manager/IManager.java | 5 +-
.../manager/consensus/ConsensusManager.java | 58 +-
.../iotdb/confignode/manager/cq/CQManager.java | 11 -
.../iotdb/confignode/manager/node/NodeManager.java | 18 +-
.../pipe/coordinator/runtime/PipeMetaSyncer.java | 10 -
.../thrift/ConfigNodeRPCServiceProcessor.java | 9 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 5 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 19 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 24 +-
.../protocol/writeback/WriteBackConnector.java | 2 +-
.../legacy/IoTDBLegacyPipeReceiverAgent.java | 2 +-
.../receiver/legacy/loader/DeletionLoader.java | 2 +-
.../pipe/receiver/legacy/loader/TsFileLoader.java | 2 +-
.../receiver/thrift/IoTDBThriftReceiverV1.java | 2 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 47 +-
.../protocol/rest/filter/AuthorizationFilter.java | 2 +-
.../iotdb/db/protocol/session/SessionManager.java | 11 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 8 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 16 +-
.../db/queryengine/common/MPPQueryContext.java | 5 +
.../iotdb/db/queryengine/common/SessionInfo.java | 16 +-
.../fragment/FragmentInstanceContext.java | 2 +-
.../operator/process/ProjectOperator.java | 101 ++++
.../db/queryengine/plan/analyze/Analysis.java | 15 +
.../queryengine/plan/analyze/AnalyzeVisitor.java | 87 +--
.../plan/analyze/ExpressionTypeAnalyzer.java | 10 +
.../queryengine/plan/analyze/ExpressionUtils.java | 11 +
.../queryengine/plan/analyze/PredicateUtils.java | 155 +++---
.../queryengine/plan/analyze/TemplatedAnalyze.java | 20 +-
.../db/queryengine/plan/analyze/TemplatedInfo.java | 168 +++---
.../queryengine/plan/execution/QueryExecution.java | 6 +-
.../plan/execution/config/ConfigTaskVisitor.java | 18 +-
.../config/executor/ClusterConfigTaskExecutor.java | 34 +-
.../config/executor/IConfigTaskExecutor.java | 4 +-
...epairDataTask.java => StartRepairDataTask.java} | 12 +-
...RepairDataTask.java => StopRepairDataTask.java} | 12 +-
.../memory/StatementMemorySourceVisitor.java | 8 +-
.../plan/expression/ExpressionFactory.java | 35 ++
.../plan/expression/leaf/ConstantOperand.java | 15 +-
.../plan/expression/ternary/BetweenExpression.java | 6 +-
.../BindTypeForTimeSeriesOperandVisitor.java | 13 -
.../visitor/ExpressionNormalizeVisitor.java | 13 +-
.../visitor/GetMeasurementExpressionVisitor.java | 18 -
.../visitor/LowercaseNormalizeVisitor.java | 13 +-
.../expression/visitor/ReconstructVisitor.java | 33 ++
.../ReplaceRawPathWithGroupedPathVisitor.java | 23 -
.../visitor/ReplaceSubTreeWithViewVisitor.java | 16 -
.../visitor/logical/LogicalAndVisitor.java | 70 +++
.../visitor/logical/LogicalOrVisitor.java | 70 +++
.../PredicateCanPushDownToSourceChecker.java} | 21 +-
.../visitor/logical/TimeFilterExistChecker.java} | 17 +-
.../predicate/ConvertPredicateToFilterVisitor.java | 406 ++++++++++++++
.../ConvertPredicateToTimeFilterVisitor.java | 30 +-
.../predicate/PredicatePushIntoScanChecker.java | 171 ++++++
.../visitor/predicate/PredicateSimplifier.java | 279 ++++++++++
.../plan/optimization/PredicatePushDown.java | 468 ++++++++++++++++
.../db/queryengine/plan/parser/ASTVisitor.java | 35 +-
.../plan/parser/StatementGenerator.java | 13 +-
.../plan/planner/LocalExecutionPlanContext.java | 14 +
.../plan/planner/LogicalPlanBuilder.java | 61 +--
.../plan/planner/LogicalPlanVisitor.java | 15 +-
.../queryengine/plan/planner/LogicalPlanner.java | 7 +-
.../plan/planner/OperatorTreeGenerator.java | 386 ++++++-------
.../plan/planner/SubPlanTypeExtractor.java | 11 +
.../plan/planner/TemplatedLogicalPlan.java | 7 +-
.../plan/planner/TemplatedLogicalPlanBuilder.java | 4 +-
.../planner/distribution/ExchangeNodeAdder.java | 6 +
.../plan/planner/distribution/SourceRewriter.java | 27 +-
.../plan/planner/plan/node/PlanGraphPrinter.java | 49 +-
.../plan/planner/plan/node/PlanNodeType.java | 2 +
.../plan/planner/plan/node/PlanVisitor.java | 9 +-
.../plan/planner/plan/node/process/FilterNode.java | 16 +-
.../planner/plan/node/process/ProjectNode.java | 25 +-
.../planner/plan/node/process/TransformNode.java | 25 +-
.../plan/node/process/join/InnerTimeJoinNode.java | 24 +-
.../node/process/join/LeftOuterTimeJoinNode.java | 4 +-
.../plan/node/source/AlignedSeriesScanNode.java | 128 +----
.../planner/plan/node/source/SeriesScanNode.java | 119 +---
...riesScanNode.java => SeriesScanSourceNode.java} | 163 ++----
.../queryengine/plan/statement/StatementType.java | 3 +-
.../plan/statement/StatementVisitor.java | 11 +-
.../plan/statement/component/SelectComponent.java | 11 +-
.../metadata/CreateContinuousQueryStatement.java | 4 -
.../plan/statement/sys/ShowQueriesStatement.java | 11 -
...tatement.java => StartRepairDataStatement.java} | 6 +-
...Statement.java => StopRepairDataStatement.java} | 6 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 4 +
.../metrics/IoTDBInternalLocalReporter.java | 2 +-
.../iotdb/db/storageengine/StorageEngine.java | 48 +-
.../db/storageengine/dataregion/DataRegion.java | 98 ++--
.../CompactionValidationFailedException.java | 15 +
.../execute/task/AbstractCompactionTask.java | 62 ++-
.../task/InsertionCrossSpaceCompactionTask.java | 4 +-
.../compaction/repair/RepairDataFileScanUtil.java | 271 ++++++++++
.../dataregion/compaction/repair/RepairLogger.java | 146 +++--
.../RepairProgress.java} | 23 +-
.../repair/RepairTaskRecoverLogParser.java | 30 +-
.../compaction/repair/RepairTaskStatus.java} | 13 +-
.../compaction/repair/RepairTimePartition.java | 4 +
.../repair/RepairTimePartitionScanTask.java | 202 +++++++
.../repair/UnsortedFileRepairTaskScheduler.java | 316 ++++-------
.../schedule/CompactionScheduleTaskManager.java | 317 +++++++++++
.../schedule/CompactionScheduleTaskWorker.java | 69 +++
.../compaction/schedule/CompactionScheduler.java | 101 ++--
.../compaction/schedule/CompactionTaskManager.java | 6 +-
.../compaction/schedule/CompactionTaskQueue.java | 101 ++++
.../compaction/schedule/CompactionWorker.java | 60 +-
.../dataregion/utils/TsFileResourceUtils.java | 2 +-
.../db/storageengine/rescon/memory/SystemInfo.java | 88 ++-
.../datastructure/FixedPriorityBlockingQueue.java | 12 +-
.../predicate/PredicatePushIntoScanTest.java | 140 +++++
.../predicate/PredicateRemoveNotTest.java | 2 +-
.../expression/predicate/PredicateUtilsTest.java | 77 +++
.../db/queryengine/plan/analyze/AnalyzeTest.java | 10 +-
.../optimization/ColumnInjectionPushDownTest.java | 51 +-
.../plan/optimization/LimitOffsetPushDownTest.java | 38 +-
.../plan/optimization/OptimizationTestUtil.java | 71 ++-
.../plan/optimization/PredicatePushDownTest.java | 449 +++++++++++++++
.../plan/optimization/TestPlanBuilder.java | 94 +++-
.../plan/parser/StatementGeneratorTest.java | 6 +-
.../plan/planner/FragmentInstanceSerdeTest.java | 3 +-
.../queryengine/plan/planner/QueryPlannerTest.java | 106 ----
.../distribution/AggregationAlignByDeviceTest.java | 34 +-
.../AlignByDeviceOrderByLimitOffsetTest.java | 242 ++++-----
.../planner/distribution/AlignedByDeviceTest.java | 375 +++++++++++--
.../plan/planner/distribution/Util.java | 2 +-
.../plan/planner/distribution/Util2.java | 2 +-
.../DataQueryLogicalPlannerTest.java} | 217 +++++---
.../logical/LogicalPlannerTestUtil.java} | 53 +-
.../SchemaQueryLogicalPlannerTest.java} | 116 +---
.../planner/node/process/FilterNodeSerdeTest.java | 2 -
.../node/process/TimeJoinNodeSerdeTest.java | 19 +-
.../plan/relational/analyzer/AnalyzerTest.java | 2 +-
.../builder/EvaluationDAGBuilderTest.java | 3 +-
.../compaction/CompactionOverlapCheckTest.java | 14 +
.../compaction/CompactionWorkerTest.java | 103 +++-
.../FastCrossCompactionPerformerTest.java | 22 +-
.../cross/CrossSpaceCompactionSelectorTest.java | 38 +-
.../cross/InsertionCrossSpaceCompactionTest.java | 13 +-
.../compaction/inner/InnerCompactionLogTest.java | 2 +-
.../inner/InnerCompactionMoreDataTest.java | 3 +-
.../SizeTieredCompactionSelectorTest.java | 38 --
.../compaction/repair/AbstractRepairDataTest.java | 66 +++
.../repair/RepairDataFileScanUtilTest.java | 138 +++++
.../compaction/repair/RepairLoggerTest.java | 73 ++-
.../repair/RepairUnsortedFileCompactionTest.java | 84 ++-
.../compaction/utils/CompactionTaskQueueTest.java | 201 +++++++
.../resources/conf/iotdb-common.properties | 4 +
.../iotdb/commons/concurrent/ThreadName.java | 1 +
.../threadpool/WrappedThreadPoolExecutor.java | 2 +-
.../org/apache/iotdb/commons/path/AlignedPath.java | 8 +
.../apache/iotdb/commons/service/ServiceType.java | 2 +
.../tsfile/read/filter/basic/ValueFilter.java | 4 +
.../tsfile/read/filter/factory/ValueFilterApi.java | 8 +
.../src/main/thrift/confignode.thrift | 5 +-
.../src/main/thrift/datanode.thrift | 4 +-
168 files changed, 7234 insertions(+), 2388 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDown2IT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDown3IT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDown4IT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBPredicatePushDownIT.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ProjectOperator.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/{RepairDataTask.java
=> StartRepairDataTask.java} (75%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/{RepairDataTask.java
=> StopRepairDataTask.java} (75%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/logical/LogicalAndVisitor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/logical/LogicalOrVisitor.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{storageengine/dataregion/compaction/execute/exception/CompactionValidationFailedException.java
=>
queryengine/plan/expression/visitor/logical/PredicateCanPushDownToSourceChecker.java}
(54%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{storageengine/dataregion/compaction/execute/exception/CompactionValidationFailedException.java
=> queryengine/plan/expression/visitor/logical/TimeFilterExistChecker.java}
(61%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToFilterVisitor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/PredicatePushIntoScanChecker.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/PredicateSimplifier.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDown.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/{SeriesScanNode.java
=> SeriesScanSourceNode.java} (51%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/{RepairDataStatement.java
=> StartRepairDataStatement.java} (89%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/{RepairDataStatement.java
=> StopRepairDataStatement.java} (89%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/{execute/exception/CompactionValidationFailedException.java
=> repair/RepairProgress.java} (62%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{exception/sql/MeasurementNotExistException.java
=> storageengine/dataregion/compaction/repair/RepairTaskStatus.java} (75%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartitionScanTask.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleTaskManager.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleTaskWorker.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskQueue.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/PredicatePushIntoScanTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/PredicateUtilsTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDownTest.java
delete mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/QueryPlannerTest.java
rename
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/{QueryLogicalPlanUtil.java
=> logical/DataQueryLogicalPlannerTest.java} (90%)
copy
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/{optimization/OptimizationTestUtil.java
=> planner/logical/LogicalPlannerTestUtil.java} (53%)
rename
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/{LogicalPlannerTest.java
=> logical/SchemaQueryLogicalPlannerTest.java} (87%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/AbstractRepairDataTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtilTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTaskQueueTest.java
