This is an automated email from the ASF dual-hosted git repository.
lancelly pushed a change to branch support_uncorrelated_quantified_comparison
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 44dab8ecc27 remove todo and merge master
add d87f9342fab Pipe: Fix the problem that the Pipe indicator is
inaccurate and negative (#14627)
add 992f18e0639 Fix that place-holder tag column are not Binary[] (#14621)
add c2673ffd031 Reduce some deletion-related log levels (#14631)
add 7bbf27df0ac Make show-queries can execute on more status node
add eb5d1df9a4e Load: Convert TsFiles into Tablets when the target regions
are unavailable (#14626)
add 2af3d029a7a Add aggregate function count_if for relational table
add 95451e45583 Pipe IT: Add table model testPipeAfterDataRegionLeaderStop
retry logic and extend IoTDBPipeTypeConversionIT retry time (#14628)
add 51c92c3620c Compaction task memory is incorrect when
compaction_max_aligned_series_num_in_one_batch <= 0 (#14603)
add 4c3f64581ce Pipe IT: Save IT operation logs to the cluster-logs
directory (#14637)
add cfc15dec90a Update Python examples (#14640)
add 5df241c044b [Table Model] Refactor UDSF/UDAF interface
add 2669666d8d2 [To dev/1.3]Script adaptation to 4 new data types (#14296)
(#14630)
add f007e8a1407 Pipe: Support 'format'='tsfile' option in table model data
sync sinks (#14108)
add 6db2249c12c Load: Add LoadWithConvertOnTypeMismatchIT on Table Model
&& Fix TSInsertTabletReq.writeToTable is not set True in
PipeTransferTabletRawReq.toTPipeTransferRawReq when inserting a Tablet in Table
mode (#14611)
add 487a1a21ce9 Pipe & Load: Retry when a file operation causes an
exception (#14598)
add d60c3d52cb1 IoTConsensusV2: Fix adoption for table-deletion #14648
add 90cdf3bbc94 Load: Consider isGeneratedByPipe mark when executing
tsfile-tablet conversion in analysis stage (#14651)
add d9214b31465 Rename interface name of FunctionArguments
add 5f8b2c08ac9 Adjust default wal_throttle_threshold_in_byte and
region_migration_speed_limit_bytes_per_second (#14653)
add 815ea9c069d Implemented query for schema related Information schema &
Fixed the bug that the deviceEntry get from cache does not reserve query memory
add a4ce6f7a1a3 Modify the condition to skip compaction schdule after
insertion compaction task selection (#14644)
add c087c24435c Do not add the source path of a view when it can be
matched by the deletion pattern (#14658)
add c60520aa41a Load: Add a new configuration to enable datatype
conversion when converting into Tablets & Refactor LoadAnalyzeException.
(#14646)
add debdb527d89 Reduced the error log cause by batch activate template
when some devices are already activated (#14661)
add a54573a2afb Use DataNodeDevicePathCache when insert by sql (#14662)
add 5f01571f4c6 Pipe: tree/table model isolation for
alter/start/stop/drop/show pipe operations & support 'mode.double-living'
(#14386)
add 7dc5179def6 Fix select-distinct when there is no order by and select
items contains expression
add e92be6f63ab [Table Model Subquery] Support uncorrelated in predicate
(#14438)
new 981fdb6a58a fix conflicts
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/pipe-it-2cluster.yml | 18 +-
example/udf/pom.xml | 6 +-
.../apache/iotdb/udf/AggregateFunctionExample.java | 35 +-
.../apache/iotdb/udf/ScalarFunctionExample.java | 57 +--
.../db/query/udf/example/relational/AllSum.java | 49 +-
.../query/udf/example/relational/ContainNull.java | 21 +-
.../relational/{DatePlusOne.java => DatePlus.java} | 31 +-
.../query/udf/example/relational/FirstTwoSum.java | 33 +-
.../db/query/udf/example/relational/MyAvg.java | 32 +-
.../db/query/udf/example/relational/MyCount.java | 19 +-
.../iotdb/it/utils/TsFileTableGenerator.java | 202 ++++++++
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 80 ++-
.../iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java | 3 +-
.../it/autocreate/IoTDBPipeSwitchStatusIT.java | 23 +-
.../pipe/it/manual/IoTDBPipeTypeConversionIT.java | 2 +-
.../iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java | 30 +-
.../pipe/it/tablemodel/IoTDBPipeClusterIT.java | 290 ++++++-----
.../pipe/it/tablemodel/IoTDBPipeDataSinkIT.java | 429 +++++++++++++++-
.../it/tablemodel/IoTDBPipeDoubleLivingIT.java | 329 ++++++++++++
.../pipe/it/tablemodel/IoTDBPipeIsolationIT.java | 377 ++++++++++++++
.../pipe/it/tablemodel/IoTDBPipeNullValueIT.java | 4 +-
.../it/tablemodel/IoTDBPipeSwitchStatusIT.java | 23 +-
.../pipe/it/tablemodel/IoTDBPipeSyntaxIT.java | 33 +-
.../pipe/it/tablemodel/IoTDBPipeWithLoadIT.java | 8 +-
.../iotdb/pipe/it/tablemodel/TableModelUtils.java | 563 +++++++++++++++++----
.../it/db/it/udf/IoTDBSQLFunctionManagementIT.java | 19 +-
.../it/udf/IoTDBUserDefinedScalarFunctionIT.java | 2 +-
.../it/query/recent/IoTDBTableAggregationIT.java | 339 +++++++++++++
.../IoTDBUncorrelatedInPredicateSubqueryIT.java | 2 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 41 ++
.../it/session/IoTDBSessionRelationalIT.java | 156 ++++--
.../analysis/AggregateFunctionAnalysis.java | 62 +++
.../api/customizer/analysis/FunctionAnalysis.java | 15 +-
.../analysis/ScalarFunctionAnalysis.java | 51 ++
...ctionParameters.java => FunctionArguments.java} | 28 +-
.../exception/UDFArgumentNotValidException.java | 10 +-
.../udf/api/relational/AggregateFunction.java | 48 +-
.../iotdb/udf/api/relational/ScalarFunction.java | 47 +-
.../apache/iotdb/tool/data/AbstractDataTool.java | 32 +-
.../org/apache/iotdb/tool/data/ExportData.java | 43 +-
.../org/apache/iotdb/tool/data/ImportData.java | 24 +-
.../client-py/table_model_session_example.py | 12 +-
.../client-py/table_model_session_pool_example.py | 8 +-
.../payload/SubscriptionFileHandler.java | 24 +-
.../org/apache/iotdb/session/util/RetryUtils.java | 30 +-
.../consensus/request/ConfigPhysicalPlanType.java | 2 +
.../table/DescTable4InformationSchemaPlan.java} | 12 +-
.../table/ShowTable4InformationSchemaPlan.java} | 11 +-
.../response/pipe/task/PipeTableResp.java | 7 +
.../table/DescTable4InformationSchemaResp.java | 42 ++
.../table/ShowTable4InformationSchemaResp.java | 44 ++
.../iotdb/confignode/manager/ConfigManager.java | 32 +-
.../apache/iotdb/confignode/manager/IManager.java | 31 +-
.../iotdb/confignode/manager/load/LoadManager.java | 11 +
.../confignode/manager/load/cache/LoadCache.java | 17 +
.../iotdb/confignode/manager/node/NodeManager.java | 13 +
.../pipe/coordinator/task/PipeTaskCoordinator.java | 74 ++-
.../manager/schema/ClusterSchemaManager.java | 32 ++
.../persistence/executor/ConfigPlanExecutor.java | 4 +
.../confignode/persistence/pipe/PipeTaskInfo.java | 21 +-
.../persistence/schema/ClusterSchemaInfo.java | 77 +++
.../confignode/persistence/schema/ConfigMTree.java | 16 +
.../impl/pipe/task/AlterPipeProcedureV2.java | 4 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 32 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +-
.../schemaregion/SchemaExecutionVisitor.java | 15 +-
.../LoadAnalyzeException.java} | 8 +-
.../LoadAnalyzeTableColumnDisorderException.java} | 7 +-
.../LoadAnalyzeTypeMismatchException.java} | 6 +-
.../batch/PipeTabletEventTsFileBatch.java | 384 +++-----------
.../request/PipeTransferTabletRawReq.java | 4 +-
.../request/PipeTransferTabletRawReqV2.java | 9 +-
.../connector/protocol/opcua/OpcUaNameSpace.java | 8 +-
.../request/PipeConsensusDeleteNodeReq.java | 4 +-
.../async/IoTDBDataRegionAsyncConnector.java | 10 +-
.../PipeTransferTabletBatchEventHandler.java | 2 +-
.../async/handler/PipeTransferTsFileHandler.java | 13 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 19 +-
.../util/builder/PipeTableModeTsFileBuilder.java | 273 ++++++++++
.../util/builder/PipeTreeModelTsFileBuilder.java | 268 ++++++++++
.../connector/util/builder/PipeTsFileBuilder.java | 162 ++++++
.../util/{ => cacher}/LeaderCacheUtils.java | 2 +-
.../sorter/PipeTableModelTabletEventSorter.java | 271 ++++++++++
.../util/{ => sorter}/PipeTabletEventSorter.java | 107 +---
.../sorter/PipeTreeModelTabletEventSorter.java | 121 +++++
.../dataregion/IoTDBDataRegionExtractor.java | 20 +-
.../PipeDataNodeRemainingEventAndTimeOperator.java | 20 +-
.../pipeconsensus/PipeConsensusReceiver.java | 31 +-
.../protocol/thrift/IoTDBDataNodeReceiver.java | 7 +-
.../PipeConvertedInsertTabletStatement.java | 4 +
...leStatementDataTypeConvertExecutionVisitor.java | 7 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 32 +-
.../InformationSchemaContentSupplierFactory.java | 372 ++++++++++++--
.../relational/aggregation/AccumulatorFactory.java | 23 +-
.../relational/aggregation/CountIfAccumulator.java | 101 ++++
.../UserDefinedAggregateFunctionAccumulator.java | 27 +-
.../grouped/GroupedCountIfAccumulator.java | 79 +++
.../relational/ColumnTransformerBuilder.java | 14 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 4 +
.../db/queryengine/plan/analyze/Analysis.java | 10 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 57 ++-
.../plan/analyze/load/LoadTsFileAnalyzer.java | 44 +-
.../analyze/load/LoadTsFileTableSchemaCache.java | 14 +-
.../load/LoadTsFileToTableModelAnalyzer.java | 27 +-
.../load/LoadTsFileToTreeModelAnalyzer.java | 17 +-
.../load/TreeSchemaAutoCreatorAndVerifier.java | 30 +-
.../config/executor/ClusterConfigTaskExecutor.java | 20 +-
.../config/metadata/relational/ShowDBTask.java | 4 +-
.../execution/config/sys/pipe/AlterPipeTask.java | 8 +-
.../execution/config/sys/pipe/DropPipeTask.java | 7 +-
.../execution/config/sys/pipe/ShowPipeTask.java | 3 +-
.../execution/config/sys/pipe/StartPipeTask.java | 7 +-
.../execution/config/sys/pipe/StopPipeTask.java | 7 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 9 +-
.../plan/planner/LogicalPlanBuilder.java | 2 +-
.../plan/planner/TableOperatorGenerator.java | 35 +-
.../plan/planner/plan/node/PlanVisitor.java | 2 +-
.../relational/analyzer/StatementAnalyzer.java | 12 +-
.../plan/relational/metadata/Metadata.java | 4 +-
.../relational/metadata/TableMetadataImpl.java | 42 +-
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 10 +-
.../fetcher/TableHeaderSchemaValidator.java | 8 +-
.../plan/relational/planner/QueryPlanner.java | 2 +-
.../plan/relational/planner/node/SemiJoinNode.java | 28 +
.../DataNodeLocationSupplierFactory.java | 51 +-
.../sql/ast/AbstractQueryDeviceWithCache.java | 2 +
.../plan/relational/sql/ast/AlterPipe.java | 22 +-
.../plan/relational/sql/ast/AstVisitor.java | 4 +
.../sql/ast/{DropPipe.java => CountStatement.java} | 55 +-
.../plan/relational/sql/ast/DropPipe.java | 10 +-
.../plan/relational/sql/ast/LoadTsFile.java | 14 +-
.../plan/relational/sql/ast/ShowPipes.java | 10 +-
.../plan/relational/sql/ast/StartPipe.java | 8 +-
.../plan/relational/sql/ast/StopPipe.java | 8 +-
.../plan/relational/sql/parser/AstBuilder.java | 36 +-
.../plan/relational/sql/rewrite/ShowRewrite.java | 51 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 80 ++-
.../plan/statement/crud/LoadTsFileStatement.java | 9 +
.../metadata/pipe/AlterPipeStatement.java | 31 +-
.../statement/metadata/pipe/DropPipeStatement.java | 23 +-
.../metadata/pipe/ShowPipesStatement.java | 18 +-
.../metadata/pipe/StartPipeStatement.java | 17 +-
.../statement/metadata/pipe/StopPipeStatement.java | 17 +-
.../schemaengine/table/InformationSchemaUtils.java | 9 +-
.../db/storageengine/dataregion/DataRegion.java | 135 ++---
.../CompactionSourceFileDeletedException.java} | 13 +-
.../execute/task/InnerSpaceCompactionTask.java | 6 +-
.../schedule/CompactionScheduleContext.java | 21 +
.../compaction/schedule/CompactionScheduler.java | 60 ++-
.../estimator/AbstractCrossSpaceEstimator.java | 7 +-
.../estimator/AbstractInnerSpaceEstimator.java | 8 +-
.../estimator/CompactionEstimateUtils.java | 35 +-
.../FastCompactionInnerCompactionEstimator.java | 16 +-
.../FastCrossSpaceCompactionEstimator.java | 17 +-
.../selector/estimator/MetadataInfo.java | 42 ++
.../ReadChunkInnerCompactionEstimator.java | 15 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 16 +-
.../db/storageengine/load/LoadTsFileManager.java | 21 +-
.../load/active/ActiveLoadDirScanner.java | 27 +-
.../load/active/ActiveLoadTsFileLoader.java | 13 +-
.../LoadConvertedInsertTabletStatement.java | 18 +-
...leStatementDataTypeConvertExecutionVisitor.java | 10 +-
...eeStatementDataTypeConvertExecutionVisitor.java | 3 +-
.../converter/LoadTsFileDataTypeConverter.java | 103 ++--
.../load/metrics/LoadTsFileCostMetricsSet.java | 13 +-
.../batch/SubscriptionPipeTsFileEventBatch.java | 11 +-
.../org/apache/iotdb/db/utils/CommonUtils.java | 6 -
.../pipe/connector/PipeTabletEventSorterTest.java | 210 +++++++-
.../cross/CrossSpaceCompactionSelectorTest.java | 102 +---
.../InsertionCrossSpaceCompactionSelectorTest.java | 4 +-
.../cross/InsertionCrossSpaceCompactionTest.java | 64 ++-
.../utils/CompactionTaskMemCostEstimatorTest.java | 48 ++
.../conf/iotdb-system.properties.template | 6 +-
.../apache/iotdb/commons/cluster/NodeStatus.java | 13 +
.../commons/pipe/agent/task/meta/PipeMeta.java | 40 ++
.../pipe/agent/task/meta/PipeMetaKeeper.java | 8 +
.../config/constant/PipeExtractorConstant.java | 3 +
.../pipe/datastructure/pattern/TablePattern.java | 21 +-
.../pipe/datastructure/pattern/TreePattern.java | 22 +-
.../commons/pipe/extractor/IoTDBExtractor.java | 65 ++-
.../commons/pipe/receiver/IoTDBFileReceiver.java | 17 +-
.../commons/pipe/receiver/IoTDBReceiverAgent.java | 7 +-
.../snapshot/PipeSnapshotResourceManager.java | 9 +-
.../TableBuiltinAggregationFunction.java | 2 +
.../org/apache/iotdb/commons/utils/FileUtils.java | 49 ++
.../apache/iotdb/commons/utils/RetryUtils.java} | 30 +-
.../src/main/thrift/confignode.thrift | 42 +-
187 files changed, 7047 insertions(+), 1825 deletions(-)
rename
integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/{DatePlusOne.java
=> DatePlus.java} (55%)
create mode 100644
integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileTableGenerator.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java
create mode 100644
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/AggregateFunctionAnalysis.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java
=>
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/FunctionAnalysis.java
(72%)
create mode 100644
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/analysis/ScalarFunctionAnalysis.java
rename
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/parameter/{FunctionParameters.java
=> FunctionArguments.java} (73%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java
=>
iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/exception/UDFArgumentNotValidException.java
(73%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java
=>
iotdb-client/session/src/main/java/org/apache/iotdb/session/util/RetryUtils.java
(58%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java
=>
confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/DescTable4InformationSchemaPlan.java}
(66%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java
=>
confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTable4InformationSchemaPlan.java}
(66%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/DescTable4InformationSchemaResp.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/ShowTable4InformationSchemaResp.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/{VerifyMetadataException.java
=> load/LoadAnalyzeException.java} (82%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/{VerifyMetadataTypeMismatchException.java
=> load/LoadAnalyzeTableColumnDisorderException.java} (78%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/{VerifyMetadataTypeMismatchException.java
=> load/LoadAnalyzeTypeMismatchException.java} (83%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModeTsFileBuilder.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTreeModelTsFileBuilder.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTsFileBuilder.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/{ =>
cacher}/LeaderCacheUtils.java (97%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTableModelTabletEventSorter.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/{ =>
sorter}/PipeTabletEventSorter.java (56%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTreeModelTabletEventSorter.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountIfAccumulator.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedCountIfAccumulator.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{DropPipe.java
=> CountStatement.java} (51%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{exception/VerifyMetadataTypeMismatchException.java
=>
storageengine/dataregion/compaction/execute/exception/CompactionSourceFileDeletedException.java}
(72%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/MetadataInfo.java
rename
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java
=> node-commons/src/main/java/org/apache/iotdb/commons/utils/RetryUtils.java}
(58%)
