This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch TableModelIngestion in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a8cb529600282a7f3562006fef3a7c15282209a1 Merge: 6c24a6f9ec6 50d8708f2ec Author: DESKTOP-L0L5GPJ\jt <[email protected]> AuthorDate: Wed Jun 12 14:58:07 2024 +0800 Merge branch 'master' into ty/TableModelGrammarTest # Conflicts: # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java distribution/src/assembly/all.xml | 9 + distribution/src/assembly/confignode.xml | 9 + distribution/src/assembly/datanode.xml | 9 + .../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 2 +- .../iotdb/db/it/IoTDBSetConfigurationIT.java | 89 +++++++ .../java/org/apache/iotdb/db/it/IoTDBSettleIT.java | 2 +- .../org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java | 5 +- .../db/it/aggregation/IoTDBAggregationIT.java | 15 +- .../db/it/aggregation/maxby/IoTDBMaxByIT.java | 3 +- .../db/it/aggregation/minby/IoTDBMinByIT.java | 3 +- .../iotdb/db/it/udf/IoTDBUDFWindowQuery2IT.java | 216 ++++++++++++++++ .../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 4 +- .../java/org/apache/iotdb/cli/AbstractCli.java | 34 ++- .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 8 + .../org/apache/iotdb/rpc/IoTDBJDBCDataSet.java | 4 +- .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 4 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 11 +- .../assembly/resources/sbin/stop-confignode.bat | 19 +- .../confignode/client/ConfigNodeRequestType.java | 4 +- .../confignode/client/DataNodeRequestType.java | 2 + .../client/async/AsyncDataNodeClientPool.java | 7 + .../client/sync/SyncConfigNodeClientPool.java | 3 + .../client/sync/SyncDataNodeClientPool.java | 8 +- .../confignode/conf/ConfigNodeDescriptor.java | 6 +- .../iotdb/confignode/manager/ConfigManager.java | 52 ++++ .../apache/iotdb/confignode/manager/IManager.java | 8 + .../iotdb/confignode/manager/node/NodeManager.java | 107 +++++++- .../manager/partition/PartitionManager.java | 2 +- .../persistence/pipe/PipePluginInfo.java | 39 +-- .../procedure/env/ConfigNodeProcedureEnv.java | 22 +- .../procedure/env/RegionMaintainHandler.java | 78 +++--- .../impl/sync/AuthOperationProcedure.java | 11 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 12 + .../src/assembly/resources/sbin/stop-datanode.bat | 24 +- .../org/apache/iotdb/db/audit/AuditLogger.java | 1 + .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 13 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 14 ++ .../protocol/thrift/impl/ClientRPCServiceImpl.java | 71 +++++- .../impl/DataNodeInternalRPCServiceImpl.java | 22 ++ .../db/queryengine/plan/analyze/Analysis.java | 11 + .../db/queryengine/plan/analyze/IAnalysis.java | 15 ++ .../plan/analyze/LoadTsfileAnalyzer.java | 8 + .../plan/execution/config/ConfigTaskVisitor.java | 8 + .../config/executor/ClusterConfigTaskExecutor.java | 87 +++++-- .../config/executor/IConfigTaskExecutor.java | 3 + .../execution/config/sys/SetConfigurationTask.java | 45 ++++ .../db/queryengine/plan/parser/ASTVisitor.java | 20 ++ .../distribution/WriteFragmentParallelPlanner.java | 11 +- .../plan/planner/plan/node/WritePlanNode.java | 4 +- .../plan/node/load/LoadSingleTsFileNode.java | 4 +- .../planner/plan/node/load/LoadTsFileNode.java | 9 +- .../plan/node/load/LoadTsFilePieceNode.java | 4 +- .../node/metedata/write/ActivateTemplateNode.java | 4 +- .../node/metedata/write/AlterTimeSeriesNode.java | 4 +- .../metedata/write/BatchActivateTemplateNode.java | 4 +- .../write/CreateAlignedTimeSeriesNode.java | 4 +- .../metedata/write/CreateMultiTimeSeriesNode.java | 4 +- .../node/metedata/write/CreateTimeSeriesNode.java | 4 +- .../write/InternalBatchActivateTemplateNode.java | 4 +- .../write/InternalCreateMultiTimeSeriesNode.java | 4 +- .../write/InternalCreateTimeSeriesNode.java | 4 +- .../metedata/write/view/CreateLogicalViewNode.java | 4 +- .../plan/node/pipe/PipeEnrichedDeleteDataNode.java | 4 +- .../plan/node/pipe/PipeEnrichedInsertNode.java | 4 +- .../plan/node/pipe/PipeEnrichedWritePlanNode.java | 4 +- .../planner/plan/node/write/DeleteDataNode.java | 5 +- .../plan/node/write/InsertMultiTabletsNode.java | 4 +- .../planner/plan/node/write/InsertRowNode.java | 4 +- .../planner/plan/node/write/InsertRowsNode.java | 4 +- .../plan/node/write/InsertRowsOfOneDeviceNode.java | 4 +- .../planner/plan/node/write/InsertTabletNode.java | 4 +- .../plan/relational/analyzer/Analysis.java | 28 +++ .../metadata/ITableDeviceSchemaValidation.java | 66 +++++ .../plan/relational/metadata/Metadata.java | 29 +++ .../relational/metadata/TableMetadataImpl.java | 13 + .../distribute/TableDistributionPlanner.java | 5 +- ...or.java => TableModelQueryFragmentPlanner.java} | 7 +- .../queryengine/plan/statement/StatementType.java | 1 + .../plan/statement/StatementVisitor.java | 5 + .../statement/sys/SetConfigurationStatement.java | 72 ++++++ .../dag/intermediate/MultiInputLayer.java | 5 +- .../SingleInputMultiReferenceLayer.java | 12 +- .../SingleInputSingleReferenceLayer.java | 12 +- .../iotdb/db/storageengine/StorageEngine.java | 42 ++++ .../org/apache/iotdb/db/utils/DateTimeUtils.java | 4 +- .../plan/relational/analyzer/TestMatadata.java | 14 ++ iotdb-core/node-commons/pom.xml | 76 ++++++ .../resources/conf/generate_properties.bat | 64 +++++ .../assembly/resources/conf/generate_properties.sh | 59 +++++ .../resources/conf/iotdb-system.properties | 2 +- .../iotdb/commons/conf/CommonDescriptor.java | 5 +- .../commons/conf/ConfigFileAutoUpdateTool.java | 126 ---------- .../iotdb/commons/conf/ConfigurationFileUtils.java | 275 +++++++++++++++++++++ .../commons/partition/SeriesPartitionTable.java | 28 +-- .../pipe/connector/protocol/IoTDBConnector.java | 28 ++- .../thrift-commons/src/main/thrift/common.thrift | 18 +- .../src/main/thrift/confignode.thrift | 6 + .../thrift-datanode/src/main/thrift/client.thrift | 4 + .../src/main/thrift/datanode.thrift | 4 + 99 files changed, 1886 insertions(+), 382 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java index 823a056df22,4adcd7e298c..97d29a81c7d --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java @@@ -19,14 -19,18 +19,19 @@@ package org.apache.iotdb.db.queryengine.plan.analyze; + import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.partition.DataPartition; + import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; + import org.apache.iotdb.db.queryengine.plan.planner.plan.TimePredicate; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.tsfile.read.common.block.TsBlock; + import java.util.List; + public interface IAnalysis { boolean isFailed(); @@@ -45,13 -49,13 +50,23 @@@ String getStatementType(); + void setFinishQueryAfterAnalyze(boolean b); + + void setFailStatus(TSStatus status); + + boolean isFinishQueryAfterAnalyze(); + + default void setRealStatement(Statement realStatement) {} + + void setDataPartitionInfo(DataPartition dataPartition); ++ + SchemaPartition getSchemaPartitionInfo(); + + DataPartition getDataPartitionInfo(); + + void setRedirectNodeList(List<TEndPoint> redirectNodeList); + + void addEndPointToRedirectNodeList(TEndPoint endPoint); + + TimePredicate getCovertedTimePredicate(); } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java index 1411338c070,dc66581b74c..166a2e4c78b --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java @@@ -73,30 -69,32 +73,59 @@@ public interface Metadata List<Expression> expressionList, List<String> attributeColumns); + /** + * This method is used for table column validation and should be invoked before device validation. + * + * <p>This method return all the existing column schemas in the target table. + * + * <p>When table or column is missing, this method will execute auto creation. + * + * <p>When using SQL, the columnSchemaList could be null and there won't be any validation. + * + * <p>When the input dataType or category of one column is null, the column cannot be auto + * created. + * + * <p>If validation failed, a SemanticException will be thrown. + */ + TableSchema validateTableHeaderSchema( + String database, TableSchema tableSchema, MPPQueryContext context); + + /** + * This method is used for table device validation and should be invoked after column validation. + * + * <p>When device id is missing, this method will execute auto creation. + * + * <p>When device attribute is missing or different from that stored in IoTDB, the attribute will + * be auto upsert. + * + * <p>If validation failed, a SemanticException will be thrown. + */ + void validateDeviceSchema(ITableDeviceSchemaValidation schemaValidation, MPPQueryContext context); ++ + /** + * Fetch and compute the schema of target timeseries, with device and measurement defined in given + * schemaComputationWithAutoCreation. The computation defined in given + * schemaComputationWithAutoCreation will be executed during scanning the fetched schema. If some + * target timeseries doesn't exist, they will be auto created. + * + * @param schemaComputationWithAutoCreationList define the target devices, measurements and + * computation + */ + default void fetchAndComputeSchemaWithAutoCreate( + List<? extends ISchemaComputationWithAutoCreation> schemaComputationWithAutoCreationList, + MPPQueryContext context) { + throw new UnsupportedOperationException(); + } + + /** + * Get or create data partition, used in cluster write scenarios. if enableAutoCreateSchema is + * true and database/series/time slots not exists, then automatically create. + * + * @param dataPartitionQueryParams the list of DataPartitionQueryParams + * @param userName + */ + default DataPartition getOrCreateDataPartition( + List<DataPartitionQueryParam> dataPartitionQueryParams, String userName) { + throw new UnsupportedOperationException(); + } }
