This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch force_ci/alter_column_datatype in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f677dda16551c9a0e19b9b0acc7036158ef41274 Merge: 5575a3d8b11 174b2cb968f Author: Tian Jiang <[email protected]> AuthorDate: Mon Mar 3 11:01:01 2025 +0800 Merge branch 'master' into force_ci/alter_column_datatype # Conflicts: # iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java # iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java .github/workflows/vulnerability-check.yml | 7 +- dependencies.json | 1 + .../server/CustomizedJsonPayloadFormatter.java | 8 +- .../org/apache/iotdb/mqtt/server/MyMessage.java | 80 --- .../java/org/apache/iotdb/mqtt/MQTTClient.java | 2 +- .../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 2 + .../relational/it/db/it/IoTDBInsertTableIT.java | 11 +- .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java | 12 +- .../relational/it/mqtt/IoTDBMQTTServiceIT.java | 2 +- .../IoTDBCorrelatedExistsSubqueryIT.java | 428 +++++++++++++ .../IoTDBUncorrelatedExistsSubqueryIT.java | 391 ++++++++++++ .../it/session/IoTDBSessionRelationalIT.java | 34 - .../iotdb/session/it/IoTDBSessionSimpleIT.java | 48 -- .../org/apache/iotdb/tool/common/OptionsUtil.java | 54 +- .../org/apache/iotdb/tool/data/ExportData.java | 4 +- .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 25 +- .../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 31 +- .../IoTDBRelationalDatabaseMetadata.java | 52 +- .../java/org/apache/iotdb/session/Session.java | 17 +- .../SubscriptionTablePushConsumerBuilder.java | 3 +- iotdb-core/confignode/pom.xml | 4 + .../iotdb/confignode/conf/ConfigNodeConfig.java | 45 ++ .../confignode/conf/ConfigNodeDescriptor.java | 30 + .../iotdb/confignode/manager/ConfigManager.java | 9 + .../apache/iotdb/confignode/manager/IManager.java | 3 + .../iotdb/confignode/manager/ModelManager.java | 5 + .../manager/load/balancer/RouteBalancer.java | 154 +++-- .../manager/load/cache/AbstractLoadCache.java | 26 +- .../manager/load/cache/IFailureDetector.java} | 24 +- .../manager/load/cache/detector/FixedDetector.java | 58 ++ .../load/cache/detector/PhiAccrualDetector.java | 181 ++++++ .../load/cache/node/AINodeHeartbeatCache.java | 15 +- .../load/cache/node/ConfigNodeHeartbeatCache.java | 12 +- .../load/cache/node/DataNodeHeartbeatCache.java | 11 +- .../manager/load/cache/region/RegionCache.java | 12 +- .../iotdb/confignode/manager/node/NodeManager.java | 8 + .../manager/schema/ClusterSchemaManager.java | 12 +- .../iotdb/confignode/persistence/ModelInfo.java | 2 +- .../iotdb/confignode/persistence/TTLInfo.java | 2 +- .../persistence/schema/ClusterSchemaInfo.java | 20 + .../procedure/PartitionTableAutoCleaner.java | 14 +- .../procedure/impl/StateMachineProcedure.java | 9 +- .../impl/pipe/AbstractOperatePipeProcedureV2.java | 2 + .../impl/pipe/task/AlterPipeProcedureV2.java | 1 + .../manager/load/cache/detector/DetectorTest.java | 179 ++++++ .../common/request/IConsensusRequest.java | 5 + .../common/request/IndexedConsensusRequest.java | 8 +- .../consensus/config/PipeConsensusConfig.java | 21 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 12 +- .../consensus/pipe/PipeConsensusServerImpl.java | 21 +- ...xManager.java => ReplicateProgressManager.java} | 8 +- .../pipe/metric/PipeConsensusSyncLagManager.java | 71 ++- iotdb-core/datanode/pom.xml | 8 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- .../db/consensus/DataRegionConsensusImpl.java | 4 +- .../PipeDataRegionConnectorConstructor.java | 4 + .../agent/task/connection/PipeEventCollector.java | 13 +- .../agent/task/stage/PipeTaskProcessorStage.java | 5 +- .../connector/protocol/opcda/OpcDaConnector.java | 138 ++++ .../pipe/connector/protocol/opcda/OpcDaHeader.java | 200 ++++++ .../protocol/opcda/OpcDaServerHandle.java | 390 ++++++++++++ .../connector/protocol/opcua/OpcUaConnector.java | 50 +- .../connector/protocol/opcua/OpcUaNameSpace.java | 2 +- .../pipeconsensus/PipeConsensusAsyncConnector.java | 16 +- .../pipeconsensus/PipeConsensusSyncConnector.java | 9 +- .../PipeConsensusTabletInsertionEventHandler.java | 8 +- .../PipeConsensusTsFileInsertionEventHandler.java | 8 +- .../PipeConsensusTransferBatchReqBuilder.java | 5 +- ....java => ReplicateProgressDataNodeManager.java} | 36 +- .../deletion/DeletionResourceManager.java | 4 +- .../deletion/persist/PageCacheDeletionBuffer.java | 5 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 17 +- .../scan/TsFileInsertionEventScanParser.java | 3 - .../table/TsFileInsertionEventTableParser.java | 95 ++- ...ileInsertionEventTableParserTabletIterator.java | 421 ++++++++++--- .../event/realtime/PipeRealtimeEventFactory.java | 71 ++- ...oricalDataRegionTsFileAndDeletionExtractor.java | 17 + .../realtime/assigner/PipeDataRegionAssigner.java | 2 + .../listener/PipeInsertionDataNodeListener.java | 16 +- .../pipeconsensus/PipeConsensusProcessor.java | 43 +- .../pipeconsensus/PipeConsensusReceiver.java | 338 +++++----- .../resource/memory/InsertNodeMemoryEstimator.java | 2 +- .../pipe/resource/memory/PipeMemoryWeightUtil.java | 65 ++ .../db/protocol/mqtt/JSONPayloadFormatter.java | 7 +- .../db/protocol/mqtt/LinePayloadFormatter.java | 7 +- .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 11 +- .../iotdb/db/protocol/mqtt/PayloadFormatter.java | 6 + .../protocol/thrift/impl/ClientRPCServiceImpl.java | 2 +- .../operator/process/AssignUniqueIdOperator.java | 142 +++++ .../operator/schema/SchemaQueryScanOperator.java | 12 +- .../operator/schema/source/ISchemaSource.java | 5 + .../schema/source/TableDeviceFetchSource.java | 21 + .../schema/source/TableDeviceQuerySource.java | 36 +- .../relational/MergeSortLeftJoinOperator.java | 163 +++++ .../aggregation/grouped/hash/HashStrategy.java | 2 +- .../plan/planner/TableOperatorGenerator.java | 53 ++ .../plan/planner/plan/node/PlanGraphPrinter.java | 10 + .../plan/planner/plan/node/PlanNodeType.java | 4 + .../plan/planner/plan/node/PlanVisitor.java | 5 + .../plan/planner/plan/node/write/InsertNode.java | 3 +- .../planner/plan/node/write/InsertRowNode.java | 3 - .../PredicatePushIntoMetadataChecker.java | 7 +- .../fetcher/TableHeaderSchemaValidator.java | 4 + .../distribute/TableDistributedPlanGenerator.java | 58 +- .../iterative/rule/AggregationDecorrelation.java | 83 +++ .../iterative/rule/PruneAssignUniqueIdColumns.java | 47 ++ .../iterative/rule/RemoveRedundantExists.java | 104 +++ ...orrelatedDistinctAggregationWithProjection.java | 186 ++++++ ...elatedDistinctAggregationWithoutProjection.java | 167 +++++ ...mCorrelatedGlobalAggregationWithProjection.java | 318 ++++++++++ ...rrelatedGlobalAggregationWithoutProjection.java | 302 +++++++++ ...CorrelatedGroupedAggregationWithProjection.java | 252 ++++++++ ...relatedGroupedAggregationWithoutProjection.java | 233 +++++++ .../rule/TransformCorrelatedJoinToJoin.java | 99 +++ .../rule/TransformExistsApplyToCorrelatedJoin.java | 214 +++++++ .../relational/planner/node/AssignUniqueId.java | 97 +++ .../plan/relational/planner/node/Patterns.java | 27 +- .../planner/optimizations/JoinUtils.java | 249 +++++--- .../optimizations/LogicalOptimizeFactory.java | 40 +- .../optimizations/PlanNodeDecorrelator.java | 699 +++++++++++++++++++++ .../optimizations/PushPredicateIntoTableScan.java | 143 ++++- .../optimizations/UnaliasSymbolReferences.java | 13 + .../relational/planner/optimizations/Util.java | 19 + .../plan/relational/sql/ast/InsertRows.java | 3 - .../plan/statement/crud/InsertBaseStatement.java | 28 +- .../plan/statement/crud/InsertRowStatement.java | 3 - .../rescon/ISchemaRegionStatistics.java | 2 + .../rescon/MemSchemaRegionStatistics.java | 16 + .../attribute/DeviceAttributeStore.java | 48 +- .../attribute/IDeviceAttributeStore.java | 16 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 36 +- .../iotdb/db/storageengine/StorageEngine.java | 19 +- .../dataregion/memtable/AbstractMemTable.java | 85 ++- .../memtable/AlignedWritableMemChunk.java | 13 + .../memtable/AlignedWritableMemChunkGroup.java | 15 +- .../dataregion/memtable/IMemTable.java | 2 + .../memtable/IWritableMemChunkGroup.java | 2 + .../dataregion/memtable/TsFileProcessor.java | 41 +- .../dataregion/memtable/WritableMemChunk.java | 8 + .../dataregion/memtable/WritableMemChunkGroup.java | 20 + .../dataregion/tsfile/TsFileResource.java | 8 +- .../broker/SubscriptionPrefetchingQueue.java | 10 +- .../broker/SubscriptionPrefetchingTabletQueue.java | 2 +- .../broker/SubscriptionPrefetchingTsFileQueue.java | 2 +- .../db/subscription/event/SubscriptionEvent.java | 4 +- .../event/batch/SubscriptionPipeEventBatch.java | 2 +- .../event/batch/SubscriptionPipeEventBatches.java | 2 +- .../batch/SubscriptionPipeTabletEventBatch.java | 6 +- .../SubscriptionPipeTabletIterationSnapshot.java | 18 +- .../batch/SubscriptionPipeTsFileEventBatch.java | 2 +- .../event/pipe/SubscriptionPipeEmptyEvent.java | 2 +- .../event/pipe/SubscriptionPipeEvents.java | 2 +- .../pipe/SubscriptionPipeTabletBatchEvents.java | 11 +- .../pipe/SubscriptionPipeTsFileBatchEvents.java | 4 +- .../pipe/SubscriptionPipeTsFilePlainEvent.java | 2 +- .../db/tools/schema/SRStatementGenerator.java | 2 + .../db/utils/datastructure/AlignedTVList.java | 44 +- .../iotdb/db/utils/datastructure/BinaryTVList.java | 38 +- .../db/utils/datastructure/BooleanTVList.java | 38 +- .../iotdb/db/utils/datastructure/DoubleTVList.java | 38 +- .../iotdb/db/utils/datastructure/FloatTVList.java | 38 +- .../iotdb/db/utils/datastructure/IntTVList.java | 38 +- .../iotdb/db/utils/datastructure/LongTVList.java | 38 +- .../iotdb/db/utils/datastructure/TVList.java | 70 ++- .../db/protocol/mqtt/PayloadFormatManagerTest.java | 2 +- .../execution/operator/OperatorMemoryTest.java | 2 - .../relational/analyzer/GroupByLargeDataTest.java | 179 ++++++ .../relational/planner/CorrelatedSubqueryTest.java | 153 +++++ ...ueryTest.java => UncorrelatedSubqueryTest.java} | 104 ++- .../planner/assertions/AssignUniqueIdMatcher.java} | 32 +- .../planner/assertions/PlanMatchPattern.java | 6 + .../dataregion/memtable/TsFileProcessorTest.java | 36 +- .../conf/iotdb-system.properties.template | 25 +- .../iotdb/commons/path/ExtendedPartialPath.java | 4 + .../agent/plugin/builtin/BuiltinPipePlugin.java | 5 + .../builtin/connector/opcda/OpcDaConnector.java} | 21 +- .../config/constant/PipeConnectorConstant.java | 8 +- .../connector/protocol/IoTDBAirGapConnector.java | 21 +- .../iotdb/commons/pipe/event/EnrichedEvent.java | 17 +- .../apache/iotdb/commons/schema/table/TsTable.java | 2 +- .../thrift-commons/src/main/thrift/common.thrift | 1 + .../src/main/thrift/pipeconsensus.thrift | 5 +- pom.xml | 8 +- 183 files changed, 8293 insertions(+), 1162 deletions(-) diff --cc iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 3748980d2ad,ab4b88fb78c..fe0289176ed --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@@ -105,7 -105,7 +105,8 @@@ import org.apache.iotdb.mpp.rpc.thrift. import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; + import org.apache.tsfile.annotations.TableModel; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; diff --cc iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index b4fec9efc5e,e313c6d93a0..01969873a7f --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@@ -93,7 -91,7 +93,8 @@@ import org.apache.iotdb.db.schemaengine import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; + import org.apache.tsfile.annotations.TableModel; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java index c5b91f4dcdd,36c8470af2c..7d06d1f7e32 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java @@@ -170,8 -173,11 +175,16 @@@ public class AlignedWritableMemChunkGro return memChunkGroup; } + @Override + public void checkDataType(InsertNode node) throws DataTypeInconsistentException { + memChunk.checkDataType(node); + } ++ + protected static AlignedWritableMemChunkGroup deserializeSingleTVListMemChunks( + DataInputStream stream, boolean isTableModel) throws IOException { + AlignedWritableMemChunkGroup memChunkGroup = new AlignedWritableMemChunkGroup(); + memChunkGroup.memChunk = + AlignedWritableMemChunk.deserializeSingleTVListMemChunks(stream, isTableModel); + return memChunkGroup; + } } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java index a37475e2e0b,382573e7450..a61407d4e48 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java @@@ -60,7 -58,7 +60,9 @@@ public interface IWritableMemChunkGrou long getMeasurementSize(String measurement); + IWritableMemChunk getWritableMemChunk(String measurement); + long getMaxTime(); + + void checkDataType(InsertNode node) throws DataTypeInconsistentException; } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java index deb73559b19,e47debd9d73..824b5c7a958 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java @@@ -197,19 -202,15 +205,31 @@@ public class WritableMemChunkGroup impl return memChunkGroup; } + @Override + public void checkDataType(InsertNode node) throws DataTypeInconsistentException { + for (MeasurementSchema incomingSchema : node.getMeasurementSchemas()) { + if (incomingSchema == null) { + continue; + } + + IWritableMemChunk memChunk = memChunkMap.get(incomingSchema.getMeasurementName()); + if (memChunk != null + && memChunk.getWorkingTVList().getDataType() != incomingSchema.getType()) { + throw new DataTypeInconsistentException( + memChunk.getWorkingTVList().getDataType(), incomingSchema.getType()); + } + } + } ++ + public static WritableMemChunkGroup deserializeSingleTVListMemChunks(DataInputStream stream) + throws IOException { + WritableMemChunkGroup memChunkGroup = new WritableMemChunkGroup(); + int memChunkMapSize = stream.readInt(); + for (int i = 0; i < memChunkMapSize; ++i) { + String measurement = ReadWriteIOUtils.readString(stream); + IWritableMemChunk memChunk = WritableMemChunk.deserializeSingleTVListMemChunks(stream); + memChunkGroup.memChunkMap.put(measurement, memChunk); + } + return memChunkGroup; + } }
