This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch codex/table-load-auto-create-schema in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 34edcbe3ae612fb87a34002784eb61d5a85dfea1 Merge: d72f71ad8fa b896b6f5186 Author: Caideyipi <[email protected]> AuthorDate: Thu Nov 5 10:12:53 2026 +0800 Merge origin/master into codex/table-load-auto-create-schema integration-test/AGENTS.md | 5 + integration-test/pom.xml | 2 +- .../it/env/cluster/node/ConfigNodeWrapper.java | 8 + .../relational/it/db/it/IoTDBDeletionTableIT.java | 8 +- .../relational/it/db/it/IoTDBInsertTableIT.java | 24 ++- .../java/org/apache/iotdb/session/Session.java | 47 +++++- .../java/org/apache/iotdb/session/SessionTest.java | 75 +++++++++ .../confignode/persistence/pipe/PipeTaskInfo.java | 10 +- .../impl/region/AddRegionPeerProcedure.java | 17 ++ .../impl/region/AddRegionPeerProcedureTest.java | 122 ++++++++++++++ .../apache/iotdb/db/i18n/DataNodePipeMessages.java | 2 + .../apache/iotdb/db/i18n/DataNodePipeMessages.java | 2 + .../task/execution/PipeSubtaskExecutorManager.java | 27 ++- .../pipe/metric/overview/PipeResourceMetrics.java | 11 +- .../realtime/epoch/TsFileEpochManager.java | 5 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 13 ++ .../load/TreeSchemaAutoCreatorAndVerifier.java | 19 +++ .../pipe/agent/task/PipeDataNodeTaskAgentTest.java | 40 +++++ .../agent/task/PipeSinkSubtaskExecutorTest.java | 182 +++++++++++++++++++++ .../pipe/agent/task/PipeSubtaskExecutorTest.java | 24 +++ .../execution/PipeSubtaskExecutorManagerTest.java | 66 ++++++++ .../realtime/epoch/TsFileEpochManagerTest.java | 11 -- .../load/LoadTsFileAnalyzerDiagnosticsTest.java | 124 +++++++++++++- .../apache/iotdb/commons/i18n/PipeMessages.java | 15 ++ .../apache/iotdb/commons/i18n/PipeMessages.java | 14 ++ .../commons/pipe/agent/task/PipeTaskAgent.java | 35 +++- .../agent/task/execution/PipeSubtaskExecutor.java | 29 +++- .../task/subtask/PipeAbstractSinkSubtask.java | 33 ++-- 28 files changed, 923 insertions(+), 47 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index f24b783e210,813aff6fedc..51975f5ffc4 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@@ -890,20 -897,20 +891,32 @@@ public class LoadTsFileAnalyzer impleme setRealStatement(analysis); } + private LoadTsFile createTableConversionStatement(final File tsFile) { + final Map<String, String> attributes = + Collections.singletonMap( + AUTO_CREATE_SCHEMA_KEY, String.valueOf(isAutoCreateSchemaRequested)); + return (isGeneratedByPipe + ? LoadTsFile.createForPipe(null, tsFile.getPath(), attributes) + : LoadTsFile.createUnchecked(null, tsFile.getPath(), attributes)) + .setDatabase(databaseForTableData) + .setDeleteAfterLoad(isDeleteAfterLoad) + .setConvertOnTypeMismatch(isConvertOnTypeMismatch); + } + private boolean setTemporaryUnavailableStatusIfNecessary( final IAnalysis analysis, final Throwable throwable) { + // Region unavailability can persist beyond recovery. Retain the file for retry; permanent + // region loss still requires operator intervention. + if (throwable instanceof IoTDBRuntimeException + && ((IoTDBRuntimeException) throwable).getErrorCode() + == TSStatusCode.NO_AVAILABLE_REGION_GROUP.getStatusCode()) { + analysis.setFinishQueryAfterAnalyze(true); + analysis.setFailStatus( + RpcUtils.getStatus( + TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION, throwable.getMessage())); + setRealStatement(analysis); + return true; + } if (isTemporaryUnavailableDueToPipeSchemaNotReady(throwable)) { setFailAnalysisForTemporaryUnavailablePipeSchema(analysis, throwable); return true; diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerDiagnosticsTest.java index 8a29bdc2cce,3dade21bb99..3513e7037fe --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerDiagnosticsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerDiagnosticsTest.java @@@ -21,10 -21,9 +21,13 @@@ package org.apache.iotdb.db.queryengine import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.conf.IoTDBConstant; + import org.apache.iotdb.commons.exception.IoTDBException; + import org.apache.iotdb.commons.exception.IoTDBRuntimeException; + import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.queryengine.common.SqlDialect; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.load.LoadAnalyzeException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; @@@ -32,15 -31,16 +35,19 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; + import org.apache.iotdb.db.queryengine.plan.analyze.schema.SchemaValidator; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; +import org.apache.iotdb.db.storageengine.load.active.ActiveLoadPathHelper; import org.apache.iotdb.db.storageengine.load.converter.LoadTsFileDataTypeConverter; +import org.apache.iotdb.db.storageengine.load.util.LoadUtil; import org.apache.iotdb.rpc.TSStatusCode; + import org.apache.tsfile.enums.TSDataType; + import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.read.TsFileSequenceReader; + import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@@ -56,18 -55,11 +63,19 @@@ import org.powermock.modules.junit4.Pow import org.powermock.reflect.Whitebox; import java.io.File; + import java.io.FileOutputStream; import java.io.IOException; +import java.time.ZoneOffset; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.ASYNC_LOAD_KEY; +import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.AUTO_CREATE_SCHEMA_KEY; +import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.DATABASE_NAME_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyList; @@@ -79,7 -68,7 +87,7 @@@ import static org.mockito.Mockito.when @RunWith(PowerMockRunner.class) @PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*"}) - @PrepareForTest({LoadTsFileAnalyzer.class, LoadUtil.class}) -@PrepareForTest({LoadTsFileAnalyzer.class, SchemaValidator.class}) ++@PrepareForTest({LoadTsFileAnalyzer.class, LoadUtil.class, SchemaValidator.class}) public class LoadTsFileAnalyzerDiagnosticsTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
