This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch codex/table-load-auto-create-schema in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d72f71ad8fab0672692aa2ae44d0c338430f762d Author: Caideyipi <[email protected]> AuthorDate: Thu Nov 5 10:08:10 2026 +0800 Honor auto-create-schema settings for table-model LOAD --- .../plan/analyze/load/LoadTsFileAnalyzer.java | 48 ++-- .../analyze/load/LoadTsFileTableSchemaCache.java | 19 +- .../plan/node/load/LoadSingleTsFileNode.java | 28 +++ .../planner/plan/node/load/LoadTsFileNode.java | 6 +- .../plan/relational/metadata/Metadata.java | 3 +- .../relational/metadata/TableMetadataImpl.java | 4 +- .../fetcher/TableHeaderSchemaValidator.java | 8 +- .../plan/scheduler/load/LoadFallbackHandler.java | 9 +- .../converter/LoadTsFileDataTypeConverter.java | 27 ++- .../load/LoadTsFileAnalyzerDiagnosticsTest.java | 173 ++++++++++++- .../plan/analyze/load/LoadTsFileAnalyzerTest.java | 269 +++++++++++++++++++++ .../scheduler/load/LoadFallbackHandlerTest.java | 104 ++++++++ .../LoadTsFileDataTypeConverterSchemaTest.java | 237 ++++++++++++++++++ 13 files changed, 895 insertions(+), 40 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 802e4a1697e..f24b783e210 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -81,6 +81,7 @@ import java.util.Objects; import java.util.Optional; import static org.apache.iotdb.db.queryengine.plan.execution.config.TableConfigTaskVisitor.DATABASE_NOT_SPECIFIED; +import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.AUTO_CREATE_SCHEMA_KEY; import static org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet.ANALYSIS; import static org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet.ANALYSIS_ASYNC_MOVE; @@ -316,13 +317,19 @@ public class LoadTsFileAnalyzer implements AutoCloseable { } else { databaseName = null; } + // The active loader uses a new session, so carry the request session's restriction with it. + final boolean allowAutoCreateSchema = + isAutoCreateSchemaRequested + && (Objects.isNull(context) + || Objects.isNull(context.getSession()) + || context.getSession().isEnableAutoCreateSchema()); final Map<String, String> activeLoadAttributes = ActiveLoadPathHelper.buildAttributes( databaseName, databaseLevel, isConvertOnTypeMismatch, isVerifySchema, - isAutoCreateSchemaRequested, + allowAutoCreateSchema, tabletConversionThresholdBytes, isGeneratedByPipe, Objects.nonNull(context) && Objects.nonNull(context.getSession()) @@ -478,8 +485,10 @@ public class LoadTsFileAnalyzer implements AutoCloseable { sessionInfo.getSessionId(), sessionInfo.getUserEntity(), sessionInfo.getZoneId(), + sessionInfo.getVersion(), sessionInfo.getDatabaseName().orElse(null), - SqlDialect.TABLE); + SqlDialect.TABLE, + sessionInfo.isEnableAutoCreateSchema()); context.setSession(newSessionInfo); doAnalyzeSingleTableFile(tsFile, reader, timeseriesMetadataIterator, tableSchemaMap); } else { @@ -530,15 +539,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable { final TSStatus status = isTableModelTsFile.get(i) ? loadTsFileDataTypeConverter - .convertForTableModel( - (isGeneratedByPipe - ? LoadTsFile.createForPipe( - null, tsFiles.get(i).getPath(), Collections.emptyMap()) - : LoadTsFile.createUnchecked( - null, tsFiles.get(i).getPath(), Collections.emptyMap())) - .setDatabase(databaseForTableData) - .setDeleteAfterLoad(isDeleteAfterLoad) - .setConvertOnTypeMismatch(isConvertOnTypeMismatch)) + .convertForTableModel(createTableConversionStatement(tsFiles.get(i))) .orElse(null) : loadTsFileDataTypeConverter .convertForTreeModel( @@ -713,7 +714,8 @@ public class LoadTsFileAnalyzer implements AutoCloseable { private LoadTsFileTableSchemaCache getOrCreateTableSchemaCache() { if (tableSchemaCache == null) { tableSchemaCache = - new LoadTsFileTableSchemaCache(metadata, context, isAutoCreateDatabase, isVerifySchema); + new LoadTsFileTableSchemaCache( + metadata, context, isAutoCreateDatabase, isVerifySchema, isAutoCreateSchemaRequested); } return tableSchemaCache; } @@ -834,15 +836,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable { final TSStatus status = isTableModelTsFile.get(i) ? loadTsFileDataTypeConverter - .convertForTableModel( - (isGeneratedByPipe - ? LoadTsFile.createForPipe( - null, tsFiles.get(i).getPath(), Collections.emptyMap()) - : LoadTsFile.createUnchecked( - null, tsFiles.get(i).getPath(), Collections.emptyMap())) - .setDatabase(databaseForTableData) - .setDeleteAfterLoad(isDeleteAfterLoad) - .setConvertOnTypeMismatch(isConvertOnTypeMismatch)) + .convertForTableModel(createTableConversionStatement(tsFiles.get(i))) .orElse(null) : loadTsFileDataTypeConverter .convertForTreeModel( @@ -896,6 +890,18 @@ public class LoadTsFileAnalyzer implements AutoCloseable { setRealStatement(analysis); } + private LoadTsFile createTableConversionStatement(final File tsFile) { + final Map<String, String> attributes = + Collections.singletonMap( + AUTO_CREATE_SCHEMA_KEY, String.valueOf(isAutoCreateSchemaRequested)); + return (isGeneratedByPipe + ? LoadTsFile.createForPipe(null, tsFile.getPath(), attributes) + : LoadTsFile.createUnchecked(null, tsFile.getPath(), attributes)) + .setDatabase(databaseForTableData) + .setDeleteAfterLoad(isDeleteAfterLoad) + .setConvertOnTypeMismatch(isConvertOnTypeMismatch); + } + private boolean setTemporaryUnavailableStatusIfNecessary( final IAnalysis analysis, final Throwable throwable) { if (isTemporaryUnavailableDueToPipeSchemaNotReady(throwable)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java index 26e1ba8cb9f..577e4ead742 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java @@ -100,6 +100,7 @@ public class LoadTsFileTableSchemaCache { private final Metadata metadata; private final MPPQueryContext context; private final boolean shouldVerifyDataType; + private final boolean allowAutoCreateSchema; private Map<String, Set<IDeviceID>> currentBatchTable2Devices; @@ -123,12 +124,23 @@ public class LoadTsFileTableSchemaCache { final boolean needToCreateDatabase, final boolean shouldVerifyDataType) throws LoadRuntimeOutOfMemoryException { + this(metadata, context, needToCreateDatabase, shouldVerifyDataType, true); + } + + public LoadTsFileTableSchemaCache( + final Metadata metadata, + final MPPQueryContext context, + final boolean needToCreateDatabase, + final boolean shouldVerifyDataType, + final boolean allowAutoCreateSchema) + throws LoadRuntimeOutOfMemoryException { this.block = LoadTsFileMemoryManager.getInstance() .allocateMemoryBlock(ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES); this.metadata = metadata; this.context = context; this.shouldVerifyDataType = shouldVerifyDataType; + this.allowAutoCreateSchema = allowAutoCreateSchema; this.currentBatchTable2Devices = new HashMap<>(); this.currentModifications = PatternTreeMapFactory.getModsPatternTreeMap(); this.needToCreateDatabase = needToCreateDatabase; @@ -316,7 +328,12 @@ public class LoadTsFileTableSchemaCache { final TableSchema realSchema = metadata .validateTableHeaderSchema4TsFile( - database, fileSchema, context, true, true, needDecode4DifferentTimeColumn) + database, + fileSchema, + context, + allowAutoCreateSchema, + true, + needDecode4DifferentTimeColumn) .orElse(null); if (Objects.isNull(realSchema)) { throw new LoadAnalyzeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index ecaf0aea6ec..4bdcdc15674 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -62,6 +62,7 @@ public class LoadSingleTsFileNode extends WritePlanNode { private final boolean isTableModel; private final String database; private final boolean deleteAfterLoad; + private final boolean autoCreateSchema; private final long writePointCount; private boolean needDecodeTsFile; @@ -75,12 +76,33 @@ public class LoadSingleTsFileNode extends WritePlanNode { final boolean deleteAfterLoad, final long writePointCount, final boolean needDecodeTsFile) { + this( + id, + resource, + isTableModel, + database, + deleteAfterLoad, + writePointCount, + needDecodeTsFile, + true); + } + + public LoadSingleTsFileNode( + final PlanNodeId id, + final TsFileResource resource, + final boolean isTableModel, + final String database, + final boolean deleteAfterLoad, + final long writePointCount, + final boolean needDecodeTsFile, + final boolean autoCreateSchema) { super(id); this.tsFile = resource.getTsFile(); this.resource = resource; this.isTableModel = isTableModel; this.database = database; this.deleteAfterLoad = deleteAfterLoad; + this.autoCreateSchema = autoCreateSchema; this.writePointCount = writePointCount; this.needDecodeTsFile = needDecodeTsFile; } @@ -163,6 +185,10 @@ public class LoadSingleTsFileNode extends WritePlanNode { return isTableModel; } + public boolean isAutoCreateSchema() { + return autoCreateSchema; + } + public long getWritePointCount() { return writePointCount; } @@ -278,6 +304,7 @@ public class LoadSingleTsFileNode extends WritePlanNode { && Objects.equals(database, loadSingleTsFileNode.database) && Objects.equals(needDecodeTsFile, loadSingleTsFileNode.needDecodeTsFile) && Objects.equals(deleteAfterLoad, loadSingleTsFileNode.deleteAfterLoad) + && autoCreateSchema == loadSingleTsFileNode.autoCreateSchema && Objects.equals(localRegionReplicaSet, loadSingleTsFileNode.localRegionReplicaSet); } @@ -290,6 +317,7 @@ public class LoadSingleTsFileNode extends WritePlanNode { database, needDecodeTsFile, deleteAfterLoad, + autoCreateSchema, localRegionReplicaSet); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java index 6b1a6b464f1..929f4a1fed1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java @@ -130,7 +130,8 @@ public class LoadTsFileNode extends WritePlanNode { database, statement.isDeleteAfterLoad(), statement.getWritePointCount(i), - needDecode4TimeColumn)); + needDecode4TimeColumn, + statement.isAutoCreateSchema())); } return res; } @@ -153,7 +154,8 @@ public class LoadTsFileNode extends WritePlanNode { database, statement.isDeleteAfterLoad(), statement.getWritePointCount(i), - needDecode4TimeColumn)); + needDecode4TimeColumn, + statement.isAutoCreateSchema())); } else { throw new IllegalStateException( DataNodeQueryMessages.LOADTSFILE_STATEMENT_IS_NULL_DURING_TABLE_MODEL_SPLIT); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java index 04db571083f..58a4c146b17 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java @@ -116,6 +116,7 @@ public interface Metadata extends ITypeMetadata, ITableFunctionFactory { * <p>The caller need to recheck the dataType of measurement columns to decide whether to do * partial insert * + * @param allowAutoCreateSchema whether to allow automatic creation of missing tables and columns * @param isStrictTagColumn if {@code true}, when the table already exists, the tag columns in the * existing table should be the prefix of those in the input tableSchema, or input tag columns * be the prefix of existing tag columns. @@ -129,7 +130,7 @@ public interface Metadata extends ITypeMetadata, ITableFunctionFactory { final String database, final TableSchema tableSchema, final MPPQueryContext context, - final boolean allowCreateTable, + final boolean allowAutoCreateSchema, final boolean isStrictTagColumn, final AtomicBoolean needDecode4DifferentTimeColumn) throws LoadAnalyzeTableColumnDisorderException; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 04dcc93b731..3fc7d57bdab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -1705,7 +1705,7 @@ public class TableMetadataImpl implements Metadata { final String database, final TableSchema tableSchema, final MPPQueryContext context, - final boolean allowCreateTable, + final boolean allowAutoCreateSchema, final boolean isStrictTagColumn, final AtomicBoolean needDecode4DifferentTimeColumn) throws LoadAnalyzeTableColumnDisorderException { @@ -1714,7 +1714,7 @@ public class TableMetadataImpl implements Metadata { database, tableSchema, context, - allowCreateTable, + allowAutoCreateSchema, isStrictTagColumn, needDecode4DifferentTimeColumn); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java index 01c2555608a..79708549120 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java @@ -103,7 +103,7 @@ public class TableHeaderSchemaValidator { final String database, final TableSchema tableSchema, final MPPQueryContext context, - final boolean allowCreateTable, + final boolean allowAutoCreateSchema, final boolean isStrictTagColumn, final @Nonnull AtomicBoolean needDecode4DifferentTimeColumn) throws LoadAnalyzeTableColumnDisorderException { @@ -124,13 +124,15 @@ public class TableHeaderSchemaValidator { final List<ColumnSchema> missingColumnList = new ArrayList<>(); final boolean isAutoCreateSchemaEnabled = - IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled(); + allowAutoCreateSchema + && IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled() + && context.getSession().isEnableAutoCreateSchema(); // first round validate, check existing schema if (table == null) { // auto create missing table // it's ok that many write requests concurrently auto create same table, the thread safety // will be guaranteed by ProcedureManager.createTable in CN - if (allowCreateTable && isAutoCreateSchemaEnabled) { + if (isAutoCreateSchemaEnabled) { autoCreateTable(context, database, tableSchema); table = DataNodeTableCache.getInstance().getTable(database, tableSchema.getTableName(), false); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadFallbackHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadFallbackHandler.java index 4d0752d3f86..6db5c03eb0e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadFallbackHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadFallbackHandler.java @@ -38,9 +38,12 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.AUTO_CREATE_SCHEMA_KEY; + /** * Fallback execution handler for the LOAD scheduler. When direct chunk staging fails, converts * failed TsFiles into In-Memory Tablets, retries insertion through the regular write pipeline, and @@ -190,10 +193,12 @@ public class LoadFallbackHandler { final String filePath, final boolean isGeneratedByPipe, final boolean deleteAfterLoad) { + final Map<String, String> attributes = + Collections.singletonMap(AUTO_CREATE_SCHEMA_KEY, String.valueOf(node.isAutoCreateSchema())); final LoadTsFile statement = (isGeneratedByPipe - ? LoadTsFile.createForPipe(null, filePath, Collections.emptyMap()) - : LoadTsFile.createUnchecked(null, filePath, Collections.emptyMap())) + ? LoadTsFile.createForPipe(null, filePath, attributes) + : LoadTsFile.createUnchecked(null, filePath, attributes)) .setDatabase(node.getDatabase()) .setDeleteAfterLoad(deleteAfterLoad) .setConvertOnTypeMismatch(true); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java index 20d2802e0fd..4ecdb8cf575 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.audit.UserEntity; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeOutOfMemoryCriticalException; import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.commons.queryengine.common.SessionInfo; import org.apache.iotdb.commons.queryengine.common.SqlDialect; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -157,8 +158,6 @@ public class LoadTsFileDataTypeConverter { private final MPPQueryContext context; private final SqlParser relationalSqlParser = new SqlParser(); - private final LoadTableStatementDataTypeConvertExecutionVisitor - tableStatementDataTypeConvertExecutionVisitor; private final LoadTreeStatementDataTypeConvertExecutionVisitor treeStatementDataTypeConvertExecutionVisitor; @@ -167,8 +166,6 @@ public class LoadTsFileDataTypeConverter { this.context = context; this.isGeneratedByPipe = isGeneratedByPipe; - tableStatementDataTypeConvertExecutionVisitor = - new LoadTableStatementDataTypeConvertExecutionVisitor(this::executeForTableModel); treeStatementDataTypeConvertExecutionVisitor = new LoadTreeStatementDataTypeConvertExecutionVisitor(this::executeForTreeModel); } @@ -179,7 +176,11 @@ public class LoadTsFileDataTypeConverter { getTabletConversionSemaphore().acquire(); isPermitAcquired = true; return loadTsFileTableStatement.accept( - tableStatementDataTypeConvertExecutionVisitor, loadTsFileTableStatement.getDatabase()); + new LoadTableStatementDataTypeConvertExecutionVisitor( + (statement, databaseName) -> + executeForTableModel( + statement, databaseName, loadTsFileTableStatement.isAutoCreateSchema())), + loadTsFileTableStatement.getDatabase()); } catch (final InterruptedException e) { return getInterruptedConversionStatus(e); } catch (Exception e) { @@ -240,12 +241,16 @@ public class LoadTsFileDataTypeConverter { session.setClientVersion(IoTDBConstant.ClientVersion.V_1_0); session.setZoneId(ZoneId.systemDefault()); session.setSqlDialect(sqlDialect); + if (Objects.nonNull(context) && Objects.nonNull(context.getSession())) { + session.setEnableAutoCreateSchema(context.getSession().isEnableAutoCreateSchema()); + } SESSION_MANAGER.registerSession(session); return session; } - private TSStatus executeForTableModel(final Statement statement, final String databaseName) { + private TSStatus executeForTableModel( + final Statement statement, final String databaseName, final boolean allowAutoCreateSchema) { final IClientSession currentSession = SESSION_MANAGER.getCurrSession(); final boolean needToCreateSession = Objects.isNull(currentSession); final IClientSession session = @@ -257,7 +262,15 @@ public class LoadTsFileDataTypeConverter { relationalSqlParser, session, SESSION_MANAGER.requestQueryId(), - SESSION_MANAGER.getSessionInfoOfPipeReceiver(session, databaseName), + new SessionInfo( + session.getId(), + new UserEntity( + session.getUserId(), session.getUsername(), session.getClientAddress()), + ZoneId.systemDefault(), + session.getClientVersion(), + databaseName, + SqlDialect.TABLE, + allowAutoCreateSchema && session.isEnableAutoCreateSchema()), "", LocalExecutionPlanner.getInstance().metadata, IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold()) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerDiagnosticsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerDiagnosticsTest.java index 65eb3d27710..8a29bdc2cce 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerDiagnosticsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerDiagnosticsTest.java @@ -21,6 +21,10 @@ package org.apache.iotdb.db.queryengine.plan.analyze.load; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.queryengine.common.SqlDialect; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.load.LoadAnalyzeException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; @@ -28,8 +32,11 @@ import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; +import org.apache.iotdb.db.storageengine.load.active.ActiveLoadPathHelper; import org.apache.iotdb.db.storageengine.load.converter.LoadTsFileDataTypeConverter; +import org.apache.iotdb.db.storageengine.load.util.LoadUtil; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.file.metadata.TableSchema; @@ -41,6 +48,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -49,16 +57,29 @@ import org.powermock.reflect.Whitebox; import java.io.File; import java.io.IOException; +import java.time.ZoneOffset; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.ASYNC_LOAD_KEY; +import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.AUTO_CREATE_SCHEMA_KEY; +import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.DATABASE_NAME_KEY; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(PowerMockRunner.class) @PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*"}) -@PrepareForTest(LoadTsFileAnalyzer.class) +@PrepareForTest({LoadTsFileAnalyzer.class, LoadUtil.class}) public class LoadTsFileAnalyzerDiagnosticsTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -177,6 +198,156 @@ public class LoadTsFileAnalyzerDiagnosticsTest { Assert.assertTrue(status.getMessage().contains("missing WRITE_DATA privilege")); } + @Test + public void testMiniFileTableConversionPreservesAutoCreateSchema() throws Exception { + assertTableConversionPreservesAutoCreateSchema(true); + } + + @Test + public void testFallbackTableConversionPreservesAutoCreateSchema() throws Exception { + assertTableConversionPreservesAutoCreateSchema(false); + } + + @Test + public void testAsyncTableSqlLoadPreservesSessionSchemaCreationSetting() throws Exception { + assertAsyncLoadPreservesSessionSchemaCreationSetting(true); + } + + @Test + public void testAsyncTreeSqlLoadPreservesSessionSchemaCreationSetting() throws Exception { + assertAsyncLoadPreservesSessionSchemaCreationSetting(false); + } + + private void assertAsyncLoadPreservesSessionSchemaCreationSetting(final boolean tableModelRequest) + throws Exception { + final AtomicReference<Map<String, String>> handedOffAttributes = new AtomicReference<>(); + PowerMockito.mockStatic(LoadUtil.class); + when(LoadUtil.loadTsFileAsyncToActiveDir(anyList(), anyMap(), anyBoolean())) + .thenAnswer( + invocation -> { + handedOffAttributes.set(new HashMap<>(invocation.getArgument(1))); + return true; + }); + final File file = temporaryFolder.newFile("async-schema.tsfile"); + final File pendingDir = temporaryFolder.newFolder("pending"); + for (final boolean sessionEnabled : new boolean[] {false, true, false}) { + for (final boolean requested : new boolean[] {false, true, false}) { + handedOffAttributes.set(null); + final Map<String, String> attributes = new HashMap<>(); + attributes.put(ASYNC_LOAD_KEY, "true"); + attributes.put(DATABASE_NAME_KEY, "db"); + if (!requested) { + attributes.put(AUTO_CREATE_SCHEMA_KEY, "false"); + } + final LoadTsFile tableStatement = + LoadTsFile.createUnchecked(null, file.getAbsolutePath(), attributes); + final LoadTsFileStatement treeStatement = + LoadTsFileStatement.createUnchecked(file.getAbsolutePath()); + treeStatement.setLoadAttributes(attributes); + final SessionInfo sessionInfo = + new SessionInfo( + 0, + new UserEntity(0, "root", ""), + ZoneOffset.UTC, + IoTDBConstant.ClientVersion.V_1_0, + "db", + tableModelRequest ? SqlDialect.TABLE : SqlDialect.TREE, + sessionEnabled); + final MPPQueryContext context = new MPPQueryContext(new QueryId("async_load_schema")); + context.setSession(sessionInfo); + try (final LoadTsFileAnalyzer analyzer = + tableModelRequest + ? new LoadTsFileAnalyzer(tableStatement, false, context) + : new LoadTsFileAnalyzer(treeStatement, false, context)) { + final Analysis analysis = new Analysis(); + analyzer.analyzeFileByFile(analysis); + Assert.assertTrue(analysis.isFinishQueryAfterAnalyze()); + Assert.assertFalse(analysis.isFailed()); + } + + Assert.assertNotNull(handedOffAttributes.get()); + // Reconstruct the request as the active loader does with its own internal session. + final File queuedFile = + new File( + ActiveLoadPathHelper.resolveTargetDir(pendingDir, handedOffAttributes.get()), + file.getName()); + final Map<String, String> parsedAttributes = + ActiveLoadPathHelper.parseAttributes(queuedFile, pendingDir); + final LoadTsFileStatement activeStatement = + LoadTsFileStatement.createUnchecked(file.getAbsolutePath()); + ActiveLoadPathHelper.applyAttributesToStatement(parsedAttributes, activeStatement, false); + Assert.assertEquals(requested && sessionEnabled, activeStatement.isAutoCreateSchema()); + Assert.assertEquals("db", activeStatement.getDatabase()); + Assert.assertSame(sessionInfo, context.getSession()); + Assert.assertEquals(sessionEnabled, sessionInfo.isEnableAutoCreateSchema()); + Assert.assertEquals(requested, tableStatement.isAutoCreateSchema()); + Assert.assertEquals(requested, treeStatement.isAutoCreateSchema()); + Assert.assertTrue(file.exists()); + } + } + } + + private void assertTableConversionPreservesAutoCreateSchema(final boolean miniFile) + throws Exception { + when(converter.convertForTableModel(any())) + .thenReturn(Optional.of(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()))); + for (final boolean tableModelRequest : new boolean[] {false, true}) { + for (final boolean requested : new boolean[] {false, true}) { + for (final boolean generatedByPipe : new boolean[] {false, true}) { + clearInvocations(converter); + final File file = + temporaryFolder.newFile( + "conversion-" + + tableModelRequest + + "-" + + requested + + "-" + + generatedByPipe + + ".tsfile"); + final LoadTsFile tableStatement = + LoadTsFile.createUnchecked( + null, + file.getAbsolutePath(), + requested + ? Collections.emptyMap() + : Collections.singletonMap("auto-create-schema", "false")) + .setDatabase("db"); + final LoadTsFileStatement treeStatement = + LoadTsFileStatement.createUnchecked(file.getAbsolutePath()); + treeStatement.setAutoCreateSchema(requested); + treeStatement.setDatabase("db"); + final MPPQueryContext context = + new MPPQueryContext(new QueryId("load_conversion_attributes")); + try (final LoadTsFileAnalyzer analyzer = + tableModelRequest + ? new LoadTsFileAnalyzer(tableStatement, generatedByPipe, context) + : new LoadTsFileAnalyzer(treeStatement, generatedByPipe, context)) { + final List<Boolean> tableModelFiles = + Whitebox.getInternalState(analyzer, "isTableModelTsFile"); + tableModelFiles.set(0, true); + Whitebox.setInternalState(analyzer, "isTableModelTsFileReliableIndex", 0); + if (miniFile) { + Assert.assertTrue( + (Boolean) Whitebox.invokeMethod(analyzer, "handleSingleMiniFile", 0)); + } else { + final Analysis analysis = new Analysis(); + Whitebox.invokeMethod( + analyzer, + "executeTabletConversionOnException", + (IAnalysis) analysis, + new LoadAnalyzeException("original analysis failure")); + Assert.assertTrue(analysis.isFinishQueryAfterAnalyze()); + } + final ArgumentCaptor<LoadTsFile> statement = ArgumentCaptor.forClass(LoadTsFile.class); + verify(converter).convertForTableModel(statement.capture()); + Assert.assertEquals(requested, statement.getValue().isAutoCreateSchema()); + Assert.assertEquals("db", statement.getValue().getDatabase()); + } + } + } + } + } + private void mockModelReader(final boolean tableModel) throws Exception { final TsFileSequenceReader reader = mock(TsFileSequenceReader.class); when(reader.getTableSchemaMap()) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java index 1e9aae08428..2c907895016 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java @@ -19,9 +19,19 @@ package org.apache.iotdb.db.queryengine.plan.analyze.load; +import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.exception.table.TableNotExistsException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.queryengine.common.SqlDialect; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.load.LoadAnalyzeException; import org.apache.iotdb.db.exception.load.LoadAnalyzeInvalidPathException; @@ -33,9 +43,14 @@ import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; +import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.schemaengine.table.ITableCache; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; @@ -59,11 +74,20 @@ import java.io.File; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.time.ZoneId; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class LoadTsFileAnalyzerTest { @@ -80,6 +104,228 @@ public class LoadTsFileAnalyzerTest { IoTDBDescriptor.getInstance().getConfig().setDataNodeId(dataNodeId); } + @Test + public void testTableLoadDoesNotAutoCreateSchemaWhenRequestDisablesIt() throws Exception { + for (final boolean tableModelStatement : new boolean[] {true, false}) { + assertTableLoadDoesNotAutoCreateSchema(tableModelStatement, true, false, true); + } + } + + @Test + public void testTableLoadDoesNotAutoCreateSchemaWhenGlobalConfigDisablesIt() throws Exception { + for (final boolean tableModelStatement : new boolean[] {true, false}) { + assertTableLoadDoesNotAutoCreateSchema(tableModelStatement, false, true, true); + } + } + + @Test + public void testTableLoadDoesNotAutoCreateSchemaWhenSessionDisablesIt() throws Exception { + for (final boolean tableModelStatement : new boolean[] {true, false}) { + assertTableLoadDoesNotAutoCreateSchema(tableModelStatement, true, true, false); + } + } + + @Test + public void testTableTsFileAnalysisPreservesSessionAutoCreateSchemaSetting() throws Exception { + final boolean originalAutoCreateSchemaEnabled = + IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled(); + final long originalTabletConversionThresholdBytes = + IoTDBDescriptor.getInstance().getConfig().getLoadTabletConversionThresholdBytes(); + final File tsFile = File.createTempFile("table-load-session-schema", ".tsfile"); + writeTableTsFileWithMixedDevices(tsFile); + + try { + IoTDBDescriptor.getInstance().getConfig().setAutoCreateSchemaEnabled(true); + IoTDBDescriptor.getInstance().getConfig().setLoadTabletConversionThresholdBytes(-1); + for (final boolean tableModelStatement : new boolean[] {true, false}) { + final String database = "load_session_schema_" + tableModelStatement; + final MPPQueryContext context = + createLoadQueryContext( + "load_session_schema_" + tableModelStatement, + database, + tableModelStatement ? SqlDialect.TABLE : SqlDialect.TREE, + false); + final LoadTsFile tableStatement = + LoadTsFile.createUnchecked(null, tsFile.getAbsolutePath(), Collections.emptyMap()) + .setDatabase(database); + tableStatement.setAutoCreateDatabase(false); + final LoadTsFileStatement treeStatement = + LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath()); + treeStatement.setDatabase(database); + treeStatement.setAutoCreateSchema(true); + treeStatement.setAutoCreateDatabase(false); + final boolean[] schemaCreationEnabledDuringValidation = {true}; + final SqlDialect[] validationDialect = {SqlDialect.TREE}; + final Metadata metadata = mock(Metadata.class); + when(metadata.validateTableHeaderSchema4TsFile( + anyString(), + any( + org.apache.iotdb.commons.queryengine.plan.relational.metadata.TableSchema + .class), + any(MPPQueryContext.class), + anyBoolean(), + anyBoolean(), + any(AtomicBoolean.class))) + .thenAnswer( + invocation -> { + final MPPQueryContext validationContext = invocation.getArgument(2); + schemaCreationEnabledDuringValidation[0] = + validationContext.getSession().isEnableAutoCreateSchema(); + validationDialect[0] = validationContext.getSession().getSqlDialect(); + return java.util.Optional.empty(); + }); + + try (final LoadTsFileAnalyzer analyzer = + tableModelStatement + ? new LoadTsFileAnalyzer(tableStatement, false, context) + : new LoadTsFileAnalyzer(treeStatement, false, context)) { + final Field metadataField = LoadTsFileAnalyzer.class.getDeclaredField("metadata"); + metadataField.setAccessible(true); + metadataField.set(analyzer, metadata); + final Method analyzeSingleTsFileMethod = + LoadTsFileAnalyzer.class.getDeclaredMethod( + "analyzeSingleTsFile", File.class, int.class); + analyzeSingleTsFileMethod.setAccessible(true); + final InvocationTargetException exception = + Assert.assertThrows( + InvocationTargetException.class, + () -> analyzeSingleTsFileMethod.invoke(analyzer, tsFile, 0)); + Assert.assertTrue(exception.getCause() instanceof LoadAnalyzeException); + } finally { + DataNodeSchemaLockManager.getInstance().releaseReadLock(context); + } + + Assert.assertFalse(schemaCreationEnabledDuringValidation[0]); + Assert.assertEquals(SqlDialect.TABLE, validationDialect[0]); + Assert.assertFalse(context.getSession().isEnableAutoCreateSchema()); + } + } finally { + IoTDBDescriptor.getInstance() + .getConfig() + .setAutoCreateSchemaEnabled(originalAutoCreateSchemaEnabled); + IoTDBDescriptor.getInstance() + .getConfig() + .setLoadTabletConversionThresholdBytes(originalTabletConversionThresholdBytes); + Assert.assertTrue(tsFile.delete()); + } + } + + private void assertTableLoadDoesNotAutoCreateSchema( + final boolean tableModelStatement, + final boolean globalAutoCreateSchemaEnabled, + final boolean requestAutoCreateSchema, + final boolean sessionAutoCreateSchemaEnabled) + throws Exception { + final boolean originalAutoCreateSchemaEnabled = + IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled(); + final boolean originalEnablePartialInsert = + IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert(); + final File tsFile = File.createTempFile("table-load-auto-create-schema", ".tsfile"); + final String database = "load_auto_create_schema_test"; + final MPPQueryContext context = + createTableQueryContext( + "load_auto_create_schema", database, sessionAutoCreateSchemaEnabled); + final ITableCache tableCache = DataNodeTableCache.getInstance(); + + try { + IoTDBDescriptor.getInstance() + .getConfig() + .setAutoCreateSchemaEnabled(globalAutoCreateSchemaEnabled); + IoTDBDescriptor.getInstance().getConfig().setEnablePartialInsert(false); + tableCache.invalid(database); + final TsTable existingTable = new TsTable("table1"); + existingTable.addColumnSchema(new TagColumnSchema("tag1", TSDataType.STRING)); + existingTable.addColumnSchema(new FieldColumnSchema("s1", TSDataType.INT64)); + tableCache.preUpdateTable(database, existingTable, null); + tableCache.commitUpdateTable(database, "table1", null, null); + + final LoadTsFile tableStatement = + LoadTsFile.createUnchecked( + null, + tsFile.getAbsolutePath(), + requestAutoCreateSchema + ? Collections.emptyMap() + : Collections.singletonMap("auto-create-schema", "false")) + .setDatabase(database); + tableStatement.setAutoCreateDatabase(false); + final LoadTsFileStatement treeStatement = + LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath()); + treeStatement.setAutoCreateSchema(requestAutoCreateSchema); + treeStatement.setAutoCreateDatabase(false); + + try (final LoadTsFileAnalyzer analyzer = + tableModelStatement + ? new LoadTsFileAnalyzer(tableStatement, false, context) + : new LoadTsFileAnalyzer(treeStatement, false, context)) { + final Method getOrCreateTableSchemaCacheMethod = + LoadTsFileAnalyzer.class.getDeclaredMethod("getOrCreateTableSchemaCache"); + getOrCreateTableSchemaCacheMethod.setAccessible(true); + final LoadTsFileTableSchemaCache schemaCache = + (LoadTsFileTableSchemaCache) getOrCreateTableSchemaCacheMethod.invoke(analyzer); + schemaCache.setDatabase(database); + + schemaCache.setTableSchemaMap( + new HashMap<>( + Collections.singletonMap( + "missing_table", createTsFileTableSchema("missing_table", false)))); + final SemanticException missingTableException = + Assert.assertThrows( + SemanticException.class, + () -> schemaCache.createTableAndDatabaseIfNecessary("missing_table")); + Assert.assertTrue(missingTableException.getCause() instanceof TableNotExistsException); + Assert.assertNull(tableCache.getTableInCache(database, "missing_table")); + + schemaCache.setTableSchemaMap( + new HashMap<>( + Collections.singletonMap("table1", createTsFileTableSchema("table1", true)))); + final SemanticException missingColumnException = + Assert.assertThrows( + SemanticException.class, + () -> schemaCache.createTableAndDatabaseIfNecessary("table1")); + Assert.assertEquals( + TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode(), missingColumnException.getErrorCode()); + Assert.assertNull(tableCache.getTableInCache(database, "table1").getColumnSchema("s2")); + + // Existing partial-insert behavior still applies without creating the missing field. + IoTDBDescriptor.getInstance().getConfig().setEnablePartialInsert(true); + schemaCache.setTableSchemaMap( + new HashMap<>( + Collections.singletonMap("table1", createTsFileTableSchema("table1", true)))); + schemaCache.createTableAndDatabaseIfNecessary("table1"); + Assert.assertNull(tableCache.getTableInCache(database, "table1").getColumnSchema("s2")); + + schemaCache.setTableSchemaMap( + new HashMap<>( + Collections.singletonMap("table1", createTsFileTableSchema("table1", false)))); + schemaCache.createTableAndDatabaseIfNecessary("table1"); + } + } finally { + DataNodeSchemaLockManager.getInstance().releaseReadLock(context); + tableCache.invalid(database); + IoTDBDescriptor.getInstance() + .getConfig() + .setAutoCreateSchemaEnabled(originalAutoCreateSchemaEnabled); + IoTDBDescriptor.getInstance().getConfig().setEnablePartialInsert(originalEnablePartialInsert); + Assert.assertTrue(tsFile.delete()); + } + } + + private TableSchema createTsFileTableSchema(final String tableName, final boolean extraField) { + return new TableSchema( + tableName, + extraField + ? Arrays.asList( + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT64), + new MeasurementSchema("s2", TSDataType.DOUBLE)) + : Arrays.asList( + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT64)), + extraField + ? Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD) + : Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + } + @Test public void testAnalyzeSingleTableFileShouldNotCountTimestampInPointCount() throws Exception { final File tsFile = new File("load-table-mixed-null-device.tsfile"); @@ -490,6 +736,29 @@ public class LoadTsFileAnalyzerTest { .anyMatch(device -> Arrays.equals(device.getSegments(), expectedSegments)); } + private MPPQueryContext createTableQueryContext( + final String queryId, final String database, final boolean enableAutoCreateSchema) { + return createLoadQueryContext(queryId, database, SqlDialect.TABLE, enableAutoCreateSchema); + } + + private MPPQueryContext createLoadQueryContext( + final String queryId, + final String database, + final SqlDialect dialect, + final boolean enableAutoCreateSchema) { + final MPPQueryContext context = new MPPQueryContext(new QueryId(queryId)); + context.setSession( + new SessionInfo( + 0, + new UserEntity(AuthorityChecker.SUPER_USER_ID, AuthorityChecker.SUPER_USER, ""), + ZoneId.systemDefault(), + IoTDBConstant.ClientVersion.V_1_0, + database, + dialect, + enableAutoCreateSchema)); + return context; + } + private static class TrackingLoadTsFileTableSchemaCache extends LoadTsFileTableSchemaCache { private final Set<List<Object>> verifiedDevices = new HashSet<>(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadFallbackHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadFallbackHandlerTest.java index b765405277a..0acf692e67a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadFallbackHandlerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadFallbackHandlerTest.java @@ -20,14 +20,22 @@ package org.apache.iotdb.db.queryengine.plan.scheduler.load; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.execution.QueryState; import org.apache.iotdb.db.queryengine.execution.QueryStateMachine; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadSingleTsFileNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched; +import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; +import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.load.converter.LoadTsFileDataTypeConverter; +import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -38,6 +46,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -51,7 +60,9 @@ import java.util.List; import java.util.Optional; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(PowerMockRunner.class) @@ -134,6 +145,99 @@ public class LoadFallbackHandlerTest { Assert.assertTrue(failedIndexes.isEmpty()); } + @Test + public void testTableFallbackPreservesRequestSchemaCreationSetting() throws Exception { + assertTableConversionPreservesRequestSchemaCreationSetting(false); + } + + @Test + public void testTableRewritePreservesRequestSchemaCreationSetting() throws Exception { + assertTableConversionPreservesRequestSchemaCreationSetting(true); + } + + private void assertTableConversionPreservesRequestSchemaCreationSetting(final boolean rewrite) + throws Exception { + when(converter.convertForTableModel(any())).thenReturn(Optional.of(RpcUtils.SUCCESS_STATUS)); + final File file = temporaryFolder.newFile("table-schema.tsfile"); + for (final boolean tableModelRequest : new boolean[] {false, true}) { + for (final boolean generatedByPipe : new boolean[] {false, true}) { + for (final boolean requested : new boolean[] {false, true, false}) { + clearInvocations(converter); + final LoadSingleTsFileNode node = + splitTableLoadNode(file, tableModelRequest, generatedByPipe, requested); + if (rewrite) { + Assert.assertTrue( + new TabletRewriteLoadStrategy( + converter, generatedByPipe, mock(LoadTsFileCostMetricsSet.class)) + .execute(node)); + } else { + final List<Integer> failedIndexes = new ArrayList<>(Collections.singletonList(0)); + final QueryStateMachine stateMachine = newStateMachine(); + new LoadFallbackHandler( + mock(MPPQueryContext.class), + generatedByPipe, + Collections.singletonList(node), + failedIndexes, + stateMachine) + .convertFailedTsFilesToTablets(); + Assert.assertEquals(QueryState.FINISHED, stateMachine.getState()); + Assert.assertTrue(failedIndexes.isEmpty()); + } + final ArgumentCaptor<LoadTsFile> statement = ArgumentCaptor.forClass(LoadTsFile.class); + verify(converter).convertForTableModel(statement.capture()); + Assert.assertEquals(requested, statement.getValue().isAutoCreateSchema()); + Assert.assertEquals("test", statement.getValue().getDatabase()); + Assert.assertEquals(generatedByPipe, statement.getValue().isGeneratedByPipe()); + Assert.assertEquals(!rewrite, statement.getValue().isDeleteAfterLoad()); + Assert.assertTrue(file.exists()); + } + } + } + } + + private LoadSingleTsFileNode splitTableLoadNode( + final File file, + final boolean tableModelRequest, + final boolean generatedByPipe, + final boolean requested) + throws Exception { + final IAnalysis analysis; + if (tableModelRequest) { + final LoadTsFile statement = + LoadTsFile.createUnchecked( + null, + file.getAbsolutePath(), + requested + ? Collections.emptyMap() + : Collections.singletonMap("auto-create-schema", "false")) + .setDatabase("test") + .setDeleteAfterLoad(true); + statement.addWritePointCount(1); + final org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis tableAnalysis = + new org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis( + generatedByPipe ? new PipeEnriched(statement) : statement, Collections.emptyMap()); + analysis = tableAnalysis; + } else { + final LoadTsFileStatement statement = + LoadTsFileStatement.createUnchecked(file.getAbsolutePath()).setDeleteAfterLoad(true); + statement.setAutoCreateSchema(requested); + statement.addWritePointCount(1); + final org.apache.iotdb.db.queryengine.plan.analyze.Analysis treeAnalysis = + new org.apache.iotdb.db.queryengine.plan.analyze.Analysis(); + treeAnalysis.setRealStatement( + generatedByPipe ? new PipeEnrichedStatement(statement) : statement); + analysis = treeAnalysis; + } + final LoadTsFileNode loadNode = + new LoadTsFileNode( + new PlanNodeId("load_schema"), + Collections.singletonList(new TsFileResource(file)), + Collections.singletonList(true), + "test", + false); + return (LoadSingleTsFileNode) loadNode.splitByPartition(analysis).get(0); + } + private LoadSingleTsFileNode newNode(final String name, final boolean tableModel) throws Exception { final File file = temporaryFolder.newFile(name); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverterSchemaTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverterSchemaTest.java new file mode 100644 index 00000000000..15b10d90c0d --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverterSchemaTest.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.load.converter; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.queryengine.common.SqlDialect; +import org.apache.iotdb.db.auth.AuthorityChecker; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.protocol.session.InternalClientSession; +import org.apache.iotdb.db.protocol.session.SessionManager; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.plan.Coordinator; +import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.schema.Schema; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.io.File; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*"}) +@PrepareForTest({Coordinator.class, AuthorityChecker.class}) +public class LoadTsFileDataTypeConverterSchemaTest { + + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private final List<SessionInfo> conversionSessions = new ArrayList<>(); + private InternalClientSession session; + private int originalDataNodeId; + private File tsFile; + + @Before + public void setUp() throws Exception { + originalDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0); + session = new InternalClientSession("load_schema_conversion"); + session.setUserId(0); + session.setUsername("root"); + session.setClientVersion(IoTDBConstant.ClientVersion.V_1_0); + session.setZoneId(ZoneOffset.UTC); + session.setSqlDialect(SqlDialect.TREE); + SessionManager.getInstance().registerSession(session); + + PowerMockito.mockStatic(Coordinator.class); + PowerMockito.mockStatic(AuthorityChecker.class); + PowerMockito.when(AuthorityChecker.getUserId(anyString())).thenReturn(Optional.of(0L)); + final Coordinator coordinator = mock(Coordinator.class); + when(Coordinator.getInstance()).thenReturn(coordinator); + when(coordinator.executeForTableModel( + any(Statement.class), + any(SqlParser.class), + any(IClientSession.class), + anyLong(), + any(SessionInfo.class), + anyString(), + any(Metadata.class), + anyLong())) + .thenAnswer( + invocation -> { + conversionSessions.add(invocation.getArgument(4)); + return new ExecutionResult( + new QueryId("load_schema_conversion"), + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + }); + tsFile = new File(temporaryFolder.getRoot(), "table.tsfile"); + writeTableTsFile(); + } + + @After + public void tearDown() { + SessionManager.getInstance().removeCurrSession(); + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(originalDataNodeId); + } + + @Test + public void testTableConversionHonorsRequestWithoutChangingClientSession() { + final LoadTsFileDataTypeConverter converter = + new LoadTsFileDataTypeConverter(new MPPQueryContext(new QueryId("load_schema")), false); + for (final boolean requested : new boolean[] {false, true, false}) { + assertConversionSchemaFlag(converter, requested, requested); + Assert.assertTrue(session.isEnableAutoCreateSchema()); + Assert.assertEquals(SqlDialect.TREE, session.getSqlDialect()); + Assert.assertSame(session, SessionManager.getInstance().getCurrSession()); + } + } + + @Test + public void testTableConversionPreservesDisabledClientSessionSchemaCreation() { + session.setEnableAutoCreateSchema(false); + final LoadTsFileDataTypeConverter converter = + new LoadTsFileDataTypeConverter(new MPPQueryContext(new QueryId("load_schema")), false); + assertConversionSchemaFlag(converter, true, false); + Assert.assertFalse(session.isEnableAutoCreateSchema()); + } + + @Test + public void testTableConversionPreservesDisabledQuerySessionWithoutClientSession() { + SessionManager.getInstance().removeCurrSession(); + final MPPQueryContext context = new MPPQueryContext(new QueryId("load_schema")); + context.setSession( + new SessionInfo( + session.getId(), + new UserEntity(session.getUserId(), session.getUsername(), session.getClientAddress()), + ZoneOffset.UTC, + session.getClientVersion(), + "db", + SqlDialect.TABLE, + false)); + final LoadTsFileDataTypeConverter converter = new LoadTsFileDataTypeConverter(context, false); + + assertConversionFlagWithoutClientSession(converter); + + Assert.assertNull(SessionManager.getInstance().getCurrSession()); + } + + private void assertConversionSchemaFlag( + final LoadTsFileDataTypeConverter converter, + final boolean requested, + final boolean expected) { + conversionSessions.clear(); + final LoadTsFile statement = + LoadTsFile.createUnchecked( + null, + tsFile.getAbsolutePath(), + requested + ? Collections.emptyMap() + : Collections.singletonMap("auto-create-schema", "false")) + .setDatabase("db"); + final Optional<TSStatus> status = converter.convertForTableModel(statement); + Assert.assertTrue(status.isPresent()); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.get().getCode()); + Assert.assertFalse(conversionSessions.isEmpty()); + for (final SessionInfo conversionSession : conversionSessions) { + Assert.assertEquals(expected, conversionSession.isEnableAutoCreateSchema()); + Assert.assertEquals("db", conversionSession.getDatabaseName().orElse(null)); + Assert.assertEquals(SqlDialect.TABLE, conversionSession.getSqlDialect()); + Assert.assertEquals("root", conversionSession.getUserName()); + } + } + + private void assertConversionFlagWithoutClientSession( + final LoadTsFileDataTypeConverter converter) { + conversionSessions.clear(); + final LoadTsFile statement = + LoadTsFile.createUnchecked( + null, + tsFile.getAbsolutePath(), + Collections.singletonMap("auto-create-schema", "true")) + .setDatabase("db"); + final Optional<TSStatus> status = converter.convertForTableModel(statement); + Assert.assertTrue(status.isPresent()); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.get().getCode()); + Assert.assertFalse(conversionSessions.isEmpty()); + for (final SessionInfo conversionSession : conversionSessions) { + Assert.assertFalse(conversionSession.isEnableAutoCreateSchema()); + } + } + + private void writeTableTsFile() throws Exception { + final List<IMeasurementSchema> columns = + Arrays.asList( + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT64)); + final Schema schema = new Schema(); + schema.registerTableSchema( + new TableSchema( + "table1", columns, Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD))); + try (final TsFileIOWriter writer = new TsFileIOWriter(tsFile)) { + writer.setSchema(schema); + writer.startChunkGroup(new StringArrayDeviceID(new String[] {"table1", "tagA"})); + final AlignedChunkWriterImpl chunkWriter = + new AlignedChunkWriterImpl(columns.subList(1, columns.size())); + chunkWriter.getTimeChunkWriter().write(1); + chunkWriter.getValueChunkWriterByIndex(0).write(1, 1L, false); + chunkWriter.writeToFileWriter(writer); + writer.endChunkGroup(); + writer.endFile(); + } + } +}
