This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch codex/load-compatible-tsfile in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7d9e71850553770365f12beb941ad0745c692647 Author: Caideyipi <[email protected]> AuthorDate: Mon Nov 16 14:27:57 2026 +0800 [Load] Support compatible TsFile data types in LOAD --- .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 34 ++++++++++++ .../relational/it/db/it/IoTDBLoadTsFileIT.java | 38 ++++++++++++++ .../plan/analyze/load/LoadTsFileAnalyzer.java | 22 ++++++++ .../analyze/load/LoadTsFileTableSchemaCache.java | 40 ++++++++++----- .../load/TreeSchemaAutoCreatorAndVerifier.java | 22 ++++---- .../LoadConvertedInsertTabletStatement.java | 4 ++ .../PipeConvertedInsertTabletStatementTest.java | 32 ++++++++++++ .../plan/analyze/load/LoadTsFileAnalyzerTest.java | 60 ++++++++++++++++++++-- 8 files changed, 226 insertions(+), 26 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index b17c6a19b57..c35b13871db 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -75,6 +75,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -631,6 +632,39 @@ public class IoTDBLoadTsFileIT { } } + @Test + public void testLoadWithCompatibleDataType() throws Exception { + registerSchema(); + + final String device = "root.sg.test_0.compatible_type"; + final MeasurementSchema fileSchema = new MeasurementSchema("s", TSDataType.INT32); + final MeasurementSchema targetSchema = new MeasurementSchema("s", TSDataType.INT64); + final long seed = 42L; + final long expectedValue = new Random(seed).nextInt(); + final File file = new File(tmpDir, "1-0-0-0.tsfile"); + + try (final TsFileGenerator generator = new TsFileGenerator(file)) { + generator.registerTimeseries(device, Collections.singletonList(fileSchema)); + generator.resetRandom(seed); + generator.generateData(device, 1, 1000L, false); + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(convert2SQL(device, targetSchema)); + statement.execute( + String.format( + "load \"%s\" with ('tablet-conversion-threshold'='-1')", file.getAbsolutePath())); + + try (final ResultSet resultSet = + statement.executeQuery(String.format("select s from %s where time=1000", device))) { + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(expectedValue, resultSet.getLong(1)); + Assert.assertFalse(resultSet.next()); + } + } + } + // Loading aligned and non-aligned devices extends the bound template and preserves every point. @Test public void testLoadWithExtendTemplate() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java index f36117f70c8..993ba719ade 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileIT.java @@ -67,6 +67,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Objects; +import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -288,6 +289,43 @@ public class IoTDBLoadTsFileIT { } } + @Test + public void testLoadWithCompatibleDataType() throws Exception { + final String tableName = "load_compatible_type"; + final MeasurementSchema fileSchema = new MeasurementSchema("f", TSDataType.INT32); + final MeasurementSchema targetSchema = new MeasurementSchema("f", TSDataType.INT64); + final List<ColumnCategory> columnCategories = Collections.singletonList(ColumnCategory.FIELD); + final long seed = 42L; + final long expectedValue = new Random(seed).nextInt(); + final File file = new File(tmpDir, "1-0-0-0.tsfile"); + + try (final TsFileTableGenerator generator = new TsFileTableGenerator(file)) { + generator.registerTable(tableName, Collections.singletonList(fileSchema), columnCategories); + generator.resetRandom(seed); + generator.generateData(tableName, 1, 1000L); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(String.format("create database if not exists %s", database)); + statement.execute(String.format("use %s", database)); + statement.execute( + convert2TableSQL(tableName, Collections.singletonList(targetSchema), columnCategories)); + statement.execute( + String.format( + "load '%s' with ('database'='%s', 'tablet-conversion-threshold'='-1')", + file.getAbsolutePath(), database)); + + try (final ResultSet resultSet = + statement.executeQuery(String.format("select f from %s order by time", tableName))) { + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(expectedValue, resultSet.getLong(1)); + Assert.assertFalse(resultSet.next()); + } + } + } + /** * Generate two tables with 10,000 rows each and delete all rows from the second through TsFile * mods; after loading, expect 10,000 rows in the first table and only that table to be visible. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 5193a352acb..09421edc643 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -128,6 +128,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable { private final boolean isDeleteAfterLoad; private final boolean isConvertOnTypeMismatch; private final long tabletConversionThresholdBytes; + private boolean hasCompatibleDataTypeMismatch; // Schema creators for tree and table private TreeSchemaAutoCreatorAndVerifier treeSchemaAutoCreatorAndVerifier; @@ -224,6 +225,15 @@ public class LoadTsFileAnalyzer implements AutoCloseable { return isConvertOnTypeMismatch; } + void markCompatibleDataTypeMismatch() { + hasCompatibleDataTypeMismatch = true; + } + + boolean hasCompatibleDataTypeMismatch() { + return hasCompatibleDataTypeMismatch + || (tableSchemaCache != null && tableSchemaCache.hasCompatibleDataTypeMismatch()); + } + public IAnalysis analyzeFileByFile(IAnalysis analysis) { if (!checkBeforeAnalyzeFileByFile(analysis)) { return analysis; @@ -270,6 +280,11 @@ public class LoadTsFileAnalyzer implements AutoCloseable { return analysis; } + if (hasCompatibleDataTypeMismatch()) { + executeTabletConversion(analysis); + return analysis; + } + if (isGeneratedByPipe) { LOGGER.debug(DataNodeQueryMessages.LOAD_ANALYSIS_STAGE_ALL_TSFILES_HAVE_BEEN_ANALYZED); } else { @@ -671,6 +686,9 @@ public class LoadTsFileAnalyzer implements AutoCloseable { } getOrCreateTableSchemaCache().flush(); + if (getOrCreateTableSchemaCache().hasCompatibleDataTypeMismatch()) { + markCompatibleDataTypeMismatch(); + } if (getOrCreateTableSchemaCache().isNeedDecode4DifferentTimeColumn()) { if (isTableModelStatement) { loadTsFileTableStatement.enableNeedDecode4TimeColumn(); @@ -807,6 +825,10 @@ public class LoadTsFileAnalyzer implements AutoCloseable { return; } + executeTabletConversion(analysis); + } + + private void executeTabletConversion(final IAnalysis analysis) { if (isTableModelTsFileReliableIndex < tsFiles.size() - 1) { try { getFileModelInfoBeforeTabletConversion(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java index bfe316d8f18..065f66e1a07 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java @@ -58,6 +58,7 @@ import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.utils.Pair; @@ -110,6 +111,7 @@ public class LoadTsFileTableSchemaCache { private final MPPQueryContext context; private final boolean shouldVerifyDataType; private final boolean allowAutoCreateSchema; + private boolean hasCompatibleDataTypeMismatch; private Map<String, Set<IDeviceID>> currentBatchTable2Devices; @@ -418,6 +420,10 @@ public class LoadTsFileTableSchemaCache { return needDecode4DifferentTimeColumn.get(); } + boolean hasCompatibleDataTypeMismatch() { + return hasCompatibleDataTypeMismatch; + } + private org.apache.tsfile.file.metadata.TableSchema resolveTableSchemaFromCache( final String tableName) { final TsTable table = DataNodeTableCache.getInstance().getTable(database, tableName, false); @@ -504,19 +510,27 @@ public class LoadTsFileTableSchemaCache { } else if (fileColumn.getColumnCategory() == TsTableColumnCategory.FIELD) { ColumnSchema realColumn = fieldColumnNameToSchema.get(fileColumn.getName()); if (realColumn != null && !fileColumn.getType().equals(realColumn.getType())) { - final String message = - String.format( - DataNodeQueryMessages - .MESSAGE_DATA_TYPE_MISMATCH_FOR_COLUMN_ARG_IN_TABLE_ARG_TYPE_IN_TSFILE_ARG_TYPE_IN_IOTDB_ARG_ECAD0C63, - fileColumn.getName(), - realSchema.getTableName(), - fileColumn.getType(), - realColumn.getType()); - if (shouldVerifyDataType) { - throw new LoadAnalyzeTypeMismatchException(message); - } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(message); + final TSDataType realDataType = + TSDataType.valueOf(realColumn.getType().getTypeEnum().name()); + final TSDataType fileDataType = + TSDataType.valueOf(fileColumn.getType().getTypeEnum().name()); + if (realDataType.isCompatible(fileDataType)) { + hasCompatibleDataTypeMismatch |= shouldVerifyDataType; + } else { + final String message = + String.format( + DataNodeQueryMessages + .MESSAGE_DATA_TYPE_MISMATCH_FOR_COLUMN_ARG_IN_TABLE_ARG_TYPE_IN_TSFILE_ARG_TYPE_IN_IOTDB_ARG_ECAD0C63, + fileColumn.getName(), + realSchema.getTableName(), + fileColumn.getType(), + realColumn.getType()); + if (shouldVerifyDataType) { + throw new LoadAnalyzeTypeMismatchException(message); + } + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(message); + } } } else if (LOGGER.isDebugEnabled() && realColumn == null) { LOGGER.debug( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java index d27da400dfa..0d813b7f1aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java @@ -605,15 +605,19 @@ public class TreeSchemaAutoCreatorAndVerifier { // check datatype if (!tsFileSchema.getType().equals(iotdbSchema.getType())) { - throw new LoadAnalyzeTypeMismatchException( - String.format( - DataNodeQueryMessages - .EXCEPTION_DATA_TYPE_MISMATCH_FOR_MEASUREMENT_ARGARGARG_TYPE_IN_TSFILE_ARG_TYPE_IN_IOTDB_ARG_C5BA7DBD, - device, - TsFileConstant.PATH_SEPARATOR, - iotdbSchema.getMeasurementName(), - tsFileSchema.getType(), - iotdbSchema.getType())); + if (iotdbSchema.getType().isCompatible(tsFileSchema.getType())) { + loadTsFileAnalyzer.markCompatibleDataTypeMismatch(); + } else { + throw new LoadAnalyzeTypeMismatchException( + String.format( + DataNodeQueryMessages + .EXCEPTION_DATA_TYPE_MISMATCH_FOR_MEASUREMENT_ARGARGARG_TYPE_IN_TSFILE_ARG_TYPE_IN_IOTDB_ARG_C5BA7DBD, + device, + TsFileConstant.PATH_SEPARATOR, + iotdbSchema.getMeasurementName(), + tsFileSchema.getType(), + iotdbSchema.getType())); + } } // check encoding diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java index 3edf9e150b8..a5abc3ccc98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java @@ -53,6 +53,10 @@ public class LoadConvertedInsertTabletStatement extends PipeConvertedInsertTable return false; } + if (dataType.isCompatible(dataTypes[columnIndex])) { + return originalCheckAndCastDataType(columnIndex, dataType); + } + LOGGER.info( StorageEngineMessages.STORAGE_LOG_LOAD_INSERTING_TABLET_TO_CASTING_TYPE_FROM_TO_AE808A8B, devicePath, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatementTest.java index a6bd537bee4..05432fee358 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatementTest.java @@ -113,6 +113,38 @@ public class PipeConvertedInsertTabletStatementTest { Assert.assertArrayEquals(new double[] {1.0, 2.0}, (double[]) tablet.getValues()[0], 0.0); } + @Test + public void testLoadCompatibleTypeUsesInsertTabletCasting() throws Exception { + final InsertTabletStatement source = new InsertTabletStatement(); + source.setDevicePath(new PartialPath("root.sg.d1")); + source.setMeasurements(new String[] {"s1"}); + source.setDataTypes(new TSDataType[] {TSDataType.INT32}); + source.setMeasurementSchemas( + new MeasurementSchema[] {new MeasurementSchema("s1", TSDataType.INT64)}); + source.setTimes(new long[] {1L, 2L}); + source.setColumns(new Object[] {new int[] {Integer.MIN_VALUE, Integer.MAX_VALUE}}); + source.setRowCount(2); + + final InsertTabletStatement ordinaryInsert = new InsertTabletStatement(); + ordinaryInsert.setDevicePath(new PartialPath("root.sg.d1")); + ordinaryInsert.setMeasurements(new String[] {"s1"}); + ordinaryInsert.setDataTypes(new TSDataType[] {TSDataType.INT32}); + ordinaryInsert.setMeasurementSchemas( + new MeasurementSchema[] {new MeasurementSchema("s1", TSDataType.INT64)}); + ordinaryInsert.setTimes(new long[] {1L, 2L}); + ordinaryInsert.setColumns(new Object[] {new int[] {Integer.MIN_VALUE, Integer.MAX_VALUE}}); + ordinaryInsert.setRowCount(2); + ordinaryInsert.selfCheckDataTypes(0); + + final LoadConvertedInsertTabletStatement loadInsert = + new LoadConvertedInsertTabletStatement(source, true); + loadInsert.selfCheckDataTypes(0); + + Assert.assertEquals(TSDataType.INT64, loadInsert.getDataTypes()[0]); + Assert.assertArrayEquals( + (long[]) ordinaryInsert.getColumns()[0], (long[]) loadInsert.getColumns()[0]); + } + private static InsertTabletStatement createPartiallyFailedStatementWithMissingColumn() throws Exception { final InsertTabletStatement source = new InsertTabletStatement(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java index 9ef5f7b7f62..1e14055ba3e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java @@ -487,8 +487,8 @@ public class LoadTsFileAnalyzerTest { getVerifyTableDataTypeMethod() .invoke( schemaCache, - createTableSchema(TSDataType.INT64), - createTableSchema(TSDataType.DOUBLE))); + createTableSchema(TSDataType.INT32), + createTableSchema(TSDataType.BOOLEAN))); Assert.assertTrue(exception.getCause() instanceof LoadAnalyzeTypeMismatchException); } finally { @@ -496,6 +496,21 @@ public class LoadTsFileAnalyzerTest { } } + @Test + public void testTableSchemaCacheShouldAllowCompatibleDataType() throws Exception { + final LoadTsFileTableSchemaCache schemaCache = createTableSchemaCache(true); + try { + getVerifyTableDataTypeMethod() + .invoke( + schemaCache, + createTableSchema(TSDataType.INT32), + createTableSchema(TSDataType.INT64)); + Assert.assertTrue(schemaCache.hasCompatibleDataTypeMismatch()); + } finally { + schemaCache.close(); + } + } + @Test public void testTableSchemaCacheShouldNotThrowMismatchWhenSkippingDataTypeVerification() throws Exception { @@ -504,8 +519,9 @@ public class LoadTsFileAnalyzerTest { getVerifyTableDataTypeMethod() .invoke( schemaCache, - createTableSchema(TSDataType.INT64), - createTableSchema(TSDataType.DOUBLE)); + createTableSchema(TSDataType.INT32), + createTableSchema(TSDataType.BOOLEAN)); + Assert.assertFalse(schemaCache.hasCompatibleDataTypeMismatch()); } finally { schemaCache.close(); } @@ -556,6 +572,42 @@ public class LoadTsFileAnalyzerTest { } } + @Test + public void testTreeSchemaVerifierShouldAllowCompatibleDataType() throws Exception { + final File tsFile = File.createTempFile("load-tree-compatible-type", ".tsfile"); + + try (final LoadTsFileAnalyzer analyzer = + new LoadTsFileAnalyzer( + LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath()), + false, + new MPPQueryContext(new QueryId("load_tree_compatible_type_test")))) { + final TreeSchemaAutoCreatorAndVerifier verifier = + new TreeSchemaAutoCreatorAndVerifier(analyzer); + try { + final IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d1"); + getTreeSchemaCache(verifier) + .addTimeSeries(device, new MeasurementSchema("s1", TSDataType.INT32)); + getTreeSchemaCache(verifier).addIsAlignedCache(device, true, true); + + final ClusterSchemaTree schemaTree = new ClusterSchemaTree(); + schemaTree.appendSingleMeasurement( + new PartialPath("root.sg.d1.s1"), + new MeasurementSchema("s1", TSDataType.INT64), + null, + null, + null, + true); + + getVerifyTreeSchemaMethod().invoke(verifier, schemaTree); + Assert.assertTrue(analyzer.hasCompatibleDataTypeMismatch()); + } finally { + verifier.close(); + } + } finally { + Assert.assertTrue(tsFile.delete()); + } + } + @Test public void testTreeSchemaVerifierShouldRejectDeviceWithEmptyPathNode() throws Exception { final File tsFile = File.createTempFile("load-tree-illegal-device", ".tsfile");
