This is an automated email from the ASF dual-hosted git repository. justinchen pushed a commit to branch UserDefineTime-TsFile in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1459d61f836782471874bac8357e457f0f87c9ad Author: Caideyipi <[email protected]> AuthorDate: Thu Jan 22 16:14:52 2026 +0800 Partial --- .../plan/analyze/load/LoadTsFileAnalyzer.java | 3 +++ .../plan/analyze/load/LoadTsFileTableSchemaCache.java | 10 ++++++---- .../queryengine/plan/planner/LogicalPlanVisitor.java | 3 ++- .../planner/plan/node/load/LoadSingleTsFileNode.java | 14 ++++++++------ .../plan/planner/plan/node/load/LoadTsFileNode.java | 14 +++++++++++--- .../plan/relational/metadata/Metadata.java | 4 +++- .../plan/relational/metadata/TableMetadataImpl.java | 19 +++++++++++++------ .../metadata/fetcher/TableHeaderSchemaValidator.java | 6 ++++-- .../plan/relational/planner/RelationPlanner.java | 6 +++++- .../plan/relational/sql/ast/LoadTsFile.java | 11 ++++++++++- .../plan/planner/node/load/LoadTsFileNodeTest.java | 2 +- .../plan/relational/analyzer/AnalyzerTest.java | 4 +++- .../plan/relational/analyzer/TSBSMetadata.java | 4 +++- .../plan/relational/analyzer/TestMetadata.java | 4 +++- .../plan/statement/InsertStatementTest.java | 10 +++++++--- 15 files changed, 82 insertions(+), 32 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 8202de5c499..1bcb6a26426 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -573,6 +573,9 @@ public class LoadTsFileAnalyzer implements AutoCloseable { } getOrCreateTableSchemaCache().flush(); + if (getOrCreateTableSchemaCache().isNeedDecode4DifferentTimeColumn()) { + loadTsFileTableStatement.enableNeedDecode4TimeColumn(); + } getOrCreateTableSchemaCache().clearTagColumnMapper(); TimestampPrecisionUtils.checkTimestampPrecision(tsFileResource.getFileEndTime()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java index 84dec4eca01..79ede0f459e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java @@ -68,6 +68,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import static org.apache.iotdb.commons.schema.MemUsageUtil.computeStringMemUsage; @@ -111,7 +112,7 @@ public class LoadTsFileTableSchemaCache { private long currentTimeIndexMemoryUsageSizeInBytes = 0; private int currentBatchDevicesCount = 0; - private boolean needDecode4DifferentTimeColumn; + private final AtomicBoolean needDecode4DifferentTimeColumn = new AtomicBoolean(false); public LoadTsFileTableSchemaCache( final Metadata metadata, final MPPQueryContext context, final boolean needToCreateDatabase) @@ -300,7 +301,8 @@ public class LoadTsFileTableSchemaCache { tableName, schema); final TableSchema realSchema = metadata - .validateTableHeaderSchema4TsFile(database, fileSchema, context, true, true) + .validateTableHeaderSchema4TsFile( + database, fileSchema, context, true, true, needDecode4DifferentTimeColumn) .orElse(null); if (Objects.isNull(realSchema)) { throw new LoadAnalyzeException( @@ -312,7 +314,7 @@ public class LoadTsFileTableSchemaCache { } public boolean isNeedDecode4DifferentTimeColumn() { - return needDecode4DifferentTimeColumn; + return needDecode4DifferentTimeColumn.get(); } private void autoCreateTableDatabaseIfAbsent(final String database) throws LoadAnalyzeException { @@ -456,7 +458,7 @@ public class LoadTsFileTableSchemaCache { currentBatchTable2Devices = null; tableTagColumnMapper = null; - needDecode4DifferentTimeColumn = false; + needDecode4DifferentTimeColumn.set(false); } private void clearDevices() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 2274869c355..a36564878df 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -548,7 +548,8 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte context.getQueryId().genPlanNodeId(), loadTsFileStatement.getResources(), isTableModel, - loadTsFileStatement.getDatabase()); + loadTsFileStatement.getDatabase(), + false); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 79db4f10785..c8170a4880a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -66,12 +66,13 @@ public class LoadSingleTsFileNode extends WritePlanNode { private TRegionReplicaSet localRegionReplicaSet; public LoadSingleTsFileNode( - PlanNodeId id, - TsFileResource resource, - boolean isTableModel, - String database, - boolean deleteAfterLoad, - long writePointCount) { + final PlanNodeId id, + final TsFileResource resource, + final boolean isTableModel, + final String database, + final boolean deleteAfterLoad, + final long writePointCount, + final boolean needDecodeTsFile) { super(id); this.tsFile = resource.getTsFile(); this.resource = resource; @@ -79,6 +80,7 @@ public class LoadSingleTsFileNode extends WritePlanNode { this.database = database; this.deleteAfterLoad = deleteAfterLoad; this.writePointCount = writePointCount; + this.needDecodeTsFile = needDecodeTsFile; } public boolean isTsFileEmpty() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java index 3588b6ddbb0..25ad9f3a5c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java @@ -46,13 +46,19 @@ public class LoadTsFileNode extends WritePlanNode { private final List<TsFileResource> resources; private final List<Boolean> isTableModel; private final String database; + private final boolean needDecode4TimeColumn; public LoadTsFileNode( - PlanNodeId id, List<TsFileResource> resources, List<Boolean> isTableModel, String database) { + final PlanNodeId id, + final List<TsFileResource> resources, + final List<Boolean> isTableModel, + final String database, + final boolean needDecode4TimeColumn) { super(id); this.resources = resources; this.isTableModel = isTableModel; this.database = database; + this.needDecode4TimeColumn = needDecode4TimeColumn; } @Override @@ -121,7 +127,8 @@ public class LoadTsFileNode extends WritePlanNode { isTableModel.get(i), database, statement.isDeleteAfterLoad(), - statement.getWritePointCount(i))); + statement.getWritePointCount(i), + false)); } return res; } @@ -143,7 +150,8 @@ public class LoadTsFileNode extends WritePlanNode { isTableModel.get(i), database, statement.isDeleteAfterLoad(), - statement.getWritePointCount(i))); + statement.getWritePointCount(i), + needDecode4TimeColumn)); } } return res; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java index 7c0b6aae877..82ee3323614 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java @@ -43,6 +43,7 @@ import org.apache.tsfile.read.common.type.Type; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; // All the input databases shall not contain "root" public interface Metadata { @@ -125,7 +126,8 @@ public interface Metadata { final TableSchema tableSchema, final MPPQueryContext context, final boolean allowCreateTable, - final boolean isStrictTagColumn) + final boolean isStrictTagColumn, + final AtomicBoolean needDecode4DifferentTimeColumn) throws LoadAnalyzeTableColumnDisorderException; void validateInsertNodeMeasurements( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 3e8bbf12890..ed0be5b10d0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -74,6 +74,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import static org.apache.iotdb.db.queryengine.transformation.dag.column.FailFunctionColumnTransformer.FAIL_FUNCTION_NAME; @@ -1397,15 +1398,21 @@ public class TableMetadataImpl implements Metadata { @Override public Optional<TableSchema> validateTableHeaderSchema4TsFile( - String database, - TableSchema tableSchema, - MPPQueryContext context, - boolean allowCreateTable, - boolean isStrictTagColumn) + final String database, + final TableSchema tableSchema, + final MPPQueryContext context, + final boolean allowCreateTable, + final boolean isStrictTagColumn, + final AtomicBoolean needDecode4DifferentTimeColumn) throws LoadAnalyzeTableColumnDisorderException { return TableHeaderSchemaValidator.getInstance() .validateTableHeaderSchema4TsFile( - database, tableSchema, context, allowCreateTable, isStrictTagColumn); + database, + tableSchema, + context, + allowCreateTable, + isStrictTagColumn, + needDecode4DifferentTimeColumn); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java index f1cf4b48987..adc99807210 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java @@ -68,6 +68,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; @@ -99,7 +100,8 @@ public class TableHeaderSchemaValidator { final TableSchema tableSchema, final MPPQueryContext context, final boolean allowCreateTable, - final boolean isStrictTagColumn) + final boolean isStrictTagColumn, + final AtomicBoolean needDecode4DifferentTimeColumn) throws LoadAnalyzeTableColumnDisorderException { // The schema cache R/W and fetch operation must be locked together thus the cache clean // operation executed by delete timeSeries will be effective. @@ -212,7 +214,7 @@ public class TableHeaderSchemaValidator { noField = false; } } else { - // leave measurement columns' dataType checking to the caller, then the caller can decide + // leave field columns' dataType checking to the caller, then the caller can decide // whether to do partial insert // only check column category diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index d442d51a214..ded699588c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -1388,7 +1388,11 @@ public class RelationPlanner extends AstVisitor<RelationPlan, Void> { } return new RelationPlan( new LoadTsFileNode( - idAllocator.genPlanNodeId(), node.getResources(), isTableModel, node.getDatabase()), + idAllocator.genPlanNodeId(), + node.getResources(), + isTableModel, + node.getDatabase(), + node.isNeedDecode4TimeColumn()), analysis.getRootScope(), Collections.emptyList(), outerContext); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java index 9b7cd372ee9..8deb97c2e5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java @@ -58,12 +58,13 @@ public class LoadTsFile extends Statement { private boolean isGeneratedByPipe = false; - private Map<String, String> loadAttributes; + private final Map<String, String> loadAttributes; private List<File> tsFiles; private List<TsFileResource> resources; private List<Long> writePointCountList; private List<Boolean> isTableModel; + private boolean needDecode4TimeColumn; public LoadTsFile(NodeLocation location, String filePath, Map<String, String> loadAttributes) { super(location); @@ -167,6 +168,14 @@ public class LoadTsFile extends Statement { this.isTableModel = isTableModel; } + public boolean isNeedDecode4TimeColumn() { + return needDecode4TimeColumn; + } + + public void enableNeedDecode4TimeColumn() { + this.needDecode4TimeColumn = true; + } + public List<File> getTsFiles() { return tsFiles; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java index e425c709815..a22ab8f6739 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java @@ -41,7 +41,7 @@ public class LoadTsFileNodeTest { TsFileResource resource = new TsFileResource(new File("1")); String database = "root.db"; LoadSingleTsFileNode node = - new LoadSingleTsFileNode(new PlanNodeId(""), resource, false, database, true, 0L); + new LoadSingleTsFileNode(new PlanNodeId(""), resource, false, database, true, 0L, false); Assert.assertTrue(node.isDeleteAfterLoad()); Assert.assertEquals(resource, node.getTsFileResource()); Assert.assertEquals(database, node.getDatabase()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java index c789490bb10..36bed6932a8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java @@ -96,6 +96,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -1063,7 +1064,8 @@ public class AnalyzerTest { TableSchema schema, MPPQueryContext context, boolean allowCreateTable, - boolean isStrictTagColumn) { + boolean isStrictTagColumn, + final AtomicBoolean needDecode4DifferentTimeColumn) { TableSchema tableSchema = StatementTestUtils.genTableSchema(); assertEquals(tableSchema, schema); return Optional.of(tableSchema); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java index 48dd9409d63..49f23e0358c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java @@ -64,6 +64,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.MockTSBSDataPartition.T1_DEVICE_1; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.MockTSBSDataPartition.T1_DEVICE_2; @@ -347,7 +348,8 @@ public class TSBSMetadata implements Metadata { TableSchema tableSchema, MPPQueryContext context, boolean allowCreateTable, - boolean isStrictTagColumn) { + boolean isStrictTagColumn, + final AtomicBoolean needDecode4DifferentTimeColumn) { throw new UnsupportedOperationException(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java index 551dbc5a961..722212b78b7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java @@ -80,6 +80,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import static org.apache.iotdb.commons.schema.table.InformationSchema.INFORMATION_DATABASE; @@ -486,7 +487,8 @@ public class TestMetadata implements Metadata { final TableSchema tableSchema, final MPPQueryContext context, final boolean allowCreateTable, - final boolean isStrictTagColumn) { + final boolean isStrictTagColumn, + final AtomicBoolean needDecode4DifferentTimeColumn) { throw new UnsupportedOperationException(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java index cb23ee0ec04..dbbadd1be91 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java @@ -51,6 +51,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertThrows; @@ -105,7 +106,8 @@ public class InsertStatementTest { any(TableSchema.class), any(MPPQueryContext.class), any(Boolean.class), - any(Boolean.class))) + any(Boolean.class), + any(AtomicBoolean.class))) .thenReturn(Optional.of(tableSchema)); doAnswer( @@ -257,7 +259,8 @@ public class InsertStatementTest { any(TableSchema.class), any(MPPQueryContext.class), any(Boolean.class), - any(Boolean.class))) + any(Boolean.class), + any(AtomicBoolean.class))) .thenReturn(Optional.of(tableSchema)); assertThrows( @@ -292,7 +295,8 @@ public class InsertStatementTest { any(TableSchema.class), any(MPPQueryContext.class), any(Boolean.class), - any(Boolean.class))) + any(Boolean.class), + any(AtomicBoolean.class))) .thenReturn(Optional.of(tableSchema)); assertThrows(
