This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new fbe70759892 Load: Add a new configuration to enable datatype
conversion when converting into Tablets & Refactor LoadAnalyzeException.
(#14646) (#14674)
fbe70759892 is described below
commit fbe70759892988f0ca82b95491c31843bf8789b5
Author: Itami Sho <[email protected]>
AuthorDate: Mon Jan 13 15:41:14 2025 +0800
Load: Add a new configuration to enable datatype conversion when converting
into Tablets & Refactor LoadAnalyzeException. (#14646) (#14674)
---
...ataException.java => LoadAnalyzeException.java} | 6 +--
....java => LoadAnalyzeTypeMismatchException.java} | 4 +-
.../PipeConvertedInsertTabletStatement.java | 4 ++
.../plan/analyze/LoadTsFileAnalyzer.java | 46 +++++++++++-----------
.../LoadConvertedInsertTabletStatement.java | 11 +++++-
...eeStatementDataTypeConvertExecutionVisitor.java | 3 +-
6 files changed, 44 insertions(+), 30 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/LoadAnalyzeException.java
similarity index 85%
rename from
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java
rename to
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/LoadAnalyzeException.java
index 78c6d8d4d5c..f0620725979 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/LoadAnalyzeException.java
@@ -22,13 +22,13 @@ package org.apache.iotdb.db.exception;
import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.rpc.TSStatusCode;
-public class VerifyMetadataException extends IoTDBException {
+public class LoadAnalyzeException extends IoTDBException {
- public VerifyMetadataException(String message) {
+ public LoadAnalyzeException(String message) {
super(message, TSStatusCode.VERIFY_METADATA_ERROR.getStatusCode());
}
- public VerifyMetadataException(String message, int errorCode) {
+ public LoadAnalyzeException(String message, int errorCode) {
super(message, errorCode);
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/LoadAnalyzeTypeMismatchException.java
similarity index 86%
rename from
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java
rename to
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/LoadAnalyzeTypeMismatchException.java
index 209fb83bcdb..744eab2841a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/LoadAnalyzeTypeMismatchException.java
@@ -21,9 +21,9 @@ package org.apache.iotdb.db.exception;
import org.apache.iotdb.rpc.TSStatusCode;
-public class VerifyMetadataTypeMismatchException extends
VerifyMetadataException {
+public class LoadAnalyzeTypeMismatchException extends LoadAnalyzeException {
- public VerifyMetadataTypeMismatchException(String message) {
+ public LoadAnalyzeTypeMismatchException(String message) {
super(message, TSStatusCode.VERIFY_METADATA_ERROR.getStatusCode());
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java
index a5013725061..5f0d1f4f80c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java
@@ -62,4 +62,8 @@ public class PipeConvertedInsertTabletStatement extends
InsertTabletStatement {
dataTypes[columnIndex] = dataType;
return true;
}
+
+ protected boolean originalCheckAndCastDataType(int columnIndex, TSDataType
dataType) {
+ return super.checkAndCastDataType(columnIndex, dataType);
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java
index 26e7146c7a7..5604b687068 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java
@@ -35,8 +35,8 @@ import
org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.exception.VerifyMetadataException;
-import org.apache.iotdb.db.exception.VerifyMetadataTypeMismatchException;
+import org.apache.iotdb.db.exception.LoadAnalyzeException;
+import org.apache.iotdb.db.exception.LoadAnalyzeTypeMismatchException;
import org.apache.iotdb.db.exception.load.LoadEmptyFileException;
import org.apache.iotdb.db.exception.load.LoadFileException;
import org.apache.iotdb.db.exception.load.LoadReadOnlyException;
@@ -181,8 +181,8 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
}
} catch (AuthException e) {
return createFailAnalysisForAuthException(e);
- } catch (VerifyMetadataTypeMismatchException e) {
- executeDataTypeConversionOnTypeMismatch(analysis, e);
+ } catch (LoadAnalyzeTypeMismatchException e) {
+ executeTabletConversion(analysis, e);
// just return false to STOP the analysis process,
// the real result on the conversion will be set in the analysis.
return analysis;
@@ -202,8 +202,8 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
schemaAutoCreatorAndVerifier.flush();
} catch (AuthException e) {
return createFailAnalysisForAuthException(e);
- } catch (VerifyMetadataTypeMismatchException e) {
- executeDataTypeConversionOnTypeMismatch(analysis, e);
+ } catch (LoadAnalyzeTypeMismatchException e) {
+ executeTabletConversion(analysis, e);
// just return false to STOP the analysis process,
// the real result on the conversion will be set in the analysis.
return analysis;
@@ -226,12 +226,12 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
return analysis;
}
- private void executeDataTypeConversionOnTypeMismatch(
- final Analysis analysis, final VerifyMetadataTypeMismatchException e) {
+ private void executeTabletConversion(final Analysis analysis, final
LoadAnalyzeException e) {
final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
new
LoadTsFileDataTypeConverter(loadTsFileStatement.isGeneratedByPipe());
final TSStatus status =
- loadTsFileStatement.isConvertOnTypeMismatch()
+ (!(e instanceof LoadAnalyzeTypeMismatchException)
+ || loadTsFileStatement.isConvertOnTypeMismatch())
?
loadTsFileDataTypeConverter.convertForTreeModel(loadTsFileStatement).orElse(null)
: null;
@@ -252,7 +252,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
}
private void analyzeSingleTsFile(final File tsFile, final boolean
isDeleteAfterLoad)
- throws IOException, AuthException, VerifyMetadataTypeMismatchException {
+ throws IOException, AuthException, LoadAnalyzeTypeMismatchException {
try (final TsFileSequenceReader reader = new
TsFileSequenceReader(tsFile.getAbsolutePath())) {
// can be reused when constructing tsfile resource
final TsFileSequenceReaderTimeseriesMetadataIterator
timeseriesMetadataIterator =
@@ -342,7 +342,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
public void autoCreateAndVerify(
TsFileSequenceReader reader,
Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadataList)
- throws IOException, AuthException, VerifyMetadataTypeMismatchException
{
+ throws IOException, AuthException, LoadAnalyzeTypeMismatchException {
for (final Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry :
device2TimeseriesMetadataList.entrySet()) {
final IDeviceID device = entry.getKey();
@@ -444,14 +444,14 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
schemaCache.clearDeviceIsAlignedCacheIfNecessary();
}
- public void flush() throws AuthException,
VerifyMetadataTypeMismatchException {
+ public void flush() throws AuthException, LoadAnalyzeTypeMismatchException
{
doAutoCreateAndVerify();
schemaCache.clearTimeSeries();
}
private void doAutoCreateAndVerify()
- throws SemanticException, AuthException,
VerifyMetadataTypeMismatchException {
+ throws SemanticException, AuthException,
LoadAnalyzeTypeMismatchException {
if (schemaCache.getDevice2TimeSeries().isEmpty()) {
return;
}
@@ -472,7 +472,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
if (loadTsFileStatement.isVerifySchema()) {
verifySchema(schemaTree);
}
- } catch (AuthException | VerifyMetadataTypeMismatchException e) {
+ } catch (AuthException | LoadAnalyzeTypeMismatchException e) {
throw e;
} catch (Exception e) {
LOGGER.warn("Auto create or verify schema error.", e);
@@ -483,7 +483,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
}
}
- private void makeSureNoDuplicatedMeasurementsInDevices() throws
VerifyMetadataException {
+ private void makeSureNoDuplicatedMeasurementsInDevices() throws
LoadAnalyzeException {
for (final Map.Entry<IDeviceID, Set<MeasurementSchema>> entry :
schemaCache.getDevice2TimeSeries().entrySet()) {
final IDeviceID device = entry.getKey();
@@ -491,7 +491,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
for (final MeasurementSchema timeseriesSchema : entry.getValue()) {
final String measurement = timeseriesSchema.getMeasurementId();
if (measurement2Schema.containsKey(measurement)) {
- throw new VerifyMetadataException(
+ throw new LoadAnalyzeException(
String.format("Duplicated measurements %s in device %s.",
measurement, device));
}
measurement2Schema.put(measurement, timeseriesSchema);
@@ -500,7 +500,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
}
private void autoCreateDatabase()
- throws VerifyMetadataException, LoadFileException,
IllegalPathException, AuthException {
+ throws LoadAnalyzeException, LoadFileException, IllegalPathException,
AuthException {
final int databasePrefixNodesLength =
loadTsFileStatement.getDatabaseLevel() + 1;
final Set<PartialPath> databasesNeededToBeSet = new HashSet<>();
@@ -509,7 +509,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
final String[] devicePrefixNodes = devicePath.getNodes();
if (devicePrefixNodes.length < databasePrefixNodesLength) {
- throw new VerifyMetadataException(
+ throw new LoadAnalyzeException(
String.format(
"Database level %d is longer than device %s.",
databasePrefixNodesLength, device));
@@ -633,7 +633,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
}
private void verifySchema(ISchemaTree schemaTree)
- throws VerifyMetadataException, IllegalPathException,
VerifyMetadataTypeMismatchException {
+ throws LoadAnalyzeException, IllegalPathException,
LoadAnalyzeTypeMismatchException {
for (final Map.Entry<IDeviceID, Set<MeasurementSchema>> entry :
schemaCache.getDevice2TimeSeries().entrySet()) {
final IDeviceID device = entry.getKey();
@@ -646,7 +646,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
.collect(Collectors.toList()));
if (iotdbDeviceSchemaInfo == null) {
- throw new VerifyMetadataException(
+ throw new LoadAnalyzeException(
String.format(
"Device %s does not exist in IoTDB and can not be created. "
+ "Please check weather auto-create-schema is enabled.",
@@ -657,7 +657,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
final boolean isAlignedInTsFile =
schemaCache.getDeviceIsAligned(device);
final boolean isAlignedInIoTDB = iotdbDeviceSchemaInfo.isAligned();
if (isAlignedInTsFile != isAlignedInIoTDB) {
- throw new VerifyMetadataException(
+ throw new LoadAnalyzeException(
String.format(
"Device %s in TsFile is %s, but in IoTDB is %s.",
device,
@@ -672,7 +672,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
final MeasurementSchema tsFileSchema =
tsfileTimeseriesSchemas.get(i);
final MeasurementSchema iotdbSchema = iotdbTimeseriesSchemas.get(i);
if (iotdbSchema == null) {
- throw new VerifyMetadataException(
+ throw new LoadAnalyzeException(
String.format(
"Measurement %s does not exist in IoTDB and can not be
created. "
+ "Please check weather auto-create-schema is
enabled.",
@@ -681,7 +681,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
// check datatype
if (!tsFileSchema.getType().equals(iotdbSchema.getType())) {
- throw new VerifyMetadataTypeMismatchException(
+ throw new LoadAnalyzeTypeMismatchException(
String.format(
"Measurement %s%s%s datatype not match, TsFile: %s, IoTDB:
%s",
device,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java
index 1a814442402..97a71dbb94b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java
@@ -32,12 +32,21 @@ public class LoadConvertedInsertTabletStatement extends
PipeConvertedInsertTable
private static final Logger LOGGER =
LoggerFactory.getLogger(LoadConvertedInsertTabletStatement.class);
- public LoadConvertedInsertTabletStatement(final InsertTabletStatement
insertTabletStatement) {
+ private final boolean shouldConvertOnTypeMismatch;
+
+ public LoadConvertedInsertTabletStatement(
+ final InsertTabletStatement insertTabletStatement,
+ final boolean shouldConvertOnTypeMismatch) {
super(insertTabletStatement);
+ this.shouldConvertOnTypeMismatch = shouldConvertOnTypeMismatch;
}
@Override
protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType)
{
+ if (!shouldConvertOnTypeMismatch) {
+ return originalCheckAndCastDataType(columnIndex, dataType);
+ }
+
LOGGER.info(
"Load: Inserting tablet to {}.{}. Casting type from {} to {}.",
devicePath,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
index e6c35d384df..222fdb5d93f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
@@ -76,7 +76,8 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
new LoadConvertedInsertTabletStatement(
PipeTransferTabletRawReq.toTPipeTransferRawReq(
tabletWithIsAligned.getLeft(),
tabletWithIsAligned.getRight())
- .constructStatement());
+ .constructStatement(),
+ loadTsFileStatement.isConvertOnTypeMismatch());
TSStatus result;
try {
