This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new db0a9b4d756 Fix batch partial insert failure reporting (#17925)
db0a9b4d756 is described below
commit db0a9b4d756cc2558ca466895590ee58427c2c6c
Author: Caideyipi <[email protected]>
AuthorDate: Mon Nov 9 09:38:22 2026 +0800
Fix batch partial insert failure reporting (#17925)
* Fix batch partial insert failure reporting
* Fix IT
* Preserve batch insert failures when invalid rows are removed
* Localize batch partial insert failure reporting
---
.../iotdb/db/i18n/DataNodeQueryMessages.java | 2 +
.../iotdb/db/i18n/DataNodeQueryMessages.java | 2 +
.../db/queryengine/plan/analyze/AnalyzeUtils.java | 34 +++++-
.../crud/InsertMultiTabletsStatement.java | 46 ++++++--
.../crud/InsertRowsOfOneDeviceStatement.java | 46 +++++++-
.../plan/statement/crud/InsertRowsStatement.java | 49 ++++++++-
.../crud/InsertStatementPartialInsertTest.java | 117 +++++++++++++++++++++
7 files changed, 279 insertions(+), 17 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index c6a839aff0f..b84bebec93d 100644
---
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -20,6 +20,8 @@
package org.apache.iotdb.db.i18n;
public final class DataNodeQueryMessages {
+ public static final String
MESSAGE_FAIL_TO_INSERT_MEASUREMENTS_ARG_CAUSED_BY_ARG_B9858E6B =
+ "Fail to insert measurements %s caused by %s";
public static final String
EXCEPTION_UNSUPPORTED_JOIN_CRETERIA_ARG_AFTER_PREDICATE_PUSH_DOWN_92BDB23B =
"Unsupported Join creteria [%s] after predicate push down";
diff --git
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index f78ee27beaa..1aef0814d92 100644
---
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -20,6 +20,8 @@
package org.apache.iotdb.db.i18n;
public final class DataNodeQueryMessages {
+ public static final String
MESSAGE_FAIL_TO_INSERT_MEASUREMENTS_ARG_CAUSED_BY_ARG_B9858E6B =
+ "测点 %s 插入失败,原因:%s";
public static final String
EXCEPTION_UNSUPPORTED_JOIN_CRETERIA_ARG_AFTER_PREDICATE_PUSH_DOWN_92BDB23B =
"谓词下推后存在不支持的 Join 条件 [%s]";
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
index 4e2a9de0cd4..00ad7004ce2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
@@ -86,6 +86,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
@@ -338,18 +339,43 @@ public class AnalyzeUtils {
boolean hasFailedMeasurement = insertStatement.hasFailedMeasurements();
String partialInsertMessage;
if (hasFailedMeasurement && !analysis.isFinishQueryAfterAnalyze()) {
+ final List<String> failedMeasurements = new ArrayList<>();
+ final List<String> failedMessages = new ArrayList<>();
+ collectDistinctFailedMeasurementMessages(
+ insertStatement.getFailedMeasurements(),
+ insertStatement.getFailedMessages(),
+ failedMeasurements,
+ failedMessages);
partialInsertMessage =
String.format(
- DataNodeQueryMessages
-
.EXCEPTION_FAIL_TO_INSERT_MEASUREMENTS_ARG_CAUSED_BY_ARG_B9858E6B,
- insertStatement.getFailedMeasurements(),
- insertStatement.getFailedMessages());
+
DataNodeQueryMessages.MESSAGE_FAIL_TO_INSERT_MEASUREMENTS_ARG_CAUSED_BY_ARG_B9858E6B,
+ failedMeasurements,
+ failedMessages);
LOGGER.warn(partialInsertMessage);
analysis.setFailStatus(
RpcUtils.getStatus(TSStatusCode.METADATA_ERROR.getStatusCode(),
partialInsertMessage));
}
}
+ private static void collectDistinctFailedMeasurementMessages(
+ List<String> rawFailedMeasurements,
+ List<String> rawFailedMessages,
+ List<String> failedMeasurements,
+ List<String> failedMessages) {
+ final Set<List<String>> distinctFailures = new LinkedHashSet<>();
+ for (int i = 0; i < rawFailedMeasurements.size(); i++) {
+ final String failedMeasurement = rawFailedMeasurements.get(i);
+ final String failedMessage = i < rawFailedMessages.size() ?
rawFailedMessages.get(i) : null;
+ final List<String> failureKey = new ArrayList<>(2);
+ failureKey.add(failedMeasurement);
+ failureKey.add(failedMessage);
+ if (distinctFailures.add(failureKey)) {
+ failedMeasurements.add(failedMeasurement);
+ failedMessages.add(failedMessage);
+ }
+ }
+ }
+
public static InsertBaseStatement removeLogicalView(
IAnalysis analysis, InsertBaseStatement insertBaseStatement,
IAuditEntity context) {
try {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertMultiTabletsStatement.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertMultiTabletsStatement.java
index 6adea132496..677d8cb36fb 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertMultiTabletsStatement.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertMultiTabletsStatement.java
@@ -23,7 +23,6 @@ import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.exception.SemanticException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
-import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaValidation;
import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
@@ -122,12 +121,45 @@ public class InsertMultiTabletsStatement extends
InsertBaseStatement {
}
@Override
- public void updateAfterSchemaValidation(MPPQueryContext context) {
- for (InsertTabletStatement insertTabletStatement :
insertTabletStatementList) {
- if (!this.hasFailedMeasurements() &&
insertTabletStatement.hasFailedMeasurements()) {
- this.failedMeasurementIndex2Info =
insertTabletStatement.failedMeasurementIndex2Info;
- }
- }
+ public boolean hasFailedMeasurements() {
+ return insertTabletStatementList != null
+ &&
insertTabletStatementList.stream().anyMatch(InsertBaseStatement::hasFailedMeasurements);
+ }
+
+ @Override
+ public int getFailedMeasurementNumber() {
+ return insertTabletStatementList == null
+ ? 0
+ : insertTabletStatementList.stream()
+ .mapToInt(InsertBaseStatement::getFailedMeasurementNumber)
+ .sum();
+ }
+
+ @Override
+ public List<String> getFailedMeasurements() {
+ return insertTabletStatementList == null
+ ? Collections.emptyList()
+ : insertTabletStatementList.stream()
+ .flatMap(statement -> statement.getFailedMeasurements().stream())
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List<Exception> getFailedExceptions() {
+ return insertTabletStatementList == null
+ ? Collections.emptyList()
+ : insertTabletStatementList.stream()
+ .flatMap(statement -> statement.getFailedExceptions().stream())
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List<String> getFailedMessages() {
+ return insertTabletStatementList == null
+ ? Collections.emptyList()
+ : insertTabletStatementList.stream()
+ .flatMap(statement -> statement.getFailedMessages().stream())
+ .collect(Collectors.toList());
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java
index 80dd52eca14..271cca7cede 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java
@@ -37,6 +37,7 @@ import org.apache.tsfile.exception.NotImplementedException;
import org.apache.tsfile.utils.RamUsageEstimator;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
@@ -141,12 +142,51 @@ public class InsertRowsOfOneDeviceStatement extends
InsertBaseStatement {
public void updateAfterSchemaValidation(MPPQueryContext context) throws
QueryProcessException {
for (InsertRowStatement insertRowStatement : insertRowStatementList) {
insertRowStatement.updateAfterSchemaValidation(context);
- if (!this.hasFailedMeasurements() &&
insertRowStatement.hasFailedMeasurements()) {
- this.failedMeasurementIndex2Info =
insertRowStatement.failedMeasurementIndex2Info;
- }
}
}
+ @Override
+ public boolean hasFailedMeasurements() {
+ return insertRowStatementList != null
+ &&
insertRowStatementList.stream().anyMatch(InsertBaseStatement::hasFailedMeasurements);
+ }
+
+ @Override
+ public int getFailedMeasurementNumber() {
+ return insertRowStatementList == null
+ ? 0
+ : insertRowStatementList.stream()
+ .mapToInt(InsertBaseStatement::getFailedMeasurementNumber)
+ .sum();
+ }
+
+ @Override
+ public List<String> getFailedMeasurements() {
+ return insertRowStatementList == null
+ ? Collections.emptyList()
+ : insertRowStatementList.stream()
+ .flatMap(statement -> statement.getFailedMeasurements().stream())
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List<Exception> getFailedExceptions() {
+ return insertRowStatementList == null
+ ? Collections.emptyList()
+ : insertRowStatementList.stream()
+ .flatMap(statement -> statement.getFailedExceptions().stream())
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List<String> getFailedMessages() {
+ return insertRowStatementList == null
+ ? Collections.emptyList()
+ : insertRowStatementList.stream()
+ .flatMap(statement -> statement.getFailedMessages().stream())
+ .collect(Collectors.toList());
+ }
+
@Override
protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType)
{
return false;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java
index e89d32231c5..6aded255733 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsStatement.java
@@ -53,6 +53,8 @@ public class InsertRowsStatement extends InsertBaseStatement {
/** the InsertRowsStatement list */
private List<InsertRowStatement> insertRowStatementList;
+ private List<InsertRowStatement> schemaValidatedRowStatementList;
+
/**
* Whether every row of this statement targets the same table and carries
the same column set.
* Only producers that build all rows from a single (table, column)
definition set this, e.g. a
@@ -161,10 +163,51 @@ public class InsertRowsStatement extends
InsertBaseStatement {
public void updateAfterSchemaValidation(MPPQueryContext context) throws
QueryProcessException {
for (InsertRowStatement insertRowStatement : insertRowStatementList) {
insertRowStatement.updateAfterSchemaValidation(context);
- if (!this.hasFailedMeasurements() &&
insertRowStatement.hasFailedMeasurements()) {
- this.failedMeasurementIndex2Info =
insertRowStatement.failedMeasurementIndex2Info;
- }
}
+ // Table schema validation can remove rows without valid fields. Keep
their failures for
+ // reporting.
+ schemaValidatedRowStatementList = insertRowStatementList;
+ }
+
+ private Stream<InsertRowStatement> getRowsForFailureReporting() {
+ final List<InsertRowStatement> rows =
+ schemaValidatedRowStatementList == null
+ ? insertRowStatementList
+ : schemaValidatedRowStatementList;
+ return rows == null ? Stream.empty() : rows.stream();
+ }
+
+ @Override
+ public boolean hasFailedMeasurements() {
+ return
getRowsForFailureReporting().anyMatch(InsertBaseStatement::hasFailedMeasurements);
+ }
+
+ @Override
+ public int getFailedMeasurementNumber() {
+ return getRowsForFailureReporting()
+ .mapToInt(InsertBaseStatement::getFailedMeasurementNumber)
+ .sum();
+ }
+
+ @Override
+ public List<String> getFailedMeasurements() {
+ return getRowsForFailureReporting()
+ .flatMap(statement -> statement.getFailedMeasurements().stream())
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List<Exception> getFailedExceptions() {
+ return getRowsForFailureReporting()
+ .flatMap(statement -> statement.getFailedExceptions().stream())
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List<String> getFailedMessages() {
+ return getRowsForFailureReporting()
+ .flatMap(statement -> statement.getFailedMessages().stream())
+ .collect(Collectors.toList());
}
@Override
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java
index a37407636c9..6dc3b52e1d5 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java
@@ -22,6 +22,9 @@ package org.apache.iotdb.db.queryengine.plan.statement.crud;
import org.apache.iotdb.commons.path.PartialPath;
import
org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
+import org.apache.iotdb.db.queryengine.plan.analyze.Analysis;
+import org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeUtils;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.type.TypeFactory;
@@ -315,6 +318,120 @@ public class InsertStatementPartialInsertTest {
Assert.assertEquals("s1",
result.get(statement.getDevicePath()).get(0).left);
}
+ @Test
+ public void testInsertRowsStatementAggregatesFailedMeasurements() throws
Exception {
+ final InsertRowStatement statement1 = createInsertRowStatement();
+ final InsertRowStatement statement2 = createInsertRowStatement();
+ statement1.markFailedMeasurement(0, new RuntimeException("failed1"));
+ statement2.markFailedMeasurement(1, new RuntimeException("failed2"));
+
+ final InsertRowsStatement rowsStatement = new InsertRowsStatement();
+ rowsStatement.setInsertRowStatementList(Arrays.asList(statement1,
statement2));
+
+ Assert.assertTrue(rowsStatement.hasFailedMeasurements());
+ Assert.assertEquals(2, rowsStatement.getFailedMeasurementNumber());
+ Assert.assertEquals(Arrays.asList("s1", "s2"),
rowsStatement.getFailedMeasurements());
+ Assert.assertEquals(Arrays.asList("failed1", "failed2"),
rowsStatement.getFailedMessages());
+ }
+
+ @Test
+ public void testInsertRowsStatementRetainsFailuresAfterRemovingInvalidRows()
throws Exception {
+ final InsertRowStatement validRow = createInsertRowStatement();
+ final InsertRowStatement invalidRow = createInsertRowStatement();
+ final InsertRowStatement partiallyValidRow = createInsertRowStatement();
+ final RuntimeException failure1 = new RuntimeException("failed1");
+ final RuntimeException failure2 = new RuntimeException("failed2");
+ final RuntimeException failure3 = new RuntimeException("failed3");
+ invalidRow.markFailedMeasurement(0, failure1);
+ invalidRow.markFailedMeasurement(1, failure2);
+ partiallyValidRow.markFailedMeasurement(0, failure3);
+
+ final InsertRowsStatement rowsStatement = new InsertRowsStatement();
+ rowsStatement.setInsertRowStatementList(Arrays.asList(validRow,
invalidRow, partiallyValidRow));
+ rowsStatement.updateAfterSchemaValidation(null);
+
+ final Analysis analysis = new Analysis();
+ AnalyzeUtils.validateSchema(
+ analysis,
+ rowsStatement,
+ () -> rowsStatement.setInsertRowStatementList(Arrays.asList(validRow,
partiallyValidRow)));
+
+ Assert.assertEquals(
+ Arrays.asList(validRow, partiallyValidRow),
rowsStatement.getInsertRowStatementList());
+ Assert.assertTrue(rowsStatement.hasFailedMeasurements());
+ Assert.assertEquals(3, rowsStatement.getFailedMeasurementNumber());
+ Assert.assertEquals(Arrays.asList("s1", "s2", "s1"),
rowsStatement.getFailedMeasurements());
+ Assert.assertEquals(
+ Arrays.asList(failure1, failure2, failure3),
rowsStatement.getFailedExceptions());
+ Assert.assertEquals(
+ Arrays.asList("failed1", "failed2", "failed3"),
rowsStatement.getFailedMessages());
+ Assert.assertEquals(
+ String.format(
+
DataNodeQueryMessages.MESSAGE_FAIL_TO_INSERT_MEASUREMENTS_ARG_CAUSED_BY_ARG_B9858E6B,
+ "[s1, s2, s1]",
+ "[failed1, failed2, failed3]"),
+ analysis.getFailStatus().getMessage());
+ }
+
+ @Test
+ public void testValidateSchemaDeduplicatesFailedMeasurementMessage() throws
Exception {
+ final InsertRowStatement statement1 = createInsertRowStatement();
+ final InsertRowStatement statement2 = createInsertRowStatement();
+ final InsertRowStatement statement3 = createInsertRowStatement();
+ statement1.markFailedMeasurement(0, new RuntimeException("failed1"));
+ statement2.markFailedMeasurement(0, new RuntimeException("failed1"));
+ statement3.markFailedMeasurement(1, new RuntimeException("failed2"));
+
+ final InsertRowsStatement rowsStatement = new InsertRowsStatement();
+ rowsStatement.setInsertRowStatementList(Arrays.asList(statement1,
statement2, statement3));
+
+ final Analysis analysis = new Analysis();
+ AnalyzeUtils.validateSchema(analysis, rowsStatement, () -> {});
+
+ Assert.assertEquals(
+ String.format(
+
DataNodeQueryMessages.MESSAGE_FAIL_TO_INSERT_MEASUREMENTS_ARG_CAUSED_BY_ARG_B9858E6B,
+ "[s1, s2]",
+ "[failed1, failed2]"),
+ analysis.getFailStatus().getMessage());
+ }
+
+ @Test
+ public void testInsertRowsOfOneDeviceStatementAggregatesFailedMeasurements()
throws Exception {
+ final InsertRowStatement statement1 = createInsertRowStatement();
+ final InsertRowStatement statement2 = createInsertRowStatement();
+ statement1.markFailedMeasurement(0, new RuntimeException("failed1"));
+ statement2.markFailedMeasurement(1, new RuntimeException("failed2"));
+
+ final InsertRowsOfOneDeviceStatement rowsOfOneDeviceStatement =
+ new InsertRowsOfOneDeviceStatement();
+
rowsOfOneDeviceStatement.setInsertRowStatementList(Arrays.asList(statement1,
statement2));
+
+ Assert.assertTrue(rowsOfOneDeviceStatement.hasFailedMeasurements());
+ Assert.assertEquals(2,
rowsOfOneDeviceStatement.getFailedMeasurementNumber());
+ Assert.assertEquals(
+ Arrays.asList("s1", "s2"),
rowsOfOneDeviceStatement.getFailedMeasurements());
+ Assert.assertEquals(
+ Arrays.asList("failed1", "failed2"),
rowsOfOneDeviceStatement.getFailedMessages());
+ }
+
+ @Test
+ public void testInsertMultiTabletsStatementAggregatesFailedMeasurements()
throws Exception {
+ final InsertTabletStatement statement1 = createInsertTabletStatement();
+ final InsertTabletStatement statement2 = createInsertTabletStatement();
+ statement1.markFailedMeasurement(0, new RuntimeException("failed1"));
+ statement2.markFailedMeasurement(1, new RuntimeException("failed2"));
+
+ final InsertMultiTabletsStatement multiTabletsStatement = new
InsertMultiTabletsStatement();
+
multiTabletsStatement.setInsertTabletStatementList(Arrays.asList(statement1,
statement2));
+
+ Assert.assertTrue(multiTabletsStatement.hasFailedMeasurements());
+ Assert.assertEquals(2, multiTabletsStatement.getFailedMeasurementNumber());
+ Assert.assertEquals(Arrays.asList("s1", "s2"),
multiTabletsStatement.getFailedMeasurements());
+ Assert.assertEquals(
+ Arrays.asList("failed1", "failed2"),
multiTabletsStatement.getFailedMessages());
+ }
+
private static InsertRowStatement createInsertRowStatement() throws
Exception {
final InsertRowStatement statement = new InsertRowStatement();
statement.setDevicePath(new PartialPath("root.sg.d1"));