This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch fix/table-delete-attribute-no-tag in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cd3eef58f7aea7e0a8e643eddfa1f1185587e113 Author: Caideyipi <[email protected]> AuthorDate: Thu Jul 9 18:36:43 2026 +0800 Fix no-tag table attribute deletion --- .../relational/it/db/it/IoTDBDeletionTableIT.java | 18 ++++ .../metadata/fetcher/SchemaPredicateUtil.java | 3 +- .../fetcher/TableDeviceSchemaValidator.java | 107 +++++++++++++-------- .../plan/relational/sql/ast/InsertRows.java | 91 +++++++++++++++++- .../schemaRegion/SchemaRegionTableDeviceTest.java | 26 +++++ 5 files changed, 205 insertions(+), 40 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 3ac335df1ad..82b6c1a8ff6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -653,6 +653,24 @@ public class IoTDBDeletionTableIT { + "(3, 'blue', 'large', 13)," + "(4, 'red', 'small', 14)"); + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from delete_by_attr_no_tag"), + "attr1,attr2,", + Collections.singleton("red,small,")); + TestUtils.assertResultSetEqual( + statement.executeQuery( + "show devices from delete_by_attr_no_tag " + + "where attr1 = 'red' AND attr2 = 'small'"), + "attr1,attr2,", + Collections.singleton("red,small,")); + statement.execute( + "INSERT INTO delete_by_attr_no_tag(time, attr1, attr2, s1) VALUES " + + "(2, 'blue', 'large', 12)," + + "(4, 'red', 'small', 14)"); + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from delete_by_attr_no_tag"), + "attr1,attr2,", + Collections.singleton("red,small,")); statement.execute( "DELETE FROM delete_by_attr_no_tag " + "WHERE time >= 2 AND time <= 4 AND attr1 = 'red' AND attr2 = 'small'"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/SchemaPredicateUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/SchemaPredicateUtil.java index 0037e2f1ac9..8a39ad9d266 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/SchemaPredicateUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/SchemaPredicateUtil.java @@ -226,7 +226,8 @@ public class SchemaPredicateUtil { final int tagCount = tableInstance.getTagNum(); for (int i = 0; i < index2FilterMapList.size(); i++) { final Map<Integer, List<SchemaFilter>> filterMap = index2FilterMapList.get(i); - if (filterMap.size() == tagCount + if (tagCount > 0 + && filterMap.size() == tagCount && filterMap.values().stream() .allMatch( filterList -> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java index 4ae7ce549c6..2f9813ab95f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java @@ -42,9 +42,12 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher.convertTagValuesToDeviceID; @@ -123,20 +126,27 @@ public class TableDeviceSchemaValidator { final List<String> attributeKeyList, final List<Object[]> attributeValueList) { final ValidateResult result = new ValidateResult(); + final Map<IDeviceID, Map<String, Binary>> validatedDeviceSchemaMap = new HashMap<>(); for (int i = 0, size = deviceIdList.size(); i < size; i++) { - final Map<String, Binary> attributeMap = - TableDeviceSchemaCache.getInstance() - .getDeviceAttribute( - schemaValidation.getDatabase(), - convertTagValuesToDeviceID( - schemaValidation.getTableName(), (String[]) deviceIdList.get(i))); + final IDeviceID deviceID = + convertTagValuesToDeviceID( + schemaValidation.getTableName(), (String[]) deviceIdList.get(i)); + Map<String, Binary> attributeMap = validatedDeviceSchemaMap.get(deviceID); if (attributeMap == null) { - result.missingDeviceIndexList.add(i); - } else { - if (isAttributeUpdateRequired(attributeKeyList, attributeValueList.get(i), attributeMap)) { - result.attributeUpdateDeviceIndexList.add(i); + final Map<String, Binary> cachedAttributeMap = + TableDeviceSchemaCache.getInstance() + .getDeviceAttribute(schemaValidation.getDatabase(), deviceID); + if (cachedAttributeMap == null) { + result.missingDeviceIndexList.add(i); + continue; } + attributeMap = new HashMap<>(cachedAttributeMap); + validatedDeviceSchemaMap.put(deviceID, attributeMap); + } + if (isAttributeUpdateRequired(attributeKeyList, attributeValueList.get(i), attributeMap)) { + result.attributeUpdateDeviceIndexList.add(i); + applyAttributeValues(attributeKeyList, attributeValueList.get(i), attributeMap); } } return result; @@ -160,36 +170,41 @@ public class TableDeviceSchemaValidator { context); final ValidateResult result = new ValidateResult(); + final Map<IDeviceID, Map<String, Binary>> validatedDeviceSchemaMap = new HashMap<>(); + final Set<IDeviceID> missingDeviceSet = new HashSet<>(); + final Set<IDeviceID> selectedMissingDeviceSet = new HashSet<>(); for (final int index : previousValidateResult.missingDeviceIndexList) { + final IDeviceID deviceID = + convertTagValuesToDeviceID( + schemaValidation.getTableName(), (String[]) deviceIdList.get(index)); final Map<String, Binary> attributeMap = - fetchedDeviceSchema.get( - convertTagValuesToDeviceID( - schemaValidation.getTableName(), (String[]) deviceIdList.get(index))); - if (attributeMap == null) { + validatedDeviceSchemaMap.computeIfAbsent( + deviceID, + ignored -> { + final Map<String, Binary> fetchedAttributeMap = fetchedDeviceSchema.get(deviceID); + if (fetchedAttributeMap == null) { + missingDeviceSet.add(deviceID); + return new HashMap<>(); + } + return new HashMap<>(fetchedAttributeMap); + }); + if (missingDeviceSet.contains(deviceID) && selectedMissingDeviceSet.add(deviceID)) { result.missingDeviceIndexList.add(index); - } else { - constructAttributeUpdateDeviceIndexList( - attributeKeyList, attributeValueList, result, index, attributeMap); + applyAttributeValues(attributeKeyList, attributeValueList.get(index), attributeMap); + continue; + } + if (isAttributeUpdateRequired( + attributeKeyList, attributeValueList.get(index), attributeMap)) { + result.attributeUpdateDeviceIndexList.add(index); + applyAttributeValues(attributeKeyList, attributeValueList.get(index), attributeMap); } } result.attributeUpdateDeviceIndexList.addAll( previousValidateResult.attributeUpdateDeviceIndexList); - return result; } - private void constructAttributeUpdateDeviceIndexList( - final List<String> attributeKeyList, - final List<Object[]> attributeValueList, - final ValidateResult result, - final int index, - final Map<String, Binary> attributeMap) { - if (isAttributeUpdateRequired(attributeKeyList, attributeValueList.get(index), attributeMap)) { - result.attributeUpdateDeviceIndexList.add(index); - } - } - static boolean isAttributeUpdateRequired( final List<String> attributeKeyList, final Object[] deviceAttributeValueList, @@ -207,6 +222,22 @@ public class TableDeviceSchemaValidator { return false; } + private static void applyAttributeValues( + final List<String> attributeKeyList, + final Object[] deviceAttributeValueList, + final Map<String, Binary> attributeMap) { + for (int j = 0, size = attributeKeyList.size(); j < size; j++) { + if (deviceAttributeValueList.length <= j) { + break; + } + final Object inputValue = deviceAttributeValueList[j]; + if (inputValue == null || inputValue == Constants.NONE) { + continue; + } + attributeMap.put(attributeKeyList.get(j), (Binary) inputValue); + } + } + private List<Object[]> normalizeNullAttributeValuesForInsert( final List<Object[]> attributeValueList) { List<Object[]> result = null; @@ -256,17 +287,17 @@ public class TableDeviceSchemaValidator { final List<Object[]> deviceIdList = new ArrayList<>(size); final List<Object[]> attributeValueList = new ArrayList<>(size); + final boolean[] deviceSchemaUpdateRequired = new boolean[inputDeviceIdList.size()]; previousValidateResult.missingDeviceIndexList.forEach( - index -> { - deviceIdList.add(inputDeviceIdList.get(index)); - attributeValueList.add(intPutAttributeValueList.get(index)); - }); - + index -> deviceSchemaUpdateRequired[index] = true); previousValidateResult.attributeUpdateDeviceIndexList.forEach( - index -> { - deviceIdList.add(inputDeviceIdList.get(index)); - attributeValueList.add(intPutAttributeValueList.get(index)); - }); + index -> deviceSchemaUpdateRequired[index] = true); + for (int index = 0; index < deviceSchemaUpdateRequired.length; index++) { + if (deviceSchemaUpdateRequired[index]) { + deviceIdList.add(inputDeviceIdList.get(index)); + attributeValueList.add(intPutAttributeValueList.get(index)); + } + } final ExecutionResult executionResult = coordinator.executeForTableModel( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java index 4eb49df7183..9fc72a65cd6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java @@ -31,11 +31,15 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Constants; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; public class InsertRows extends WrappedInsertStatement { @@ -100,10 +104,27 @@ public class InsertRows extends WrappedInsertStatement { @Override public void validateDeviceSchema(Metadata metadata, MPPQueryContext context) { + // Preserve row order for duplicate devices in one INSERT and avoid stale async attribute cache. + final Map<String, Map<String, BatchedDeviceSchemaValidation>> validationMap = + new LinkedHashMap<>(); for (InsertRowStatement insertRowStatement : getInnerTreeStatement().getInsertRowStatementList()) { - metadata.validateDeviceSchema(createTableSchemaValidation(insertRowStatement), context); + final ITableDeviceSchemaValidation rowValidation = + createTableSchemaValidation(insertRowStatement); + validationMap + .computeIfAbsent(rowValidation.getDatabase(), k -> new LinkedHashMap<>()) + .computeIfAbsent( + rowValidation.getTableName(), + tableName -> + new BatchedDeviceSchemaValidation(rowValidation.getDatabase(), tableName)) + .add( + rowValidation.getDeviceIdList().get(0), + rowValidation.getAttributeColumnNameList(), + rowValidation.getAttributeValueList().get(0)); } + validationMap.values().stream() + .flatMap(tableValidationMap -> tableValidationMap.values().stream()) + .forEach(validation -> metadata.validateDeviceSchema(validation, context)); } protected ITableDeviceSchemaValidation createTableSchemaValidation( @@ -157,4 +178,72 @@ public class InsertRows extends WrappedInsertStatement { } }; } + + private static class BatchedDeviceSchemaValidation implements ITableDeviceSchemaValidation { + + private final String database; + private final String tableName; + private final List<Object[]> deviceIdList = new ArrayList<>(); + private final List<String> attributeColumnNameList = new ArrayList<>(); + private final Map<String, Integer> attributeColumnIndexMap = new LinkedHashMap<>(); + private final List<Map<Integer, Object>> rowAttributeValueMapList = new ArrayList<>(); + + private BatchedDeviceSchemaValidation(final String database, final String tableName) { + this.database = database; + this.tableName = tableName; + } + + private void add( + final Object[] deviceId, + final List<String> attributeColumnNames, + final Object[] attributeValues) { + deviceIdList.add(deviceId); + final Map<Integer, Object> rowAttributeValueMap = new HashMap<>(); + for (int i = 0; i < attributeColumnNames.size(); i++) { + final int attributeColumnIndex = + attributeColumnIndexMap.computeIfAbsent( + attributeColumnNames.get(i), + attributeColumnName -> { + attributeColumnNameList.add(attributeColumnName); + return attributeColumnNameList.size() - 1; + }); + if (i < attributeValues.length) { + rowAttributeValueMap.put(attributeColumnIndex, attributeValues[i]); + } + } + rowAttributeValueMapList.add(rowAttributeValueMap); + } + + @Override + public String getDatabase() { + return database; + } + + @Override + public String getTableName() { + return tableName; + } + + @Override + public List<Object[]> getDeviceIdList() { + return deviceIdList; + } + + @Override + public List<String> getAttributeColumnNameList() { + return attributeColumnNameList; + } + + @Override + public List<Object[]> getAttributeValueList() { + final List<Object[]> attributeValueList = new ArrayList<>(rowAttributeValueMapList.size()); + for (final Map<Integer, Object> rowAttributeValueMap : rowAttributeValueMapList) { + final Object[] attributeValues = new Object[attributeColumnNameList.size()]; + Arrays.fill(attributeValues, Constants.NONE); + rowAttributeValueMap.forEach((index, value) -> attributeValues[index] = value); + attributeValueList.add(attributeValues); + } + return attributeValueList; + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTableDeviceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTableDeviceTest.java index 9b5a5df72fc..7fcbe0710da 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTableDeviceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTableDeviceTest.java @@ -276,6 +276,32 @@ public class SchemaRegionTableDeviceTest extends AbstractSchemaRegionTest { Assert.assertEquals(2, deviceSchemaInfoList.size()); } + @Test + public void testDeviceQueryWithoutIdColumn() throws Exception { + if (!testParams.getTestModeName().equals("MemoryMode")) { + return; + } + final ISchemaRegion schemaRegion = getSchemaRegion("db", 0); + final String tableName = "t"; + + final Map<String, String> attributeMap = new HashMap<>(); + attributeMap.put("type", "new"); + attributeMap.put("cycle", "monthly"); + SchemaRegionTestUtil.createTableDevice(schemaRegion, tableName, new String[] {}, attributeMap); + + final List<IDeviceSchemaInfo> deviceSchemaInfoList = + SchemaRegionTestUtil.getTableDevice(schemaRegion, tableName, 0, Collections.emptyList()); + Assert.assertEquals(1, deviceSchemaInfoList.size()); + Assert.assertArrayEquals( + new String[] {"root", "db", tableName}, deviceSchemaInfoList.get(0).getRawNodes()); + Assert.assertEquals( + new Binary("new", TSFileConfig.STRING_CHARSET), + deviceSchemaInfoList.get(0).getAttributeValue("type")); + Assert.assertEquals( + new Binary("monthly", TSFileConfig.STRING_CHARSET), + deviceSchemaInfoList.get(0).getAttributeValue("cycle")); + } + @Test public void testDeviceWithDifferentIdLength() throws Exception { if (!testParams.getTestModeName().equals("MemoryMode")) {
