This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/ValidateSensorBug in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 500e26df9d7818c653bb9dc985f69abdc12e62b7 Author: JackieTien97 <[email protected]> AuthorDate: Fri Jul 5 17:41:04 2024 +0800 Fix some bugs in query schema fetch --- .../plan/relational/metadata/Metadata.java | 3 +- .../relational/metadata/TableMetadataImpl.java | 6 +- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 112 ++++++++++--------- .../fetcher/TableDeviceSchemaValidator.java | 119 ++++++++++++--------- .../fetcher/cache/TableDeviceCacheEntry.java | 25 +++-- .../planner/node/CreateTableDeviceNode.java | 44 ++++++-- .../optimizations/PushPredicateIntoTableScan.java | 3 +- .../attribute/DeviceAttributeStore.java | 28 ++--- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 4 +- .../read/resp/info/impl/ShowDevicesResult.java | 3 +- .../plan/relational/analyzer/TestMatadata.java | 3 +- 11 files changed, 205 insertions(+), 145 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java index 6eba3f8a196..62eee0d46a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java @@ -75,7 +75,8 @@ public interface Metadata { List<DeviceEntry> indexScan( QualifiedObjectName tableName, List<Expression> expressionList, - List<String> attributeColumns); + List<String> attributeColumns, + MPPQueryContext context); /** * This method is used for table column validation and should be invoked before device validation. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 45f5708e8ff..369b29fded3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -302,13 +302,15 @@ public class TableMetadataImpl implements Metadata { public List<DeviceEntry> indexScan( QualifiedObjectName tableName, List<Expression> expressionList, - List<String> attributeColumns) { + List<String> attributeColumns, + MPPQueryContext context) { return TableDeviceSchemaFetcher.getInstance() .fetchDeviceSchemaForDataQuery( tableName.getDatabaseName(), tableName.getObjectName(), expressionList, - attributeColumns); + attributeColumns, + context); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index 5ce6001c618..fd85de5572b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -41,6 +41,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.Ta import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.rpc.TSStatusCode; @@ -65,6 +66,8 @@ import java.util.stream.Collectors; public class TableDeviceSchemaFetcher { + private final SqlParser relationSqlParser = new SqlParser(); + private static final Logger LOGGER = LoggerFactory.getLogger(TableDeviceSchemaFetcher.class); private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -101,12 +104,12 @@ public class TableDeviceSchemaFetcher { ExecutionResult executionResult = coordinator.executeForTableModel( statement, - null, + relationSqlParser, SessionManager.getInstance().getCurrSession(), queryId, SessionManager.getInstance() .getSessionInfo(SessionManager.getInstance().getCurrSession()), - "", + "Fetch Device for insert", LocalExecutionPlanner.getInstance().metadata, config.getQueryTimeoutThreshold()); if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -135,26 +138,11 @@ public class TableDeviceSchemaFetcher { Column[] columns = tsBlock.get().getValueColumns(); for (int i = 0; i < tsBlock.get().getPositionCount(); i++) { String[] nodes = new String[idLength]; - int idIndex = 0; Map<String, String> attributeMap = new HashMap<>(); - for (int j = 0; j < columnHeaderList.size(); j++) { - TsTableColumnSchema columnSchema = - tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName()); - if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { - if (columns[j].isNull(i)) { - nodes[idIndex] = null; - } else { - nodes[idIndex] = columns[j].getBinary(i).toString(); - } - idIndex++; - } else { - if (columns[j].isNull(i)) { - attributeMap.put(columnSchema.getColumnName(), null); - } else { - attributeMap.put(columnSchema.getColumnName(), columns[j].getBinary(i).toString()); - } - } - } + + constructNodsArrayAndAttributeMap( + attributeMap, nodes, 0, columnHeaderList, columns, tableInstance, i); + fetchedDeviceSchema.put(new TableDeviceId(nodes), attributeMap); } } @@ -171,7 +159,8 @@ public class TableDeviceSchemaFetcher { String database, String table, List<Expression> expressionList, - List<String> attributeColumns) { + List<String> attributeColumns, + MPPQueryContext queryContext) { List<DeviceEntry> deviceEntryList = new ArrayList<>(); TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table); @@ -252,7 +241,9 @@ public class TableDeviceSchemaFetcher { idPredicateForFetch, compactedIdFuzzyPredicate, deviceEntryList, - idSingleMatchIndexList.size() == idPredicateList.size()); + // only cache those exact device query + idSingleMatchIndexList.size() == idPredicateList.size(), + queryContext); } // TODO table metadata: implement deduplicate during schemaRegion execution @@ -307,7 +298,8 @@ public class TableDeviceSchemaFetcher { List<List<Expression>> idDeterminedPredicateList, Expression idFuzzyPredicate, List<DeviceEntry> deviceEntryList, - boolean cacheFetchedDevice) { + boolean cacheFetchedDevice, + MPPQueryContext mppQueryContext) { String table = tableInstance.getTableName(); @@ -317,12 +309,14 @@ public class TableDeviceSchemaFetcher { ExecutionResult executionResult = coordinator.executeForTableModel( statement, - null, + relationSqlParser, SessionManager.getInstance().getCurrSession(), queryId, SessionManager.getInstance() .getSessionInfo(SessionManager.getInstance().getCurrSession()), - "", + String.format( + "fetch device for query %s : %s", + mppQueryContext.getQueryId(), mppQueryContext.getSql()), LocalExecutionPlanner.getInstance().metadata, config.getQueryTimeoutThreshold()); if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -334,8 +328,6 @@ public class TableDeviceSchemaFetcher { List<ColumnHeader> columnHeaderList = coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders(); int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums(); - Map<String, String> attributeMap; - Throwable t = null; try { while (coordinator.getQueryExecution(queryId).hasNextResult()) { @@ -353,31 +345,11 @@ public class TableDeviceSchemaFetcher { for (int i = 0; i < tsBlock.get().getPositionCount(); i++) { String[] nodes = new String[idLength + 1]; nodes[0] = table; - int idIndex = 0; - attributeMap = new HashMap<>(); - for (int j = 0; j < columnHeaderList.size(); j++) { - TsTableColumnSchema columnSchema = - tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName()); - if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { - if (columns[j].isNull(i)) { - nodes[idIndex + 1] = null; - } else { - nodes[idIndex + 1] = - columns[j].getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET); - } - idIndex++; - } else { - if (columns[j].isNull(i)) { - attributeMap.put(columnSchema.getColumnName(), null); - } else { - attributeMap.put( - columnSchema.getColumnName(), - columns[j].getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET)); - } - } - } + Map<String, String> attributeMap = new HashMap<>(); + constructNodsArrayAndAttributeMap( + attributeMap, nodes, 1, columnHeaderList, columns, tableInstance, i); IDeviceID deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create(nodes); - // TODO table metadata: add memory control + // TODO table metadata: add memory control in query deviceEntryList.add( new DeviceEntry( deviceID, @@ -394,4 +366,40 @@ public class TableDeviceSchemaFetcher { coordinator.cleanupQueryExecution(queryId, null, t); } } + + private void constructNodsArrayAndAttributeMap( + Map<String, String> attributeMap, + String[] nodes, + int startIndex, + List<ColumnHeader> columnHeaderList, + Column[] columns, + TsTable tableInstance, + int rowIndex) { + for (int j = 0; j < columnHeaderList.size(); j++) { + TsTableColumnSchema columnSchema = + tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName()); + // means that TsTable tableInstance which previously fetched is outdated, but it's ok that we + // ignore that newly added column here + if (columnSchema == null) { + continue; + } + if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { + if (columns[j].isNull(rowIndex)) { + nodes[startIndex] = null; + } else { + nodes[startIndex] = + columns[j].getBinary(rowIndex).getStringValue(TSFileConfig.STRING_CHARSET); + } + startIndex++; + } else { + if (columns[j].isNull(rowIndex)) { + attributeMap.put(columnSchema.getColumnName(), null); + } else { + attributeMap.put( + columnSchema.getColumnName(), + columns[j].getBinary(rowIndex).getStringValue(TSFileConfig.STRING_CHARSET)); + } + } + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java index dd9ccbe4765..8dd2f5885c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java @@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.Coordinator; @@ -32,16 +33,19 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.ITableDeviceSche import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceId; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; public class TableDeviceSchemaValidator { + private final SqlParser relationSqlParser = new SqlParser(); private static final Logger LOGGER = LoggerFactory.getLogger(TableDeviceSchemaValidator.class); @@ -91,24 +95,22 @@ public class TableDeviceSchemaValidator { Map<String, String> attributeMap = fetcher .getTableDeviceCache() - .getDeviceAttribute(database, tableName, parseArray(deviceIdList.get(i))); + .getDeviceAttribute(database, tableName, parseDeviceIdArray(deviceIdList.get(i))); if (attributeMap == null) { result.missingDeviceIndexList.add(i); - continue; - } - Object[] deviceAttributeValueList = attributeValueList.get(i); - for (int j = 0; j < attributeKeyList.size(); j++) { - String value = attributeMap.get(attributeKeyList.get(j)); - if (deviceAttributeValueList[j] == null) { - continue; - } - if (value == null) { - result.attributeMissingInCacheDeviceIndexList.add(i); - break; - } else { - if (!value.equals(String.valueOf(deviceAttributeValueList[j]))) { - result.attributeUpdateDeviceIndexList.add(i); - break; + } else { + Object[] deviceAttributeValueList = attributeValueList.get(i); + for (int j = 0, attributeSize = attributeKeyList.size(); j < attributeSize; j++) { + if (deviceAttributeValueList[j] != null) { + String key = attributeKeyList.get(j); + String value = attributeMap.get(key); + if (value == null) { + result.attributeMissingInCacheDeviceIndexList.add(i); + break; + } else if (!value.equals(deviceAttributeValueList[j])) { + result.attributeUpdateDeviceIndexList.add(i); + break; + } } } } @@ -116,12 +118,22 @@ public class TableDeviceSchemaValidator { return result; } - private String[] parseArray(Object[] objects) { + // we need to truncate the tailing null + private String[] parseDeviceIdArray(Object[] objects) { String[] strings = new String[objects.length]; + int lastNonNullIndex = -1; for (int i = 0; i < objects.length; i++) { - strings[i] = objects[i] == null ? null : String.valueOf(objects[i]); + if (objects[i] != null) { + strings[i] = (String) objects[i]; + lastNonNullIndex = i; + } + } + if (lastNonNullIndex == -1) { + throw new SemanticException("We don't support all IDColumns are null."); } - return strings; + return lastNonNullIndex == objects.length - 1 + ? strings + : Arrays.copyOf(strings, lastNonNullIndex + 1); } private ValidateResult fetchAndValidateDeviceSchema( @@ -155,48 +167,31 @@ public class TableDeviceSchemaValidator { entry.getValue()); } + List<String> attributeKeyList = schemaValidation.getAttributeColumnNameList(); + List<Object[]> attributeValueList = schemaValidation.getAttributeValueList(); + ValidateResult result = new ValidateResult(); for (int index : previousValidateResult.missingDeviceIndexList) { Object[] deviceId = schemaValidation.getDeviceIdList().get(index); Map<String, String> attributeMap = - fetchedDeviceSchema.get(new TableDeviceId(parseArray(deviceId))); + fetchedDeviceSchema.get(new TableDeviceId(parseDeviceIdArray(deviceId))); if (attributeMap == null) { result.missingDeviceIndexList.add(index); } else { - Object[] deviceAttributeValueList = schemaValidation.getAttributeValueList().get(index); - for (int j = 0; j < schemaValidation.getAttributeColumnNameList().size(); j++) { - String key = schemaValidation.getAttributeColumnNameList().get(j); - String value = attributeMap.get(key); - if (value == null) { - if (deviceAttributeValueList[j] != null) { - result.attributeUpdateDeviceIndexList.add(index); - } - } else { - if (deviceAttributeValueList[j] == null - || !value.equals(String.valueOf(deviceAttributeValueList[j]))) { - result.attributeUpdateDeviceIndexList.add(index); - } - } - } + constructAttributeUpdateDeviceIndexList( + attributeKeyList, attributeValueList, result, index, attributeMap); } } for (int index : previousValidateResult.attributeMissingInCacheDeviceIndexList) { Object[] deviceId = schemaValidation.getDeviceIdList().get(index); Map<String, String> attributeMap = - fetchedDeviceSchema.get(new TableDeviceId(parseArray(deviceId))); + fetchedDeviceSchema.get(new TableDeviceId(parseDeviceIdArray(deviceId))); if (attributeMap == null) { throw new IllegalStateException("Device shall exist but not exist."); } else { - for (int j = 0; j < schemaValidation.getAttributeColumnNameList().size(); j++) { - String key = schemaValidation.getAttributeColumnNameList().get(j); - String value = attributeMap.get(key); - if (value == null - || !value.equals(schemaValidation.getAttributeValueList().get(index)[j])) { - result.attributeUpdateDeviceIndexList.add(index); - break; - } - } + constructAttributeUpdateDeviceIndexList( + attributeKeyList, attributeValueList, result, index, attributeMap); } } @@ -206,15 +201,35 @@ public class TableDeviceSchemaValidator { return result; } + private void constructAttributeUpdateDeviceIndexList( + List<String> attributeKeyList, + List<Object[]> attributeValueList, + ValidateResult result, + int index, + Map<String, String> attributeMap) { + Object[] deviceAttributeValueList = attributeValueList.get(index); + for (int j = 0, size = attributeKeyList.size(); j < size; j++) { + if (deviceAttributeValueList[j] != null) { + String key = attributeKeyList.get(j); + String value = attributeMap.get(key); + + if (!deviceAttributeValueList[j].equals(value)) { + result.attributeUpdateDeviceIndexList.add(index); + break; + } + } + } + } + private void autoCreateDeviceSchema( ITableDeviceSchemaValidation schemaValidation, ValidateResult previousValidateResult, MPPQueryContext context) { - List<Object[]> deviceIdList = - new ArrayList<>( - previousValidateResult.missingDeviceIndexList.size() - + previousValidateResult.attributeUpdateDeviceIndexList.size()); - List<Object[]> attributeValueList = new ArrayList<>(deviceIdList.size()); + int size = + previousValidateResult.missingDeviceIndexList.size() + + previousValidateResult.attributeUpdateDeviceIndexList.size(); + List<Object[]> deviceIdList = new ArrayList<>(size); + List<Object[]> attributeValueList = new ArrayList<>(size); for (int index : previousValidateResult.missingDeviceIndexList) { deviceIdList.add(schemaValidation.getDeviceIdList().get(index)); attributeValueList.add(schemaValidation.getAttributeValueList().get(index)); @@ -234,12 +249,12 @@ public class TableDeviceSchemaValidator { ExecutionResult executionResult = coordinator.executeForTableModel( statement, - null, + relationSqlParser, SessionManager.getInstance().getCurrSession(), SessionManager.getInstance().requestQueryId(), SessionManager.getInstance() .getSessionInfo(SessionManager.getInstance().getCurrSession()), - "", + "Create device or update device attribute for insert", LocalExecutionPlanner.getInstance().metadata, context == null || context.getQueryType().equals(QueryType.WRITE) ? config.getQueryTimeoutThreshold() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java index a6d6f3f5e42..7ca4dba0ef0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java @@ -19,22 +19,25 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache; -import org.apache.iotdb.commons.schema.MemUsageUtil; +import com.google.common.collect.ImmutableMap; +import org.apache.tsfile.utils.RamUsageEstimator; -import java.util.HashMap; import java.util.Map; public class TableDeviceCacheEntry { - // should use a Map implementation that allows null for value - private final Map<String, String> attributeMap; + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(TableDeviceCacheEntry.class); - public TableDeviceCacheEntry() { - attributeMap = new HashMap<>(); - } + // the cached attributeMap may not be the latest, but there won't be any correctness problems + // because when missing getting the key-value from this attributeMap, caller will try to get or + // create from remote + // there may exist key is not null, but value is null in this map, which means that the key's + // corresponding value is null, doesn't mean that the key doesn't exist + private final ImmutableMap<String, String> attributeMap; public TableDeviceCacheEntry(Map<String, String> attributeMap) { - this.attributeMap = new HashMap<>(attributeMap); + this.attributeMap = ImmutableMap.copyOf(attributeMap); } public String getAttribute(String key) { @@ -46,10 +49,6 @@ public class TableDeviceCacheEntry { } public int estimateSize() { - int size = 8; // map reference - for (Map.Entry<String, String> entry : attributeMap.entrySet()) { - size += (int) MemUsageUtil.computeKVMemUsageInMap(entry.getKey(), entry.getValue()); - } - return size; + return (int) (INSTANCE_SIZE + RamUsageEstimator.sizeOfMap(attributeMap)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java index 13209c2274f..f3888bc042a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java @@ -34,6 +34,8 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -68,11 +70,35 @@ public class CreateTableDeviceNode extends WritePlanNode { super(id); this.database = database; this.tableName = tableName; - this.deviceIdList = deviceIdList; + // truncate the tailing null + this.deviceIdList = truncateTailingNull(deviceIdList); this.attributeNameList = attributeNameList; this.attributeValueList = attributeValueList; } + private static List<Object[]> truncateTailingNull(List<Object[]> deviceIdList) { + List<Object[]> res = new ArrayList<>(deviceIdList.size()); + for (Object[] device : deviceIdList) { + if (device == null || device.length == 0) { + throw new IllegalArgumentException("DeviceID's length should be larger than 0."); + } + int lastNonNullIndex = -1; + for (int i = device.length - 1; i >= 0; i--) { + if (device[i] != null) { + lastNonNullIndex = i; + break; + } + } + if (lastNonNullIndex == -1) { + throw new IllegalArgumentException("DeviceID shouldn't be all nulls."); + } + res.add(Arrays.copyOf(device, lastNonNullIndex + 1)); + } + return res; + } + + // in this constructor, we don't need to truncate tailing nulls for deviceIdList, because this + // constructor can only be generated from another CreateTableDeviceNode public CreateTableDeviceNode( PlanNodeId id, TRegionReplicaSet regionReplicaSet, @@ -122,27 +148,29 @@ public class CreateTableDeviceNode extends WritePlanNode { public List<IDeviceID> getPartitionKeyList() { if (partitionKeyList == null) { - List<IDeviceID> partitionKeyList = new ArrayList<>(); + List<IDeviceID> tmpPartitionKeyList = new ArrayList<>(); for (Object[] rawId : deviceIdList) { String[] partitionKey = new String[rawId.length + 1]; partitionKey[0] = tableName; for (int i = 0; i < rawId.length; i++) { - partitionKey[i + 1] = Objects.toString(rawId[i].toString()); + partitionKey[i + 1] = (String) rawId[i]; } - partitionKeyList.add(IDeviceID.Factory.DEFAULT_FACTORY.create(partitionKey)); + tmpPartitionKeyList.add(IDeviceID.Factory.DEFAULT_FACTORY.create(partitionKey)); } - this.partitionKeyList = partitionKeyList; + this.partitionKeyList = tmpPartitionKeyList; } return partitionKeyList; } @Override public List<PlanNode> getChildren() { - return new ArrayList<>(); + return Collections.emptyList(); } @Override - public void addChild(PlanNode child) {} + public void addChild(PlanNode child) { + throw new UnsupportedOperationException(); + } @Override public PlanNode clone() { @@ -163,7 +191,7 @@ public class CreateTableDeviceNode extends WritePlanNode { @Override public List<String> getOutputColumnNames() { - return null; + throw new UnsupportedOperationException(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index a3215c00953..98b10ab23ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -251,7 +251,8 @@ public class PushPredicateIntoTableScan implements TablePlanOptimizer { .map(Symbol::getName) .collect(Collectors.toList()); List<DeviceEntry> deviceEntries = - metadata.indexScan(node.getQualifiedObjectName(), metadataExpressions, attributeColumns); + metadata.indexScan( + node.getQualifiedObjectName(), metadataExpressions, attributeColumns, queryContext); node.setDeviceEntries(deviceEntries); if (deviceEntries.isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java index 70125fe2831..457b48e0a6c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -123,9 +123,11 @@ public class DeviceAttributeStore implements IDeviceAttributeStore { Map<String, String> attributeMap = new HashMap<>(); String value; for (int i = 0; i < nameList.size(); i++) { - value = valueList[i] == null ? null : valueList[i].toString(); - attributeMap.put(nameList.get(i), value); - memUsage += MemUsageUtil.computeKVMemUsageInMap(nameList.get(i), value); + value = valueList[i] == null ? null : (String) valueList[i]; + if (value != null) { + attributeMap.put(nameList.get(i), value); + memUsage += MemUsageUtil.computeKVMemUsageInMap(nameList.get(i), value); + } } deviceAttributeList.add(attributeMap); requestMemory(memUsage); @@ -142,15 +144,17 @@ public class DeviceAttributeStore implements IDeviceAttributeStore { String value; for (int i = 0; i < nameList.size(); i++) { String key = nameList.get(i); - originMemUsage = - attributeMap.containsKey(key) - ? 0 - : MemUsageUtil.computeKVMemUsageInMap(key, attributeMap.get(key)); - - value = valueList[i] == null ? null : valueList[i].toString(); - attributeMap.put(key, value); - updatedMemUsage = MemUsageUtil.computeKVMemUsageInMap(key, value); - memUsageDelta += updatedMemUsage - originMemUsage; + value = valueList[i] == null ? null : (String) valueList[i]; + if (value != null) { + originMemUsage = + attributeMap.containsKey(key) + ? 0 + : MemUsageUtil.computeKVMemUsageInMap(key, attributeMap.get(key)); + + attributeMap.put(key, value); + updatedMemUsage = MemUsageUtil.computeKVMemUsageInMap(key, value); + memUsageDelta += (updatedMemUsage - originMemUsage); + } } requestMemory(memUsageDelta); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 6663d8867a4..22fc44c4b43 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -1510,8 +1510,8 @@ public class MTreeBelowSGMemoryImpl { cur = child; String nodeName; - for (int i = 0; i < devicePath.length; i++) { - nodeName = devicePath[i] == null ? null : devicePath[i].toString(); + for (Object o : devicePath) { + nodeName = o == null ? null : (String) o; child = cur.getChild(nodeName); if (child == null) { child = store.addChild(cur, nodeName, nodeFactory.createInternalMNode(cur, nodeName)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java index b71dab7121b..dd275ca16ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchem import java.util.Objects; import java.util.function.Function; +import java.util.function.UnaryOperator; public class ShowDevicesResult extends ShowSchemaResult implements IDeviceSchemaInfo { private Boolean isAligned; @@ -53,7 +54,7 @@ public class ShowDevicesResult extends ShowSchemaResult implements IDeviceSchema return templateId; } - public void setAttributeProvider(Function<String, String> attributeProvider) { + public void setAttributeProvider(UnaryOperator<String> attributeProvider) { this.attributeProvider = attributeProvider; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java index bf30dc00718..0873c14c885 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java @@ -205,7 +205,8 @@ public class TestMatadata implements Metadata { public List<DeviceEntry> indexScan( QualifiedObjectName tableName, List<Expression> expressionList, - List<String> attributeColumns) { + List<String> attributeColumns, + MPPQueryContext context) { return Arrays.asList( new DeviceEntry(new StringArrayDeviceID(DEVICE_4.split("\\.")), DEVICE_4_ATTRIBUTES), new DeviceEntry(new StringArrayDeviceID(DEVICE_1.split("\\.")), DEVICE_1_ATTRIBUTES),
