This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/TableModelGrammar
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/ty/TableModelGrammar by this
push:
new 99bc163d83a Fix some bugs in query schema fetch
99bc163d83a is described below
commit 99bc163d83a90c696023f2ebc631e60edfd73a3a
Author: Jackie Tien <[email protected]>
AuthorDate: Sun Jul 7 11:25:11 2024 +0800
Fix some bugs in query schema fetch
---
.../plan/relational/metadata/Metadata.java | 3 +-
.../relational/metadata/TableMetadataImpl.java | 6 +-
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 112 ++++++++++---------
.../fetcher/TableDeviceSchemaValidator.java | 119 ++++++++++++---------
.../fetcher/cache/TableDeviceCacheEntry.java | 25 +++--
.../planner/node/CreateTableDeviceNode.java | 47 ++++++--
.../optimizations/PushPredicateIntoTableScan.java | 3 +-
.../attribute/DeviceAttributeStore.java | 28 ++---
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 4 +-
.../read/resp/info/impl/ShowDevicesResult.java | 3 +-
.../plan/relational/analyzer/TestMatadata.java | 3 +-
11 files changed, 208 insertions(+), 145 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java
index 6eba3f8a196..62eee0d46a2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java
@@ -75,7 +75,8 @@ public interface Metadata {
List<DeviceEntry> indexScan(
QualifiedObjectName tableName,
List<Expression> expressionList,
- List<String> attributeColumns);
+ List<String> attributeColumns,
+ MPPQueryContext context);
/**
* This method is used for table column validation and should be invoked
before device validation.
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
index 45f5708e8ff..369b29fded3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
@@ -302,13 +302,15 @@ public class TableMetadataImpl implements Metadata {
public List<DeviceEntry> indexScan(
QualifiedObjectName tableName,
List<Expression> expressionList,
- List<String> attributeColumns) {
+ List<String> attributeColumns,
+ MPPQueryContext context) {
return TableDeviceSchemaFetcher.getInstance()
.fetchDeviceSchemaForDataQuery(
tableName.getDatabaseName(),
tableName.getObjectName(),
expressionList,
- attributeColumns);
+ attributeColumns,
+ context);
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
index 5ce6001c618..fd85de5572b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
@@ -41,6 +41,7 @@ import
org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.Ta
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -65,6 +66,8 @@ import java.util.stream.Collectors;
public class TableDeviceSchemaFetcher {
+ private final SqlParser relationSqlParser = new SqlParser();
+
private static final Logger LOGGER =
LoggerFactory.getLogger(TableDeviceSchemaFetcher.class);
private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
@@ -101,12 +104,12 @@ public class TableDeviceSchemaFetcher {
ExecutionResult executionResult =
coordinator.executeForTableModel(
statement,
- null,
+ relationSqlParser,
SessionManager.getInstance().getCurrSession(),
queryId,
SessionManager.getInstance()
.getSessionInfo(SessionManager.getInstance().getCurrSession()),
- "",
+ "Fetch Device for insert",
LocalExecutionPlanner.getInstance().metadata,
config.getQueryTimeoutThreshold());
if (executionResult.status.getCode() !=
TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
@@ -135,26 +138,11 @@ public class TableDeviceSchemaFetcher {
Column[] columns = tsBlock.get().getValueColumns();
for (int i = 0; i < tsBlock.get().getPositionCount(); i++) {
String[] nodes = new String[idLength];
- int idIndex = 0;
Map<String, String> attributeMap = new HashMap<>();
- for (int j = 0; j < columnHeaderList.size(); j++) {
- TsTableColumnSchema columnSchema =
-
tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName());
- if
(columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) {
- if (columns[j].isNull(i)) {
- nodes[idIndex] = null;
- } else {
- nodes[idIndex] = columns[j].getBinary(i).toString();
- }
- idIndex++;
- } else {
- if (columns[j].isNull(i)) {
- attributeMap.put(columnSchema.getColumnName(), null);
- } else {
- attributeMap.put(columnSchema.getColumnName(),
columns[j].getBinary(i).toString());
- }
- }
- }
+
+ constructNodsArrayAndAttributeMap(
+ attributeMap, nodes, 0, columnHeaderList, columns,
tableInstance, i);
+
fetchedDeviceSchema.put(new TableDeviceId(nodes), attributeMap);
}
}
@@ -171,7 +159,8 @@ public class TableDeviceSchemaFetcher {
String database,
String table,
List<Expression> expressionList,
- List<String> attributeColumns) {
+ List<String> attributeColumns,
+ MPPQueryContext queryContext) {
List<DeviceEntry> deviceEntryList = new ArrayList<>();
TsTable tableInstance =
DataNodeTableCache.getInstance().getTable(database, table);
@@ -252,7 +241,9 @@ public class TableDeviceSchemaFetcher {
idPredicateForFetch,
compactedIdFuzzyPredicate,
deviceEntryList,
- idSingleMatchIndexList.size() == idPredicateList.size());
+ // only cache those exact device query
+ idSingleMatchIndexList.size() == idPredicateList.size(),
+ queryContext);
}
// TODO table metadata: implement deduplicate during schemaRegion
execution
@@ -307,7 +298,8 @@ public class TableDeviceSchemaFetcher {
List<List<Expression>> idDeterminedPredicateList,
Expression idFuzzyPredicate,
List<DeviceEntry> deviceEntryList,
- boolean cacheFetchedDevice) {
+ boolean cacheFetchedDevice,
+ MPPQueryContext mppQueryContext) {
String table = tableInstance.getTableName();
@@ -317,12 +309,14 @@ public class TableDeviceSchemaFetcher {
ExecutionResult executionResult =
coordinator.executeForTableModel(
statement,
- null,
+ relationSqlParser,
SessionManager.getInstance().getCurrSession(),
queryId,
SessionManager.getInstance()
.getSessionInfo(SessionManager.getInstance().getCurrSession()),
- "",
+ String.format(
+ "fetch device for query %s : %s",
+ mppQueryContext.getQueryId(), mppQueryContext.getSql()),
LocalExecutionPlanner.getInstance().metadata,
config.getQueryTimeoutThreshold());
if (executionResult.status.getCode() !=
TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
@@ -334,8 +328,6 @@ public class TableDeviceSchemaFetcher {
List<ColumnHeader> columnHeaderList =
coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders();
int idLength = DataNodeTableCache.getInstance().getTable(database,
table).getIdNums();
- Map<String, String> attributeMap;
-
Throwable t = null;
try {
while (coordinator.getQueryExecution(queryId).hasNextResult()) {
@@ -353,31 +345,11 @@ public class TableDeviceSchemaFetcher {
for (int i = 0; i < tsBlock.get().getPositionCount(); i++) {
String[] nodes = new String[idLength + 1];
nodes[0] = table;
- int idIndex = 0;
- attributeMap = new HashMap<>();
- for (int j = 0; j < columnHeaderList.size(); j++) {
- TsTableColumnSchema columnSchema =
-
tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName());
- if
(columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) {
- if (columns[j].isNull(i)) {
- nodes[idIndex + 1] = null;
- } else {
- nodes[idIndex + 1] =
-
columns[j].getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET);
- }
- idIndex++;
- } else {
- if (columns[j].isNull(i)) {
- attributeMap.put(columnSchema.getColumnName(), null);
- } else {
- attributeMap.put(
- columnSchema.getColumnName(),
-
columns[j].getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET));
- }
- }
- }
+ Map<String, String> attributeMap = new HashMap<>();
+ constructNodsArrayAndAttributeMap(
+ attributeMap, nodes, 1, columnHeaderList, columns,
tableInstance, i);
IDeviceID deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create(nodes);
- // TODO table metadata: add memory control
+ // TODO table metadata: add memory control in query
deviceEntryList.add(
new DeviceEntry(
deviceID,
@@ -394,4 +366,40 @@ public class TableDeviceSchemaFetcher {
coordinator.cleanupQueryExecution(queryId, null, t);
}
}
+
+ private void constructNodsArrayAndAttributeMap(
+ Map<String, String> attributeMap,
+ String[] nodes,
+ int startIndex,
+ List<ColumnHeader> columnHeaderList,
+ Column[] columns,
+ TsTable tableInstance,
+ int rowIndex) {
+ for (int j = 0; j < columnHeaderList.size(); j++) {
+ TsTableColumnSchema columnSchema =
+
tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName());
+ // means that TsTable tableInstance which previously fetched is
outdated, but it's ok that we
+ // ignore that newly added column here
+ if (columnSchema == null) {
+ continue;
+ }
+ if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) {
+ if (columns[j].isNull(rowIndex)) {
+ nodes[startIndex] = null;
+ } else {
+ nodes[startIndex] =
+
columns[j].getBinary(rowIndex).getStringValue(TSFileConfig.STRING_CHARSET);
+ }
+ startIndex++;
+ } else {
+ if (columns[j].isNull(rowIndex)) {
+ attributeMap.put(columnSchema.getColumnName(), null);
+ } else {
+ attributeMap.put(
+ columnSchema.getColumnName(),
+
columns[j].getBinary(rowIndex).getStringValue(TSFileConfig.STRING_CHARSET));
+ }
+ }
+ }
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
index dd9ccbe4765..8dd2f5885c1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
@@ -22,6 +22,7 @@ package
org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher;
import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.protocol.session.SessionManager;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.Coordinator;
@@ -32,16 +33,19 @@ import
org.apache.iotdb.db.queryengine.plan.relational.metadata.ITableDeviceSche
import
org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceId;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDevice;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
import org.apache.iotdb.rpc.TSStatusCode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
public class TableDeviceSchemaValidator {
+ private final SqlParser relationSqlParser = new SqlParser();
private static final Logger LOGGER =
LoggerFactory.getLogger(TableDeviceSchemaValidator.class);
@@ -91,24 +95,22 @@ public class TableDeviceSchemaValidator {
Map<String, String> attributeMap =
fetcher
.getTableDeviceCache()
- .getDeviceAttribute(database, tableName,
parseArray(deviceIdList.get(i)));
+ .getDeviceAttribute(database, tableName,
parseDeviceIdArray(deviceIdList.get(i)));
if (attributeMap == null) {
result.missingDeviceIndexList.add(i);
- continue;
- }
- Object[] deviceAttributeValueList = attributeValueList.get(i);
- for (int j = 0; j < attributeKeyList.size(); j++) {
- String value = attributeMap.get(attributeKeyList.get(j));
- if (deviceAttributeValueList[j] == null) {
- continue;
- }
- if (value == null) {
- result.attributeMissingInCacheDeviceIndexList.add(i);
- break;
- } else {
- if (!value.equals(String.valueOf(deviceAttributeValueList[j]))) {
- result.attributeUpdateDeviceIndexList.add(i);
- break;
+ } else {
+ Object[] deviceAttributeValueList = attributeValueList.get(i);
+ for (int j = 0, attributeSize = attributeKeyList.size(); j <
attributeSize; j++) {
+ if (deviceAttributeValueList[j] != null) {
+ String key = attributeKeyList.get(j);
+ String value = attributeMap.get(key);
+ if (value == null) {
+ result.attributeMissingInCacheDeviceIndexList.add(i);
+ break;
+ } else if (!value.equals(deviceAttributeValueList[j])) {
+ result.attributeUpdateDeviceIndexList.add(i);
+ break;
+ }
}
}
}
@@ -116,12 +118,22 @@ public class TableDeviceSchemaValidator {
return result;
}
- private String[] parseArray(Object[] objects) {
+ // we need to truncate the tailing null
+ private String[] parseDeviceIdArray(Object[] objects) {
String[] strings = new String[objects.length];
+ int lastNonNullIndex = -1;
for (int i = 0; i < objects.length; i++) {
- strings[i] = objects[i] == null ? null : String.valueOf(objects[i]);
+ if (objects[i] != null) {
+ strings[i] = (String) objects[i];
+ lastNonNullIndex = i;
+ }
+ }
+ if (lastNonNullIndex == -1) {
+ throw new SemanticException("We don't support all IDColumns are null.");
}
- return strings;
+ return lastNonNullIndex == objects.length - 1
+ ? strings
+ : Arrays.copyOf(strings, lastNonNullIndex + 1);
}
private ValidateResult fetchAndValidateDeviceSchema(
@@ -155,48 +167,31 @@ public class TableDeviceSchemaValidator {
entry.getValue());
}
+ List<String> attributeKeyList =
schemaValidation.getAttributeColumnNameList();
+ List<Object[]> attributeValueList =
schemaValidation.getAttributeValueList();
+
ValidateResult result = new ValidateResult();
for (int index : previousValidateResult.missingDeviceIndexList) {
Object[] deviceId = schemaValidation.getDeviceIdList().get(index);
Map<String, String> attributeMap =
- fetchedDeviceSchema.get(new TableDeviceId(parseArray(deviceId)));
+ fetchedDeviceSchema.get(new
TableDeviceId(parseDeviceIdArray(deviceId)));
if (attributeMap == null) {
result.missingDeviceIndexList.add(index);
} else {
- Object[] deviceAttributeValueList =
schemaValidation.getAttributeValueList().get(index);
- for (int j = 0; j <
schemaValidation.getAttributeColumnNameList().size(); j++) {
- String key = schemaValidation.getAttributeColumnNameList().get(j);
- String value = attributeMap.get(key);
- if (value == null) {
- if (deviceAttributeValueList[j] != null) {
- result.attributeUpdateDeviceIndexList.add(index);
- }
- } else {
- if (deviceAttributeValueList[j] == null
- || !value.equals(String.valueOf(deviceAttributeValueList[j])))
{
- result.attributeUpdateDeviceIndexList.add(index);
- }
- }
- }
+ constructAttributeUpdateDeviceIndexList(
+ attributeKeyList, attributeValueList, result, index, attributeMap);
}
}
for (int index :
previousValidateResult.attributeMissingInCacheDeviceIndexList) {
Object[] deviceId = schemaValidation.getDeviceIdList().get(index);
Map<String, String> attributeMap =
- fetchedDeviceSchema.get(new TableDeviceId(parseArray(deviceId)));
+ fetchedDeviceSchema.get(new
TableDeviceId(parseDeviceIdArray(deviceId)));
if (attributeMap == null) {
throw new IllegalStateException("Device shall exist but not exist.");
} else {
- for (int j = 0; j <
schemaValidation.getAttributeColumnNameList().size(); j++) {
- String key = schemaValidation.getAttributeColumnNameList().get(j);
- String value = attributeMap.get(key);
- if (value == null
- ||
!value.equals(schemaValidation.getAttributeValueList().get(index)[j])) {
- result.attributeUpdateDeviceIndexList.add(index);
- break;
- }
- }
+ constructAttributeUpdateDeviceIndexList(
+ attributeKeyList, attributeValueList, result, index, attributeMap);
}
}
@@ -206,15 +201,35 @@ public class TableDeviceSchemaValidator {
return result;
}
+ private void constructAttributeUpdateDeviceIndexList(
+ List<String> attributeKeyList,
+ List<Object[]> attributeValueList,
+ ValidateResult result,
+ int index,
+ Map<String, String> attributeMap) {
+ Object[] deviceAttributeValueList = attributeValueList.get(index);
+ for (int j = 0, size = attributeKeyList.size(); j < size; j++) {
+ if (deviceAttributeValueList[j] != null) {
+ String key = attributeKeyList.get(j);
+ String value = attributeMap.get(key);
+
+ if (!deviceAttributeValueList[j].equals(value)) {
+ result.attributeUpdateDeviceIndexList.add(index);
+ break;
+ }
+ }
+ }
+ }
+
private void autoCreateDeviceSchema(
ITableDeviceSchemaValidation schemaValidation,
ValidateResult previousValidateResult,
MPPQueryContext context) {
- List<Object[]> deviceIdList =
- new ArrayList<>(
- previousValidateResult.missingDeviceIndexList.size()
- +
previousValidateResult.attributeUpdateDeviceIndexList.size());
- List<Object[]> attributeValueList = new ArrayList<>(deviceIdList.size());
+ int size =
+ previousValidateResult.missingDeviceIndexList.size()
+ + previousValidateResult.attributeUpdateDeviceIndexList.size();
+ List<Object[]> deviceIdList = new ArrayList<>(size);
+ List<Object[]> attributeValueList = new ArrayList<>(size);
for (int index : previousValidateResult.missingDeviceIndexList) {
deviceIdList.add(schemaValidation.getDeviceIdList().get(index));
attributeValueList.add(schemaValidation.getAttributeValueList().get(index));
@@ -234,12 +249,12 @@ public class TableDeviceSchemaValidator {
ExecutionResult executionResult =
coordinator.executeForTableModel(
statement,
- null,
+ relationSqlParser,
SessionManager.getInstance().getCurrSession(),
SessionManager.getInstance().requestQueryId(),
SessionManager.getInstance()
.getSessionInfo(SessionManager.getInstance().getCurrSession()),
- "",
+ "Create device or update device attribute for insert",
LocalExecutionPlanner.getInstance().metadata,
context == null || context.getQueryType().equals(QueryType.WRITE)
? config.getQueryTimeoutThreshold()
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java
index a6d6f3f5e42..7ca4dba0ef0 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java
@@ -19,22 +19,25 @@
package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache;
-import org.apache.iotdb.commons.schema.MemUsageUtil;
+import com.google.common.collect.ImmutableMap;
+import org.apache.tsfile.utils.RamUsageEstimator;
-import java.util.HashMap;
import java.util.Map;
public class TableDeviceCacheEntry {
- // should use a Map implementation that allows null for value
- private final Map<String, String> attributeMap;
+ private static final long INSTANCE_SIZE =
+ RamUsageEstimator.shallowSizeOfInstance(TableDeviceCacheEntry.class);
- public TableDeviceCacheEntry() {
- attributeMap = new HashMap<>();
- }
+ // the cached attributeMap may not be the latest, but there won't be any
correctness problems
+ // because when missing getting the key-value from this attributeMap, caller
will try to get or
+ // create from remote
+ // there may exist key is not null, but value is null in this map, which
means that the key's
+ // corresponding value is null, doesn't mean that the key doesn't exist
+ private final ImmutableMap<String, String> attributeMap;
public TableDeviceCacheEntry(Map<String, String> attributeMap) {
- this.attributeMap = new HashMap<>(attributeMap);
+ this.attributeMap = ImmutableMap.copyOf(attributeMap);
}
public String getAttribute(String key) {
@@ -46,10 +49,6 @@ public class TableDeviceCacheEntry {
}
public int estimateSize() {
- int size = 8; // map reference
- for (Map.Entry<String, String> entry : attributeMap.entrySet()) {
- size += (int) MemUsageUtil.computeKVMemUsageInMap(entry.getKey(),
entry.getValue());
- }
- return size;
+ return (int) (INSTANCE_SIZE + RamUsageEstimator.sizeOfMap(attributeMap));
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java
index 13209c2274f..6de1ef903ce 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java
@@ -34,6 +34,8 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -68,11 +70,38 @@ public class CreateTableDeviceNode extends WritePlanNode {
super(id);
this.database = database;
this.tableName = tableName;
- this.deviceIdList = deviceIdList;
+ // truncate the tailing null
+ this.deviceIdList = truncateTailingNull(deviceIdList);
this.attributeNameList = attributeNameList;
this.attributeValueList = attributeValueList;
}
+ private static List<Object[]> truncateTailingNull(List<Object[]>
deviceIdList) {
+ List<Object[]> res = new ArrayList<>(deviceIdList.size());
+ for (Object[] device : deviceIdList) {
+ if (device == null || device.length == 0) {
+ throw new IllegalArgumentException("DeviceID's length should be larger
than 0.");
+ }
+ int lastNonNullIndex = -1;
+ for (int i = device.length - 1; i >= 0; i--) {
+ if (device[i] != null) {
+ lastNonNullIndex = i;
+ break;
+ }
+ }
+ if (lastNonNullIndex == -1) {
+ throw new IllegalArgumentException("DeviceID shouldn't be all nulls.");
+ }
+ res.add(
+ lastNonNullIndex == device.length - 1
+ ? device
+ : Arrays.copyOf(device, lastNonNullIndex + 1));
+ }
+ return res;
+ }
+
+ // in this constructor, we don't need to truncate tailing nulls for
deviceIdList, because this
+ // constructor can only be generated from another CreateTableDeviceNode
public CreateTableDeviceNode(
PlanNodeId id,
TRegionReplicaSet regionReplicaSet,
@@ -122,27 +151,29 @@ public class CreateTableDeviceNode extends WritePlanNode {
public List<IDeviceID> getPartitionKeyList() {
if (partitionKeyList == null) {
- List<IDeviceID> partitionKeyList = new ArrayList<>();
+ List<IDeviceID> tmpPartitionKeyList = new ArrayList<>();
for (Object[] rawId : deviceIdList) {
String[] partitionKey = new String[rawId.length + 1];
partitionKey[0] = tableName;
for (int i = 0; i < rawId.length; i++) {
- partitionKey[i + 1] = Objects.toString(rawId[i].toString());
+ partitionKey[i + 1] = (String) rawId[i];
}
-
partitionKeyList.add(IDeviceID.Factory.DEFAULT_FACTORY.create(partitionKey));
+
tmpPartitionKeyList.add(IDeviceID.Factory.DEFAULT_FACTORY.create(partitionKey));
}
- this.partitionKeyList = partitionKeyList;
+ this.partitionKeyList = tmpPartitionKeyList;
}
return partitionKeyList;
}
@Override
public List<PlanNode> getChildren() {
- return new ArrayList<>();
+ return Collections.emptyList();
}
@Override
- public void addChild(PlanNode child) {}
+ public void addChild(PlanNode child) {
+ throw new UnsupportedOperationException();
+ }
@Override
public PlanNode clone() {
@@ -163,7 +194,7 @@ public class CreateTableDeviceNode extends WritePlanNode {
@Override
public List<String> getOutputColumnNames() {
- return null;
+ throw new UnsupportedOperationException();
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
index a3215c00953..98b10ab23ce 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
@@ -251,7 +251,8 @@ public class PushPredicateIntoTableScan implements
TablePlanOptimizer {
.map(Symbol::getName)
.collect(Collectors.toList());
List<DeviceEntry> deviceEntries =
- metadata.indexScan(node.getQualifiedObjectName(),
metadataExpressions, attributeColumns);
+ metadata.indexScan(
+ node.getQualifiedObjectName(), metadataExpressions,
attributeColumns, queryContext);
node.setDeviceEntries(deviceEntries);
if (deviceEntries.isEmpty()) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java
index 70125fe2831..457b48e0a6c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java
@@ -123,9 +123,11 @@ public class DeviceAttributeStore implements
IDeviceAttributeStore {
Map<String, String> attributeMap = new HashMap<>();
String value;
for (int i = 0; i < nameList.size(); i++) {
- value = valueList[i] == null ? null : valueList[i].toString();
- attributeMap.put(nameList.get(i), value);
- memUsage += MemUsageUtil.computeKVMemUsageInMap(nameList.get(i), value);
+ value = valueList[i] == null ? null : (String) valueList[i];
+ if (value != null) {
+ attributeMap.put(nameList.get(i), value);
+ memUsage += MemUsageUtil.computeKVMemUsageInMap(nameList.get(i),
value);
+ }
}
deviceAttributeList.add(attributeMap);
requestMemory(memUsage);
@@ -142,15 +144,17 @@ public class DeviceAttributeStore implements
IDeviceAttributeStore {
String value;
for (int i = 0; i < nameList.size(); i++) {
String key = nameList.get(i);
- originMemUsage =
- attributeMap.containsKey(key)
- ? 0
- : MemUsageUtil.computeKVMemUsageInMap(key,
attributeMap.get(key));
-
- value = valueList[i] == null ? null : valueList[i].toString();
- attributeMap.put(key, value);
- updatedMemUsage = MemUsageUtil.computeKVMemUsageInMap(key, value);
- memUsageDelta += updatedMemUsage - originMemUsage;
+ value = valueList[i] == null ? null : (String) valueList[i];
+ if (value != null) {
+ originMemUsage =
+ attributeMap.containsKey(key)
+ ? 0
+ : MemUsageUtil.computeKVMemUsageInMap(key,
attributeMap.get(key));
+
+ attributeMap.put(key, value);
+ updatedMemUsage = MemUsageUtil.computeKVMemUsageInMap(key, value);
+ memUsageDelta += (updatedMemUsage - originMemUsage);
+ }
}
requestMemory(memUsageDelta);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
index 6663d8867a4..22fc44c4b43 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
@@ -1510,8 +1510,8 @@ public class MTreeBelowSGMemoryImpl {
cur = child;
String nodeName;
- for (int i = 0; i < devicePath.length; i++) {
- nodeName = devicePath[i] == null ? null : devicePath[i].toString();
+ for (Object o : devicePath) {
+ nodeName = o == null ? null : (String) o;
child = cur.getChild(nodeName);
if (child == null) {
child = store.addChild(cur, nodeName,
nodeFactory.createInternalMNode(cur, nodeName));
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java
index b71dab7121b..dd275ca16ff 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java
@@ -23,6 +23,7 @@ import
org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchem
import java.util.Objects;
import java.util.function.Function;
+import java.util.function.UnaryOperator;
public class ShowDevicesResult extends ShowSchemaResult implements
IDeviceSchemaInfo {
private Boolean isAligned;
@@ -53,7 +54,7 @@ public class ShowDevicesResult extends ShowSchemaResult
implements IDeviceSchema
return templateId;
}
- public void setAttributeProvider(Function<String, String> attributeProvider)
{
+ public void setAttributeProvider(UnaryOperator<String> attributeProvider) {
this.attributeProvider = attributeProvider;
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java
index bf30dc00718..0873c14c885 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java
@@ -205,7 +205,8 @@ public class TestMatadata implements Metadata {
public List<DeviceEntry> indexScan(
QualifiedObjectName tableName,
List<Expression> expressionList,
- List<String> attributeColumns) {
+ List<String> attributeColumns,
+ MPPQueryContext context) {
return Arrays.asList(
new DeviceEntry(new StringArrayDeviceID(DEVICE_4.split("\\.")),
DEVICE_4_ATTRIBUTES),
new DeviceEntry(new StringArrayDeviceID(DEVICE_1.split("\\.")),
DEVICE_1_ATTRIBUTES),
