This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/CollectOperator in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5ecbed2b118dffa83ac18d08b86fa159479fa714 Author: JackieTien97 <[email protected]> AuthorDate: Wed Jul 3 18:59:31 2024 +0800 fix some bugs in table schema fetcher --- .../operator/process/CollectOperator.java | 130 +++++++++++++++++++++ .../schema/source/TableDeviceQuerySource.java | 17 ++- .../node/metedata/read/TableDeviceQueryNode.java | 4 +- .../PredicatePushIntoMetadataChecker.java | 7 +- .../schema/CheckSchemaPredicateVisitor.java | 35 +++--- .../ConvertSchemaPredicateToFilterVisitor.java | 19 +-- .../fetcher/DeviceInCacheFilterVisitor.java | 17 +-- .../metadata/fetcher/SchemaPredicateUtil.java | 48 +++----- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 40 ++++--- .../fetcher/cache/TableDeviceCacheEntry.java | 4 +- .../plan/relational/sql/ast/ShowDevice.java | 6 +- .../read/req/impl/ShowTableDevicesPlan.java | 4 +- 12 files changed, 233 insertions(+), 98 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/CollectOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/CollectOperator.java new file mode 100644 index 00000000000..2f55743854e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/CollectOperator.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.queryengine.execution.operator.Operator; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.List; + +public class CollectOperator implements ProcessOperator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(CollectOperator.class); + + private final OperatorContext operatorContext; + private final List<Operator> children; + + private int currentIndex; + + public CollectOperator(OperatorContext operatorContext, List<Operator> children) { + this.operatorContext = operatorContext; + this.children = children; + this.currentIndex = 0; + } + + @Override + public boolean hasNext() throws Exception { + return currentIndex < children.size(); + } + + @Override + public TsBlock next() throws Exception { + if (children.get(currentIndex).hasNextWithTimer()) { + return children.get(currentIndex).nextWithTimer(); + } else { + closeCurrentChild(currentIndex); + currentIndex++; + return null; + } + } + + private void closeCurrentChild(int index) throws Exception { + children.get(index).close(); + children.set(index, null); + } + + @Override + public ListenableFuture<?> isBlocked() { + return children.get(currentIndex).isBlocked(); + } + + @Override + public boolean isFinished() throws Exception { + return currentIndex >= children.size(); + } + + @Override + public OperatorContext getOperatorContext() { + return operatorContext; + } + + @Override + public void close() throws Exception { + for (int i = currentIndex, n = children.size(); i < n; i++) { + Operator currentChild = children.get(i); + if (currentChild != null) { + closeCurrentChild(i); + } + } + } + + @Override + public long calculateMaxPeekMemory() { + long maxPeekMemory = 0; + for (Operator child : children) { + maxPeekMemory = Math.max(maxPeekMemory, child.calculateMaxPeekMemoryWithCounter()); + } + return maxPeekMemory; + } + + @Override + public long calculateMaxReturnSize() { + long maxReturnSize = 0; + for (Operator child : children) { + maxReturnSize = Math.max(maxReturnSize, child.calculateMaxReturnSize()); + } + return maxReturnSize; + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + return 0L; + } + + @TestOnly + public List<Operator> getChildren() { + return children; + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + children.stream() + .mapToLong(MemoryEstimationHelper::getEstimatedSizeOfAccountableObject) + .sum() + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java index 553da4d21b1..8dc1ce33ed8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java @@ -46,17 +46,17 @@ import java.util.NoSuchElementException; public class TableDeviceQuerySource implements ISchemaSource<IDeviceSchemaInfo> { - private String database; + private final String database; - private String tableName; + private final String tableName; - private List<List<Expression>> idDeterminedPredicateList; + private final List<List<Expression>> idDeterminedPredicateList; - private Expression idFuzzyPredicate; + private final Expression idFuzzyPredicate; - private List<ColumnHeader> columnHeaderList; + private final List<ColumnHeader> columnHeaderList; - private TsTable table; + private final TsTable table; public TableDeviceQuerySource( String database, @@ -182,9 +182,8 @@ public class TableDeviceQuerySource implements ISchemaSource<IDeviceSchemaInfo> private SchemaFilter getExecutableIdFuzzyFilter(Expression idFuzzyExpression) { ConvertSchemaPredicateToFilterVisitor visitor = new ConvertSchemaPredicateToFilterVisitor(); - ConvertSchemaPredicateToFilterVisitor.Context context = - new ConvertSchemaPredicateToFilterVisitor.Context(table); - return visitor.process(idFuzzyExpression); + return visitor.process( + idFuzzyExpression, new ConvertSchemaPredicateToFilterVisitor.Context(table)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceQueryNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceQueryNode.java index 6caf198e024..f1f781fc918 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceQueryNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TableDeviceQueryNode.java @@ -46,10 +46,10 @@ public class TableDeviceQueryNode extends TableDeviceSourceNode { * <p>Each inner list represents a device pattern and each expression of it represents one * condition on some id column. */ - private List<List<Expression>> idDeterminedPredicateList; + private final List<List<Expression>> idDeterminedPredicateList; /** filters/conditions involving non-id columns and concat by OR to id column filters */ - private Expression idFuzzyPredicate; + private final Expression idFuzzyPredicate; public TableDeviceQueryNode( PlanNodeId planNodeId, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoMetadataChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoMetadataChecker.java index 5b3eeff0768..9c47c5a2cf6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoMetadataChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoMetadataChecker.java @@ -40,6 +40,11 @@ import java.util.Set; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference; +/** + * only the following predicate will return true: 1. tagColumn = 'XXX' 2. 'XXX' = tagColumn 3. + * attributeColumn = 'XXX' 4. 'XXX' = attributeColumn 5. tagColumn/attributeColumn IS NULL 6. using + * or to combine the above expression + */ public class PredicatePushIntoMetadataChecker extends PredicateVisitor<Boolean, Void> { private final Set<String> idOrAttributeColumnNames; @@ -64,7 +69,7 @@ public class PredicatePushIntoMetadataChecker extends PredicateVisitor<Boolean, @Override protected Boolean visitIsNullPredicate(IsNullPredicate node, Void context) { - return Boolean.FALSE; + return isIdOrAttributeColumn(node.getValue()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/CheckSchemaPredicateVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/CheckSchemaPredicateVisitor.java index ae8011217a6..91eaf0cd651 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/CheckSchemaPredicateVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/CheckSchemaPredicateVisitor.java @@ -24,13 +24,12 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicateVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BetweenPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IsNotNullPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IsNullPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; @@ -38,6 +37,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpre import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; +import static org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.SchemaPredicateUtil.getColumnName; + // return whether input expression has attribute column predicate public class CheckSchemaPredicateVisitor extends PredicateVisitor<Boolean, CheckSchemaPredicateVisitor.Context> { @@ -59,7 +60,12 @@ public class CheckSchemaPredicateVisitor @Override protected Boolean visitIsNotNullPredicate(IsNotNullPredicate node, Context context) { - return visitExpression(node, context); + String columnName = ((SymbolReference) node.getValue()).getName(); + return context + .table + .getColumnSchema(columnName) + .getColumnCategory() + .equals(TsTableColumnCategory.ATTRIBUTE); } @Override @@ -69,8 +75,12 @@ public class CheckSchemaPredicateVisitor @Override protected Boolean visitLogicalExpression(LogicalExpression node, Context context) { - return node.getTerms().get(0).accept(this, context) - || node.getTerms().get(1).accept(this, context); + for (Expression expression : node.getTerms()) { + if (this.process(expression, context)) { + return true; + } + } + return false; } @Override @@ -80,20 +90,7 @@ public class CheckSchemaPredicateVisitor @Override protected Boolean visitComparisonExpression(ComparisonExpression node, Context context) { - String columnName; - if (node.getLeft() instanceof Literal) { - if (node.getRight() instanceof Identifier) { - columnName = ((Identifier) (node.getRight())).getValue(); - } else { - columnName = ((SymbolReference) (node.getRight())).getName(); - } - } else { - if (node.getLeft() instanceof Identifier) { - columnName = ((Identifier) (node.getLeft())).getValue(); - } else { - columnName = ((SymbolReference) (node.getLeft())).getName(); - } - } + String columnName = getColumnName(node); return context .table .getColumnSchema(columnName) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java index 5d4c058ce37..0c45b92b540 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java @@ -29,7 +29,6 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicateVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BetweenPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IsNotNullPredicate; @@ -82,6 +81,10 @@ public class ConvertSchemaPredicateToFilterVisitor @Override protected SchemaFilter visitLogicalExpression(LogicalExpression node, Context context) { + if (node.getOperator() != LogicalExpression.Operator.OR || node.getTerms().size() != 2) { + throw new IllegalStateException( + "Operator is " + node.getOperator() + ", operand size is " + node.getTerms().size()); + } // the operator of the logical expression shall be OR return new OrFilter( node.getTerms().get(0).accept(this, context), node.getTerms().get(1).accept(this, context)); @@ -98,18 +101,16 @@ public class ConvertSchemaPredicateToFilterVisitor String value; if (node.getLeft() instanceof Literal) { value = ((StringLiteral) (node.getLeft())).getValue(); - if (node.getRight() instanceof Identifier) { - columnName = ((Identifier) (node.getRight())).getValue(); - } else { - columnName = ((SymbolReference) (node.getRight())).getName(); + if (!(node.getRight() instanceof SymbolReference)) { + throw new IllegalStateException("Can only be SymbolReference, now is " + node.getRight()); } + columnName = ((SymbolReference) (node.getRight())).getName(); } else { value = ((StringLiteral) (node.getRight())).getValue(); - if (node.getLeft() instanceof Identifier) { - columnName = ((Identifier) (node.getLeft())).getValue(); - } else { - columnName = ((SymbolReference) (node.getLeft())).getName(); + if (!(node.getLeft() instanceof SymbolReference)) { + throw new IllegalStateException("Can only be SymbolReference, now is " + node.getRight()); } + columnName = ((SymbolReference) (node.getLeft())).getName(); } if (context .table diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/DeviceInCacheFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/DeviceInCacheFilterVisitor.java index 88d27e0c1d3..f64f68d5153 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/DeviceInCacheFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/DeviceInCacheFilterVisitor.java @@ -30,6 +30,7 @@ import org.apache.tsfile.file.metadata.IDeviceID; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; public class DeviceInCacheFilterVisitor extends SchemaFilterVisitor<DeviceEntry> { @@ -49,18 +50,20 @@ public class DeviceInCacheFilterVisitor extends SchemaFilterVisitor<DeviceEntry> @Override public boolean visitDeviceIdFilter(DeviceIdFilter filter, DeviceEntry deviceEntry) { IDeviceID deviceID = deviceEntry.getDeviceID(); - // the first segment is "db.table", skip it - if (deviceID.segmentNum() < filter.getIndex() + 1) { - return false; + // the first segment is "tableName", skip it + int index = filter.getIndex() + 1; + // if index out of array bound, means that value will be null + if (deviceID.segmentNum() <= index) { + return filter.getValue() == null; } else { - return deviceID.segment(filter.getIndex() + 1).equals(filter.getValue()); + return Objects.equals(deviceID.segment(index), filter.getValue()); } } @Override public boolean visitDeviceAttributeFilter(DeviceAttributeFilter filter, DeviceEntry deviceEntry) { - return filter - .getValue() - .equals(deviceEntry.getAttributeColumnValues().get(attributeIndexMap.get(filter.getKey()))); + return Objects.equals( + deviceEntry.getAttributeColumnValues().get(attributeIndexMap.get(filter.getKey())), + filter.getValue()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/SchemaPredicateUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/SchemaPredicateUtil.java index a347a67b6b8..1d364246713 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/SchemaPredicateUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/SchemaPredicateUtil.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.schema.CheckSchemaPredicateVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; @@ -37,8 +36,14 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.IrUtils.extractPredicates; + public class SchemaPredicateUtil { + private SchemaPredicateUtil() {} + + // pair.left is Expressions only contain ID columns + // pair.right is Expressions contain at least one ATTRIBUTE column static Pair<List<Expression>, List<Expression>> separateIdDeterminedPredicate( List<Expression> expressionList, TsTable table) { List<Expression> idDeterminedList = new ArrayList<>(); @@ -66,7 +71,7 @@ public class SchemaPredicateUtil { List<Expression> schemaFilterList) { List<List<Expression>> orConcatList = schemaFilterList.stream() - .map(SchemaPredicateUtil::convertOneDeviceIdPredicateToOrConcat) + .map(expression -> extractPredicates(LogicalExpression.Operator.OR, expression)) .collect(Collectors.toList()); int orSize = orConcatList.size(); int finalResultSize = 1; @@ -96,6 +101,9 @@ public class SchemaPredicateUtil { if (!hasConflictFilter) { result.add(oneCase); + } else { + // TODO table metadata + throw new IllegalStateException("have conflict filter!"); } for (int k = orSize - 1; k >= 0; k--) { @@ -110,37 +118,19 @@ public class SchemaPredicateUtil { return result; } - private static List<Expression> convertOneDeviceIdPredicateToOrConcat(Expression schemaFilter) { - List<Expression> result = new ArrayList<>(); - if (schemaFilter instanceof LogicalExpression) { - LogicalExpression logicalExpression = (LogicalExpression) schemaFilter; - if (logicalExpression.getOperator().equals(LogicalExpression.Operator.AND)) { - throw new IllegalStateException("Input filter shall not be AND operation"); - } else if (logicalExpression.getOperator().equals(LogicalExpression.Operator.OR)) { - result.addAll(convertOneDeviceIdPredicateToOrConcat(logicalExpression.getTerms().get(0))); - result.addAll(convertOneDeviceIdPredicateToOrConcat(logicalExpression.getTerms().get(1))); - } - } else { - result.add(schemaFilter); - } - return result; - } - - private static String getColumnName(Expression expression) { - ComparisonExpression comparisonExpression = (ComparisonExpression) expression; + public static String getColumnName(Expression expression) { + ComparisonExpression node = (ComparisonExpression) expression; String columnName; - if (comparisonExpression.getLeft() instanceof Literal) { - if (comparisonExpression.getRight() instanceof Identifier) { - columnName = ((Identifier) (comparisonExpression.getRight())).getValue(); - } else { - columnName = ((SymbolReference) (comparisonExpression.getRight())).getName(); + if (node.getLeft() instanceof Literal) { + if (!(node.getRight() instanceof SymbolReference)) { + throw new IllegalStateException("Can only be SymbolReference, now is " + node.getRight()); } + columnName = ((SymbolReference) (node.getRight())).getName(); } else { - if (comparisonExpression.getLeft() instanceof Identifier) { - columnName = ((Identifier) (comparisonExpression.getLeft())).getValue(); - } else { - columnName = ((SymbolReference) (comparisonExpression.getLeft())).getName(); + if (!(node.getLeft() instanceof SymbolReference)) { + throw new IllegalStateException("Can only be SymbolReference, now is " + node.getLeft()); } + columnName = ((SymbolReference) (node.getLeft())).getName(); } return columnName; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index 33c94832498..75704422210 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; @@ -44,6 +45,7 @@ import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.utils.Pair; @@ -173,10 +175,16 @@ public class TableDeviceSchemaFetcher { List<DeviceEntry> deviceEntryList = new ArrayList<>(); TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table); + if (tableInstance == null) { + throw new SemanticException(String.format("Table '%s.%s' does not exist", database, table)); + } Pair<List<Expression>, List<Expression>> separatedExpression = SchemaPredicateUtil.separateIdDeterminedPredicate(expressionList, tableInstance); List<Expression> idDeterminedPredicateList = separatedExpression.left; // and-concat List<Expression> idFuzzyPredicateList = separatedExpression.right; // and-concat + + // here we use binary tree way, because the following SchemaFilter only support binary OrFilter + // TODO table metadata: add multi way OrFilter for SchemaFilter Expression compactedIdFuzzyPredicate = SchemaPredicateUtil.compactDeviceIdFuzzyPredicate(idFuzzyPredicateList); @@ -184,8 +192,8 @@ public class TableDeviceSchemaFetcher { // expressions inner each element are and-concat representing conditions of different column List<List<Expression>> idPredicateList = SchemaPredicateUtil.convertDeviceIdPredicateToOrConcatList(idDeterminedPredicateList); - // List<Expression> in idPredicateList contains all id columns comparison which can use - // SchemaCache + // if List<Expression> in idPredicateList contains all id columns comparison which can use + // SchemaCache, we store its index. List<Integer> idSingleMatchIndexList = SchemaPredicateUtil.extractIdSingleMatchExpressionCases(idPredicateList, tableInstance); // store missing cache index in idSingleMatchIndexList @@ -197,7 +205,7 @@ public class TableDeviceSchemaFetcher { ConvertSchemaPredicateToFilterVisitor.Context context = new ConvertSchemaPredicateToFilterVisitor.Context(tableInstance); DeviceInCacheFilterVisitor filterVisitor = new DeviceInCacheFilterVisitor(attributeColumns); - SchemaFilter attributeFilter = + SchemaFilter fuzzyFilter = compactedIdFuzzyPredicate == null ? null : compactedIdFuzzyPredicate.accept(visitor, context); @@ -209,7 +217,7 @@ public class TableDeviceSchemaFetcher { idPredicateList.get(index).stream() .map(o -> o.accept(visitor, context)) .collect(Collectors.toList()), - o -> attributeFilter == null || filterVisitor.process(attributeFilter, o), + o -> fuzzyFilter == null || filterVisitor.process(fuzzyFilter, o), attributeColumns)) { idSingleMatchPredicateNotInCache.add(index); } @@ -226,7 +234,6 @@ public class TableDeviceSchemaFetcher { int idx1 = 0; int idx2 = 0; for (int i = 0; i < idPredicateList.size(); i++) { - // if (idx1 >= idSingleMatchIndexList.size() || i != idSingleMatchIndexList.get(idx1)) { idPredicateForFetch.add(idPredicateList.get(i)); } else { @@ -248,8 +255,9 @@ public class TableDeviceSchemaFetcher { idSingleMatchIndexList.size() == idPredicateList.size()); } - // todo implement deduplicate during schemaRegion execution - // todo need further process on input predicates and transform them into disjoint sets + // TODO table metadata: implement deduplicate during schemaRegion execution + // TODO table metadata: need further process on input predicates and transform them into + // disjoint sets Set<DeviceEntry> set = new LinkedHashSet<>(deviceEntryList); return new ArrayList<>(set); } @@ -276,18 +284,16 @@ public class TableDeviceSchemaFetcher { List<String> attributeValues = new ArrayList<>(attributeColumns.size()); for (String attributeKey : attributeColumns) { String value = attributeMap.get(attributeKey); - if (value == null) { - return false; - } else { - attributeValues.add(value); - } + // TODO table metadata: what if the value is null? + attributeValues.add(value); } String[] deviceIdNodes = new String[idValues.length + 1]; deviceIdNodes[0] = tableInstance.getTableName(); System.arraycopy(idValues, 0, deviceIdNodes, 1, idValues.length); DeviceEntry deviceEntry = new DeviceEntry(IDeviceID.Factory.DEFAULT_FACTORY.create(deviceIdNodes), attributeValues); - // todo process cases that selected attr columns different from those used for predicate + // TODO table metadata: process cases that selected attr columns different from those used for + // predicate if (check.test(deviceEntry)) { deviceEntryList.add(deviceEntry); } @@ -356,18 +362,22 @@ public class TableDeviceSchemaFetcher { if (columns[j].isNull(i)) { nodes[idIndex + 1] = null; } else { - nodes[idIndex + 1] = columns[j].getBinary(i).toString(); + nodes[idIndex + 1] = + columns[j].getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET); } idIndex++; } else { if (columns[j].isNull(i)) { attributeMap.put(columnSchema.getColumnName(), null); } else { - attributeMap.put(columnSchema.getColumnName(), columns[j].getBinary(i).toString()); + attributeMap.put( + columnSchema.getColumnName(), + columns[j].getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET)); } } } IDeviceID deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create(nodes); + // TODO table metadata: add memory control deviceEntryList.add( new DeviceEntry( deviceID, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java index 9c75010451d..a6d6f3f5e42 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java @@ -23,14 +23,14 @@ import org.apache.iotdb.commons.schema.MemUsageUtil; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; public class TableDeviceCacheEntry { + // should use a Map implementation that allows null for value private final Map<String, String> attributeMap; public TableDeviceCacheEntry() { - attributeMap = new ConcurrentHashMap<>(); + attributeMap = new HashMap<>(); } public TableDeviceCacheEntry(Map<String, String> attributeMap) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowDevice.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowDevice.java index b6f4ac3ad5e..3b2fee11a89 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowDevice.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowDevice.java @@ -84,14 +84,14 @@ public class ShowDevice extends Statement { public List<List<Expression>> getIdDeterminedPredicateList() { if (idDeterminedPredicateList == null) { - // todo process raw expression input by show device sql + // TODO table metadata: process raw expression input by show device sql } return idDeterminedPredicateList; } public Expression getIdFuzzyPredicate() { if (idFuzzyPredicate == null) { - // todo process raw expression input by show device sql + // TODO table metadata: process raw expression input by show device sql } return idFuzzyPredicate; } @@ -106,7 +106,7 @@ public class ShowDevice extends Statement { public List<IDeviceID> getPartitionKeyList() { if (partitionKeyList == null) { - // todo parse idDeterminedFilterList to IDeviceID list + // TODO table metadata: parse idDeterminedFilterList to IDeviceID list } return partitionKeyList; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java index b328f28c2c2..092cacf4ec1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTableDevicesPlan.java @@ -28,9 +28,9 @@ import java.util.List; public class ShowTableDevicesPlan { - private PartialPath devicePattern; + private final PartialPath devicePattern; - private SchemaFilter attributeFilter; + private final SchemaFilter attributeFilter; public ShowTableDevicesPlan(PartialPath devicePattern, SchemaFilter attributeFilter) { this.devicePattern = devicePattern;
