This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 74fb62206b8 Fixed the bug that schema related query fails when get
from cache with complicated filter
74fb62206b8 is described below
commit 74fb62206b889202856a9e114dd9db30d95cef18
Author: Caideyipi <[email protected]>
AuthorDate: Mon Sep 9 21:10:56 2024 +0800
Fixed the bug that schema related query fails when get from cache with
complicated filter
---
.../iotdb/relational/it/schema/IoTDBDeviceIT.java | 6 ++
.../ConvertSchemaPredicateToFilterVisitor.java | 82 ++++++++++++++--------
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 26 ++++---
3 files changed, 78 insertions(+), 36 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java
index c4f8bcafb29..1650497e2a8 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java
@@ -141,6 +141,12 @@ public class IoTDBDeviceIT {
statement.executeQuery("count devices from table0 where region_id >=
'2'"),
"count(devices),",
Collections.singleton("0,"));
+ // Test cache with complicated filter
+ TestUtils.assertResultSetEqual(
+ statement.executeQuery(
+ "show devices from table0 where region_id = '1' and plant_id in
('3', '5') and device_id = '3' and device_id between region_id and plant_id"),
+ "region_id,plant_id,device_id,model,",
+ Collections.singleton("1,5,3,A,"));
try {
statement.executeQuery("show devices from table2");
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java
index 6130f507215..d78170b64bb 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java
@@ -51,25 +51,36 @@ import
org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpress
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference;
+import javax.annotation.Nullable;
+
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.stream.Collectors;
import static com.google.common.base.Preconditions.checkArgument;
import static
org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference;
import static org.apache.tsfile.utils.RegexUtils.parseLikePatternToRegex;
+/**
+ * The {@link ConvertSchemaPredicateToFilterVisitor} will convert a predicate
to {@link
+ * SchemaFilter}. For the predicates which can not be converted, this will
return {@code null}.
+ * However, for IdDeterminedPredicate, this visitor shall never return {@code
null}.
+ */
public class ConvertSchemaPredicateToFilterVisitor
extends PredicateVisitor<SchemaFilter,
ConvertSchemaPredicateToFilterVisitor.Context> {
@Override
- protected SchemaFilter visitInPredicate(final InPredicate node, final
Context context) {
+ protected @Nullable SchemaFilter visitInPredicate(final InPredicate node,
final Context context) {
final Expression valueList = node.getValueList();
checkArgument(valueList instanceof InListExpression);
final List<Expression> values = ((InListExpression) valueList).getValues();
for (final Expression value : values) {
- checkArgument(value instanceof Literal);
+ if (!(value instanceof Literal)) {
+ return null;
+ }
}
return wrapIdOrAttributeFilter(
@@ -97,7 +108,13 @@ public class ConvertSchemaPredicateToFilterVisitor
}
@Override
- protected SchemaFilter visitLikePredicate(final LikePredicate node, final
Context context) {
+ protected @Nullable SchemaFilter visitLikePredicate(
+ final LikePredicate node, final Context context) {
+ // TODO: Support stringLiteral like id/attr?
+ if (!(node.getValue() instanceof SymbolReference)
+ || !(node.getPattern() instanceof StringLiteral)) {
+ return null;
+ }
return wrapIdOrAttributeFilter(
new LikeFilter(parseLikePatternToRegex(((StringLiteral)
node.getPattern()).getValue())),
((SymbolReference) node.getValue()).getName(),
@@ -105,24 +122,31 @@ public class ConvertSchemaPredicateToFilterVisitor
}
@Override
- protected SchemaFilter visitLogicalExpression(
+ protected @Nullable SchemaFilter visitLogicalExpression(
final LogicalExpression node, final Context context) {
- final List<SchemaFilter> children =
- node.getTerms().stream()
- .map(expression -> expression.accept(this, context))
- .collect(Collectors.toList());
+ final List<SchemaFilter> children = new ArrayList<>();
+ for (final Expression term : node.getTerms()) {
+ final SchemaFilter childResult = term.accept(this, context);
+ if (Objects.nonNull(childResult)) {
+ children.add(childResult);
+ } else {
+ return null;
+ }
+ }
return node.getOperator() == LogicalExpression.Operator.OR
? new OrFilter(children)
: new AndFilter(children);
}
@Override
- protected SchemaFilter visitNotExpression(final NotExpression node, final
Context context) {
- return new NotFilter(node.getValue().accept(this, context));
+ protected @Nullable SchemaFilter visitNotExpression(
+ final NotExpression node, final Context context) {
+ final SchemaFilter result = node.getValue().accept(this, context);
+ return Objects.nonNull(result) ? new NotFilter(result) : null;
}
@Override
- protected SchemaFilter visitComparisonExpression(
+ protected @Nullable SchemaFilter visitComparisonExpression(
final ComparisonExpression node, final Context context) {
final String columnName;
final String value;
@@ -132,11 +156,13 @@ public class ConvertSchemaPredicateToFilterVisitor
checkArgument(isSymbolReference(node.getRight()));
columnName = ((SymbolReference) (node.getRight())).getName();
isOrdered = false;
- } else {
+ } else if (node.getRight() instanceof Literal) {
value = ((StringLiteral) (node.getRight())).getValue();
checkArgument(isSymbolReference(node.getLeft()));
columnName = ((SymbolReference) (node.getLeft())).getName();
isOrdered = true;
+ } else {
+ return null;
}
return wrapIdOrAttributeFilter(
@@ -175,41 +201,41 @@ public class ConvertSchemaPredicateToFilterVisitor
}
@Override
- protected SchemaFilter visitSimpleCaseExpression(SimpleCaseExpression node,
Context context) {
+ protected SchemaFilter visitSimpleCaseExpression(
+ final SimpleCaseExpression node, final Context context) {
return visitExpression(node, context);
}
@Override
- protected SchemaFilter visitSearchedCaseExpression(SearchedCaseExpression
node, Context context) {
+ protected SchemaFilter visitSearchedCaseExpression(
+ final SearchedCaseExpression node, final Context context) {
return visitExpression(node, context);
}
@Override
- protected SchemaFilter visitIfExpression(IfExpression node, Context context)
{
+ protected SchemaFilter visitIfExpression(final IfExpression node, final
Context context) {
return visitExpression(node, context);
}
@Override
- protected SchemaFilter visitNullIfExpression(NullIfExpression node, Context
context) {
+ protected SchemaFilter visitNullIfExpression(final NullIfExpression node,
final Context context) {
return visitExpression(node, context);
}
@Override
- protected SchemaFilter visitBetweenPredicate(BetweenPredicate node, Context
context) {
+ protected SchemaFilter visitBetweenPredicate(final BetweenPredicate node,
final Context context) {
return visitExpression(node, context);
}
private SchemaFilter wrapIdOrAttributeFilter(
final SchemaFilter filter, final String columnName, final Context
context) {
- if (context
- .table
- .getColumnSchema(columnName)
- .getColumnCategory()
- .equals(TsTableColumnCategory.ID)) {
- return new IdFilter(filter, context.idColumnIndexMap.get(columnName));
- } else {
- return new AttributeFilter(filter, columnName);
- }
+ return context
+ .table
+ .getColumnSchema(columnName)
+ .getColumnCategory()
+ .equals(TsTableColumnCategory.ID)
+ ? new IdFilter(filter, context.idColumnIndexMap.get(columnName))
+ : new AttributeFilter(filter, columnName);
}
public static class Context {
@@ -217,12 +243,12 @@ public class ConvertSchemaPredicateToFilterVisitor
private final TsTable table;
private final Map<String, Integer> idColumnIndexMap;
- public Context(TsTable table) {
+ public Context(final TsTable table) {
this.table = table;
this.idColumnIndexMap = getIdColumnIndex(table);
}
- private Map<String, Integer> getIdColumnIndex(TsTable table) {
+ private Map<String, Integer> getIdColumnIndex(final TsTable table) {
Map<String, Integer> map = new HashMap<>();
List<TsTableColumnSchema> columnSchemaList = table.getColumnList();
int idIndex = 0;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
index a84cb77dcba..9222aa1f102 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
@@ -238,17 +238,24 @@ public class TableDeviceSchemaFetcher {
new ConvertSchemaPredicateToFilterVisitor.Context(tableInstance);
final DeviceInCacheFilterVisitor filterVisitor =
new DeviceInCacheFilterVisitor(attributeColumns);
- final SchemaFilter fuzzyFilter =
- compactedIdFuzzyPredicate == null
- ? null
- : compactedIdFuzzyPredicate.accept(visitor, context);
+
+ final Predicate<DeviceEntry> check;
+ if (Objects.isNull(compactedIdFuzzyPredicate)) {
+ check = o -> true;
+ } else {
+ final SchemaFilter fuzzyFilter =
compactedIdFuzzyPredicate.accept(visitor, context);
+ // Currently if a predicate cannot be converted to schema filter, we
abandon cache
+ // and just fetch remote. Later cache will be a memory source and
combine filter
+ check = Objects.nonNull(fuzzyFilter) ? o ->
filterVisitor.process(fuzzyFilter, o) : null;
+ }
+
for (final int index : idSingleMatchIndexList) {
if (!tryGetDeviceInCache(
deviceEntryList,
database,
tableInstance,
index2FilterMapList.get(index),
- o -> fuzzyFilter == null || filterVisitor.process(fuzzyFilter, o),
+ check,
attributeColumns,
fetchPaths,
isDirectDeviceQuery)) {
@@ -316,9 +323,12 @@ public class TableDeviceSchemaFetcher {
final IDeviceID deviceID = convertIdValuesToDeviceID(idValues,
tableInstance);
- // DeviceEntryList == null means that this is update statement
- // Shall not get from cache and shall reach the SchemaRegion to update
- if (Objects.isNull(attributeMap) || Objects.isNull(deviceEntryList)) {
+ // 1. AttributeMap == null means cache miss
+ // 2. DeviceEntryList == null means that this is update statement, shall
not get from cache and
+ // shall reach the SchemaRegion to update
+ // 3. Check == null means that the fuzzyPredicate cannot be parsed into
schema filter,
+ // and currently we shall push it to schema region
+ if (Objects.isNull(attributeMap) || Objects.isNull(deviceEntryList) ||
Objects.isNull(check)) {
if (Objects.nonNull(fetchPaths)) {
fetchPaths.add(deviceID);
}
