morrySnow commented on code in PR #54405:
URL: https://github.com/apache/doris/pull/54405#discussion_r2260092519
##########
fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java:
##########
@@ -624,22 +627,35 @@ private static TFetchSchemaTableDataResult
viewDependencyMetadataResult(TSchemaT
if (!params.isSetCurrentUserIdent()) {
return errorResult("current user ident is not set.");
}
+ List<Expression> conjuncts = Collections.EMPTY_LIST;
+ if (params.isSetFrontendConjuncts()) {
+ conjuncts =
FrontendConjunctsUtils.convertToExpression(params.getFrontendConjuncts());
+ }
Collection<DatabaseIf<? extends TableIf>> allDbs =
Env.getCurrentEnv().getInternalCatalog().getAllDbs();
TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
List<TRow> dataBatch = Lists.newArrayList();
ConnectContext ctx = new ConnectContext();
ctx.setEnv(Env.getCurrentEnv());
for (DatabaseIf<? extends TableIf> db : allDbs) {
- List<? extends TableIf> tables = db.getTables();
String dbName = db.getFullName();
+ if (FrontendConjunctsUtils.isFiltered(conjuncts, "VIEW_SCHEMA",
dbName)) {
Review Comment:
should add a conjunct filter to remove the conjunct that do not contain slot
named `VIEW_SCHEMA` to avoid do useless filter for each tuple.
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterIntoSchemaScan.java:
##########
@@ -22,56 +22,101 @@
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
+import org.apache.doris.nereids.trees.expressions.InPredicate;
+import org.apache.doris.nereids.trees.expressions.LessThan;
+import org.apache.doris.nereids.trees.expressions.LessThanEqual;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import java.util.List;
import java.util.Optional;
/**
* Used to push down catalog/db/table name to schema scan node.
*/
public class PushDownFilterIntoSchemaScan extends OneRewriteRuleFactory {
+ public static ImmutableSet<String> SUPPOPRT_FRONTEND_CONJUNCTS_TABLES =
+ ImmutableSet.of("view_dependency");
+
@Override
public Rule build() {
return logicalFilter(logicalSchemaScan()).when(p ->
!p.child().isFilterPushed()).thenApply(ctx -> {
LogicalFilter<LogicalSchemaScan> filter = ctx.root;
LogicalSchemaScan scan = filter.child();
- Optional<String> schemaCatalog = Optional.empty();
- Optional<String> schemaDatabase = Optional.empty();
- Optional<String> schemaTable = Optional.empty();
- for (Expression expression : filter.getConjuncts()) {
- if (!(expression instanceof EqualTo)) {
- continue;
- }
- Expression slot = expression.child(0);
- if (!(slot instanceof SlotReference)) {
- continue;
- }
- Optional<Column> column = ((SlotReference)
slot).getOriginalColumn();
- if (!column.isPresent()) {
- continue;
- }
- String columnName = column.get().getName();
- Expression slotValue = expression.child(1);
- if (!(slotValue instanceof VarcharLiteral)) {
- continue;
- }
- String columnValue = ((VarcharLiteral) slotValue).getValue();
- if ("TABLE_CATALOG".equals(columnName)) {
- schemaCatalog = Optional.of(columnValue);
- } else if ("TABLE_SCHEMA".equals(columnName)) {
- schemaDatabase = Optional.of(columnValue);
- } else if ("TABLE_NAME".equals(columnName)) {
- schemaTable = Optional.of(columnValue);
- }
- }
- LogicalSchemaScan rewrittenScan =
scan.withSchemaIdentifier(schemaCatalog, schemaDatabase, schemaTable);
+ List<Optional<String>> fixedFilter = getFixedFilter(filter);
+ List<Expression> commonFilter = getCommonFilter(filter);
+ LogicalSchemaScan rewrittenScan =
scan.withFrontendConjuncts(fixedFilter.get(0), fixedFilter.get(1),
+ fixedFilter.get(2), commonFilter);
return filter.withChildren(ImmutableList.of(rewrittenScan));
}).toRule(RuleType.PUSH_FILTER_INTO_SCHEMA_SCAN);
}
+
+ private List<Optional<String>>
getFixedFilter(LogicalFilter<LogicalSchemaScan> filter) {
+ Optional<String> schemaCatalog = Optional.empty();
+ Optional<String> schemaDatabase = Optional.empty();
+ Optional<String> schemaTable = Optional.empty();
+ for (Expression expression : filter.getConjuncts()) {
+ if (!(expression instanceof EqualTo)) {
+ continue;
+ }
+ Expression slot = expression.child(0);
+ if (!(slot instanceof SlotReference)) {
+ continue;
+ }
+ Optional<Column> column = ((SlotReference)
slot).getOriginalColumn();
+ if (!column.isPresent()) {
+ continue;
+ }
+ String columnName = column.get().getName();
+ Expression slotValue = expression.child(1);
+ if (!(slotValue instanceof VarcharLiteral)) {
+ continue;
+ }
+ String columnValue = ((VarcharLiteral) slotValue).getValue();
+ if ("TABLE_CATALOG".equals(columnName)) {
+ schemaCatalog = Optional.of(columnValue);
+ } else if ("TABLE_SCHEMA".equals(columnName)) {
+ schemaDatabase = Optional.of(columnValue);
+ } else if ("TABLE_NAME".equals(columnName)) {
+ schemaTable = Optional.of(columnValue);
+ }
+ }
+ return Lists.newArrayList(schemaCatalog, schemaDatabase, schemaTable);
+ }
+
+ private List<Expression> getCommonFilter(LogicalFilter<LogicalSchemaScan>
filter) {
+ List<Expression> res = Lists.newArrayList();
+ if
(!SUPPOPRT_FRONTEND_CONJUNCTS_TABLES.contains(filter.child().getTable().getName().toLowerCase()))
{
+ return res;
+ }
+ for (Expression expression : filter.getConjuncts()) {
+ if (!supportOnFe(expression)) {
+ continue;
+ }
+ res.add(expression);
+ }
+ return res;
+ }
+
+ private boolean supportOnFe(Expression expression) {
+ return expression instanceof EqualTo
+ || expression instanceof Or
+ || expression instanceof InPredicate
+ || expression instanceof LessThan
+ || expression instanceof LessThanEqual
+ || expression instanceof GreaterThan
+ || expression instanceof GreaterThanEqual
Review Comment:
why not support NullSafeEqual
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java:
##########
@@ -1069,6 +1069,14 @@ public PlanFragment
visitPhysicalSchemaScan(PhysicalSchemaScan schemaScan, PlanT
return planFragment;
}
+ private List<Expr> translateToExprs(List<Expression> expressions) {
+ List<Expr> exprs = Lists.newArrayListWithCapacity(expressions.size());
+ for (Expression expression : expressions) {
+ exprs.add(ExpressionTranslator.translate(expression, context));
Review Comment:
pass context through parameter
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/util/FrontendConjunctsUtils.java:
##########
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.util;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.common.collect.Maps;
+import com.google.gson.reflect.TypeToken;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+/**
+ * FrontendConjunctsUtils
+ */
+public class FrontendConjunctsUtils {
+ private static final Logger LOG =
LogManager.getLogger(FrontendConjunctsUtils.class);
+
+ public static List<Expression> convertToExpression(String conjuncts) {
+ List<Expr> exprs = GsonUtils.GSON.fromJson(conjuncts, new
TypeToken<List<Expr>>() {
+ }.getType());
+ return exprs.stream()
+ .map(expr -> exprToExpression(expr))
+ .collect(Collectors.toList());
+ }
+
+ public static Expression exprToExpression(Expr expr) {
+ NereidsParser nereidsParser = new NereidsParser();
+ return nereidsParser.parseExpression(expr.toSql());
+ }
Review Comment:
we'd better write a translator to trans a Expr to expression to avoid use
toSql and parse again. let us do it later
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/util/FrontendConjunctsUtils.java:
##########
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.util;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.common.collect.Maps;
+import com.google.gson.reflect.TypeToken;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+/**
+ * FrontendConjunctsUtils
+ */
+public class FrontendConjunctsUtils {
+ private static final Logger LOG =
LogManager.getLogger(FrontendConjunctsUtils.class);
+
+ public static List<Expression> convertToExpression(String conjuncts) {
+ List<Expr> exprs = GsonUtils.GSON.fromJson(conjuncts, new
TypeToken<List<Expr>>() {
+ }.getType());
+ return exprs.stream()
+ .map(expr -> exprToExpression(expr))
+ .collect(Collectors.toList());
+ }
+
+ public static Expression exprToExpression(Expr expr) {
+ NereidsParser nereidsParser = new NereidsParser();
+ return nereidsParser.parseExpression(expr.toSql());
+ }
+
+ public static boolean isFiltered(List<Expression> expressions, String
columnName, Object value) {
+ HashMap<String, Object> values = Maps.newHashMapWithExpectedSize(1);
+ values.put(columnName.toLowerCase(), value);
+ return isFiltered(expressions, values);
+ }
+
+ /**
+ * isFiltered
+ *
+ * @param expressions expressions
+ * @param values values
+ * @return isFiltered
+ */
+ public static boolean isFiltered(List<Expression> expressions, Map<String,
Object> values) {
+ for (Expression expression : expressions) {
+ if (isFiltered(expression, values)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * isFiltered
+ *
+ * @param expression expression
+ * @param values values
+ * @return isFiltered
+ */
+ public static boolean isFiltered(Expression expression, Map<String,
Object> values) {
+ try {
+ AtomicBoolean containsAllColumn = new AtomicBoolean(true);
+ Expression rewrittenExpr = expression.rewriteUp(expr -> {
+ if (expr instanceof UnboundSlot) {
+ List<String> nameParts = ((UnboundSlot)
expr).getNameParts();
+ if (!CollectionUtils.isEmpty(nameParts) &&
nameParts.size() == 1) {
+ String name = nameParts.get(0).toLowerCase();
+ if (values.containsKey(name)) {
+ return Literal.of(values.get(name));
+ } else {
+ containsAllColumn.set(false);
+ }
+ }
+ }
+ return expr;
+ });
+ // expression is: c1=v1 or c2=v2,
+ // if values is {c1=v3}
+ // we should not return true, because c2 may equals v2
+ if (!containsAllColumn.get()) {
+ return false;
+ }
+ Expression evaluate = FoldConstantRuleOnFE.evaluate(rewrittenExpr,
null);
Review Comment:
should process NullLiteral and return true for it
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/util/FrontendConjunctsUtils.java:
##########
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.util;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.common.collect.Maps;
+import com.google.gson.reflect.TypeToken;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+/**
+ * FrontendConjunctsUtils
+ */
+public class FrontendConjunctsUtils {
+ private static final Logger LOG =
LogManager.getLogger(FrontendConjunctsUtils.class);
+
+ public static List<Expression> convertToExpression(String conjuncts) {
+ List<Expr> exprs = GsonUtils.GSON.fromJson(conjuncts, new
TypeToken<List<Expr>>() {
+ }.getType());
+ return exprs.stream()
+ .map(expr -> exprToExpression(expr))
+ .collect(Collectors.toList());
+ }
+
+ public static Expression exprToExpression(Expr expr) {
+ NereidsParser nereidsParser = new NereidsParser();
+ return nereidsParser.parseExpression(expr.toSql());
+ }
+
+ public static boolean isFiltered(List<Expression> expressions, String
columnName, Object value) {
+ HashMap<String, Object> values = Maps.newHashMapWithExpectedSize(1);
+ values.put(columnName.toLowerCase(), value);
+ return isFiltered(expressions, values);
+ }
+
+ /**
+ * isFiltered
+ *
+ * @param expressions expressions
+ * @param values values
+ * @return isFiltered
+ */
+ public static boolean isFiltered(List<Expression> expressions, Map<String,
Object> values) {
+ for (Expression expression : expressions) {
+ if (isFiltered(expression, values)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * isFiltered
+ *
+ * @param expression expression
+ * @param values values
+ * @return isFiltered
+ */
+ public static boolean isFiltered(Expression expression, Map<String,
Object> values) {
+ try {
+ AtomicBoolean containsAllColumn = new AtomicBoolean(true);
+ Expression rewrittenExpr = expression.rewriteUp(expr -> {
+ if (expr instanceof UnboundSlot) {
+ List<String> nameParts = ((UnboundSlot)
expr).getNameParts();
+ if (!CollectionUtils.isEmpty(nameParts) &&
nameParts.size() == 1) {
Review Comment:
why need `nameParts.size() == 1`? maybe just get the last part from name is
enough
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java:
##########
@@ -62,17 +65,20 @@ public LogicalSchemaScan(RelationId id, TableIf table,
List<String> qualifier) {
* @param virtualColumns List of virtual columns to be included in the scan
* @param groupExpression Optional group expression for memo representation
* @param logicalProperties Optional logical properties for this plan node
+ * @param frontendConjuncts conjuncts needed by FrontendService
*/
public LogicalSchemaScan(RelationId id, TableIf table, List<String>
qualifier, boolean filterPushed,
Optional<String> schemaCatalog, Optional<String> schemaDatabase,
Optional<String> schemaTable,
List<NamedExpression> virtualColumns,
- Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties) {
+ Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties,
+ List<Expression> frontendConjuncts) {
Review Comment:
put it before groupExpression in parameter list
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/util/FrontendConjunctsUtils.java:
##########
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.util;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.common.collect.Maps;
+import com.google.gson.reflect.TypeToken;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+/**
+ * FrontendConjunctsUtils
+ */
+public class FrontendConjunctsUtils {
+ private static final Logger LOG =
LogManager.getLogger(FrontendConjunctsUtils.class);
+
+ public static List<Expression> convertToExpression(String conjuncts) {
+ List<Expr> exprs = GsonUtils.GSON.fromJson(conjuncts, new
TypeToken<List<Expr>>() {
+ }.getType());
+ return exprs.stream()
+ .map(expr -> exprToExpression(expr))
+ .collect(Collectors.toList());
+ }
+
+ public static Expression exprToExpression(Expr expr) {
+ NereidsParser nereidsParser = new NereidsParser();
+ return nereidsParser.parseExpression(expr.toSql());
+ }
+
+ public static boolean isFiltered(List<Expression> expressions, String
columnName, Object value) {
+ HashMap<String, Object> values = Maps.newHashMapWithExpectedSize(1);
+ values.put(columnName.toLowerCase(), value);
+ return isFiltered(expressions, values);
+ }
+
+ /**
+ * isFiltered
+ *
+ * @param expressions expressions
+ * @param values values
+ * @return isFiltered
+ */
+ public static boolean isFiltered(List<Expression> expressions, Map<String,
Object> values) {
+ for (Expression expression : expressions) {
+ if (isFiltered(expression, values)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * isFiltered
+ *
+ * @param expression expression
+ * @param values values
+ * @return isFiltered
+ */
+ public static boolean isFiltered(Expression expression, Map<String,
Object> values) {
Review Comment:
use a case insensitive map to avoid call toLowerCase everywhere
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
