voonhous commented on code in PR #20540:
URL: https://github.com/apache/hudi/pull/20540#discussion_r4520658303
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ExpressionIndexSupport.scala:
##########
@@ -388,111 +379,44 @@ class ExpressionIndexSupport(spark: SparkSession,
metadataConfig.isEnabled && metaClient.getIndexMetadata.isPresent &&
!metaClient.getIndexMetadata.get().getIndexDefinitions.isEmpty
}
- private def filterQueriesWithFunctionalFilterKey(queryFilters:
Seq[Expression], sourceFieldOpt: Option[String],
- attributeFetcher:
Function1[Expression, Expression]): List[Tuple2[Expression, List[String]]] = {
- var expressionIndexQueries: List[Tuple2[Expression, List[String]]] =
List.empty
- for (query <- queryFilters) {
- filterQueryWithRecordKey(query, sourceFieldOpt,
attributeFetcher).foreach({
- case (exp: Expression, literals: List[String]) =>
- expressionIndexQueries = expressionIndexQueries :+ Tuple2.apply(exp,
literals)
- })
- }
-
- expressionIndexQueries
- }
-
/**
- * Searches for an index partition based on the specified index function and
target column name.
+ * Each expression index some query conjunct reads, with those conjuncts. A
conjunct reads an index when one of its
+ * terms is the index's `f(col)` ([[IndexedExpression.matches]]).
*
- * This method looks up the index definitions available in the metadata of a
`metaClient` instance
- * and attempts to find an index partition where the index function and the
source fields match
- * the provided arguments. If a matching index definition is found, the
partition identifier for
- * that index is returned.
- *
- * @param queryFilters A sequence of `Expression` objects to analyze. Each
expression should involve a single column
- * for the method to consider it (expressions involving
multiple columns are skipped).
- * @return An `Option` containing the index partition identifier if a
matching index definition is found.
- * Returns `None` if no matching index definition is found.
+ * Only `=` and `IN` conjuncts read an expression index for now, as before;
ranges follow with #20537.
*/
- private def getExpressionIndexPartitionAndLiterals(queryFilters:
Seq[Expression]): Option[Tuple3[String, Expression, List[String]]] = {
- val indexDefinitions =
metaClient.getIndexMetadata.get().getIndexDefinitions.asScala
- if (indexDefinitions.nonEmpty) {
- val functionDefinitions = indexDefinitions.values
- .filter(definition =>
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
- .toList
- var indexPartitionAndLiteralsOpt: Option[Tuple3[String, Expression,
List[String]]] = Option.empty
- functionDefinitions.foreach(indexDefinition => {
- val queryInfoOpt = extractQueryAndLiterals(queryFilters,
indexDefinition)
- if (queryInfoOpt.isDefined) {
- indexPartitionAndLiteralsOpt =
Option.apply(Tuple3.apply(indexDefinition.getIndexName, queryInfoOpt.get._1,
queryInfoOpt.get._2))
- }
- })
- indexPartitionAndLiteralsOpt
- } else {
- Option.empty
- }
+ private def findIndexUses(queryFilters: Seq[Expression]): Seq[IndexUse] = {
+ val conjuncts =
queryFilters.flatMap(SparkBaseIndexSupport.splitConjuncts).filter(isEqualityOrIn)
+
metaClient.getIndexMetadata.get().getIndexDefinitions.values().asScala.toSeq
+ .filter(definition =>
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
+ .flatMap { definition =>
+ val indexed = IndexedExpression(definition)
+ val uses = conjuncts.filter(_.find(indexed.matches).isDefined)
+ if (uses.isEmpty) None else Some(IndexUse(definition, indexed, uses))
+ }
}
+ private def isEqualityOrIn(conjunct: Expression): Boolean =
Leaf.parse(conjunct).exists(_.op match {
+ case Op.Eq | Op.In | Op.InSet(_) => true
+ case _ => false
+ })
+
/**
- * Extracts mappings from function names to column names from a sequence of
expressions.
- *
- * This method iterates over a given sequence of Spark SQL expressions and
identifies expressions
- * that contain function calls corresponding to keys in the
ExpressionIndexFunction. It supports only
- * expressions that are simple binary expressions involving a single column.
If an expression contains
- * one of the functions and operates on a single column, this method maps
the function name to the
- * column name.
+ * Bloom filter keys: the values of `f(col) = v` and `f(col) IN (...)`, as
strings like the writer added them. A file
+ * passes if it may hold any key, which is conservative across conjuncts. A
conjunct with a value that is not a
+ * non-null literal gives no keys, and is skipped.
*/
- private def extractQueryAndLiterals(queryFilters: Seq[Expression],
indexDefinition: HoodieIndexDefinition): Option[(Expression, List[String])] = {
- val attributeFetcher = (expr: Expression) => {
- expr match {
- case expression: UnaryExpression => expression.child
- case expression: DateFormatClass if
expression.right.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption())
=>
- expression.left
- case expression: FromUnixTime
- if
expression.format.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption(TimestampFormatter.defaultPattern()))
=>
- expression.sec
- case expression: UnixTimestamp if
expression.right.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption(TimestampFormatter.defaultPattern()))
=>
- expression.timeExp
- case expression: ParseToDate if (expression.format.isEmpty &&
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexFormatOption()))
- || (expression.format.isDefined &&
expression.format.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption()))
=>
- expression.left
- case expression: ParseToTimestamp if (expression.format.isEmpty &&
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexFormatOption()))
- || (expression.format.isDefined &&
expression.format.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption()))
=>
- expression.left
- case expression: DateAdd if expression.days.isInstanceOf[Literal] &&
expression.days.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexDaysOption)
=>
- expression.startDate
- case expression: DateSub if expression.days.isInstanceOf[Literal] &&
expression.days.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexDaysOption)
=>
- expression.startDate
- case expression: Substring if
expression.pos.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPositionOption)
- &&
expression.len.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexLengthOption)=>
- expression.str
- case expression: StringTrim if (expression.trimStr.isEmpty &&
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexTrimStringOption))
- || (expression.trimStr.isDefined &&
expression.trimStr.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexTrimStringOption))
=> expression.srcStr
- case expression: StringTrimLeft if (expression.trimStr.isEmpty &&
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexTrimStringOption))
- || (expression.trimStr.isDefined &&
expression.trimStr.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexTrimStringOption))
=> expression.srcStr
- case expression: StringTrimRight if (expression.trimStr.isEmpty &&
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexTrimStringOption))
- || (expression.trimStr.isDefined &&
expression.trimStr.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexTrimStringOption))
=> expression.srcStr
- case expression: RegExpReplace if
expression.pos.asInstanceOf[Literal].value.toString.equals("1")
- &&
expression.regexp.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPatternOption)
- &&
expression.rep.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexReplacementOption)
=>
- expression.subject
- case expression: RegExpExtract if
expression.regexp.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPatternOption)
- &&
expression.idx.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexIndexOption)
=>
- expression.subject
- case expression: StringSplit if
expression.limit.asInstanceOf[Literal].value.toString.equals("-1")
- &&
expression.regex.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPatternOption)
=>
- expression.str
- case other => other
+ private def bloomFilterKeys(use: IndexUse): List[String] =
Review Comment:
Master builds the same keys (`lit.value.toString`), so this isn't new here.
Leaving it out of this PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]