voonhous commented on code in PR #20540:
URL: https://github.com/apache/hudi/pull/20540#discussion_r4520656489


##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ExpressionIndexSupport.scala:
##########
@@ -388,111 +379,44 @@ class ExpressionIndexSupport(spark: SparkSession,
     metadataConfig.isEnabled && metaClient.getIndexMetadata.isPresent && 
!metaClient.getIndexMetadata.get().getIndexDefinitions.isEmpty
   }
 
-  private def filterQueriesWithFunctionalFilterKey(queryFilters: 
Seq[Expression], sourceFieldOpt: Option[String],
-                                                   attributeFetcher: 
Function1[Expression, Expression]): List[Tuple2[Expression, List[String]]] = {
-    var expressionIndexQueries: List[Tuple2[Expression, List[String]]] = 
List.empty
-    for (query <- queryFilters) {
-      filterQueryWithRecordKey(query, sourceFieldOpt, 
attributeFetcher).foreach({
-        case (exp: Expression, literals: List[String]) =>
-          expressionIndexQueries = expressionIndexQueries :+ Tuple2.apply(exp, 
literals)
-      })
-    }
-
-    expressionIndexQueries
-  }
-
   /**
-   * Searches for an index partition based on the specified index function and 
target column name.
+   * Each expression index some query conjunct reads, with those conjuncts. A 
conjunct reads an index when one of its
+   * terms is the index's `f(col)` ([[IndexedExpression.matches]]).
    *
-   * This method looks up the index definitions available in the metadata of a 
`metaClient` instance
-   * and attempts to find an index partition where the index function and the 
source fields match
-   * the provided arguments. If a matching index definition is found, the 
partition identifier for
-   * that index is returned.
-   *
-   * @param queryFilters A sequence of `Expression` objects to analyze. Each 
expression should involve a single column
-   *                     for the method to consider it (expressions involving 
multiple columns are skipped).
-   * @return An `Option` containing the index partition identifier if a 
matching index definition is found.
-   *         Returns `None` if no matching index definition is found.
+   * Only `=` and `IN` conjuncts read an expression index for now, as before; 
ranges follow with #20537.
    */
-  private def getExpressionIndexPartitionAndLiterals(queryFilters: 
Seq[Expression]): Option[Tuple3[String, Expression, List[String]]] = {
-    val indexDefinitions = 
metaClient.getIndexMetadata.get().getIndexDefinitions.asScala
-    if (indexDefinitions.nonEmpty) {
-      val functionDefinitions = indexDefinitions.values
-        .filter(definition => 
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
-        .toList
-      var indexPartitionAndLiteralsOpt: Option[Tuple3[String, Expression, 
List[String]]] = Option.empty
-      functionDefinitions.foreach(indexDefinition => {
-        val queryInfoOpt = extractQueryAndLiterals(queryFilters, 
indexDefinition)
-        if (queryInfoOpt.isDefined) {
-          indexPartitionAndLiteralsOpt = 
Option.apply(Tuple3.apply(indexDefinition.getIndexName, queryInfoOpt.get._1, 
queryInfoOpt.get._2))
-        }
-      })
-      indexPartitionAndLiteralsOpt
-    } else {
-      Option.empty
-    }
+  private def findIndexUses(queryFilters: Seq[Expression]): Seq[IndexUse] = {
+    val conjuncts = 
queryFilters.flatMap(SparkBaseIndexSupport.splitConjuncts).filter(isEqualityOrIn)
+    
metaClient.getIndexMetadata.get().getIndexDefinitions.values().asScala.toSeq
+      .filter(definition => 
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
+      .flatMap { definition =>
+        val indexed = IndexedExpression(definition)
+        val uses = conjuncts.filter(_.find(indexed.matches).isDefined)
+        if (uses.isEmpty) None else Some(IndexUse(definition, indexed, uses))
+      }
   }
 
+  private def isEqualityOrIn(conjunct: Expression): Boolean = 
Leaf.parse(conjunct).exists(_.op match {
+    case Op.Eq | Op.In | Op.InSet(_) => true

Review Comment:
   Fixed, `InSet` stays out of the expression index, as on master.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to