lokeshj1703 commented on code in PR #12336:
URL: https://github.com/apache/hudi/pull/12336#discussion_r1858966527
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkBaseIndexSupport.scala:
##########
@@ -155,32 +158,59 @@ abstract class SparkBaseIndexSupport(spark: SparkSession,
(List.empty, List.empty)
} else {
var recordKeyQueries: List[Expression] = List.empty
- var recordKeys: List[String] = List.empty
- for (query <- queryFilters) {
- val recordKeyOpt = getRecordKeyConfig
- RecordLevelIndexSupport.filterQueryWithRecordKey(query,
recordKeyOpt).foreach({
- case (exp: Expression, recKeys: List[String]) =>
- recordKeys = recordKeys ++ recKeys
- recordKeyQueries = recordKeyQueries :+ exp
- })
+ var compositeRecordKeys: List[String] = List.empty
+ val recordKeyOpt = getRecordKeyConfig
+ val isComplexRecordKey = recordKeyOpt.map(recordKeys =>
recordKeys.length).getOrElse(0) > 1
+ recordKeyOpt.foreach { recordKeysArray =>
+ // Handle composite record keys
+ breakable {
+ for (recordKey <- recordKeysArray) {
+ var recordKeys: List[String] = List.empty
+ for (query <- queryFilters) {
+ {
+ RecordLevelIndexSupport.filterQueryWithRecordKey(query,
Option.apply(recordKey), isComplexRecordKey).foreach {
+ case (exp: Expression, recKeys: List[String]) =>
+ exp match {
+ // For IN, add each element individually to recordKeys
+ case _: In => recordKeys = recordKeys ++ recKeys
+
+ // For other cases, basically EqualTo, concatenate
recKeys with the default separator
+ case _ => recordKeys = recordKeys ++ recKeys
+ }
+ recordKeyQueries = recordKeyQueries :+ exp
+ }
+ }
+ }
+
+ if (recordKeys.isEmpty) {
+ recordKeyQueries = List.empty
+ compositeRecordKeys = List.empty
+ break()
+ } else if (!isComplexRecordKey || compositeRecordKeys.isEmpty) {
+ compositeRecordKeys = recordKeys
+ } else {
+ var tempCompositeRecordKeys: List[String] = List.empty
+ for (compRecKey <- compositeRecordKeys) {
Review Comment:
Added test cases. These cases are handled.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
