voonhous commented on code in PR #20524: URL: https://github.com/apache/hudi/pull/20524#discussion_r4476239965
########## hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/Leaf.scala: ########## @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.dataskipping + +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualNullSafe, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, InSet, IsNotNull, IsNull, LessThan, LessThanOrEqual, Literal, Not, StartsWith, SubqueryExpression} +import org.apache.spark.unsafe.types.UTF8String + +/** + * Stage 2 (Leaf): is this predicate a single `term OP constant` the index can answer? + * + * Invariants: + * - The term references exactly one column and holds no subquery; the values reference no column. + * - The term is put on the left in exactly one place (`comparison`); nothing downstream flips. + * - IS NULL / IS NOT NULL only take a bare column. + * - The only NOTs understood here are NotEq, NotIn and NotStartsWith; any other NOT is not a leaf. + */ +sealed trait Op + +object Op { + case object Eq extends Op + case object NotEq extends Op + case object Lt extends Op + case object LtEq extends Op + case object Gt extends Op + case object GtEq extends Op + case object In extends Op + case object NotIn extends Op + case object StartsWith extends Op + case object NotStartsWith extends Op + case object IsNull extends Op + case object IsNotNull extends Op +} + +case class Leaf(term: Expression, op: Op, values: Seq[Expression]) + +object Leaf { + + def parse(e: Expression): Option[Leaf] = e match { + case EqualTo(left, right) => comparison(left, Op.Eq, right) + case Not(EqualTo(left, right)) => comparison(left, Op.NotEq, right) + + // "colA <=> null" is "colA IS NULL" + case EqualNullSafe(column: AttributeReference, Literal(null, _)) => Some(Leaf(column, Op.IsNull, Nil)) + + case LessThan(left, right) => comparison(left, Op.Lt, right) + case LessThanOrEqual(left, right) => comparison(left, Op.LtEq, right) + case GreaterThan(left, right) => comparison(left, Op.Gt, right) + case GreaterThanOrEqual(left, right) => comparison(left, Op.GtEq, right) + + case IsNull(column: AttributeReference) => Some(Leaf(column, Op.IsNull, Nil)) + case IsNotNull(column: AttributeReference) => Some(Leaf(column, Op.IsNotNull, Nil)) + + case In(term, list) if isTerm(term) && list.forall(isValueExpression) => Some(Leaf(term, Op.In, list)) + // NOTE: [[InSet]] is the optimized [[In]] over static values. [[Literal#apply]] does not accept + // [[UTF8String]], so strings are converted first + case InSet(term, hset) if isTerm(term) => Some(Leaf(term, Op.In, hset.toSeq.map(toLiteral))) Review Comment: Confirmed: on 0935fd57, `st INSET (...)` and `arr INSET (...)` throw `SparkRuntimeException` where the base returns TRUE. Fixed in 0849bb0b7e62: `Leaf.parse` keeps the raw set as `Op.InSet(values)`, and the values only become literals in `Verdict`, which runs once `TermRange.derive` has found a range. The golden inputs now cover struct and array columns under `INSET`, `IN` and `=` (all TRUE, as on the base), and `TestDataSkippingStages` checks that parsing an `InSet` of structs builds no literals. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
