dongjoon-hyun commented on a change in pull request #28761:
URL: https://github.com/apache/spark/pull/28761#discussion_r466708125
##########
File path:
sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
##########
@@ -93,155 +92,200 @@ class OrcFilterSuite extends OrcTest with
SharedSparkSession {
}
test("filter pushdown - integer") {
- withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>
- checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
- checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate($"_1" <=> 1,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
- checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
- checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate(Literal(1) <=> $"_1",
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
- checkFilterPredicate(Literal(2) > $"_1",
PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate(Literal(3) < $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal(1) >= $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal(4) <= $"_1",
PredicateLeaf.Operator.LESS_THAN)
+ withNestedOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { case
(inputDF, colName, _) =>
+ implicit val df: DataFrame = inputDF
+
+ val intAttr = df(colName).expr
+ assert(df(colName).expr.dataType === IntegerType)
+
+ checkFilterPredicate(intAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+ checkFilterPredicate(intAttr === 1, PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(intAttr <=> 1,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+ checkFilterPredicate(intAttr < 2, PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(intAttr > 3,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(intAttr <= 1,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(intAttr >= 4, PredicateLeaf.Operator.LESS_THAN)
+
+ checkFilterPredicate(Literal(1) === intAttr,
PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(Literal(1) <=> intAttr,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+ checkFilterPredicate(Literal(2) > intAttr,
PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(Literal(3) < intAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal(1) >= intAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal(4) <= intAttr,
PredicateLeaf.Operator.LESS_THAN)
}
}
test("filter pushdown - long") {
- withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toLong)))) { implicit
df =>
- checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
- checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate($"_1" <=> 1,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
- checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
- checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate(Literal(1) <=> $"_1",
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
- checkFilterPredicate(Literal(2) > $"_1",
PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate(Literal(3) < $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal(1) >= $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal(4) <= $"_1",
PredicateLeaf.Operator.LESS_THAN)
+ withNestedOrcDataFrame(
+ (1 to 4).map(i => Tuple1(Option(i.toLong)))) { case (inputDF, colName,
_) =>
+ implicit val df: DataFrame = inputDF
+
+ val longAttr = df(colName).expr
+ assert(df(colName).expr.dataType === LongType)
+
+ checkFilterPredicate(longAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+ checkFilterPredicate(longAttr === 1, PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(longAttr <=> 1,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+ checkFilterPredicate(longAttr < 2, PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(longAttr > 3,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(longAttr <= 1,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(longAttr >= 4, PredicateLeaf.Operator.LESS_THAN)
+
+ checkFilterPredicate(Literal(1) === longAttr,
PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(Literal(1) <=> longAttr,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+ checkFilterPredicate(Literal(2) > longAttr,
PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(Literal(3) < longAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal(1) >= longAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal(4) <= longAttr,
PredicateLeaf.Operator.LESS_THAN)
}
}
test("filter pushdown - float") {
- withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toFloat)))) { implicit
df =>
- checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
- checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate($"_1" <=> 1,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
- checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
- checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate(Literal(1) <=> $"_1",
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
- checkFilterPredicate(Literal(2) > $"_1",
PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate(Literal(3) < $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal(1) >= $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal(4) <= $"_1",
PredicateLeaf.Operator.LESS_THAN)
+ withNestedOrcDataFrame(
+ (1 to 4).map(i => Tuple1(Option(i.toFloat)))) { case (inputDF,
colName, _) =>
+ implicit val df: DataFrame = inputDF
+
+ val floatAttr = df(colName).expr
+ assert(df(colName).expr.dataType === FloatType)
+
+ checkFilterPredicate(floatAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+ checkFilterPredicate(floatAttr === 1, PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(floatAttr <=> 1,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+ checkFilterPredicate(floatAttr < 2, PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(floatAttr > 3,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(floatAttr <= 1,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(floatAttr >= 4, PredicateLeaf.Operator.LESS_THAN)
+
+ checkFilterPredicate(Literal(1) === floatAttr,
PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(Literal(1) <=> floatAttr,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+ checkFilterPredicate(Literal(2) > floatAttr,
PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(Literal(3) < floatAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal(1) >= floatAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal(4) <= floatAttr,
PredicateLeaf.Operator.LESS_THAN)
}
}
test("filter pushdown - double") {
- withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toDouble)))) { implicit
df =>
- checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
- checkFilterPredicate($"_1" === 1, PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate($"_1" <=> 1,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
- checkFilterPredicate($"_1" < 2, PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate($"_1" > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" >= 4, PredicateLeaf.Operator.LESS_THAN)
-
- checkFilterPredicate(Literal(1) === $"_1", PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate(Literal(1) <=> $"_1",
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
- checkFilterPredicate(Literal(2) > $"_1",
PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate(Literal(3) < $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal(1) >= $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal(4) <= $"_1",
PredicateLeaf.Operator.LESS_THAN)
+ withNestedOrcDataFrame(
+ (1 to 4).map(i => Tuple1(Option(i.toDouble)))) { case (inputDF, colName,
_) =>
+ implicit val df: DataFrame = inputDF
+
+ val doubleAttr = df(colName).expr
+ assert(df(colName).expr.dataType === DoubleType)
+
+ checkFilterPredicate(doubleAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+ checkFilterPredicate(doubleAttr === 1, PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(doubleAttr <=> 1,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+ checkFilterPredicate(doubleAttr < 2, PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(doubleAttr > 3,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(doubleAttr <= 1,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(doubleAttr >= 4, PredicateLeaf.Operator.LESS_THAN)
+
+ checkFilterPredicate(Literal(1) === doubleAttr,
PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(Literal(1) <=> doubleAttr,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+ checkFilterPredicate(Literal(2) > doubleAttr,
PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(Literal(3) < doubleAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal(1) >= doubleAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal(4) <= doubleAttr,
PredicateLeaf.Operator.LESS_THAN)
}
}
test("filter pushdown - string") {
- withOrcDataFrame((1 to 4).map(i => Tuple1(i.toString))) { implicit df =>
- checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
- checkFilterPredicate($"_1" === "1", PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate($"_1" <=> "1",
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
- checkFilterPredicate($"_1" < "2", PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate($"_1" > "3",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" <= "1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" >= "4", PredicateLeaf.Operator.LESS_THAN)
-
- checkFilterPredicate(Literal("1") === $"_1",
PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate(Literal("1") <=> $"_1",
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
- checkFilterPredicate(Literal("2") > $"_1",
PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate(Literal("3") < $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal("1") >= $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal("4") <= $"_1",
PredicateLeaf.Operator.LESS_THAN)
+ withNestedOrcDataFrame(
+ (1 to 4).map(i => Tuple1(i.toString))) { case (inputDF, colName, _) =>
+ implicit val df: DataFrame = inputDF
+
+ val strAttr = df(colName).expr
+ assert(df(colName).expr.dataType === StringType)
+
+ checkFilterPredicate(strAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+ checkFilterPredicate(strAttr === "1", PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(strAttr <=> "1",
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+ checkFilterPredicate(strAttr < "2", PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(strAttr > "3",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(strAttr <= "1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(strAttr >= "4", PredicateLeaf.Operator.LESS_THAN)
+
+ checkFilterPredicate(Literal("1") === strAttr,
PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(Literal("1") <=> strAttr,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+ checkFilterPredicate(Literal("2") > strAttr,
PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(Literal("3") < strAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal("1") >= strAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal("4") <= strAttr,
PredicateLeaf.Operator.LESS_THAN)
}
}
test("filter pushdown - boolean") {
- withOrcDataFrame((true :: false :: Nil).map(b => Tuple1.apply(Option(b))))
{ implicit df =>
- checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
- checkFilterPredicate($"_1" === true, PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate($"_1" <=> true,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
- checkFilterPredicate($"_1" < true, PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate($"_1" > false,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" <= false,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate($"_1" >= false, PredicateLeaf.Operator.LESS_THAN)
-
- checkFilterPredicate(Literal(false) === $"_1",
PredicateLeaf.Operator.EQUALS)
- checkFilterPredicate(Literal(false) <=> $"_1",
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
- checkFilterPredicate(Literal(false) > $"_1",
PredicateLeaf.Operator.LESS_THAN)
- checkFilterPredicate(Literal(true) < $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal(true) >= $"_1",
PredicateLeaf.Operator.LESS_THAN_EQUALS)
- checkFilterPredicate(Literal(true) <= $"_1",
PredicateLeaf.Operator.LESS_THAN)
+ withNestedOrcDataFrame(
+ (true :: false :: Nil).map(b => Tuple1.apply(Option(b)))) { case
(inputDF, colName, _) =>
+ implicit val df: DataFrame = inputDF
+
+ val booleanAttr = df(colName).expr
+ assert(df(colName).expr.dataType === BooleanType)
+
+ checkFilterPredicate(booleanAttr.isNull, PredicateLeaf.Operator.IS_NULL)
+
+ checkFilterPredicate(booleanAttr === true, PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(booleanAttr <=> true,
PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+ checkFilterPredicate(booleanAttr < true,
PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(booleanAttr > false,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(booleanAttr <= false,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(booleanAttr >= false,
PredicateLeaf.Operator.LESS_THAN)
+
+ checkFilterPredicate(Literal(false) === booleanAttr,
PredicateLeaf.Operator.EQUALS)
+ checkFilterPredicate(Literal(false) <=> booleanAttr,
+ PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+ checkFilterPredicate(Literal(false) > booleanAttr,
PredicateLeaf.Operator.LESS_THAN)
+ checkFilterPredicate(Literal(true) < booleanAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal(true) >= booleanAttr,
PredicateLeaf.Operator.LESS_THAN_EQUALS)
+ checkFilterPredicate(Literal(true) <= booleanAttr,
PredicateLeaf.Operator.LESS_THAN)
}
}
test("filter pushdown - decimal") {
- withOrcDataFrame((1 to 4).map(i => Tuple1.apply(BigDecimal.valueOf(i)))) {
implicit df =>
- checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
+ withNestedOrcDataFrame((1 to 4)
Review comment:
This format looks inconsistent from your other code change. Is this
intentional due to some limitation?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
