This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new d829bae [KYUUBI #939][FOLLOWUP][TEST] Add string column test for
Z-Order
d829bae is described below
commit d829bae5260511d171c8b371b8c5899948105bc1
Author: Min Zhao <[email protected]>
AuthorDate: Sun Sep 26 22:23:52 2021 +0800
[KYUUBI #939][FOLLOWUP][TEST] Add string column test for Z-Order
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines:
https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in
https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your
PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g.,
'[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
Added string column test for zorder.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #1157 from zhaomin1423/zorder_test.
Closes #939
59543776 [Min Zhao] add string column test for zorder
Authored-by: Min Zhao <[email protected]>
Signed-off-by: ulysses-you <[email protected]>
---
.../scala/org/apache/spark/sql/ZorderSuite.scala | 54 ++++++++++++++++------
1 file changed, 41 insertions(+), 13 deletions(-)
diff --git
a/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/ZorderSuite.scala
b/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/ZorderSuite.scala
index e5ab071..42431a1 100644
---
a/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/ZorderSuite.scala
+++
b/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/ZorderSuite.scala
@@ -357,21 +357,22 @@ trait ZorderSuite extends KyuubiSparkSQLExtensionTest
with ExpressionEvalHelper
checkEvaluation(zorder, expected, InternalRow.fromSeq(children))
}
+ private def checkSort(input: DataFrame, expected: Seq[Row]): Unit = {
+ withTempDir { dir =>
+
input.repartition(3).write.mode("overwrite").format("json").save(dir.getCanonicalPath)
+ val df = spark.read.format("json")
+ .load(dir.getCanonicalPath)
+ .repartition(1)
+ val exprs = Seq("c1", "c2").map(col).map(_.expr)
+ val sortOrder = SortOrder(Zorder(exprs), Ascending, NullsLast, Seq.empty)
+ val zorderSort = Sort(Seq(sortOrder), true, df.logicalPlan)
+ val result = Dataset.ofRows(spark, zorderSort)
+ checkAnswer(result, expected)
+ }
+ }
+
test("sort with zorder -- int column") {
// TODO: add more datatype unit test
- def checkSort(input: DataFrame, expected: Seq[Row]): Unit = {
- withTempDir { dir =>
-
input.repartition(3).write.mode("overwrite").format("json").save(dir.getCanonicalPath)
- val df = spark.read.format("json")
- .load(dir.getCanonicalPath)
- .repartition(1)
- val exprs = Seq("c1", "c2").map(col).map(_.expr)
- val sortOrder = SortOrder(Zorder(exprs), Ascending, NullsLast,
Seq.empty)
- val zorderSort = Sort(Seq(sortOrder), true, df.logicalPlan)
- val result = Dataset.ofRows(spark, zorderSort)
- checkAnswer(result, expected)
- }
- }
val session = spark
import session.implicits._
// generate 4 * 4 matrix
@@ -398,6 +399,33 @@ trait ZorderSuite extends KyuubiSparkSQLExtensionTest with
ExpressionEvalHelper
Row(2, 1) :: Row(0, 2) :: Row(1, 2) :: Row(2, 2) :: Nil
checkSort(input2, expected2)
}
+
+ test("sort with zorder -- string column") {
+ val schema = StructType(StructField("c1", StringType) :: StructField("c2",
StringType) :: Nil)
+ val rdd = spark.sparkContext.parallelize(Seq(
+ Row("a", "a"), Row("a", "b"), Row("a", "c"), Row("a", "d"),
+ Row("b", "a"), Row("b", "b"), Row("b", "c"), Row("b", "d"),
+ Row("c", "a"), Row("c", "b"), Row("c", "c"), Row("c", "d"),
+ Row("d", "a"), Row("d", "b"), Row("d", "c"), Row("d", "d")))
+ val input = spark.createDataFrame(rdd, schema)
+ val expected = Row("a", "a") :: Row("b", "a") :: Row("c", "a") :: Row("a",
"b") ::
+ Row("a", "c") :: Row("b", "b") :: Row("c", "b") :: Row("b", "c") ::
+ Row("c", "c") :: Row("d", "a") :: Row("d", "b") :: Row("d", "c") ::
+ Row("a", "d") :: Row("b", "d") :: Row("c", "d") :: Row("d", "d") :: Nil
+ checkSort(input, expected)
+
+ val rdd2 = spark.sparkContext.parallelize(Seq(
+ Row(null, "a"), Row("a", "b"), Row("a", "c"), Row("a", null),
+ Row("b", "a"), Row(null, "b"), Row("b", null), Row("b", "d"),
+ Row("c", "a"), Row("c", null), Row(null, "c"), Row("c", "d"),
+ Row("d", null), Row("d", "b"), Row("d", "c"), Row(null, "d"), Row(null,
null)))
+ val input2 = spark.createDataFrame(rdd2, schema)
+ val expected2 = Row(null, null) :: Row("a", null) :: Row("b", null) ::
Row("c", null) ::
+ Row("d", null) :: Row(null, "a") :: Row(null, "b") :: Row(null, "c") ::
+ Row(null, "d") :: Row("b", "a") :: Row("c", "a") :: Row("a", "b") ::
+ Row("a", "c") :: Row("d", "b") :: Row("d", "c") :: Row("b", "d") ::
Row("c", "d") :: Nil
+ checkSort(input2, expected2)
+ }
}
class ZorderWithCodegenEnabledSuite extends ZorderSuite {
