This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 1dd341f [KYUUBI #1307] Fix zorder boolean column default value
1dd341f is described below
commit 1dd341fda2b1214ce8f32133cff6d1d9845cdc4d
Author: Fu Chen <[email protected]>
AuthorDate: Thu Oct 28 20:26:23 2021 +0800
[KYUUBI #1307] Fix zorder boolean column default value
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines:
https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in
https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your
PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g.,
'[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
As `true > false` and optimize command using nulls last as null ordering
strategy, we should set true as the default value of boolean column instead of
false
```sql
select true > false
```
```
+--------------+
|(true > false)|
+--------------+
| true|
+--------------+
```
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #1307 from cfmcgrady/boolean-zorder.
Closes #1307
b4ca5a66 [Fu Chen] fix zorder boolean column default value
Authored-by: Fu Chen <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.../apache/kyuubi/sql/zorder/ZorderBytesUtils.scala | 2 +-
.../test/scala/org/apache/spark/sql/ZorderSuite.scala | 18 +++++++++++++++++-
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git
a/dev/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBytesUtils.scala
b/dev/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBytesUtils.scala
index 18151b2..6b849e8 100644
---
a/dev/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBytesUtils.scala
+++
b/dev/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBytesUtils.scala
@@ -169,7 +169,7 @@ object ZorderBytesUtils {
def defaultValue(dataType: DataType): Array[Byte] = toByte {
dataType match {
case BooleanType =>
- false
+ true
case ByteType =>
Byte.MaxValue
case ShortType =>
diff --git
a/dev/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/ZorderSuite.scala
b/dev/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/ZorderSuite.scala
index 490e252..47e7839 100644
---
a/dev/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/ZorderSuite.scala
+++
b/dev/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/ZorderSuite.scala
@@ -377,7 +377,7 @@ trait ZorderSuite extends KyuubiSparkSQLExtensionTest with
ExpressionEvalHelper
0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,
0xAA, 0xAA, 0xBE, 0xAA, 0xAA, 0x8A, 0xBA, 0xAA, 0x2A, 0xEA,
0xA8, 0xAA, 0xAA, 0xA2, 0xAA, 0xAA, 0x8A, 0xAA, 0xAA, 0x2F,
- 0xEB, 0xFC)
+ 0xEB, 0xFE)
.map(_.toByte)
checkEvaluation(zorder, expected, InternalRow.fromSeq(children))
}
@@ -397,6 +397,22 @@ trait ZorderSuite extends KyuubiSparkSQLExtensionTest with
ExpressionEvalHelper
}
}
+ test("sort with zorder -- boolean column") {
+ val schema = StructType(StructField("c1", BooleanType) ::
StructField("c2", BooleanType) :: Nil)
+ val nonNullDF = spark.createDataFrame(spark.sparkContext.parallelize(
+ Seq(Row(false, false), Row(false, true), Row(true, false), Row(true,
true))
+ ), schema)
+ val expected =
+ Row(false, false) :: Row(true, false) :: Row(false, true) :: Row(true,
true) :: Nil
+ checkSort(nonNullDF, expected, Array(BooleanType, BooleanType))
+ val df = spark.createDataFrame(spark.sparkContext.parallelize(
+ Seq(Row(false, false), Row(false, null), Row(null, false), Row(null,
null))
+ ), schema)
+ val expected2 =
+ Row(false, false) :: Row(null, false) :: Row(false, null) :: Row(null,
null) :: Nil
+ checkSort(df, expected2, Array(BooleanType, BooleanType))
+ }
+
test("sort with zorder -- int column") {
// TODO: add more datatype unit test
val session = spark
