This is an automated email from the ASF dual-hosted git repository.
jackylee pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d5472d1a2b [GLUTEN-9240][VL] Write NULL value into relation in gluten
unit tests (#9241)
d5472d1a2b is described below
commit d5472d1a2bd720586d2a79cb25b4d31fb0170c9c
Author: Qian Sun <[email protected]>
AuthorDate: Tue Apr 8 19:30:59 2025 +0800
[GLUTEN-9240][VL] Write NULL value into relation in gluten unit tests
(#9241)
---
.../execution/JsonFunctionsValidateSuite.scala | 40 +++++++++++++-----
.../execution/ScalarFunctionsValidateSuite.scala | 49 ++++++++++++++++++----
2 files changed, 72 insertions(+), 17 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/JsonFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/JsonFunctionsValidateSuite.scala
index 9abbe017a6..3d7c255408 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/JsonFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/JsonFunctionsValidateSuite.scala
@@ -30,10 +30,20 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
- runQueryAndCompare(
- "SELECT l_orderkey, get_json_object('{\"a\":\"b\"}', '$.a') " +
- "from lineitem limit 1;") {
- checkGlutenOperatorMatch[ProjectExecTransformer]
+ withTempPath {
+ path =>
+ Seq[(String)](
+ ("""{"a":"b"}""")
+ )
+ .toDF("txt")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("tbl")
+
+ runQueryAndCompare("select get_json_object(txt, '$.a') from tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
}
// Invalid UTF-8 encoding.
@@ -51,12 +61,22 @@ class JsonFunctionsValidateSuite extends
FunctionsValidateSuite {
runQueryAndCompare(
s"select *, json_array_length(string_field1) " +
s"from datatab limit
5")(checkGlutenOperatorMatch[ProjectExecTransformer])
- runQueryAndCompare(
- s"select l_orderkey, json_array_length('[1,2,3,4]') " +
- s"from lineitem limit
5")(checkGlutenOperatorMatch[ProjectExecTransformer])
- runQueryAndCompare(
- s"select l_orderkey, json_array_length(null) " +
- s"from lineitem limit
5")(checkGlutenOperatorMatch[ProjectExecTransformer])
+ withTempPath {
+ path =>
+ Seq[(String)](
+ ("[1,2,3,4]"),
+ (null.asInstanceOf[String])
+ )
+ .toDF("txt")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("tbl")
+
+ runQueryAndCompare("select json_array_length(txt) from tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
}
testWithSpecifiedSparkVersion("from_json function bool", Some("3.4")) {
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 4821f4f9f5..c5cbfe7b83 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -187,8 +187,20 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
testWithSpecifiedSparkVersion("null input for array_size", Some("3.3")) {
- runQueryAndCompare("SELECT array_size(null)") {
- checkGlutenOperatorMatch[ProjectExecTransformer]
+ withTempPath {
+ path =>
+ Seq[(Array[Int])](
+ (null.asInstanceOf[Array[Int]])
+ )
+ .toDF("txt")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("tbl")
+
+ runQueryAndCompare("select array_size(txt) from tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
}
}
@@ -801,8 +813,20 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
test("Test sum/count function") {
- runQueryAndCompare("""SELECT sum(2),count(2) from lineitem""".stripMargin)
{
- checkGlutenOperatorMatch[BatchScanExecTransformer]
+ withTempPath {
+ path =>
+ Seq[(Integer, Integer)](
+ (2, 2)
+ )
+ .toDF("val1", "val2")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("tbl")
+
+ runQueryAndCompare("SELECT sum(val1),count(val2) from tbl") {
+ checkGlutenOperatorMatch[BatchScanExecTransformer]
+ }
}
}
@@ -818,9 +842,20 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
testWithSpecifiedSparkVersion("Test width_bucket function", Some("3.4")) {
- runQueryAndCompare("""SELECT width_bucket(2, 0, 4, 3), l_orderkey
- | from lineitem limit 100""".stripMargin) {
- checkGlutenOperatorMatch[ProjectExecTransformer]
+ withTempPath {
+ path =>
+ Seq[(Integer, Integer, Integer, Integer)](
+ (2, 0, 4, 3)
+ )
+ .toDF("val1", "val2", "val3", "val4")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("tbl")
+
+ runQueryAndCompare("SELECT width_bucket(val1, val2, val3, val4) from
tbl") {
+ checkGlutenOperatorMatch[BatchScanExecTransformer]
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
