This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 4b0a1c1093 [VL] Enable Spark JsonExpressionsSuite (#8099)
4b0a1c1093 is described below
commit 4b0a1c10939fa1314e5d892173dd8ed9afb3441e
Author: Zhen Li <[email protected]>
AuthorDate: Tue Dec 3 11:54:43 2024 +0800
[VL] Enable Spark JsonExpressionsSuite (#8099)
---
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 16 ++++++++++++++++
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 18 +++++++++++++++++-
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 18 +++++++++++++++++-
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 18 +++++++++++++++++-
4 files changed, 67 insertions(+), 3 deletions(-)
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 276a8a0f7a..15495270a1 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -364,6 +364,22 @@ class VeloxTestSettings extends BackendTestSettings {
// Rewrite the following two tests in GlutenDatasetSuite.
.exclude("dropDuplicates: columns with same column name")
.exclude("groupBy.as")
+ enableSuite[GlutenJsonExpressionsSuite]
+ // https://github.com/apache/incubator-gluten/issues/8102
+ .exclude("$.store.book")
+ .exclude("$")
+ .exclude("$.store.book[0]")
+ .exclude("$.store.book[*]")
+ .exclude("$.store.book[*].category")
+ .exclude("$.store.book[*].isbn")
+ .exclude("$.store.book[*].reader")
+ .exclude("$.store.basket[*]")
+ .exclude("$.store.basket[*][0]")
+ .exclude("$.store.basket[0][*]")
+ .exclude("$.store.basket[*][*]")
+ .exclude("$.store.basket[0][*].b")
+ // Exception class different.
+ .exclude("from_json - invalid data")
enableSuite[GlutenJsonFunctionsSuite]
// Velox does not support single quotes in get_json_object function.
.exclude("function get_json_object - support single quotes")
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 08fab69c84..407b9c8b95 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.utils.{BackendTestSettings,
SQLQueryTestSettings}
import org.apache.spark.GlutenSortShuffleSuite
import org.apache.spark.sql._
-import
org.apache.spark.sql.catalyst.expressions.{GlutenAnsiCastSuiteWithAnsiModeOff,
GlutenAnsiCastSuiteWithAnsiModeOn, GlutenArithmeticExpressionSuite,
GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCastSuiteWithAnsiModeOn,
GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite,
GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite,
GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite,
GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite,
GlutenIntervalExpre [...]
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.connector._
import org.apache.spark.sql.errors.{GlutenQueryCompilationErrorsDSv2Suite,
GlutenQueryExecutionErrorsSuite, GlutenQueryParsingErrorsSuite}
import org.apache.spark.sql.execution._
@@ -142,6 +142,22 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenHashExpressionsSuite]
enableSuite[GlutenHigherOrderFunctionsSuite]
enableSuite[GlutenIntervalExpressionsSuite]
+ enableSuite[GlutenJsonExpressionsSuite]
+ // https://github.com/apache/incubator-gluten/issues/8102
+ .exclude("$.store.book")
+ .exclude("$")
+ .exclude("$.store.book[0]")
+ .exclude("$.store.book[*]")
+ .exclude("$.store.book[*].category")
+ .exclude("$.store.book[*].isbn")
+ .exclude("$.store.book[*].reader")
+ .exclude("$.store.basket[*]")
+ .exclude("$.store.basket[*][0]")
+ .exclude("$.store.basket[0][*]")
+ .exclude("$.store.basket[*][*]")
+ .exclude("$.store.basket[0][*].b")
+ // Exception class different.
+ .exclude("from_json - invalid data")
enableSuite[GlutenJsonFunctionsSuite]
// Velox does not support single quotes in get_json_object function.
.exclude("function get_json_object - support single quotes")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index df79b663bc..dbb01fbe70 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.utils.{BackendTestSettings,
SQLQueryTestSettings}
import org.apache.spark.GlutenSortShuffleSuite
import org.apache.spark.sql._
-import
org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite,
GlutenBitwiseExpressionsSuite, GlutenCastSuite,
GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite,
GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite,
GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite,
GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite,
GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite,
GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, Glu [...]
+import org.apache.spark.sql.catalyst.expressions._
import
org.apache.spark.sql.connector.{GlutenDataSourceV2DataFrameSessionCatalogSuite,
GlutenDataSourceV2DataFrameSuite, GlutenDataSourceV2FunctionSuite,
GlutenDataSourceV2SQLSessionCatalogSuite, GlutenDataSourceV2SQLSuiteV1Filter,
GlutenDataSourceV2SQLSuiteV2Filter, GlutenDataSourceV2Suite,
GlutenDeleteFromTableSuite, GlutenDeltaBasedDeleteFromTableSuite,
GlutenFileDataSourceV2FallBackSuite, GlutenGroupBasedDeleteFromTableSuite,
GlutenKeyGroupedPartitioningSuite, GlutenLocalScanSuite, G [...]
import org.apache.spark.sql.errors.{GlutenQueryCompilationErrorsDSv2Suite,
GlutenQueryCompilationErrorsSuite, GlutenQueryExecutionErrorsSuite,
GlutenQueryParsingErrorsSuite}
import org.apache.spark.sql.execution.{FallbackStrategiesSuite,
GlutenBroadcastExchangeSuite, GlutenCoalesceShufflePartitionsSuite,
GlutenExchangeSuite, GlutenLocalBroadcastExchangeSuite,
GlutenReplaceHashWithSortAggSuite, GlutenReuseExchangeAndSubquerySuite,
GlutenSameResultSuite, GlutenSortSuite, GlutenSQLAggregateFunctionSuite,
GlutenSQLWindowFunctionSuite, GlutenTakeOrderedAndProjectSuite}
@@ -120,6 +120,22 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenHashExpressionsSuite]
enableSuite[GlutenHigherOrderFunctionsSuite]
enableSuite[GlutenIntervalExpressionsSuite]
+ enableSuite[GlutenJsonExpressionsSuite]
+ // https://github.com/apache/incubator-gluten/issues/8102
+ .exclude("$.store.book")
+ .exclude("$")
+ .exclude("$.store.book[0]")
+ .exclude("$.store.book[*]")
+ .exclude("$.store.book[*].category")
+ .exclude("$.store.book[*].isbn")
+ .exclude("$.store.book[*].reader")
+ .exclude("$.store.basket[*]")
+ .exclude("$.store.basket[*][0]")
+ .exclude("$.store.basket[0][*]")
+ .exclude("$.store.basket[*][*]")
+ .exclude("$.store.basket[0][*].b")
+ // Exception class different.
+ .exclude("from_json - invalid data")
enableSuite[GlutenJsonFunctionsSuite]
// Velox does not support single quotes in get_json_object function.
.exclude("function get_json_object - support single quotes")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index cb2151fe46..f5a1a07695 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.utils.{BackendTestSettings,
SQLQueryTestSettings}
import org.apache.spark.GlutenSortShuffleSuite
import org.apache.spark.sql._
-import
org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite,
GlutenBitwiseExpressionsSuite, GlutenCastSuite,
GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite,
GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite,
GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite,
GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite,
GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite,
GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, Glu [...]
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.connector._
import org.apache.spark.sql.errors.{GlutenQueryCompilationErrorsDSv2Suite,
GlutenQueryCompilationErrorsSuite, GlutenQueryExecutionErrorsSuite,
GlutenQueryParsingErrorsSuite}
import org.apache.spark.sql.execution._
@@ -121,6 +121,22 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenHashExpressionsSuite]
enableSuite[GlutenHigherOrderFunctionsSuite]
enableSuite[GlutenIntervalExpressionsSuite]
+ enableSuite[GlutenJsonExpressionsSuite]
+ // https://github.com/apache/incubator-gluten/issues/8102
+ .exclude("$.store.book")
+ .exclude("$")
+ .exclude("$.store.book[0]")
+ .exclude("$.store.book[*]")
+ .exclude("$.store.book[*].category")
+ .exclude("$.store.book[*].isbn")
+ .exclude("$.store.book[*].reader")
+ .exclude("$.store.basket[*]")
+ .exclude("$.store.basket[*][0]")
+ .exclude("$.store.basket[0][*]")
+ .exclude("$.store.basket[*][*]")
+ .exclude("$.store.basket[0][*].b")
+ // Exception class different.
+ .exclude("from_json - invalid data")
enableSuite[GlutenJsonFunctionsSuite]
// * in get_json_object expression not supported in velox
.exclude("SPARK-42782: Hive compatibility check for get_json_object")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
