This is an automated email from the ASF dual-hosted git repository.
zhangzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 43d72626f [GLUTEN-5484] add missing tests for clickhouse (#5485)
43d72626f is described below
commit 43d72626f3fdc2e8a3584c63ead8baf3c1bc7359
Author: shuai.xu <[email protected]>
AuthorDate: Wed Apr 24 08:15:03 2024 +0800
[GLUTEN-5484] add missing tests for clickhouse (#5485)
[GLUTEN-5484] add missing tests for clickhouse
---
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 +
.../gluten/utils/velox/VeloxTestSettings.scala | 3 +-
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 8 +-
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 +
.../gluten/utils/velox/VeloxTestSettings.scala | 3 +-
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 8 +-
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 +
.../gluten/utils/velox/VeloxTestSettings.scala | 3 +-
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 8 +-
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 +
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 8 +-
.../hive/execution/GlutenHiveSQLQuerySuite.scala | 115 +--------------------
.../execution/GlutenHiveSQLQuerySuiteBase.scala | 97 +++++++++++++++++
13 files changed, 139 insertions(+), 122 deletions(-)
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 7831424ed..cf6588c97 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -35,6 +35,7 @@ import org.apache.spark.sql.execution.datasources.v2._
import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
import org.apache.spark.sql.execution.joins.{GlutenExistenceJoinSuite,
GlutenInnerJoinSuite, GlutenOuterJoinSuite}
import
org.apache.spark.sql.extension.{GlutenCustomerExpressionTransformerSuite,
GlutenCustomerExtensionSuite, GlutenSessionExtensionSuite}
+import org.apache.spark.sql.hive.execution.GlutenHiveSQLQueryCHSuite
import org.apache.spark.sql.sources._
import org.apache.spark.sql.statistics.SparkFunctionStatistics
@@ -2196,6 +2197,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("SELECT structFieldComplex.Value.`value_(2)` FROM
tableWithSchema")
enableSuite[SparkFunctionStatistics]
enableSuite[GlutenSparkSessionExtensionSuite]
+ enableSuite[GlutenHiveSQLQueryCHSuite]
override def getSQLQueryTestSettings: SQLQueryTestSettings =
ClickHouseSQLQueryTestSettings
}
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 7e0ccb17c..b7196192d 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -34,7 +34,7 @@ import
org.apache.spark.sql.execution.datasources.v2.GlutenFileTableSuite
import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite,
GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
import
org.apache.spark.sql.extension.{GlutenCollapseProjectExecTransformerSuite,
GlutenCustomerExpressionTransformerSuite, GlutenCustomerExtensionSuite,
GlutenSessionExtensionSuite}
-import org.apache.spark.sql.hive.execution.{GlutenHiveSQLQueryCHSuite,
GlutenHiveSQLQuerySuite}
+import org.apache.spark.sql.hive.execution.GlutenHiveSQLQuerySuite
import
org.apache.spark.sql.sources.{GlutenBucketedReadWithoutHiveSupportSuite,
GlutenBucketedWriteWithoutHiveSupportSuite, GlutenCreateTableAsSelectSuite,
GlutenDDLSourceLoadSuite,
GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuite,
GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE,
GlutenExternalCommandRunnerSuite, GlutenFilteredScanSuite, GlutenFiltersSuite,
GlutenInsertSuite, GlutenPartitionedWriteSuite, GlutenPathOptionSuite,
GlutenPrunedScanSuite, GlutenResolve [...]
// Some settings' line length exceeds 100
@@ -1081,7 +1081,6 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenStatisticsCollectionSuite]
.exclude("SPARK-33687: analyze all tables in a specific database")
enableSuite[FallbackStrategiesSuite]
- enableSuite[GlutenHiveSQLQueryCHSuite]
enableSuite[GlutenHiveSQLQuerySuite]
enableSuite[GlutenCollapseProjectExecTransformerSuite]
enableSuite[GlutenSparkSessionExtensionSuite]
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 40569d0d9..dd14a1855 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -16,6 +16,9 @@
*/
package org.apache.spark.sql.hive.execution
+import org.apache.gluten.GlutenConfig
+import org.apache.gluten.utils.SystemParameters
+
import org.apache.spark.{DebugFilesystem, SparkConf}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.TableIdentifier
@@ -25,7 +28,9 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
override def sparkConf: SparkConf = {
defaultSparkConf
.set("spark.plugins", "org.apache.gluten.GlutenPlugin")
- .set("spark.gluten.sql.columnar.backend.lib", "ch")
+ .set(GlutenConfig.GLUTEN_LIB_PATH, SystemParameters.getClickHouseLibPath)
+ .set("spark.gluten.sql.enable.native.validation", "false")
+ .set("spark.gluten.sql.native.writer.enabled", "true")
.set("spark.sql.storeAssignmentPolicy", "legacy")
.set("spark.default.parallelism", "1")
.set("spark.memory.offHeap.enabled", "true")
@@ -99,4 +104,5 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
ignoreIfNotExists = true,
purge = false)
}
+
}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 002501097..92032bd43 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -37,6 +37,7 @@ import
org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite,
GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
import org.apache.spark.sql.extension.{GlutenCustomerExtensionSuite,
GlutenSessionExtensionSuite}
import org.apache.spark.sql.gluten.GlutenFallbackSuite
+import org.apache.spark.sql.hive.execution.GlutenHiveSQLQueryCHSuite
import org.apache.spark.sql.sources._
import org.apache.spark.sql.statistics.SparkFunctionStatistics
@@ -2107,6 +2108,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeGlutenTest("fallbackSummary with cache")
.excludeGlutenTest("fallbackSummary with cached data and shuffle")
enableSuite[GlutenSparkSessionExtensionSuite]
+ enableSuite[GlutenHiveSQLQueryCHSuite]
override def getSQLQueryTestSettings: SQLQueryTestSettings =
ClickHouseSQLQueryTestSettings
}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index d1ba0eb96..1c8d0baef 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -37,7 +37,7 @@ import
org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite,
GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
import
org.apache.spark.sql.extension.{GlutenCollapseProjectExecTransformerSuite,
GlutenSessionExtensionSuite, TestFileSourceScanExecTransformer}
import org.apache.spark.sql.gluten.GlutenFallbackSuite
-import org.apache.spark.sql.hive.execution.{GlutenHiveSQLQueryCHSuite,
GlutenHiveSQLQuerySuite}
+import org.apache.spark.sql.hive.execution.GlutenHiveSQLQuerySuite
import org.apache.spark.sql.sources._
// Some settings' line length exceeds 100
@@ -1135,7 +1135,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("cases when literal is max")
enableSuite[GlutenXPathFunctionsSuite]
enableSuite[GlutenFallbackSuite]
- enableSuite[GlutenHiveSQLQueryCHSuite]
enableSuite[GlutenHiveSQLQuerySuite]
enableSuite[GlutenImplicitsTest]
enableSuite[GlutenCollapseProjectExecTransformerSuite]
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 40569d0d9..dd14a1855 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -16,6 +16,9 @@
*/
package org.apache.spark.sql.hive.execution
+import org.apache.gluten.GlutenConfig
+import org.apache.gluten.utils.SystemParameters
+
import org.apache.spark.{DebugFilesystem, SparkConf}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.TableIdentifier
@@ -25,7 +28,9 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
override def sparkConf: SparkConf = {
defaultSparkConf
.set("spark.plugins", "org.apache.gluten.GlutenPlugin")
- .set("spark.gluten.sql.columnar.backend.lib", "ch")
+ .set(GlutenConfig.GLUTEN_LIB_PATH, SystemParameters.getClickHouseLibPath)
+ .set("spark.gluten.sql.enable.native.validation", "false")
+ .set("spark.gluten.sql.native.writer.enabled", "true")
.set("spark.sql.storeAssignmentPolicy", "legacy")
.set("spark.default.parallelism", "1")
.set("spark.memory.offHeap.enabled", "true")
@@ -99,4 +104,5 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
ignoreIfNotExists = true,
purge = false)
}
+
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 679893bb6..37e4c68f7 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -37,6 +37,7 @@ import
org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite,
GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
import org.apache.spark.sql.extension.{GlutenCustomerExtensionSuite,
GlutenSessionExtensionSuite}
import org.apache.spark.sql.gluten.GlutenFallbackSuite
+import org.apache.spark.sql.hive.execution.GlutenHiveSQLQueryCHSuite
import org.apache.spark.sql.sources._
import org.apache.spark.sql.statistics.SparkFunctionStatistics
@@ -1891,6 +1892,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("SELECT structFieldComplex.Value.`value_(2)` FROM
tableWithSchema")
enableSuite[SparkFunctionStatistics]
enableSuite[GlutenSparkSessionExtensionSuite]
+ enableSuite[GlutenHiveSQLQueryCHSuite]
override def getSQLQueryTestSettings: SQLQueryTestSettings =
ClickHouseSQLQueryTestSettings
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 4f8afe579..3cc03c882 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -37,7 +37,7 @@ import
org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite,
GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
import
org.apache.spark.sql.extension.{GlutenCollapseProjectExecTransformerSuite,
GlutenSessionExtensionSuite, TestFileSourceScanExecTransformer}
import org.apache.spark.sql.gluten.GlutenFallbackSuite
-import org.apache.spark.sql.hive.execution.{GlutenHiveSQLQueryCHSuite,
GlutenHiveSQLQuerySuite}
+import org.apache.spark.sql.hive.execution.GlutenHiveSQLQuerySuite
import
org.apache.spark.sql.sources.{GlutenBucketedReadWithoutHiveSupportSuite,
GlutenBucketedWriteWithoutHiveSupportSuite, GlutenCreateTableAsSelectSuite,
GlutenDDLSourceLoadSuite,
GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuite,
GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE,
GlutenExternalCommandRunnerSuite, GlutenFilteredScanSuite, GlutenFiltersSuite,
GlutenInsertSuite, GlutenPartitionedWriteSuite, GlutenPathOptionSuite,
GlutenPrunedScanSuite, GlutenResolve [...]
// Some settings' line length exceeds 100
@@ -1147,7 +1147,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("cases when literal is max")
enableSuite[GlutenXPathFunctionsSuite]
enableSuite[GlutenFallbackSuite]
- enableSuite[GlutenHiveSQLQueryCHSuite]
enableSuite[GlutenHiveSQLQuerySuite]
enableSuite[GlutenCollapseProjectExecTransformerSuite]
enableSuite[GlutenSparkSessionExtensionSuite]
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 40569d0d9..dd14a1855 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -16,6 +16,9 @@
*/
package org.apache.spark.sql.hive.execution
+import org.apache.gluten.GlutenConfig
+import org.apache.gluten.utils.SystemParameters
+
import org.apache.spark.{DebugFilesystem, SparkConf}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.TableIdentifier
@@ -25,7 +28,9 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
override def sparkConf: SparkConf = {
defaultSparkConf
.set("spark.plugins", "org.apache.gluten.GlutenPlugin")
- .set("spark.gluten.sql.columnar.backend.lib", "ch")
+ .set(GlutenConfig.GLUTEN_LIB_PATH, SystemParameters.getClickHouseLibPath)
+ .set("spark.gluten.sql.enable.native.validation", "false")
+ .set("spark.gluten.sql.native.writer.enabled", "true")
.set("spark.sql.storeAssignmentPolicy", "legacy")
.set("spark.default.parallelism", "1")
.set("spark.memory.offHeap.enabled", "true")
@@ -99,4 +104,5 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
ignoreIfNotExists = true,
purge = false)
}
+
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 679893bb6..37e4c68f7 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -37,6 +37,7 @@ import
org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite,
GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
import org.apache.spark.sql.extension.{GlutenCustomerExtensionSuite,
GlutenSessionExtensionSuite}
import org.apache.spark.sql.gluten.GlutenFallbackSuite
+import org.apache.spark.sql.hive.execution.GlutenHiveSQLQueryCHSuite
import org.apache.spark.sql.sources._
import org.apache.spark.sql.statistics.SparkFunctionStatistics
@@ -1891,6 +1892,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("SELECT structFieldComplex.Value.`value_(2)` FROM
tableWithSchema")
enableSuite[SparkFunctionStatistics]
enableSuite[GlutenSparkSessionExtensionSuite]
+ enableSuite[GlutenHiveSQLQueryCHSuite]
override def getSQLQueryTestSettings: SQLQueryTestSettings =
ClickHouseSQLQueryTestSettings
}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
similarity index 93%
copy from
gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
copy to
gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 40569d0d9..dd14a1855 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -16,6 +16,9 @@
*/
package org.apache.spark.sql.hive.execution
+import org.apache.gluten.GlutenConfig
+import org.apache.gluten.utils.SystemParameters
+
import org.apache.spark.{DebugFilesystem, SparkConf}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.TableIdentifier
@@ -25,7 +28,9 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
override def sparkConf: SparkConf = {
defaultSparkConf
.set("spark.plugins", "org.apache.gluten.GlutenPlugin")
- .set("spark.gluten.sql.columnar.backend.lib", "ch")
+ .set(GlutenConfig.GLUTEN_LIB_PATH, SystemParameters.getClickHouseLibPath)
+ .set("spark.gluten.sql.enable.native.validation", "false")
+ .set("spark.gluten.sql.native.writer.enabled", "true")
.set("spark.sql.storeAssignmentPolicy", "legacy")
.set("spark.default.parallelism", "1")
.set("spark.memory.offHeap.enabled", "true")
@@ -99,4 +104,5 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
ignoreIfNotExists = true,
purge = false)
}
+
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
index 15a649acf..b348f6719 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
@@ -16,83 +16,12 @@
*/
package org.apache.spark.sql.hive.execution
-import org.apache.gluten.execution.TransformSupport
-
import org.apache.spark.SparkConf
-import org.apache.spark.internal.config
-import org.apache.spark.internal.config.UI.UI_ENABLED
-import org.apache.spark.sql.{DataFrame, GlutenSQLTestsTrait, Row, SparkSession}
+import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
-import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
-import org.apache.spark.sql.hive.{HiveTableScanExecTransformer, HiveUtils}
-import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
-
-import scala.reflect.ClassTag
-
-class GlutenHiveSQLQuerySuite extends GlutenSQLTestsTrait {
- private var _spark: SparkSession = null
-
- override def beforeAll(): Unit = {
- prepareWorkDir()
- if (_spark == null) {
- _spark =
SparkSession.builder().config(sparkConf).enableHiveSupport().getOrCreate()
- }
-
- _spark.sparkContext.setLogLevel("info")
- }
-
- override protected def spark: SparkSession = _spark
-
- override def afterAll(): Unit = {
- try {
- super.afterAll()
- if (_spark != null) {
- try {
- _spark.sessionState.catalog.reset()
- } finally {
- _spark.stop()
- _spark = null
- }
- }
- } finally {
- SparkSession.clearActiveSession()
- SparkSession.clearDefaultSession()
- doThreadPostAudit()
- }
- }
-
- protected def defaultSparkConf: SparkConf = {
- val conf = new SparkConf()
- .set("spark.master", "local[1]")
- .set("spark.sql.test", "")
- .set("spark.sql.testkey", "true")
- .set(SQLConf.CODEGEN_FALLBACK.key, "false")
- .set(SQLConf.CODEGEN_FACTORY_MODE.key,
CodegenObjectFactoryMode.CODEGEN_ONLY.toString)
- .set(
- HiveUtils.HIVE_METASTORE_BARRIER_PREFIXES.key,
- "org.apache.spark.sql.hive.execution.PairSerDe")
- // SPARK-8910
- .set(UI_ENABLED, false)
- .set(config.UNSAFE_EXCEPTION_ON_MEMORY_LEAK, true)
- // Hive changed the default of
hive.metastore.disallow.incompatible.col.type.changes
- // from false to true. For details, see the JIRA HIVE-12320 and
HIVE-17764.
-
.set("spark.hadoop.hive.metastore.disallow.incompatible.col.type.changes",
"false")
- // Disable ConvertToLocalRelation for better test coverage. Test cases
built on
- // LocalRelation will exercise the optimization rules better by
disabling it as
- // this rule may potentially block testing of other optimization rules
such as
- // ConstantPropagation etc.
- .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key,
ConvertToLocalRelation.ruleName)
-
- conf.set(
- StaticSQLConf.WAREHOUSE_PATH,
- conf.get(StaticSQLConf.WAREHOUSE_PATH) + "/" + getClass.getCanonicalName)
- }
+import org.apache.spark.sql.hive.HiveTableScanExecTransformer
- def checkOperatorMatch[T <: TransformSupport](df: DataFrame)(implicit tag:
ClassTag[T]): Unit = {
- val executedPlan = getExecutedPlan(df)
- assert(executedPlan.exists(plan => plan.getClass == tag.runtimeClass))
- }
+class GlutenHiveSQLQuerySuite extends GlutenHiveSQLQuerySuiteBase {
override def sparkConf: SparkConf = {
defaultSparkConf
@@ -119,42 +48,4 @@ class GlutenHiveSQLQuerySuite extends GlutenSQLTestsTrait {
purge = false)
}
- testGluten("5182: Fix failed to parse post join filters") {
- withSQLConf(
- "spark.sql.hive.convertMetastoreParquet" -> "false",
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false") {
- sql("DROP TABLE IF EXISTS test_5128_0;")
- sql("DROP TABLE IF EXISTS test_5128_1;")
- sql(
- "CREATE TABLE test_5128_0 (from_uid STRING, vgift_typeid int, vm_count
int, " +
- "status bigint, ts bigint, vm_typeid int) " +
- "USING hive OPTIONS(fileFormat 'parquet') PARTITIONED BY (`day`
STRING);")
- sql(
- "CREATE TABLE test_5128_1 (typeid int, groupid int, ss_id bigint, " +
- "ss_start_time bigint, ss_end_time bigint) " +
- "USING hive OPTIONS(fileFormat 'parquet');")
- sql(
- "INSERT INTO test_5128_0 partition(day='2024-03-31') " +
- "VALUES('uid_1', 2, 10, 1, 11111111111, 2);")
- sql("INSERT INTO test_5128_1 VALUES(2, 1, 1, 1000000000, 2111111111);")
- val df = spark.sql(
- "select ee.from_uid as uid,day, vgift_typeid, money from " +
- "(select t_a.day, if(cast(substr(t_a.ts,1,10) as bigint) between " +
- "t_b.ss_start_time and t_b.ss_end_time, t_b.ss_id, 0) ss_id, " +
- "t_a.vgift_typeid, t_a.from_uid, vm_count money from " +
- "(select from_uid,day,vgift_typeid,vm_count,ts from test_5128_0 " +
- "where day between '2024-03-30' and '2024-03-31' and status=1 and
vm_typeid=2) t_a " +
- "left join test_5128_1 t_b on t_a.vgift_typeid=t_b.typeid " +
- "where t_b.groupid in (1,2)) ee where ss_id=1;")
- checkAnswer(df, Seq(Row("uid_1", "2024-03-31", 2, 10)))
- }
- spark.sessionState.catalog.dropTable(
- TableIdentifier("test_5128_0"),
- ignoreIfNotExists = true,
- purge = false)
- spark.sessionState.catalog.dropTable(
- TableIdentifier("test_5128_1"),
- ignoreIfNotExists = true,
- purge = false)
- }
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuiteBase.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuiteBase.scala
new file mode 100644
index 000000000..c8540647d
--- /dev/null
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuiteBase.scala
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive.execution
+
+import org.apache.gluten.execution.TransformSupport
+
+import org.apache.spark.SparkConf
+import org.apache.spark.internal.config
+import org.apache.spark.internal.config.UI.UI_ENABLED
+import org.apache.spark.sql.{DataFrame, GlutenSQLTestsTrait, SparkSession}
+import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
+import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
+import org.apache.spark.sql.hive.HiveUtils
+import org.apache.spark.sql.internal.SQLConf
+
+import scala.reflect.ClassTag
+
+abstract class GlutenHiveSQLQuerySuiteBase extends GlutenSQLTestsTrait {
+ private var _spark: SparkSession = null
+
+ override def beforeAll(): Unit = {
+ prepareWorkDir()
+ if (_spark == null) {
+ _spark =
SparkSession.builder().config(sparkConf).enableHiveSupport().getOrCreate()
+ }
+
+ _spark.sparkContext.setLogLevel("warn")
+ }
+
+ override protected def spark: SparkSession = _spark
+
+ override def afterAll(): Unit = {
+ try {
+ super.afterAll()
+ if (_spark != null) {
+ try {
+ _spark.sessionState.catalog.reset()
+ } finally {
+ _spark.stop()
+ _spark = null
+ }
+ }
+ } finally {
+ SparkSession.clearActiveSession()
+ SparkSession.clearDefaultSession()
+ doThreadPostAudit()
+ }
+ }
+
+ protected def defaultSparkConf: SparkConf = {
+ val conf = new SparkConf()
+ .set("spark.master", "local[1]")
+ .set("spark.sql.test", "")
+ .set("spark.sql.testkey", "true")
+ .set(SQLConf.CODEGEN_FALLBACK.key, "false")
+ .set(SQLConf.CODEGEN_FACTORY_MODE.key,
CodegenObjectFactoryMode.CODEGEN_ONLY.toString)
+ .set(
+ HiveUtils.HIVE_METASTORE_BARRIER_PREFIXES.key,
+ "org.apache.spark.sql.hive.execution.PairSerDe")
+ // SPARK-8910
+ .set(UI_ENABLED, false)
+ .set(config.UNSAFE_EXCEPTION_ON_MEMORY_LEAK, true)
+ // Hive changed the default of
hive.metastore.disallow.incompatible.col.type.changes
+ // from false to true. For details, see the JIRA HIVE-12320 and
HIVE-17764.
+
.set("spark.hadoop.hive.metastore.disallow.incompatible.col.type.changes",
"false")
+ // Disable ConvertToLocalRelation for better test coverage. Test cases
built on
+ // LocalRelation will exercise the optimization rules better by
disabling it as
+ // this rule may potentially block testing of other optimization rules
such as
+ // ConstantPropagation etc.
+ .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key,
ConvertToLocalRelation.ruleName)
+
+ conf.set(
+ "spark.sql.warehouse.dir",
+ getClass.getResource("/").getPath +
"/tests-working-home/spark-warehouse")
+ val metastore = getClass.getResource("/").getPath +
getClass.getCanonicalName + "/metastore_db"
+ conf.set("javax.jdo.option.ConnectionURL",
s"jdbc:derby:;databaseName=$metastore;create=true")
+ }
+
+ def checkOperatorMatch[T <: TransformSupport](df: DataFrame)(implicit tag:
ClassTag[T]): Unit = {
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.exists(plan => plan.getClass == tag.runtimeClass))
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
