This is an automated email from the ASF dual-hosted git repository.
ptoth pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 4a72b04fdfa5 [SPARK-53694][SQL][TESTS][4.0] Improve
`V1WriteHiveCommandSuite` test coverage
4a72b04fdfa5 is described below
commit 4a72b04fdfa51478eed0cdda77f7aa4be1dee6a7
Author: Cheng Pan <[email protected]>
AuthorDate: Tue Oct 21 18:38:17 2025 +0200
[SPARK-53694][SQL][TESTS][4.0] Improve `V1WriteHiveCommandSuite` test
coverage
Backport https://github.com/apache/spark/pull/52436 to branch-4.0 to
recover CI.
### What changes were proposed in this pull request?
Previously, `V1WriteHiveCommandSuite` uses `CREATE TABLE` without declaring
`USING` or `STORED AS`, which relies on
`spark.sql.legacy.createHiveTableByDefault`'s default value, SPARK-46122
changes the default value of `spark.sql.legacy.createHiveTableByDefault` from
`true` to `false`, which implicitly affects this test suite.
In addition, we should take `spark.sql.hive.convertMetastoreParquet` and
`spark.sql.hive.convertMetastoreOrc` into account in `V1WriteHiveCommandSuite`
### Why are the changes needed?
Restore and improve test coverage.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass GHA.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #52683 from pan3793/SPARK-53694-4.0.
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Peter Toth <[email protected]>
---
.../command/V1WriteHiveCommandSuite.scala | 119 ++++++++++++---------
1 file changed, 70 insertions(+), 49 deletions(-)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/V1WriteHiveCommandSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/V1WriteHiveCommandSuite.scala
index ed2eaef24feb..a3e864ee55c6 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/V1WriteHiveCommandSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/V1WriteHiveCommandSuite.scala
@@ -20,33 +20,48 @@ package org.apache.spark.sql.hive.execution.command
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.expressions.{AttributeReference,
SortOrder}
import org.apache.spark.sql.execution.datasources.V1WriteCommandSuiteBase
+import org.apache.spark.sql.hive.HiveUtils._
import org.apache.spark.sql.hive.test.TestHiveSingleton
class V1WriteHiveCommandSuite
extends QueryTest with TestHiveSingleton with V1WriteCommandSuiteBase {
+ def withCovnertMetastore(testFunc: Boolean => Any): Unit = {
+ Seq(true, false).foreach { enabled =>
+ withSQLConf(
+ CONVERT_METASTORE_PARQUET.key -> enabled.toString,
+ CONVERT_METASTORE_ORC.key -> enabled.toString) {
+ testFunc(enabled)
+ }
+ }
+ }
+
test("create hive table as select - no partition column") {
- withPlannedWrite { enabled =>
- withTable("t") {
- executeAndCheckOrdering(hasLogicalSort = false, orderingMatched =
true) {
- sql("CREATE TABLE t AS SELECT * FROM t0")
+ withCovnertMetastore { _ =>
+ withPlannedWrite { enabled =>
+ withTable("t") {
+ executeAndCheckOrdering(hasLogicalSort = false, orderingMatched =
true) {
+ sql("CREATE TABLE t STORED AS PARQUET AS SELECT * FROM t0")
+ }
}
}
}
}
test("create hive table as select") {
- withPlannedWrite { enabled =>
- withTable("t") {
- withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
- executeAndCheckOrdering(
- hasLogicalSort = enabled, orderingMatched = enabled, hasEmpty2Null
= enabled) {
- sql(
- """
- |CREATE TABLE t
- |PARTITIONED BY (k)
- |AS SELECT * FROM t0
- |""".stripMargin)
+ withCovnertMetastore { _ =>
+ withPlannedWrite { enabled =>
+ withTable("t") {
+ withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
+ executeAndCheckOrdering(
+ hasLogicalSort = enabled, orderingMatched = enabled,
hasEmpty2Null = enabled) {
+ sql(
+ """
+ |CREATE TABLE t STORED AS PARQUET
+ |PARTITIONED BY (k)
+ |AS SELECT * FROM t0
+ |""".stripMargin)
+ }
}
}
}
@@ -54,18 +69,20 @@ class V1WriteHiveCommandSuite
}
test("insert into hive table") {
- withPlannedWrite { enabled =>
- withTable("t") {
- sql(
- """
- |CREATE TABLE t (i INT, j INT)
- |PARTITIONED BY (k STRING)
- |CLUSTERED BY (i, j) SORTED BY (j) INTO 2 BUCKETS
- |""".stripMargin)
- withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
- executeAndCheckOrdering(
- hasLogicalSort = enabled, orderingMatched = enabled, hasEmpty2Null
= enabled) {
- sql("INSERT INTO t SELECT * FROM t0")
+ withCovnertMetastore { _ =>
+ withPlannedWrite { enabled =>
+ withTable("t") {
+ sql(
+ """
+ |CREATE TABLE t (i INT, j INT) STORED AS PARQUET
+ |PARTITIONED BY (k STRING)
+ |CLUSTERED BY (i, j) SORTED BY (j) INTO 2 BUCKETS
+ |""".stripMargin)
+ withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
+ executeAndCheckOrdering(
+ hasLogicalSort = enabled, orderingMatched = enabled,
hasEmpty2Null = enabled) {
+ sql("INSERT INTO t SELECT * FROM t0")
+ }
}
}
}
@@ -73,18 +90,20 @@ class V1WriteHiveCommandSuite
}
test("insert overwrite hive table") {
- withPlannedWrite { enabled =>
- withTable("t") {
- withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
- sql(
- """
- |CREATE TABLE t
- |PARTITIONED BY (k)
- |AS SELECT * FROM t0
- |""".stripMargin)
- executeAndCheckOrdering(
- hasLogicalSort = enabled, orderingMatched = enabled, hasEmpty2Null
= enabled) {
- sql("INSERT OVERWRITE t SELECT j AS i, i AS j, k FROM t0")
+ withCovnertMetastore { _ =>
+ withPlannedWrite { enabled =>
+ withTable("t") {
+ withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
+ sql(
+ """
+ |CREATE TABLE t STORED AS PARQUET
+ |PARTITIONED BY (k)
+ |AS SELECT * FROM t0
+ |""".stripMargin)
+ executeAndCheckOrdering(
+ hasLogicalSort = enabled, orderingMatched = enabled,
hasEmpty2Null = enabled) {
+ sql("INSERT OVERWRITE t SELECT j AS i, i AS j, k FROM t0")
+ }
}
}
}
@@ -92,16 +111,18 @@ class V1WriteHiveCommandSuite
}
test("insert into hive table with static partitions only") {
- withPlannedWrite { enabled =>
- withTable("t") {
- sql(
- """
- |CREATE TABLE t (i INT, j INT)
- |PARTITIONED BY (k STRING)
- |""".stripMargin)
- // No dynamic partition so no sort is needed.
- executeAndCheckOrdering(hasLogicalSort = false, orderingMatched =
true) {
- sql("INSERT INTO t PARTITION (k='0') SELECT i, j FROM t0 WHERE k =
'0'")
+ withCovnertMetastore { _ =>
+ withPlannedWrite { enabled =>
+ withTable("t") {
+ sql(
+ """
+ |CREATE TABLE t (i INT, j INT) STORED AS PARQUET
+ |PARTITIONED BY (k STRING)
+ |""".stripMargin)
+ // No dynamic partition so no sort is needed.
+ executeAndCheckOrdering(hasLogicalSort = false, orderingMatched =
true) {
+ sql("INSERT INTO t PARTITION (k='0') SELECT i, j FROM t0 WHERE k =
'0'")
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
