This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 42d33e13363 [SPARK-37888][SQL][TESTS] Unify v1 and v2 `DESCRIBE TABLE` tests 42d33e13363 is described below commit 42d33e133634e0d216cfcf914e6596e64412df07 Author: Max Gekk <max.g...@gmail.com> AuthorDate: Tue Jun 21 10:45:27 2022 +0300 [SPARK-37888][SQL][TESTS] Unify v1 and v2 `DESCRIBE TABLE` tests ### What changes were proposed in this pull request? 1. Move `DESCRIBE TABLE` parsing tests to `DescribeRelationParserSuite`. 2. Put common `DESCRIBE TABLE` tests into one trait `org.apache.spark.sql.execution.command.DescribeTableSuiteBase`, and put datasource specific tests to the `v1.DescribeTableSuite` and `v2.DescribeTableSuite`. The changes follow the approach of #30287. Closes #36671. ### Why are the changes needed? 1. The unification will allow to run common `DESCRIBE TABLE` tests for both DSv1/Hive DSv1 and DSv2 2. We can detect missing features and differences between DSv1 and DSv2 implementations. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? By running the modified test suites: ``` $ build/sbt "testOnly *DDLParserSuite" $ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *HiveDDLSuite" ``` and new test suites: ``` $ build/sbt "sql/test:testOnly *DescribeTableParserSuite" $ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *DescribeTableSuite" ``` Closes #36912 from MaxGekk/unify-describe-table-tests-4. Authored-by: Max Gekk <max.g...@gmail.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- .../spark/sql/catalyst/parser/DDLParserSuite.scala | 15 --- .../spark/sql/connector/DataSourceV2SQLSuite.scala | 110 +-------------------- .../command/DescribeTableParserSuite.scala | 39 ++++++++ .../execution/command/DescribeTableSuiteBase.scala | 87 ++++++++++++++++ .../execution/command/v1/DescribeTableSuite.scala | 106 ++++++++++++++++++++ .../execution/command/v2/DescribeTableSuite.scala | 96 ++++++++++++++++++ .../spark/sql/hive/execution/HiveDDLSuite.scala | 11 --- .../execution/command/DescribeTableSuite.scala | 85 ++++++++++++++++ 8 files changed, 414 insertions(+), 135 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala index 8dd719d3e1d..62195823bcc 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala @@ -1213,21 +1213,6 @@ class DDLParserSuite extends AnalysisTest { "The feature is not supported: DESC TABLE COLUMN for a specific partition.")) } - test("SPARK-17328 Fix NPE with EXPLAIN DESCRIBE TABLE") { - comparePlans(parsePlan("describe t"), - DescribeRelation( - UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = false)) - comparePlans(parsePlan("describe table t"), - DescribeRelation( - UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = false)) - comparePlans(parsePlan("describe table extended t"), - DescribeRelation( - UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = true)) - comparePlans(parsePlan("describe table formatted t"), - DescribeRelation( - UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = true)) - } - test("insert table: basic append") { Seq( "INSERT INTO TABLE testcat.ns1.ns2.tbl SELECT * FROM source", diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index 00bb37dcfb2..9c92c1d9a0b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} import org.apache.spark.sql.internal.SQLConf.{PARTITION_OVERWRITE_MODE, PartitionOverwriteMode, V2_SESSION_CATALOG_IMPLEMENTATION} import org.apache.spark.sql.internal.connector.SimpleTableProvider import org.apache.spark.sql.sources.SimpleScanSource -import org.apache.spark.sql.types.{BooleanType, LongType, MetadataBuilder, StringType, StructField, StructType} +import org.apache.spark.sql.types.{LongType, MetadataBuilder, StringType, StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.Utils @@ -89,71 +89,6 @@ class DataSourceV2SQLSuite checkAnswer(spark.internalCreateDataFrame(rdd, table.schema), Seq.empty) } - test("DescribeTable using v2 catalog") { - spark.sql("CREATE TABLE testcat.table_name (id bigint, data string)" + - " USING foo" + - " PARTITIONED BY (id)") - val descriptionDf = spark.sql("DESCRIBE TABLE testcat.table_name") - assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === - Seq( - ("col_name", StringType), - ("data_type", StringType), - ("comment", StringType))) - val description = descriptionDf.collect() - assert(description === Seq( - Row("id", "bigint", ""), - Row("data", "string", ""), - Row("", "", ""), - Row("# Partitioning", "", ""), - Row("Part 0", "id", ""))) - - val e = intercept[AnalysisException] { - sql("DESCRIBE TABLE testcat.table_name PARTITION (id = 1)") - } - assert(e.message.contains("DESCRIBE does not support partition for v2 tables")) - } - - test("DescribeTable with v2 catalog when table does not exist.") { - intercept[AnalysisException] { - spark.sql("DESCRIBE TABLE testcat.table_name") - } - } - - test("DescribeTable extended using v2 catalog") { - spark.sql("CREATE TABLE testcat.table_name (id bigint, data string)" + - " USING foo" + - " PARTITIONED BY (id)" + - " TBLPROPERTIES ('bar'='baz', 'password' = 'password')" + - " COMMENT 'this is a test table'" + - " LOCATION 'file:/tmp/testcat/table_name'") - val descriptionDf = spark.sql("DESCRIBE TABLE EXTENDED testcat.table_name") - assert(descriptionDf.schema.map(field => (field.name, field.dataType)) - === Seq( - ("col_name", StringType), - ("data_type", StringType), - ("comment", StringType))) - assert(descriptionDf.collect() - .map(_.toSeq) - .map(_.toArray.map(_.toString.trim)) === Array( - Array("id", "bigint", ""), - Array("data", "string", ""), - Array("", "", ""), - Array("# Partitioning", "", ""), - Array("Part 0", "id", ""), - Array("", "", ""), - Array("# Metadata Columns", "", ""), - Array("index", "int", "Metadata column used to conflict with a data column"), - Array("_partition", "string", "Partition key used to store the row"), - Array("", "", ""), - Array("# Detailed Table Information", "", ""), - Array("Name", "testcat.table_name", ""), - Array("Comment", "this is a test table", ""), - Array("Location", "file:/tmp/testcat/table_name", ""), - Array("Provider", "foo", ""), - Array(TableCatalog.PROP_OWNER.capitalize, defaultUser, ""), - Array("Table Properties", "[bar=baz,password=*********(redacted)]", ""))) - } - test("Describe column for v2 catalog") { val t = "testcat.tbl" withTable(t) { @@ -2410,49 +2345,6 @@ class DataSourceV2SQLSuite } } - test("SPARK-34561: drop/add columns to a dataset of `DESCRIBE TABLE`") { - val tbl = s"${catalogAndNamespace}tbl" - withTable(tbl) { - sql(s"CREATE TABLE $tbl (c0 INT) USING $v2Format") - val description = sql(s"DESCRIBE TABLE $tbl") - val noCommentDataset = description.drop("comment") - val expectedSchema = new StructType() - .add( - name = "col_name", - dataType = StringType, - nullable = false, - metadata = new MetadataBuilder().putString("comment", "name of the column").build()) - .add( - name = "data_type", - dataType = StringType, - nullable = false, - metadata = new MetadataBuilder().putString("comment", "data type of the column").build()) - assert(noCommentDataset.schema === expectedSchema) - val isNullDataset = noCommentDataset - .withColumn("is_null", noCommentDataset("col_name").isNull) - assert(isNullDataset.schema === expectedSchema.add("is_null", BooleanType, false)) - } - } - - test("SPARK-34576: drop/add columns to a dataset of `DESCRIBE COLUMN`") { - val tbl = s"${catalogAndNamespace}tbl" - withTable(tbl) { - sql(s"CREATE TABLE $tbl (c0 INT) USING $v2Format") - val description = sql(s"DESCRIBE TABLE $tbl c0") - val noCommentDataset = description.drop("info_value") - val expectedSchema = new StructType() - .add( - name = "info_name", - dataType = StringType, - nullable = false, - metadata = new MetadataBuilder().putString("comment", "name of the column info").build()) - assert(noCommentDataset.schema === expectedSchema) - val isNullDataset = noCommentDataset - .withColumn("is_null", noCommentDataset("info_name").isNull) - assert(isNullDataset.schema === expectedSchema.add("is_null", BooleanType, false)) - } - } - test("SPARK-34923: do not propagate metadata columns through Project") { val t1 = s"${catalogAndNamespace}table" withTable(t1) { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableParserSuite.scala new file mode 100644 index 00000000000..58c71f9563f --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableParserSuite.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedTableOrView} +import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan +import org.apache.spark.sql.catalyst.plans.logical.DescribeRelation + +class DescribeTableParserSuite extends AnalysisTest { + test("SPARK-17328: Fix NPE with EXPLAIN DESCRIBE TABLE") { + comparePlans(parsePlan("describe t"), + DescribeRelation( + UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = false)) + comparePlans(parsePlan("describe table t"), + DescribeRelation( + UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = false)) + comparePlans(parsePlan("describe table extended t"), + DescribeRelation( + UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = true)) + comparePlans(parsePlan("describe table formatted t"), + DescribeRelation( + UnresolvedTableOrView(Seq("t"), "DESCRIBE TABLE", true), Map.empty, isExtended = true)) + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableSuiteBase.scala new file mode 100644 index 00000000000..0cf062fb34e --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeTableSuiteBase.scala @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import org.apache.spark.sql.{AnalysisException, QueryTest} +import org.apache.spark.sql.types.{BooleanType, MetadataBuilder, StringType, StructType} + +/** + * This base suite contains unified tests for the `DESCRIBE TABLE` command that check V1 and V2 + * table catalogs. The tests that cannot run for all supported catalogs are located in more + * specific test suites: + * + * - V2 table catalog tests: `org.apache.spark.sql.execution.command.v2.DescribeTableSuite` + * - V1 table catalog tests: + * `org.apache.spark.sql.execution.command.v1.DescribeTableSuiteBase` + * - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.DescribeTableSuite` + * - V1 Hive External catalog: + * `org.apache.spark.sql.hive.execution.command.DescribeTableSuite` + */ +trait DescribeTableSuiteBase extends QueryTest with DDLCommandTestUtils { + override val command = "DESCRIBE TABLE" + + test("DESCRIBE TABLE in a catalog when table does not exist") { + withNamespaceAndTable("ns", "table") { tbl => + val e = intercept[AnalysisException] { + sql(s"DESCRIBE TABLE ${tbl}_non_existence") + } + assert(e.getMessage.contains(s"Table or view not found: ${tbl}_non_existence")) + } + } + + test("SPARK-34561: drop/add columns to a dataset of `DESCRIBE TABLE`") { + withNamespaceAndTable("ns", "table") { tbl => + sql(s"CREATE TABLE $tbl (c0 INT) $defaultUsing") + val description = sql(s"DESCRIBE TABLE $tbl") + val noCommentDataset = description.drop("comment") + val expectedSchema = new StructType() + .add( + name = "col_name", + dataType = StringType, + nullable = false, + metadata = new MetadataBuilder().putString("comment", "name of the column").build()) + .add( + name = "data_type", + dataType = StringType, + nullable = false, + metadata = new MetadataBuilder().putString("comment", "data type of the column").build()) + assert(noCommentDataset.schema === expectedSchema) + val isNullDataset = noCommentDataset + .withColumn("is_null", noCommentDataset("col_name").isNull) + assert(isNullDataset.schema === expectedSchema.add("is_null", BooleanType, false)) + } + } + + test("SPARK-34576: drop/add columns to a dataset of `DESCRIBE COLUMN`") { + withNamespaceAndTable("ns", "table") { tbl => + sql(s"CREATE TABLE $tbl (c0 INT) $defaultUsing") + val description = sql(s"DESCRIBE TABLE $tbl c0") + val noCommentDataset = description.drop("info_value") + val expectedSchema = new StructType() + .add( + name = "info_name", + dataType = StringType, + nullable = false, + metadata = new MetadataBuilder().putString("comment", "name of the column info").build()) + assert(noCommentDataset.schema === expectedSchema) + val isNullDataset = noCommentDataset + .withColumn("is_null", noCommentDataset("info_name").isNull) + assert(isNullDataset.schema === expectedSchema.add("is_null", BooleanType, false)) + } + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala new file mode 100644 index 00000000000..01b7aefdd78 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command.v1 + +import org.apache.spark.sql.{AnalysisException, QueryTest, Row} +import org.apache.spark.sql.execution.command +import org.apache.spark.sql.types.StringType + +/** + * This base suite contains unified tests for the `DESCRIBE TABLE` command that checks V1 + * table catalogs. The tests that cannot run for all V1 catalogs are located in more + * specific test suites: + * + * - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.DescribeTableSuite` + * - V1 Hive External catalog: + * `org.apache.spark.sql.hive.execution.command.DescribeTableSuite` + */ +trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase + with command.TestsV1AndV2Commands { + + test("DESCRIBE TABLE with non-'partitioned-by' clause") { + withNamespaceAndTable("ns", "table") { tbl => + spark.sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing") + val descriptionDf = spark.sql(s"DESCRIBE TABLE $tbl") + assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === + Seq( + ("col_name", StringType), + ("data_type", StringType), + ("comment", StringType))) + QueryTest.checkAnswer( + descriptionDf, + Seq( + Row("data", "string", null), + Row("id", "bigint", null))) + } + } + + test("Describing a partition is not supported") { + withNamespaceAndTable("ns", "table") { tbl => + spark.sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing " + + "PARTITIONED BY (id)") + val e = intercept[AnalysisException] { + sql(s"DESCRIBE TABLE $tbl PARTITION (id = 1)") + } + assert(e.message === "Partition not found in table 'table' database 'ns':\nid -> 1") + } + } +} + +/** + * The class contains tests for the `DESCRIBE TABLE` command to check V1 In-Memory + * table catalog. + */ +class DescribeTableSuite extends DescribeTableSuiteBase with CommandSuiteBase { + override def commandVersion: String = super[DescribeTableSuiteBase].commandVersion + + test("DESCRIBE TABLE EXTENDED of a partitioned table") { + withNamespaceAndTable("ns", "table") { tbl => + spark.sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing" + + " PARTITIONED BY (id)" + + " TBLPROPERTIES ('bar'='baz')" + + " COMMENT 'this is a test table'" + + " LOCATION 'file:/tmp/testcat/table_name'") + val descriptionDf = spark.sql(s"DESCRIBE TABLE EXTENDED $tbl") + assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === Seq( + ("col_name", StringType), + ("data_type", StringType), + ("comment", StringType))) + QueryTest.checkAnswer( + descriptionDf.filter("col_name != 'Created Time'"), + Seq( + Row("data", "string", null), + Row("id", "bigint", null), + Row("# Partition Information", "", ""), + Row("# col_name", "data_type", "comment"), + Row("id", "bigint", null), + Row("", "", ""), + Row("# Detailed Table Information", "", ""), + Row("Database", "ns", ""), + Row("Table", "table", ""), + Row("Last Access", "UNKNOWN", ""), + Row("Created By", "Spark 3.4.0-SNAPSHOT", ""), + Row("Type", "EXTERNAL", ""), + Row("Provider", "parquet", ""), + Row("Comment", "this is a test table", ""), + Row("Table Properties", "[bar=baz]", ""), + Row("Location", "file:/tmp/testcat/table_name", ""), + Row("Partition Provider", "Catalog", ""))) + } + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala new file mode 100644 index 00000000000..ee614b87718 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command.v2 + +import org.apache.spark.sql.{AnalysisException, QueryTest, Row} +import org.apache.spark.sql.connector.catalog.TableCatalog +import org.apache.spark.sql.execution.command +import org.apache.spark.sql.types.StringType +import org.apache.spark.util.Utils + +/** + * The class contains tests for the `DESCRIBE TABLE` command to check V2 table catalogs. + */ +class DescribeTableSuite extends command.DescribeTableSuiteBase with CommandSuiteBase { + + test("DESCRIBE TABLE with non-'partitioned-by' clause") { + withNamespaceAndTable("ns", "table") { tbl => + spark.sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing") + val descriptionDf = spark.sql(s"DESCRIBE TABLE $tbl") + assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === + Seq( + ("col_name", StringType), + ("data_type", StringType), + ("comment", StringType))) + QueryTest.checkAnswer( + descriptionDf, + Seq( + Row("data", "string", ""), + Row("id", "bigint", ""), + Row("", "", ""), + Row("# Partitioning", "", ""), + Row("Not partitioned", "", ""))) + } + } + + test("Describing a partition is not supported") { + withNamespaceAndTable("ns", "table") { tbl => + spark.sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing " + + "PARTITIONED BY (id)") + val e = intercept[AnalysisException] { + sql(s"DESCRIBE TABLE $tbl PARTITION (id = 1)") + } + assert(e.message === "DESCRIBE does not support partition for v2 tables.") + } + } + + test("DESCRIBE TABLE EXTENDED of a partitioned table") { + withNamespaceAndTable("ns", "table") { tbl => + spark.sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing" + + " PARTITIONED BY (id)" + + " TBLPROPERTIES ('bar'='baz')" + + " COMMENT 'this is a test table'" + + " LOCATION 'file:/tmp/testcat/table_name'") + val descriptionDf = spark.sql(s"DESCRIBE TABLE EXTENDED $tbl") + assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === Seq( + ("col_name", StringType), + ("data_type", StringType), + ("comment", StringType))) + QueryTest.checkAnswer( + descriptionDf, + Seq( + Row("id", "bigint", ""), + Row("data", "string", ""), + Row("", "", ""), + Row("# Partitioning", "", ""), + Row("Part 0", "id", ""), + Row("", "", ""), + Row("# Metadata Columns", "", ""), + Row("index", "int", "Metadata column used to conflict with a data column"), + Row("_partition", "string", "Partition key used to store the row"), + Row("", "", ""), + Row("# Detailed Table Information", "", ""), + Row("Name", tbl, ""), + Row("Comment", "this is a test table", ""), + Row("Location", "file:/tmp/testcat/table_name", ""), + Row("Provider", "_", ""), + Row(TableCatalog.PROP_OWNER.capitalize, Utils.getCurrentUserName(), ""), + Row("Table Properties", "[bar=baz]", ""))) + } + } +} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 5ea9a1adaa5..09c6ac63bf5 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -378,17 +378,6 @@ class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeA catalog.reset() } } - - test("Table Ownership") { - val catalog = spark.sessionState.catalog - try { - sql(s"CREATE TABLE spark_30019(k int)") - assert(sql(s"DESCRIBE TABLE EXTENDED spark_30019").where("col_name='Owner'") - .collect().head.getString(1) === Utils.getCurrentUserName()) - } finally { - catalog.reset() - } - } } @SlowHiveTest diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/DescribeTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/DescribeTableSuite.scala new file mode 100644 index 00000000000..455a2c8a307 --- /dev/null +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/DescribeTableSuite.scala @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive.execution.command + +import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.connector.catalog.TableCatalog +import org.apache.spark.sql.execution.command.v1 +import org.apache.spark.sql.types.StringType +import org.apache.spark.util.Utils + +/** + * The class contains tests for the `DESCRIBE TABLE` command to check V1 Hive external + * table catalog. + */ +class DescribeTableSuite extends v1.DescribeTableSuiteBase with CommandSuiteBase { + override def commandVersion: String = super[DescribeTableSuiteBase].commandVersion + + test("Table Ownership") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (c int) $defaultUsing") + checkHiveClientCalls(expected = 6) { + checkAnswer( + sql(s"DESCRIBE TABLE EXTENDED $t") + .where("col_name='Owner'") + .select("col_name", "data_type"), + Row("Owner", Utils.getCurrentUserName())) + } + } + } + + + test("DESCRIBE TABLE EXTENDED of a partitioned table") { + withNamespaceAndTable("ns", "table") { tbl => + spark.sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing" + + " PARTITIONED BY (id)" + + " COMMENT 'this is a test table'" + + " LOCATION 'file:/tmp/testcat/table_name'") + val descriptionDf = spark.sql(s"DESCRIBE TABLE EXTENDED $tbl") + assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === Seq( + ("col_name", StringType), + ("data_type", StringType), + ("comment", StringType))) + QueryTest.checkAnswer( + // Filter out 'Table Properties' to don't check `transient_lastDdlTime` + descriptionDf.filter("col_name != 'Created Time' and col_name != 'Table Properties'"), + Seq( + Row("data", "string", null), + Row("id", "bigint", null), + Row("# Partition Information", "", ""), + Row("# col_name", "data_type", "comment"), + Row("id", "bigint", null), + Row("", "", ""), + Row("# Detailed Table Information", "", ""), + Row("Database", "ns", ""), + Row("Table", "table", ""), + Row(TableCatalog.PROP_OWNER.capitalize, Utils.getCurrentUserName(), ""), + Row("Last Access", "UNKNOWN", ""), + Row("Created By", "Spark 3.4.0-SNAPSHOT", ""), + Row("Type", "EXTERNAL", ""), + Row("Provider", "hive", ""), + Row("Comment", "this is a test table", ""), + Row("Location", "file:/tmp/testcat/table_name", ""), + Row("Serde Library", "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", ""), + Row("InputFormat", "org.apache.hadoop.mapred.TextInputFormat", ""), + Row("OutputFormat", "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", ""), + Row("Storage Properties", "[serialization.format=1]", ""), + Row("Partition Provider", "Catalog", ""))) + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org