MaxGekk commented on code in PR #37588:
URL: https://github.com/apache/spark/pull/37588#discussion_r1372183519
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala:
##########
@@ -126,4 +127,512 @@ trait ShowTablesSuiteBase extends QueryTest with
DDLCommandTestUtils {
}
}
}
+
+ test("show table in a not existing namespace") {
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(s"SHOW TABLES IN $catalog.nonexist")
+ },
+ errorClass = "SCHEMA_NOT_FOUND",
+ parameters = Map("schemaName" -> "`nonexist`"))
+ }
+
+ test("show table extended in a not existing namespace") {
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(s"SHOW TABLE EXTENDED IN $catalog.nonexist LIKE '*tbl*'")
+ },
+ errorClass = "SCHEMA_NOT_FOUND",
+ parameters = Map("schemaName" -> "`nonexist`"))
+ }
+
+ test("show table extended in a not existing table") {
+ val namespace = "ns1"
+ val table = "nonexist"
+ withNamespaceAndTable(namespace, table, catalog) { _ =>
+ val result = sql(s"SHOW TABLE EXTENDED IN $catalog.$namespace LIKE
'*$table*'")
+ assert(result.schema.fieldNames ===
+ Seq("namespace", "tableName", "isTemporary", "information"))
+ assert(result.collect().isEmpty)
+ }
+ }
+
+ test("show table extended in non-partitioned table") {
+ val namespace = "ns1"
+ val table = "tbl"
+ withNamespaceAndTable(namespace, table, catalog) { tbl =>
+ sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing")
+ val e = intercept[AnalysisException] {
+ sql(s"SHOW TABLE EXTENDED IN $catalog.$namespace LIKE '$table'
PARTITION(id = 1)")
+ }
+ checkError(
+ exception = e,
+ errorClass = e.getErrorClass,
+ parameters = e.getErrorClass match {
+ case "_LEGACY_ERROR_TEMP_1251" =>
+ Map("action" -> "SHOW TABLE EXTENDED", "tableName" -> table) // v1
v2
Review Comment:
The comment is wrong, the case belongs to Hive:
```json
"_LEGACY_ERROR_TEMP_1251" : {
"message" : [
"<action> is not allowed on <tableName> since its partition metadata
is not stored in the Hive metastore. To import this information into the
metastore, run `msck repair table <tableName>`."
]
},
```
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala:
##########
@@ -126,4 +127,512 @@ trait ShowTablesSuiteBase extends QueryTest with
DDLCommandTestUtils {
}
}
}
+
+ test("show table in a not existing namespace") {
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(s"SHOW TABLES IN $catalog.nonexist")
+ },
+ errorClass = "SCHEMA_NOT_FOUND",
+ parameters = Map("schemaName" -> "`nonexist`"))
+ }
+
+ test("show table extended in a not existing namespace") {
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(s"SHOW TABLE EXTENDED IN $catalog.nonexist LIKE '*tbl*'")
+ },
+ errorClass = "SCHEMA_NOT_FOUND",
+ parameters = Map("schemaName" -> "`nonexist`"))
+ }
+
+ test("show table extended in a not existing table") {
+ val namespace = "ns1"
+ val table = "nonexist"
+ withNamespaceAndTable(namespace, table, catalog) { _ =>
+ val result = sql(s"SHOW TABLE EXTENDED IN $catalog.$namespace LIKE
'*$table*'")
+ assert(result.schema.fieldNames ===
+ Seq("namespace", "tableName", "isTemporary", "information"))
+ assert(result.collect().isEmpty)
+ }
+ }
+
+ test("show table extended in non-partitioned table") {
+ val namespace = "ns1"
+ val table = "tbl"
+ withNamespaceAndTable(namespace, table, catalog) { tbl =>
+ sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing")
+ val e = intercept[AnalysisException] {
+ sql(s"SHOW TABLE EXTENDED IN $catalog.$namespace LIKE '$table'
PARTITION(id = 1)")
+ }
+ checkError(
+ exception = e,
+ errorClass = e.getErrorClass,
+ parameters = e.getErrorClass match {
+ case "_LEGACY_ERROR_TEMP_1251" =>
+ Map("action" -> "SHOW TABLE EXTENDED", "tableName" -> table) // v1
v2
+ case "_LEGACY_ERROR_TEMP_1231" =>
+ Map("key" -> "id", "tblName" ->
s"`$catalog`.`$namespace`.`$table`") // hive
+ }
+ )
+ }
+ }
+
+ test("show table extended in a not existing partition") {
+ val namespace = "ns1"
+ val table = "tbl"
+ withNamespaceAndTable(namespace, table, catalog) { tbl =>
+ sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing
PARTITIONED BY (id)")
+ sql(s"ALTER TABLE $tbl ADD PARTITION (id = 1)")
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(s"SHOW TABLE EXTENDED IN $catalog.$namespace LIKE '$table'
PARTITION(id = 2)")
+ },
+ errorClass = "PARTITIONS_NOT_FOUND",
+ parameters = Map(
+ "partitionList" -> "PARTITION (`id` = 2)",
+ "tableName" -> "`ns1`.`tbl`"
+ )
+ )
+ }
+ }
+
+ test("show table extended in multi partition key table") {
+ val namespace = "ns1"
+ val table = "tbl"
+ withNamespaceAndTable(namespace, table, catalog) { tbl =>
+ sql(s"CREATE TABLE $tbl (id1 bigint, id2 bigint, data string) " +
+ s"$defaultUsing PARTITIONED BY (id1, id2)")
+ sql(s"ALTER TABLE $tbl ADD PARTITION (id1 = 1, id2 = 2)")
+
+ val result = sql(s"SHOW TABLE EXTENDED FROM $catalog.$namespace " +
+ s"LIKE '$table' PARTITION(id1 = 1, id2 = 2)")
+ assert(result.schema.fieldNames ===
+ Seq("namespace", "tableName", "isTemporary", "information"))
+ assert(result.collect()(0).length == 4)
+ assert(result.collect()(0)(0) === namespace)
+ assert(result.collect()(0)(1) === table)
+ assert(result.collect()(0)(2) === false)
+ val actualResult = exclude(result.collect()(0)(3).toString)
+ val expectedResult_v1_v2 = "Partition Values: [id1=1, id2=2]"
+ val expectedResult_hive =
+ """Partition Values: [id1=1, id2=2]
+ |Serde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ |InputFormat: org.apache.hadoop.mapred.TextInputFormat
+ |OutputFormat:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ |Storage Properties: [serialization.format=1]""".stripMargin
+ assert(actualResult === expectedResult_v1_v2 || actualResult ===
expectedResult_hive)
+
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(s"SHOW TABLE EXTENDED IN $catalog.$namespace " +
+ s"LIKE '$table' PARTITION(id1 = 1)")
+ },
+ errorClass = "_LEGACY_ERROR_TEMP_1232",
+ parameters = Map(
+ "specKeys" -> "id1",
+ "partitionColumnNames" -> "id1, id2",
+ "tableName" -> s"`$catalog`.`$namespace`.`$table`")
+ )
+ }
+ }
+
+ test("show table extended in multi tables") {
+ val namespace = "ns1"
+ val table = "tbl"
+ withNamespaceAndTable(namespace, table, catalog) { _ =>
+ sql(s"CREATE TABLE $catalog.$namespace.$table (id bigint, data string) "
+
+ s"$defaultUsing PARTITIONED BY (id)")
+ val table1 = "tbl1"
+ val table2 = "tbl2"
+ withTable(table1, table2) {
+ sql(s"CREATE TABLE $catalog.$namespace.$table1 (id1 bigint, data1
string) " +
+ s"$defaultUsing PARTITIONED BY (id1)")
+ sql(s"CREATE TABLE $catalog.$namespace.$table2 (id2 bigint, data2
string) " +
+ s"$defaultUsing PARTITIONED BY (id2)")
+
+ val result = sql(s"SHOW TABLE EXTENDED FROM $catalog.$namespace LIKE
'$table*'")
+ .sort("tableName")
+ assert(result.schema.fieldNames ===
+ Seq("namespace", "tableName", "isTemporary", "information"))
+ assert(result.collect().length == 3)
+
+ assert(result.collect()(0).length == 4)
+ assert(result.collect()(0)(1) === table)
+ assert(result.collect()(0)(2) === false)
+ val actualResult_0_3 = exclude(result.collect()(0)(3).toString)
+
+ // exclude "Created Time", "Last Access", "Created By", "Location"
+ val expectedResult_0_3_v1 =
+ s"""Catalog: $catalog
+ |Database: $namespace
+ |Table: $table
+ |Type: MANAGED
+ |Provider: parquet
+ |Partition Provider: Catalog
+ |Partition Columns: [`id`]
+ |Schema: root
+ | |-- data: string (nullable = true)
+ | |-- id: long (nullable = true)""".stripMargin
+ val expectedResult_0_3_v2 =
+ s"""Namespace: $namespace
+ |Table: $table
+ |Type: MANAGED
+ |Provider: _
+ |Owner: ${Utils.getCurrentUserName()}
+ |Partition Provider: Catalog
+ |Partition Columns: [`id`]
+ |Schema: root
+ | |-- data: string (nullable = true)
+ | |-- id: long (nullable = true)""".stripMargin
+
+ // exclude "Table Properties"
+ val expectedResult_0_3_hive =
+ s"""Catalog: $catalog
+ |Database: $namespace
+ |Table: $table
+ |Owner: ${Utils.getCurrentUserName()}
+ |Type: MANAGED
+ |Provider: hive
+ |Serde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ |InputFormat: org.apache.hadoop.mapred.TextInputFormat
+ |OutputFormat:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ |Storage Properties: [serialization.format=1]
+ |Partition Provider: Catalog
+ |Partition Columns: [`id`]
+ |Schema: root
+ | |-- data: string (nullable = true)
+ | |-- id: long (nullable = true)""".stripMargin
+ assert(actualResult_0_3 === expectedResult_0_3_v1 ||
+ actualResult_0_3 === expectedResult_0_3_v2 ||
+ actualResult_0_3 === expectedResult_0_3_hive)
+
+ assert(result.collect()(1).length == 4)
+ assert(result.collect()(1)(1) === table1)
+ assert(result.collect()(1)(2) === false)
+ val actualResult_1_3 = exclude(result.collect()(1)(3).toString)
+
+ // exclude "Created Time", "Last Access", "Created By", "Location"
+ val expectedResult_1_3_v1 =
+ s"""Catalog: $catalog
+ |Database: $namespace
+ |Table: $table1
+ |Type: MANAGED
+ |Provider: parquet
+ |Partition Provider: Catalog
+ |Partition Columns: [`id1`]
+ |Schema: root
+ | |-- data1: string (nullable = true)
+ | |-- id1: long (nullable = true)""".stripMargin
+ val expectedResult_1_3_v2 =
+ s"""Namespace: $namespace
+ |Table: $table1
+ |Type: MANAGED
+ |Provider: _
+ |Owner: ${Utils.getCurrentUserName()}
+ |Partition Provider: Catalog
+ |Partition Columns: [`id1`]
+ |Schema: root
+ | |-- data1: string (nullable = true)
+ | |-- id1: long (nullable = true)""".stripMargin
+
+ // exclude "Table Properties"
+ val expectedResult_1_3_hive =
+ s"""Catalog: $catalog
+ |Database: $namespace
+ |Table: $table1
+ |Owner: ${Utils.getCurrentUserName()}
+ |Type: MANAGED
+ |Provider: hive
+ |Serde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ |InputFormat: org.apache.hadoop.mapred.TextInputFormat
+ |OutputFormat:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ |Storage Properties: [serialization.format=1]
+ |Partition Provider: Catalog
+ |Partition Columns: [`id1`]
+ |Schema: root
+ | |-- data1: string (nullable = true)
+ | |-- id1: long (nullable = true)""".stripMargin
+ assert(actualResult_1_3 === expectedResult_1_3_v1 ||
+ actualResult_1_3 === expectedResult_1_3_v2 ||
+ actualResult_1_3 === expectedResult_1_3_hive)
Review Comment:
Don't think this good check. Let's follow existing convention, and create
separate tests in V2, V1 and Hive traits.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolvePartitionSpec.scala:
##########
@@ -50,6 +50,16 @@ object ResolvePartitionSpec extends Rule[LogicalPlan] {
}
case _ => command
}
+ case command @ ShowTableExtended(ResolvedNamespace(catalog: TableCatalog,
namespace),
Review Comment:
Could you clarify why `ShowTableExtended ` doesn't extend
`V2PartitionCommand`. And apparently why do we need this special case.
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala:
##########
@@ -126,4 +127,512 @@ trait ShowTablesSuiteBase extends QueryTest with
DDLCommandTestUtils {
}
}
}
+
+ test("show table in a not existing namespace") {
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(s"SHOW TABLES IN $catalog.nonexist")
+ },
+ errorClass = "SCHEMA_NOT_FOUND",
+ parameters = Map("schemaName" -> "`nonexist`"))
+ }
+
+ test("show table extended in a not existing namespace") {
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(s"SHOW TABLE EXTENDED IN $catalog.nonexist LIKE '*tbl*'")
+ },
+ errorClass = "SCHEMA_NOT_FOUND",
+ parameters = Map("schemaName" -> "`nonexist`"))
+ }
+
+ test("show table extended in a not existing table") {
+ val namespace = "ns1"
+ val table = "nonexist"
+ withNamespaceAndTable(namespace, table, catalog) { _ =>
+ val result = sql(s"SHOW TABLE EXTENDED IN $catalog.$namespace LIKE
'*$table*'")
+ assert(result.schema.fieldNames ===
+ Seq("namespace", "tableName", "isTemporary", "information"))
+ assert(result.collect().isEmpty)
+ }
+ }
+
+ test("show table extended in non-partitioned table") {
+ val namespace = "ns1"
+ val table = "tbl"
+ withNamespaceAndTable(namespace, table, catalog) { tbl =>
+ sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing")
+ val e = intercept[AnalysisException] {
+ sql(s"SHOW TABLE EXTENDED IN $catalog.$namespace LIKE '$table'
PARTITION(id = 1)")
+ }
+ checkError(
+ exception = e,
+ errorClass = e.getErrorClass,
+ parameters = e.getErrorClass match {
+ case "_LEGACY_ERROR_TEMP_1251" =>
+ Map("action" -> "SHOW TABLE EXTENDED", "tableName" -> table) // v1
v2
+ case "_LEGACY_ERROR_TEMP_1231" =>
+ Map("key" -> "id", "tblName" ->
s"`$catalog`.`$namespace`.`$table`") // hive
+ }
+ )
+ }
+ }
+
+ test("show table extended in a not existing partition") {
+ val namespace = "ns1"
+ val table = "tbl"
+ withNamespaceAndTable(namespace, table, catalog) { tbl =>
+ sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing
PARTITIONED BY (id)")
+ sql(s"ALTER TABLE $tbl ADD PARTITION (id = 1)")
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql(s"SHOW TABLE EXTENDED IN $catalog.$namespace LIKE '$table'
PARTITION(id = 2)")
+ },
+ errorClass = "PARTITIONS_NOT_FOUND",
+ parameters = Map(
+ "partitionList" -> "PARTITION (`id` = 2)",
+ "tableName" -> "`ns1`.`tbl`"
+ )
+ )
+ }
+ }
+
+ test("show table extended in multi partition key table") {
+ val namespace = "ns1"
+ val table = "tbl"
+ withNamespaceAndTable(namespace, table, catalog) { tbl =>
+ sql(s"CREATE TABLE $tbl (id1 bigint, id2 bigint, data string) " +
+ s"$defaultUsing PARTITIONED BY (id1, id2)")
+ sql(s"ALTER TABLE $tbl ADD PARTITION (id1 = 1, id2 = 2)")
+
+ val result = sql(s"SHOW TABLE EXTENDED FROM $catalog.$namespace " +
+ s"LIKE '$table' PARTITION(id1 = 1, id2 = 2)")
+ assert(result.schema.fieldNames ===
+ Seq("namespace", "tableName", "isTemporary", "information"))
+ assert(result.collect()(0).length == 4)
+ assert(result.collect()(0)(0) === namespace)
+ assert(result.collect()(0)(1) === table)
+ assert(result.collect()(0)(2) === false)
+ val actualResult = exclude(result.collect()(0)(3).toString)
+ val expectedResult_v1_v2 = "Partition Values: [id1=1, id2=2]"
+ val expectedResult_hive =
+ """Partition Values: [id1=1, id2=2]
+ |Serde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ |InputFormat: org.apache.hadoop.mapred.TextInputFormat
+ |OutputFormat:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ |Storage Properties: [serialization.format=1]""".stripMargin
+ assert(actualResult === expectedResult_v1_v2 || actualResult ===
expectedResult_hive)
+
+ checkError(
Review Comment:
Not clear what does this checks, and how it belongs to the test. Could you
put it to a separate test with proper title.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
