Repository: spark Updated Branches: refs/heads/master bc09a2b8c -> a1d9138ab
[SPARK-17680][SQL][TEST] Added a Testcase for Verifying Unicode Character Support for Column Names and Comments ### What changes were proposed in this pull request? Spark SQL supports Unicode characters for column names when specified within backticks(`). When the Hive support is enabled, the version of the Hive metastore must be higher than 0.12, See the JIRA: https://issues.apache.org/jira/browse/HIVE-6013 Hive metastore supports Unicode characters for column names since 0.13. In Spark SQL, table comments, and view comments always allow Unicode characters without backticks. BTW, a separate PR has been submitted for database and table name validation because we do not support Unicode characters in these two cases. ### How was this patch tested? N/A Author: gatorsmile <[email protected]> Closes #15255 from gatorsmile/unicodeSupport. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a1d9138a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a1d9138a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a1d9138a Branch: refs/heads/master Commit: a1d9138ab286dc58d7f61c27419de7ecbf5b828b Parents: bc09a2b Author: gatorsmile <[email protected]> Authored: Wed Nov 30 15:17:29 2016 +0800 Committer: Wenchen Fan <[email protected]> Committed: Wed Nov 30 15:17:29 2016 +0800 ---------------------------------------------------------------------- .../spark/sql/hive/execution/HiveDDLSuite.scala | 45 ++++++++++++++++++++ 1 file changed, 45 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/a1d9138a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 951e070..f313db6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -147,6 +147,51 @@ class HiveDDLSuite } } + test("create Hive-serde table and view with unicode columns and comment") { + val catalog = spark.sessionState.catalog + val tabName = "tab1" + val viewName = "view1" + // scalastyle:off + // non ascii characters are not allowed in the source code, so we disable the scalastyle. + val colName1 = "å" + val colName2 = "å°¼" + val comment = "åº" + // scalastyle:on + withTable(tabName) { + sql(s""" + |CREATE TABLE $tabName(`$colName1` int COMMENT '$comment') + |COMMENT '$comment' + |PARTITIONED BY (`$colName2` int) + """.stripMargin) + sql(s"INSERT OVERWRITE TABLE $tabName partition (`$colName2`=2) SELECT 1") + withView(viewName) { + sql( + s""" + |CREATE VIEW $viewName(`$colName1` COMMENT '$comment', `$colName2`) + |COMMENT '$comment' + |AS SELECT `$colName1`, `$colName2` FROM $tabName + """.stripMargin) + val tableMetadata = catalog.getTableMetadata(TableIdentifier(tabName, Some("default"))) + val viewMetadata = catalog.getTableMetadata(TableIdentifier(viewName, Some("default"))) + assert(tableMetadata.comment == Option(comment)) + assert(viewMetadata.comment == Option(comment)) + + assert(tableMetadata.schema.fields.length == 2 && viewMetadata.schema.fields.length == 2) + val column1InTable = tableMetadata.schema.fields.head + val column1InView = viewMetadata.schema.fields.head + assert(column1InTable.name == colName1 && column1InView.name == colName1) + assert(column1InTable.getComment() == Option(comment)) + assert(column1InView.getComment() == Option(comment)) + + assert(tableMetadata.schema.fields(1).name == colName2 && + viewMetadata.schema.fields(1).name == colName2) + + checkAnswer(sql(s"SELECT `$colName1`, `$colName2` FROM $tabName"), Row(1, 2) :: Nil) + checkAnswer(sql(s"SELECT `$colName1`, `$colName2` FROM $viewName"), Row(1, 2) :: Nil) + } + } + } + test("create table: partition column names exist in table definition") { val e = intercept[AnalysisException] { sql("CREATE TABLE tbl(a int) PARTITIONED BY (a string)") --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
