Github user ericl commented on a diff in the pull request:
https://github.com/apache/spark/pull/13386#discussion_r65264707
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala ---
@@ -936,7 +936,39 @@ class SparkSqlAstBuilder(conf: SQLConf) extends
AstBuilder {
comment = comment)
selectQuery match {
- case Some(q) => CreateTableAsSelectLogicalPlan(tableDesc, q,
ifNotExists)
+ case Some(q) =>
+ // Hive does not allow to use a CTAS statement to create a
partitioned table.
+ if (tableDesc.partitionColumnNames.nonEmpty) {
+ val errorMessage = "A Create Table As Select (CTAS) statement is
not allowed to " +
+ "create a partitioned table using Hive's file formats. " +
+ "Please use the syntax of \"CREATE TABLE tableName USING
dataSource " +
+ "OPTIONS (...) PARTITIONED BY ...\" to create a partitioned
table through a " +
+ "CTAS statement."
+ throw operationNotAllowed(errorMessage, ctx)
+ }
+
+ val hasStorageProperties = (ctx.createFileFormat != null) ||
(ctx.rowFormat != null)
+ if (conf.convertCTAS && !hasStorageProperties) {
+ val mode = if (ifNotExists) SaveMode.Ignore else
SaveMode.ErrorIfExists
+ val options = rowStorage.serdeProperties ++
fileStorage.serdeProperties
+ val optionsWithPath = if (location.isDefined) {
+ options + ("path" -> location.get)
+ } else {
+ options
+ }
+ CreateTableUsingAsSelect(
+ tableIdent = tableDesc.identifier,
+ provider = conf.defaultDataSourceName,
+ temporary = false,
+ partitionColumns = tableDesc.partitionColumnNames.toArray,
+ bucketSpec = None,
+ mode = mode,
+ options = optionsWithPath,
+ q
+ )
+ } else {
+ CreateTableAsSelectLogicalPlan(tableDesc, q, ifNotExists)
--- End diff --
Should this one also be renamed to `CreateHiveTableAsSelectLogicalPlan`?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
