cloud-fan commented on code in PR #43949:
URL: https://github.com/apache/spark/pull/43949#discussion_r1411876914
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala:
##########
@@ -114,10 +154,46 @@ class V2SessionCatalog(catalog: SessionCatalog)
schema: StructType,
partitions: Array[Transform],
properties: util.Map[String, String]): Table = {
- import
org.apache.spark.sql.connector.catalog.CatalogV2Implicits.TransformHelper
- val (partitionColumns, maybeBucketSpec, maybeClusterBySpec) =
- partitions.toImmutableArraySeq.convertTransforms
+ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
val provider = properties.getOrDefault(TableCatalog.PROP_PROVIDER,
conf.defaultDataSourceName)
+
+ val (newSchema, newPartitions) =
DataSourceV2Utils.getTableProvider(provider, conf) match {
+ // If the provider does not support external metadata, users should not
be allowed to
+ // specify custom schema when creating the data source table, since the
schema will not
+ // be used when loading the table.
+ case Some(p) if !p.supportsExternalMetadata() =>
+ if (schema.nonEmpty) {
+ throw new SparkUnsupportedOperationException(
+ errorClass =
"CANNOT_CREATE_DATA_SOURCE_TABLE.EXTERNAL_METADATA_UNSUPPORTED",
+ messageParameters = Map("tableName" -> ident.quoted, "provider" ->
provider))
+ }
+ // V2CreateTablePlan does not allow non-empty partitions when schema
is empty. This
+ // is checked in `PreProcessTableCreation` rule.
+ assert(partitions.isEmpty,
+ s"Partitions should be empty when the schema is empty:
${partitions.mkString(", ")}")
+ (schema, partitions)
+
+ case Some(tableProvider) =>
+ assert(tableProvider.supportsExternalMetadata())
+ lazy val dsOptions = new CaseInsensitiveStringMap(properties)
Review Comment:
do we need to put the path option?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
