maropu commented on a change in pull request #31639:
URL: https://github.com/apache/spark/pull/31639#discussion_r582448108
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
##########
@@ -874,13 +874,29 @@ class SessionCatalog(
// the extra columns that we don't require), with UpCast (to make sure the
type change is
// safe) and Alias (to respect user-specified view column names) according
to the view schema
// in the catalog.
- val projectList = viewColumnNames.zip(metadata.schema).map { case (name,
field) =>
- Alias(UpCast(UnresolvedAttribute.quoted(name), field.dataType),
field.name)(
- explicitMetadata = Some(field.metadata))
+ val projectList = if (isTempView) {
+ viewColumnNames.zip(metadata.schema).map { case (name, field) =>
+ Alias(UpCast(UnresolvedAttribute.quoted(name), field.dataType),
field.name)(
+ explicitMetadata = Some(field.metadata))
+ }
+ } else {
+ viewColumnNames.zip(metadata.schema).
+ map { case (name, field) => innerStruct(Seq(), name, field)}
}
View(desc = metadata, isTempView = isTempView, child =
Project(projectList, parsedPlan))
}
+ private def innerStruct(parent : Seq[String], name : String,
Review comment:
How about `innerStruct` -> `createNamedExpr`?
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
##########
@@ -874,13 +874,29 @@ class SessionCatalog(
// the extra columns that we don't require), with UpCast (to make sure the
type change is
// safe) and Alias (to respect user-specified view column names) according
to the view schema
// in the catalog.
- val projectList = viewColumnNames.zip(metadata.schema).map { case (name,
field) =>
- Alias(UpCast(UnresolvedAttribute.quoted(name), field.dataType),
field.name)(
- explicitMetadata = Some(field.metadata))
+ val projectList = if (isTempView) {
+ viewColumnNames.zip(metadata.schema).map { case (name, field) =>
+ Alias(UpCast(UnresolvedAttribute.quoted(name), field.dataType),
field.name)(
+ explicitMetadata = Some(field.metadata))
+ }
+ } else {
+ viewColumnNames.zip(metadata.schema).
+ map { case (name, field) => innerStruct(Seq(), name, field)}
Review comment:
nit format:
```
viewColumnNames.zip(metadata.schema).map { case (name, field) =>
innerStruct(Seq(), name, field)
}
```
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
##########
@@ -874,13 +874,29 @@ class SessionCatalog(
// the extra columns that we don't require), with UpCast (to make sure the
type change is
// safe) and Alias (to respect user-specified view column names) according
to the view schema
// in the catalog.
- val projectList = viewColumnNames.zip(metadata.schema).map { case (name,
field) =>
- Alias(UpCast(UnresolvedAttribute.quoted(name), field.dataType),
field.name)(
- explicitMetadata = Some(field.metadata))
+ val projectList = if (isTempView) {
+ viewColumnNames.zip(metadata.schema).map { case (name, field) =>
+ Alias(UpCast(UnresolvedAttribute.quoted(name), field.dataType),
field.name)(
+ explicitMetadata = Some(field.metadata))
+ }
+ } else {
+ viewColumnNames.zip(metadata.schema).
+ map { case (name, field) => innerStruct(Seq(), name, field)}
}
View(desc = metadata, isTempView = isTempView, child =
Project(projectList, parsedPlan))
}
+ private def innerStruct(parent : Seq[String], name : String,
+ field : StructField) : NamedExpression = {
Review comment:
nit format:
```
private def innerStruct(
parent : Seq[String],
name : String,
field : StructField) : NamedExpression = {
```
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
##########
@@ -874,13 +874,29 @@ class SessionCatalog(
// the extra columns that we don't require), with UpCast (to make sure the
type change is
// safe) and Alias (to respect user-specified view column names) according
to the view schema
// in the catalog.
- val projectList = viewColumnNames.zip(metadata.schema).map { case (name,
field) =>
- Alias(UpCast(UnresolvedAttribute.quoted(name), field.dataType),
field.name)(
- explicitMetadata = Some(field.metadata))
+ val projectList = if (isTempView) {
+ viewColumnNames.zip(metadata.schema).map { case (name, field) =>
+ Alias(UpCast(UnresolvedAttribute.quoted(name), field.dataType),
field.name)(
+ explicitMetadata = Some(field.metadata))
+ }
+ } else {
+ viewColumnNames.zip(metadata.schema).
+ map { case (name, field) => innerStruct(Seq(), name, field)}
}
View(desc = metadata, isTempView = isTempView, child =
Project(projectList, parsedPlan))
}
+ private def innerStruct(parent : Seq[String], name : String,
+ field : StructField) : NamedExpression = {
+ field.dataType match {
+ case structType : StructType => Alias(CreateStruct.create(structType.map
{
+ subField => innerStruct(parent :+ name, subField.name, subField)
+ }), field.name)(explicitMetadata = Some(field.metadata))
Review comment:
How about the other-related cases, e.g., `array(struct))`?
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
##########
@@ -874,13 +874,29 @@ class SessionCatalog(
// the extra columns that we don't require), with UpCast (to make sure the
type change is
// safe) and Alias (to respect user-specified view column names) according
to the view schema
// in the catalog.
- val projectList = viewColumnNames.zip(metadata.schema).map { case (name,
field) =>
- Alias(UpCast(UnresolvedAttribute.quoted(name), field.dataType),
field.name)(
- explicitMetadata = Some(field.metadata))
+ val projectList = if (isTempView) {
+ viewColumnNames.zip(metadata.schema).map { case (name, field) =>
+ Alias(UpCast(UnresolvedAttribute.quoted(name), field.dataType),
field.name)(
+ explicitMetadata = Some(field.metadata))
+ }
+ } else {
+ viewColumnNames.zip(metadata.schema).
+ map { case (name, field) => innerStruct(Seq(), name, field)}
}
View(desc = metadata, isTempView = isTempView, child =
Project(projectList, parsedPlan))
}
+ private def innerStruct(parent : Seq[String], name : String,
+ field : StructField) : NamedExpression = {
+ field.dataType match {
+ case structType : StructType => Alias(CreateStruct.create(structType.map
{
Review comment:
nit: `case structType: StructType`
##########
File path:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
##########
@@ -679,6 +687,44 @@ abstract class SessionCatalogSuite extends AnalysisTest
with Eventually {
}
}
+
+ test("SPARK-34528 - named explicitly field in struct of a view") {
Review comment:
nti: `SPARK-34528: `
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
