dtenedor commented on code in PR #40652:
URL: https://github.com/apache/spark/pull/40652#discussion_r1157749445
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveDefaultColumns.scala:
##########
@@ -271,32 +271,33 @@ case class ResolveDefaultColumns(catalog: SessionCatalog)
extends Rule[LogicalPl
/**
* Updates an inline table to generate missing default column values.
*/
- private def addMissingDefaultValuesForInsertFromInlineTable(
+ def addMissingDefaultValuesForInsertFromInlineTable(
node: LogicalPlan,
insertTableSchemaWithoutPartitionColumns: StructType,
numUserSpecifiedColumns: Int): LogicalPlan = {
val schema = insertTableSchemaWithoutPartitionColumns
val newDefaultExpressions: Seq[Expression] =
getDefaultExpressionsForInsert(schema, numUserSpecifiedColumns)
- val newNames: Seq[String] = if (numUserSpecifiedColumns > 0) {
- schema.fields.drop(numUserSpecifiedColumns).map(_.name)
- } else {
- schema.fields.map(_.name)
- }
+ val newNames: Seq[String] = schema.fields.map(_.name)
node match {
case _ if newDefaultExpressions.isEmpty => node
case table: UnresolvedInlineTable =>
table.copy(
- names = table.names ++ newNames,
+ names = newNames,
rows = table.rows.map { row => row ++ newDefaultExpressions })
case local: LocalRelation =>
// Note that we have consumed a LocalRelation but return an
UnresolvedInlineTable, because
// addMissingDefaultValuesForInsertFromProject must replace unresolved
DEFAULT references.
UnresolvedInlineTable(
- local.output.map(_.name) ++ newNames,
+ newNames,
local.data.map { row =>
val colTypes = StructType(local.output.map(col =>
StructField(col.name, col.dataType)))
- row.toSeq(colTypes).map(Literal(_)) ++ newDefaultExpressions
+ val values: Seq[Any] = row.toSeq(colTypes)
+ val dataTypes: Seq[DataType] = colTypes.map(_.dataType)
+ val literals: Seq[Literal] = values.zip(dataTypes).map {
+ case (value, dataType) => Literal(value, dataType)
Review Comment:
I was able to make this work! The challenge was that this method only added
new `UnresolvedAttribute("DEFAULT")` before, relying on later methods to
replace them. And it's not possible to add those to rows of a LocalRelation.
But I updated this to replace the default values themselves instead, and it
works. This should be simpler and safer than regenerating back an
`UnresolvedInlineTable`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
