yihua commented on code in PR #18147:
URL: https://github.com/apache/hudi/pull/18147#discussion_r2796386493
##########
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala:
##########
@@ -298,19 +298,59 @@ object HoodieAnalysis extends SparkAdapterSupport {
private object ProducesHudiMetaFields {
def unapply(plan: LogicalPlan): Option[Seq[Attribute]] = {
- val resolved = if (plan.resolved) {
- plan
- } else {
- val analyzer = spark.sessionState.analyzer
- analyzer.execute(plan)
- }
+ try {
+ val resolved = if (plan.resolved) {
+ plan
+ } else {
+ val analyzer = spark.sessionState.analyzer
+ analyzer.execute(plan)
+ }
- if (resolved.output.exists(attr => isMetaField(attr.name))) {
- Some(resolved.output)
- } else {
- None
+ if (resolved.output.exists(attr => isMetaField(attr.name))) {
Review Comment:
Instead, should we fix these places to let Spark's analysis produce a clear
error?
```
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSparkBaseAnalysis.scala
166 val sourceTable = if (sourceTableO.resolved) sourceTableO
else analyzer.execute(sourceTableO)
167 val m = mO.asInstanceOf[MergeIntoTable].copy(targetTable =
targetTable, sourceTable = sourceTable)
168 // END: custom Hudi change
169 - EliminateSubqueryAliases(targetTable) match {
169 + // If source table still has unresolved references (e.g.,
non-existent columns/tables),
170 + // return the partially resolved plan and let Spark's
CheckAnalysis produce a clear error.
171 + if (!sourceTable.resolved) {
172 + m
173 + } else EliminateSubqueryAliases(targetTable) match {
174 case r: NamedRelation if r.skipSchemaResolution =>
175 // Do not resolve the expression if the target table
accepts any schema.
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala
305 analyzer.execute(plan)
306 }
307
308 - if (resolved.output.exists(attr =>
isMetaField(attr.name))) {
308 + // If the plan is still not resolved (e.g., references
non-existent tables/columns),
309 + // skip meta-field checks and let Spark's standard
analysis produce a clear error.
310 + if (resolved.resolved && resolved.output.exists(attr =>
isMetaField(attr.name))) {
311 Some(resolved.output)
312 } else {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
