[
https://issues.apache.org/jira/browse/HUDI-8865?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Y Ethan Guo updated HUDI-8865:
------------------------------
Description:
If there are unresolved columns in MERGE INTO statement using Hudi, currently
Spark 3.5 throws the exception below which is not user-friendly:
{code:java}
Caused by: org.apache.spark.SparkException: [INTERNAL_ERROR] Cannot find main
error class '_LEGACY_ERROR_TEMP_2309'
at org.apache.spark.SparkException$.internalError(SparkException.scala:92)
at org.apache.spark.SparkException$.internalError(SparkException.scala:96)
at
org.apache.spark.ErrorClassesJsonReader.$anonfun$getMessageTemplate$1(ErrorClassesJSONReader.scala:68)
at scala.collection.immutable.HashMap$HashMap1.getOrElse0(HashMap.scala:361)
at
scala.collection.immutable.HashMap$HashTrieMap.getOrElse0(HashMap.scala:594)
at
scala.collection.immutable.HashMap$HashTrieMap.getOrElse0(HashMap.scala:589)
at scala.collection.immutable.HashMap.getOrElse(HashMap.scala:73)
at
org.apache.spark.ErrorClassesJsonReader.getMessageTemplate(ErrorClassesJSONReader.scala:68)
at
org.apache.spark.ErrorClassesJsonReader.getErrorMessage(ErrorClassesJSONReader.scala:47)
at
org.apache.spark.SparkThrowableHelper$.getMessage(SparkThrowableHelper.scala:53)
at
org.apache.spark.SparkThrowableHelper$.getMessage(SparkThrowableHelper.scala:40)
at org.apache.spark.sql.AnalysisException.<init>(AnalysisException.scala:77)
at
org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:52)
at
org.apache.spark.sql.HoodieSpark35CatalystPlanUtils$.failAnalysisForMIT(HoodieSpark35CatalystPlanUtils.scala:80)
at
org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.$anonfun$resolveMergeExprOrFail$2(HoodieSpark3Analysis.scala:263)
at
org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.$anonfun$resolveMergeExprOrFail$2$adapted(HoodieSpark3Analysis.scala:258)
at scala.collection.mutable.LinkedHashSet.foreach(LinkedHashSet.scala:95)
at
org.apache.spark.sql.catalyst.expressions.AttributeSet.foreach(AttributeSet.scala:137)
at
org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.resolveMergeExprOrFail(HoodieSpark3Analysis.scala:258)
at
org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.$anonfun$resolveAssignments$1(HoodieSpark3Analysis.scala:238)
at
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at
org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.resolveAssignments(HoodieSpark3Analysis.scala:235)
at
org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences$$anonfun$apply$1.$anonfun$applyOrElse$2(HoodieSpark3Analysis.scala:181)
at
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at
org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences$$anonfun$apply$1.applyOrElse(HoodieSpark3Analysis.scala:170)
at
org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences$$anonfun$apply$1.applyOrElse(HoodieSpark3Analysis.scala:59)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsUpWithPruning$3(AnalysisHelper.scala:138)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:76)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsUpWithPruning$1(AnalysisHelper.scala:138)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.allowInvokingTransformsInAnalyzer(AnalysisHelper.scala:323)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUpWithPruning(AnalysisHelper.scala:134)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUpWithPruning$(AnalysisHelper.scala:130)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperatorsUpWithPruning(LogicalPlan.scala:32)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUp(AnalysisHelper.scala:111)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUp$(AnalysisHelper.scala:110)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperatorsUp(LogicalPlan.scala:32)
at
org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.apply(HoodieSpark3Analysis.scala:59)
at
org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.apply(HoodieSpark3Analysis.scala:56)
{code}
was:For the unresolved MERGE INTO statement
> Fix error message of unresolved columns in MERGE INTO on Spark 3.5
> ------------------------------------------------------------------
>
> Key: HUDI-8865
> URL: https://issues.apache.org/jira/browse/HUDI-8865
> Project: Apache Hudi
> Issue Type: Sub-task
> Reporter: Y Ethan Guo
> Assignee: Y Ethan Guo
> Priority: Blocker
> Labels: pull-request-available
> Fix For: 1.0.1
>
> Time Spent: 2h
> Remaining Estimate: 0h
>
> If there are unresolved columns in MERGE INTO statement using Hudi, currently
> Spark 3.5 throws the exception below which is not user-friendly:
> {code:java}
> Caused by: org.apache.spark.SparkException: [INTERNAL_ERROR] Cannot find main
> error class '_LEGACY_ERROR_TEMP_2309'
> at org.apache.spark.SparkException$.internalError(SparkException.scala:92)
> at org.apache.spark.SparkException$.internalError(SparkException.scala:96)
> at
> org.apache.spark.ErrorClassesJsonReader.$anonfun$getMessageTemplate$1(ErrorClassesJSONReader.scala:68)
> at
> scala.collection.immutable.HashMap$HashMap1.getOrElse0(HashMap.scala:361)
> at
> scala.collection.immutable.HashMap$HashTrieMap.getOrElse0(HashMap.scala:594)
> at
> scala.collection.immutable.HashMap$HashTrieMap.getOrElse0(HashMap.scala:589)
> at scala.collection.immutable.HashMap.getOrElse(HashMap.scala:73)
> at
> org.apache.spark.ErrorClassesJsonReader.getMessageTemplate(ErrorClassesJSONReader.scala:68)
> at
> org.apache.spark.ErrorClassesJsonReader.getErrorMessage(ErrorClassesJSONReader.scala:47)
> at
> org.apache.spark.SparkThrowableHelper$.getMessage(SparkThrowableHelper.scala:53)
> at
> org.apache.spark.SparkThrowableHelper$.getMessage(SparkThrowableHelper.scala:40)
> at
> org.apache.spark.sql.AnalysisException.<init>(AnalysisException.scala:77)
> at
> org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:52)
> at
> org.apache.spark.sql.HoodieSpark35CatalystPlanUtils$.failAnalysisForMIT(HoodieSpark35CatalystPlanUtils.scala:80)
> at
> org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.$anonfun$resolveMergeExprOrFail$2(HoodieSpark3Analysis.scala:263)
> at
> org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.$anonfun$resolveMergeExprOrFail$2$adapted(HoodieSpark3Analysis.scala:258)
> at scala.collection.mutable.LinkedHashSet.foreach(LinkedHashSet.scala:95)
> at
> org.apache.spark.sql.catalyst.expressions.AttributeSet.foreach(AttributeSet.scala:137)
> at
> org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.resolveMergeExprOrFail(HoodieSpark3Analysis.scala:258)
> at
> org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.$anonfun$resolveAssignments$1(HoodieSpark3Analysis.scala:238)
> at
> scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
> at
> scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
> at
> scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
> at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
> at scala.collection.TraversableLike.map(TraversableLike.scala:286)
> at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
> at scala.collection.AbstractTraversable.map(Traversable.scala:108)
> at
> org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.resolveAssignments(HoodieSpark3Analysis.scala:235)
> at
> org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences$$anonfun$apply$1.$anonfun$applyOrElse$2(HoodieSpark3Analysis.scala:181)
> at
> scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
> at
> scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
> at
> scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
> at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
> at scala.collection.TraversableLike.map(TraversableLike.scala:286)
> at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
> at scala.collection.AbstractTraversable.map(Traversable.scala:108)
> at
> org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences$$anonfun$apply$1.applyOrElse(HoodieSpark3Analysis.scala:170)
> at
> org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences$$anonfun$apply$1.applyOrElse(HoodieSpark3Analysis.scala:59)
> at
> org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsUpWithPruning$3(AnalysisHelper.scala:138)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:76)
> at
> org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsUpWithPruning$1(AnalysisHelper.scala:138)
> at
> org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.allowInvokingTransformsInAnalyzer(AnalysisHelper.scala:323)
> at
> org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUpWithPruning(AnalysisHelper.scala:134)
> at
> org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUpWithPruning$(AnalysisHelper.scala:130)
> at
> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperatorsUpWithPruning(LogicalPlan.scala:32)
> at
> org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUp(AnalysisHelper.scala:111)
> at
> org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsUp$(AnalysisHelper.scala:110)
> at
> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperatorsUp(LogicalPlan.scala:32)
> at
> org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.apply(HoodieSpark3Analysis.scala:59)
> at
> org.apache.spark.sql.hudi.analysis.HoodieSpark3ResolveReferences.apply(HoodieSpark3Analysis.scala:56)
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
