This is an automated email from the ASF dual-hosted git repository. yao pushed a commit to branch branch-3.5 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push: new eb1e6ad13aab [SPARK-46388][SQL] HiveAnalysis misses the pattern guard of `query.resolved` eb1e6ad13aab is described below commit eb1e6ad13aab3960f1543b75bf3b75b3a7d62746 Author: Kent Yao <y...@apache.org> AuthorDate: Wed Dec 13 18:04:38 2023 +0800 [SPARK-46388][SQL] HiveAnalysis misses the pattern guard of `query.resolved` ### What changes were proposed in this pull request? This PR adds `query.resolved` as a pattern guard when HiveAnalysis converts InsertIntoStatement to InsertIntoHiveTable. ### Why are the changes needed? Due to https://github.com/apache/spark/pull/41262/files#diff-ed19f376a63eba52eea59ca71f3355d4495fad4fad4db9a3324aade0d4986a47R1080, the `table` field is resolved regardless of the query field. Before, it never got a chance to be resolved as `HiveTableRelation` and then match any rule of HiveAnalysis. But now, it gets the chance always and results in a spark-kernel bug - `Invalid call to toAttribute on unresolved object.` ``` insert into t2 select cast(a as short) from t where b=1; Invalid call to toAttribute on unresolved object ``` ### Does this PR introduce _any_ user-facing change? no, bugfix for 3.5 and later ### How was this patch tested? added new test ### Was this patch authored or co-authored using generative AI tooling? no Closes #44326 from yaooqinn/SPARK-46388. Authored-by: Kent Yao <y...@apache.org> Signed-off-by: Kent Yao <y...@apache.org> (cherry picked from commit ccc436d829cd0b07088e2864cb1ecc55ab97a491) Signed-off-by: Kent Yao <y...@apache.org> --- .../org/apache/spark/sql/hive/HiveStrategies.scala | 2 +- .../spark/sql/hive/execution/SQLQuerySuite.scala | 26 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala index 3da3d4a0eb5c..c53a6c378d45 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala @@ -161,7 +161,7 @@ object HiveAnalysis extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators { case InsertIntoStatement( r: HiveTableRelation, partSpec, _, query, overwrite, ifPartitionNotExists, _) - if DDLUtils.isHiveTable(r.tableMeta) => + if DDLUtils.isHiveTable(r.tableMeta) && query.resolved => InsertIntoHiveTable(r.tableMeta, partSpec, query, overwrite, ifPartitionNotExists, query.output.map(_.name)) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 9308d1eda146..6160c3e5f6c6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -2660,6 +2660,32 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi checkAnswer(df, Seq.empty[Row]) } } + + test("SPARK-46388: HiveAnalysis convert InsertIntoStatement to InsertIntoHiveTable " + + "iff child resolved") { + withTable("t") { + sql("CREATE TABLE t (a STRING)") + checkError( + exception = intercept[AnalysisException](sql("INSERT INTO t SELECT a*2 FROM t where b=1")), + errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", + sqlState = None, + parameters = Map("objectName" -> "`b`", "proposal" -> "`a`"), + context = ExpectedContext( + fragment = "b", + start = 38, + stop = 38) ) + checkError( + exception = intercept[AnalysisException]( + sql("INSERT INTO t SELECT cast(a as short) FROM t where b=1")), + errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", + sqlState = None, + parameters = Map("objectName" -> "`b`", "proposal" -> "`a`"), + context = ExpectedContext( + fragment = "b", + start = 51, + stop = 51)) + } + } } @SlowHiveTest --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org