zhoulii commented on code in PR #9489:
URL: https://github.com/apache/paimon/pull/9489#discussion_r3893415496
##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/MergeIntoPaimonDataEvolutionTable.scala:
##########
@@ -150,25 +151,90 @@ case class MergeIntoPaimonDataEvolutionTable(
*
* without any extra shuffle, join, or sort.
*/
- private lazy val isSelfMergeOnRowId: Boolean = {
- if (!isPaimonTable(sourceTable)) {
- false
- } else if (
-
!originalTargetRelation.name.equals(PaimonRelation.getPaimonRelation(sourceTable).name)
- ) {
- false
+ private case class SelfMergeSpec(residualCondition: Option[Expression])
+
+ private def passthroughSourceRelation(plan: LogicalPlan):
Option[DataSourceV2Relation] = {
+ EliminateSubqueryAliases(plan) match {
+ case relation: DataSourceV2Relation if
relation.table.isInstanceOf[SparkTable] =>
+ Some(relation)
+ case Project(projectList, child) if isPassthroughProject(projectList,
child) =>
+ passthroughSourceRelation(child)
+ case _ =>
+ None
+ }
+ }
+
+ private def isPassthroughProject(projectList: Seq[Expression], child:
LogicalPlan): Boolean = {
+ val childAttributes = child.output ++ child.metadataOutput
+
+ def isChildAttribute(attr: AttributeReference): Boolean =
+ childAttributes.exists(_.exprId == attr.exprId)
+
+ projectList.forall {
+ case attr: AttributeReference => isChildAttribute(attr)
+ case alias: Alias =>
+ alias.child match {
+ case attr: AttributeReference =>
+ resolver(alias.name, attr.name) && isChildAttribute(attr)
+ case _ => false
+ }
+ case _ => false
+ }
+ }
+
+ private lazy val sameSourceAndTargetTable: Boolean =
+ passthroughSourceRelation(sourceTable)
+ .exists(sourceRelation =>
originalTargetRelation.name.equals(sourceRelation.name))
Review Comment:
Thanks for catching this. Similar to the issue discussed in the comment
above, this was also a pre-existing problem in the original `_ROW_ID` shortcut,
rather than being introduced by this PR.
Following your suggestion, I made the minimal fix to disable the shortcut
for Paimon time-travel sources, so they fall back to regular MERGE and preserve
snapshot semantics. The same fix has been applied to Spark 4, with a regression
test added.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
