anuragmantri commented on code in PR #57227:
URL: https://github.com/apache/spark/pull/57227#discussion_r3607007161
##########
sql/core/src/test/scala/org/apache/spark/sql/connector/MergeIntoTableSuiteBase.scala:
##########
@@ -2851,6 +2853,127 @@ abstract class MergeIntoTableSuiteBase extends
RowLevelOperationSuiteBase
assert(catalog.lastTransaction.isClosed)
}
+ test("SPARK-58007: merge with dynamic options on the target") {
+ withTable(sourceNameAsString) {
+ createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+ """{ "pk": 1, "salary": 100, "dep": "hr" }
+ |{ "pk": 2, "salary": 200, "dep": "software" }
+ |""".stripMargin)
+ sql(s"CREATE TABLE $sourceNameAsString (pk INT NOT NULL, salary INT, dep
STRING)")
+ sql(s"INSERT INTO $sourceNameAsString VALUES (1, 150, 'hr'), (3, 300,
'hr')")
+
+ // the target options become the row-level operation options: they reach
the rewritten
+ // DataSourceV2Relation, the RowLevelOperationInfo, and the write
builder's LogicalWriteInfo
+ val Seq(qe) = withQueryExecutionsCaptured(spark) {
+ sql(
+ s"""MERGE INTO $tableNameAsString t WITH (`write.split-size` = 10)
+ |USING $sourceNameAsString s
+ |ON t.pk = s.pk
+ |WHEN MATCHED THEN UPDATE SET t.salary = s.salary
Review Comment:
I added a test for the insert only fast path and the options survived.
Thanks for the suggestion.
##########
sql/core/src/test/scala/org/apache/spark/sql/connector/MergeIntoTableSuiteBase.scala:
##########
@@ -2851,6 +2853,127 @@ abstract class MergeIntoTableSuiteBase extends
RowLevelOperationSuiteBase
assert(catalog.lastTransaction.isClosed)
}
+ test("SPARK-58007: merge with dynamic options on the target") {
+ withTable(sourceNameAsString) {
+ createAndInitTable("pk INT NOT NULL, salary INT, dep STRING",
+ """{ "pk": 1, "salary": 100, "dep": "hr" }
+ |{ "pk": 2, "salary": 200, "dep": "software" }
+ |""".stripMargin)
+ sql(s"CREATE TABLE $sourceNameAsString (pk INT NOT NULL, salary INT, dep
STRING)")
+ sql(s"INSERT INTO $sourceNameAsString VALUES (1, 150, 'hr'), (3, 300,
'hr')")
+
+ // the target options become the row-level operation options: they reach
the rewritten
+ // DataSourceV2Relation, the RowLevelOperationInfo, and the write
builder's LogicalWriteInfo
+ val Seq(qe) = withQueryExecutionsCaptured(spark) {
+ sql(
+ s"""MERGE INTO $tableNameAsString t WITH (`write.split-size` = 10)
+ |USING $sourceNameAsString s
+ |ON t.pk = s.pk
+ |WHEN MATCHED THEN UPDATE SET t.salary = s.salary
+ |WHEN NOT MATCHED THEN INSERT *
+ |""".stripMargin)
+ }
+ val writeRelation = qe.optimizedPlan.collectFirst {
+ case rd: ReplaceData => rd.table
+ case wd: WriteDelta => wd.table
+ }.getOrElse(fail("couldn't find row-level operation in optimized plan"))
+ .asInstanceOf[DataSourceV2Relation]
+ val operation =
writeRelation.table.asInstanceOf[RowLevelOperationTable].operation
+ .asInstanceOf[RowLevelOperationWithOptions]
+ assert(writeRelation.options.get("write.split-size") === "10", "relation
option")
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
