aokolnychyi commented on a change in pull request #4023:
URL: https://github.com/apache/iceberg/pull/4023#discussion_r796999380
##########
File path:
spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java
##########
@@ -75,6 +78,52 @@ public void removeTables() {
sql("DROP TABLE IF EXISTS source");
}
+ @Test
+ public void testMergeWithStaticPredicatePushDown() {
+ createAndInitTable("id BIGINT, dep STRING");
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD dep", tableName);
+
+ append(tableName,
+ "{ \"id\": 1, \"dep\": \"software\" }\n" +
+ "{ \"id\": 11, \"dep\": \"software\" }\n" +
+ "{ \"id\": 1, \"dep\": \"hr\" }");
+
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Snapshot snapshot = table.currentSnapshot();
+ String dataFilesCount =
snapshot.summary().get(SnapshotSummary.TOTAL_DATA_FILES_PROP);
+ Assert.assertEquals("Must have 2 files before MERGE", "2", dataFilesCount);
+
+ createOrReplaceView("source",
+ "{ \"id\": 1, \"dep\": \"finance\" }\n" +
+ "{ \"id\": 2, \"dep\": \"hardware\" }");
+
+ // disable dynamic pruning and rely only on static predicate pushdown
+
withSQLConf(ImmutableMap.of(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED().key(),
"false"), () -> {
+ sql("MERGE INTO %s t USING source " +
+ "ON t.id == source.id AND t.dep IN ('software') AND source.id < 10 "
+
+ "WHEN MATCHED AND source.id = 1 THEN " +
+ " UPDATE SET dep = source.dep " +
+ "WHEN NOT MATCHED THEN " +
+ " INSERT (dep, id) VALUES (source.dep, source.id)", tableName);
+ });
+
+ table.refresh();
+
+ Snapshot mergeSnapshot = table.currentSnapshot();
+ String deletedDataFilesCount =
mergeSnapshot.summary().get(SnapshotSummary.DELETED_FILES_PROP);
+ Assert.assertEquals("Must overwrite only 1 file", "1",
deletedDataFilesCount);
Review comment:
I think I missed that. Could you point me to an example?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
