danny0405 commented on code in PR #12601:
URL: https://github.com/apache/hudi/pull/12601#discussion_r1917613978
##########
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java:
##########
@@ -344,6 +350,82 @@ public void testCompactionSpecifyPartition(String regex,
List<String> expectedCo
}
}
+ @Test
+ public void testPartitionsForIncrCompaction() throws Exception {
+ HoodieWriteConfig config = getConfigBuilder()
+ .withIncrementalTableServiceEnable(true)
+ .withCompactionConfig(HoodieCompactionConfig
+ .newBuilder()
+ .withCompactionStrategy(new SmallBoundedIOCompactionStrategy())
+ .withMaxNumDeltaCommitsBeforeCompaction(2).build())
+ .build();
+ String[] partitions = {"20250115"};
+
+ HoodieWriteConfig config2 = getConfigBuilder()
+ .withIncrementalTableServiceEnable(true)
+ .withCompactionConfig(HoodieCompactionConfig
+ .newBuilder()
+ .withMaxNumDeltaCommitsBeforeCompaction(2).build())
+ .build();
+ String[] partitions2 = {"20250116"};
+
+ try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config)) {
+ prepareRecords(writeClient, config, partitions);
+ prepareRecords(writeClient, config, partitions);
+ compact(writeClient, writeClient.createNewInstantTime());
+ HoodieCompactionPlan compactionPlan1 = getLatestCompactionPlan();
+
+ List<String> affectedPartitions =
compactionPlan1.getOperations().stream()
+
.map(HoodieCompactionOperation::getPartitionPath).collect(Collectors.toList());
+ // compaction including 20250115
+ assertTrue(affectedPartitions.contains(partitions[0]));
+ List<String> missingPartitions =
compactionPlan1.getMissingSchedulePartitions();
+ // compaction missing 20250115 because not all the fileSlices are all
processed.
+ assertTrue(missingPartitions.contains(partitions[0]));
+ }
+
+ try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config2)) {
+ prepareRecords(writeClient, config2, partitions2);
+ prepareRecords(writeClient, config2, partitions2);
+ compact(writeClient, writeClient.createNewInstantTime());
+ HoodieCompactionPlan compactionPlan2 = getLatestCompactionPlan();
+ List<String> affectedPartitions2 =
compactionPlan2.getOperations().stream()
+
.map(HoodieCompactionOperation::getPartitionPath).collect(Collectors.toList());
+ // compaction including 20250115 (fetched from recorded missing
partitions)
+ assertTrue(affectedPartitions2.contains(partitions[0]));
Review Comment:
do we test the fallback for archived commits?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
