XiaoHongbo-Hope commented on code in PR #6551:
URL: https://github.com/apache/paimon/pull/6551#discussion_r2555477669
##########
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCaseBase.java:
##########
@@ -392,6 +457,466 @@ public void testRunWithMode(boolean isNamedArgument)
throws Exception {
.hasMessageContaining("Unknown mode");
}
+ @org.junit.jupiter.api.Test
+ public void testCombinedMode() throws Exception {
+ long fileCreationTime = System.currentTimeMillis();
+ FileStoreTable table1 = createTableAndWriteData("batchTable1");
+ FileStoreTable table2 = createTableAndWriteData("batchTable2");
+ FileStoreTable table3 = createTableAndWriteData("batchTable3");
+
+ FileIO fileIO1 = table1.fileIO();
+ FileIO fileIO2 = table2.fileIO();
+ FileIO fileIO3 = table3.fileIO();
+
+ Path orphanFile1Table1 = getOrphanFilePath(table1, ORPHAN_FILE_1);
+ Path orphanFile2Table1 = getOrphanFilePath(table1, ORPHAN_FILE_2);
+ Path orphanFile1Table2 = getOrphanFilePath(table2, ORPHAN_FILE_1);
+ Path orphanFile2Table2 = getOrphanFilePath(table2, ORPHAN_FILE_2);
+ Path orphanFile1Table3 = getOrphanFilePath(table3, ORPHAN_FILE_1);
+ Path orphanFile2Table3 = getOrphanFilePath(table3, ORPHAN_FILE_2);
+
+ Path[] orphanFiles = {
+ orphanFile1Table1, orphanFile2Table1,
+ orphanFile1Table2, orphanFile2Table2,
+ orphanFile1Table3, orphanFile2Table3
+ };
+ FileIO[] fileIOs = {fileIO1, fileIO1, fileIO2, fileIO2, fileIO3,
fileIO3};
+
+ Thread.sleep(2000);
+
+ long currentTime = System.currentTimeMillis();
+ long olderThanMillis = Math.max(fileCreationTime + 1000, currentTime -
1000);
+ String olderThan =
+ DateTimeUtils.formatLocalDateTime(
+ DateTimeUtils.toLocalDateTime(olderThanMillis), 3);
+
+ long expectedFileCount = 6;
+ long expectedTotalSize = 0;
+ for (int i = 0; i < orphanFiles.length; i++) {
+ if (fileIOs[i].exists(orphanFiles[i])) {
+ expectedTotalSize += fileIOs[i].getFileSize(orphanFiles[i]);
+ }
+ }
+
+ // Test divided mode
+ String dividedMode =
+ String.format(
+ "CALL sys.remove_orphan_files('%s.%s', '%s', false)",
+ database, "*", olderThan);
+ ImmutableList<Row> dividedModeResult =
ImmutableList.copyOf(executeSQL(dividedMode));
+ assertThat(dividedModeResult).hasSize(2);
+ long deletedFileCountWithDivided =
+
Long.parseLong(dividedModeResult.get(0).getField(0).toString());
+ long deletedFileTotalLenInBytesWithDivided =
+
Long.parseLong(dividedModeResult.get(1).getField(0).toString());
+ assertThat(deletedFileCountWithDivided)
+ .as("divided mode should delete 6 orphan files")
+ .isEqualTo(expectedFileCount);
+ assertThat(deletedFileTotalLenInBytesWithDivided)
+ .as("divided mode should delete files with expected total
size")
+ .isEqualTo(expectedTotalSize);
+
+ // Verify files are deleted by divided mode
+ for (int i = 0; i < orphanFiles.length; i++) {
+ assertThat(fileIOs[i].exists(orphanFiles[i]))
+ .as("Orphan file should be deleted by divided mode")
+ .isFalse();
+ }
+
+ // Recreate orphan files for combined mode test
+ long combinedFileCreationTime = System.currentTimeMillis();
+ for (int i = 0; i < orphanFiles.length; i++) {
+ fileIOs[i].writeFile(orphanFiles[i], "orphan", true);
+ }
+ Thread.sleep(2000);
+
+ long combinedCurrentTime = System.currentTimeMillis();
+ long combinedOlderThanMillis =
+ Math.max(combinedFileCreationTime + 1000, combinedCurrentTime
- 1000);
+ String combinedOlderThan =
+ DateTimeUtils.formatLocalDateTime(
+
DateTimeUtils.toLocalDateTime(combinedOlderThanMillis), 3);
+
+ // Test combined mode
+ List<String> args =
+ new ArrayList<>(
+ Arrays.asList(
+ "remove_orphan_files",
+ "--warehouse",
+ warehouse,
+ "--database",
+ database,
+ "--table",
+ "*",
+ "--mode",
+ "combined",
+ "--dry_run",
+ "false",
+ "--older_than",
+ combinedOlderThan));
+ RemoveOrphanFilesAction action1 =
createAction(RemoveOrphanFilesAction.class, args);
+ assertThatCode(action1::run).doesNotThrowAnyException();
+
+ // Verify files are deleted by combined mode (same result as divided
mode)
+ for (int i = 0; i < orphanFiles.length; i++) {
+ assertThat(fileIOs[i].exists(orphanFiles[i]))
+ .as("Orphan file should be deleted by combined mode (same
as divided mode)")
+ .isFalse();
+ }
+
+ // Verify that normal data in tables can still be read after combined
mode deletion
+ List<String> table1Data = readTableData(table1);
+ assertThat(table1Data)
+ .as("Table1 should still contain normal data after combined
mode deletion")
+ .containsExactly("+I[1, Hi]");
+
+ List<String> table2Data = readTableData(table2);
+ assertThat(table2Data)
+ .as("Table2 should still contain normal data after combined
mode deletion")
+ .containsExactly("+I[1, Hi]");
+
+ List<String> table3Data = readTableData(table3);
+ assertThat(table3Data)
+ .as("Table3 should still contain normal data after combined
mode deletion")
+ .containsExactly("+I[1, Hi]");
+ }
+
+ @org.junit.jupiter.api.Test
+ public void testCombinedModeWithBranch() throws Exception {
+ long fileCreationTime = System.currentTimeMillis();
+
+ FileStoreTable table = createTableAndWriteData("combinedBranchTable");
+
+ // Create first branch and write data
+ table.createBranch("br1");
+ FileStoreTable branchTable1 = createBranchTable(table, "br1");
+ writeToBranch(branchTable1, GenericRow.of(2L,
BinaryString.fromString("Hello"), 20));
+
+ // Create second branch and write data
+ table.createBranch("br2");
+ FileStoreTable branchTable2 = createBranchTable(table, "br2");
+ writeToBranch(branchTable2, GenericRow.of(3L,
BinaryString.fromString("World"), 30));
+
+ // Create orphan files in both branch snapshot directories
+ // This is key: same table, multiple branches - will trigger bug in
Review Comment:
> Is this comment fixed?
fixed and removed expired comment
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
