xushiyan commented on a change in pull request #2167:
URL: https://github.com/apache/hudi/pull/2167#discussion_r502854170
##########
File path:
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
##########
@@ -1098,73 +1097,82 @@ public void testCleanPreviousCorruptedCleanFiles()
throws IOException {
* @param expNumFilesDeleted Number of files deleted
*/
private void testPendingCompactions(HoodieWriteConfig config, int
expNumFilesDeleted,
- int expNumFilesUnderCompactionDeleted, boolean retryFailure) throws
IOException {
+ int expNumFilesUnderCompactionDeleted, boolean retryFailure) throws
Exception {
HoodieTableMetaClient metaClient =
HoodieTestUtils.init(hadoopConf, basePath,
HoodieTableType.MERGE_ON_READ);
- String[] instants = new String[] {"000", "001", "003", "005", "007",
"009", "011", "013"};
- String[] compactionInstants = new String[] {"002", "004", "006", "008",
"010"};
- Map<String, String> expFileIdToPendingCompaction = new HashMap<>();
- Map<String, String> fileIdToLatestInstantBeforeCompaction = new
HashMap<>();
- Map<String, List<FileSlice>> compactionInstantsToFileSlices = new
HashMap<>();
-
- for (String instant : instants) {
- HoodieTestUtils.createCommitFiles(basePath, instant);
- }
+ final String partition = "2016/03/15";
+ Map<String, String> expFileIdToPendingCompaction = new HashMap<String,
String>() {
+ {
+ put("fileId2", "004");
+ put("fileId3", "006");
+ put("fileId4", "008");
+ put("fileId5", "010");
+ }
+ };
+ Map<String, String> fileIdToLatestInstantBeforeCompaction = new
HashMap<String, String>() {
+ {
+ put("fileId1", "000");
+ put("fileId2", "000");
+ put("fileId3", "001");
+ put("fileId4", "003");
+ put("fileId5", "005");
+ put("fileId6", "009");
+ put("fileId7", "011");
+ }
+ };
// Generate 7 file-groups. First one has only one slice and no pending
compaction. File Slices (2 - 5) has
// multiple versions with pending compaction. File Slices (6 - 7) have
multiple file-slices but not under
// compactions
// FileIds 2-5 will be under compaction
- int maxNumFileIds = 7;
- String[] fileIds = new String[] {"fileId1", "fileId2", "fileId3",
"fileId4", "fileId5", "fileId6", "fileId7"};
- int maxNumFileIdsForCompaction = 4;
- for (int i = 0; i < maxNumFileIds; i++) {
- final String fileId = HoodieTestUtils.createDataFile(basePath,
- HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0],
fileIds[i]);
- HoodieTestUtils.createNewLogFile(fs, basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0],
- fileId, Option.empty());
- HoodieTestUtils.createNewLogFile(fs, basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0],
- fileId, Option.of(2));
- fileIdToLatestInstantBeforeCompaction.put(fileId, instants[0]);
- for (int j = 1; j <= i; j++) {
- if (j == i && j <= maxNumFileIdsForCompaction) {
- expFileIdToPendingCompaction.put(fileId, compactionInstants[j]);
- metaClient = HoodieTableMetaClient.reload(metaClient);
- HoodieTable table = HoodieSparkTable.create(config, context,
metaClient);
- FileSlice slice =
-
table.getSliceView().getLatestFileSlices(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)
- .filter(fs ->
fs.getFileId().equals(fileId)).findFirst().get();
- List<FileSlice> slices = new ArrayList<>();
- if
(compactionInstantsToFileSlices.containsKey(compactionInstants[j])) {
- slices = compactionInstantsToFileSlices.get(compactionInstants[j]);
- }
- slices.add(slice);
- compactionInstantsToFileSlices.put(compactionInstants[j], slices);
- // Add log-files to simulate delta-commits after pending compaction
- HoodieTestUtils.createNewLogFile(fs, basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- compactionInstants[j], fileId, Option.empty());
- HoodieTestUtils.createNewLogFile(fs, basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- compactionInstants[j], fileId, Option.of(2));
- } else {
- HoodieTestUtils.createDataFile(basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[j],
- fileId);
- HoodieTestUtils.createNewLogFile(fs, basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- instants[j], fileId, Option.empty());
- HoodieTestUtils.createNewLogFile(fs, basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- instants[j], fileId, Option.of(2));
- fileIdToLatestInstantBeforeCompaction.put(fileId, instants[j]);
- }
- }
- }
-
- // Setup pending compaction plans
- for (String instant : compactionInstants) {
- List<FileSlice> fileSliceList =
compactionInstantsToFileSlices.get(instant);
- if (null != fileSliceList) {
- HoodieTestUtils.createCompactionRequest(metaClient, instant,
fileSliceList.stream()
- .map(fs ->
Pair.of(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
fs)).collect(Collectors.toList()));
- }
- }
Review comment:
@yanghua The original logic of test prep is incredibly difficult to
understand. Please avoid reading it. I simply compared the output files in temp
directory and made sure they were equivalent before and after the change.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
