prashantwason commented on code in PR #8523:
URL: https://github.com/apache/hudi/pull/8523#discussion_r1178663188
##########
hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java:
##########
@@ -2718,4 +2595,40 @@ private HoodieLogFormat.Reader
createCorruptedFile(String fileId) throws Excepti
return reader;
}
+
+ private void checkLogBlocksAndKeys(String latestInstantTime, Schema schema,
boolean readBlocksLazily,
+ ExternalSpillableMap.DiskMapType diskMapType, boolean
isCompressionEnabled, boolean enableOptimizedLogBlocksScan, int
expectedTotalRecords,
+ int expectedTotalKeys, Option<Set<String>> expectedKeys) throws
IOException {
+ List<String> allLogFiles =
+ FSUtils.getAllLogFiles(fs, partitionPath, "test-fileid1",
HoodieLogFile.DELTA_EXTENSION, "100")
+ .map(s -> s.getPath().toString()).collect(Collectors.toList());
+
+ HoodieMergedLogRecordScanner scanner =
HoodieMergedLogRecordScanner.newBuilder()
+ .withFileSystem(fs)
+ .withBasePath(basePath)
+ .withLogFilePaths(allLogFiles)
+ .withReaderSchema(schema)
+ .withLatestInstantTime(latestInstantTime)
+ .withMaxMemorySizeInBytes(10240L)
+ .withReadBlocksLazily(readBlocksLazily)
+ .withReverseReader(false)
+ .withBufferSize(BUFFER_SIZE)
+ .withSpillableMapBasePath(spillableBasePath)
+ .withDiskMapType(diskMapType)
+ .withBitCaskDiskMapCompressionEnabled(isCompressionEnabled)
+ .withOptimizedLogBlocksScan(enableOptimizedLogBlocksScan)
+
.withRecordMerger(HoodieRecordUtils.loadRecordMerger(HoodieAvroRecordMerger.class.getName()))
+ .build();
+ assertEquals(expectedTotalRecords, scanner.getTotalLogRecords(), "There
should be " + expectedTotalRecords + " records");
+ final Set<String> readKeys = new HashSet<>();
+ scanner.forEach(s -> readKeys.add(s.getKey().getRecordKey()));
+ assertEquals(expectedTotalKeys, readKeys.size(), "Read should return
return all " + expectedTotalKeys + " keys");
+
+ if (expectedKeys.isPresent()) {
+ assertEquals(expectedKeys.get(), readKeys, "Keys read from log file
should match written keys");
+ }
+ scanner.close();
Review Comment:
Enclosed in try
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
