yihua commented on code in PR #10694:
URL: https://github.com/apache/hudi/pull/10694#discussion_r1495062939
##########
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestS3EventsHoodieIncrSource.java:
##########
@@ -287,32 +289,44 @@ public void testTwoFilesAndContinueInSameCommit() throws
IOException {
}
- @Test
- public void testTwoFilesAndContinueAcrossCommits() throws IOException {
+ @ParameterizedTest
+ @ValueSource(strings = {
+ ".json",
+ ".gz"
+ })
+ public void testTwoFilesAndContinueAcrossCommits(String extension) throws
IOException {
String commitTimeForWrites = "2";
String commitTimeForReads = "1";
Pair<String, List<HoodieRecord>> inserts =
writeS3MetadataRecords(commitTimeForReads);
inserts = writeS3MetadataRecords(commitTimeForWrites);
+ TypedProperties typedProperties = setProps(READ_UPTO_LATEST_COMMIT);
+ // In the case the extension is explicitly set to something other than the
file format.
+ if (!extension.endsWith("json")) {
+
typedProperties.setProperty(CloudSourceConfig.CLOUD_DATAFILE_EXTENSION.key(),
extension);
+ }
List<Triple<String, Long, String>> filePathSizeAndCommitTime = new
ArrayList<>();
// Add file paths and sizes to the list
- filePathSizeAndCommitTime.add(Triple.of("path/to/file1.json", 100L, "1"));
- filePathSizeAndCommitTime.add(Triple.of("path/to/file3.json", 200L, "1"));
- filePathSizeAndCommitTime.add(Triple.of("path/to/file2.json", 150L, "1"));
- filePathSizeAndCommitTime.add(Triple.of("path/to/file4.json", 50L, "2"));
- filePathSizeAndCommitTime.add(Triple.of("path/to/file5.json", 150L, "2"));
+ filePathSizeAndCommitTime.add(Triple.of(String.format("path/to/file1%s",
extension), 100L, "1"));
Review Comment:
Add a file that does not end with the extension to test the filter?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
