kl0u commented on a change in pull request #9228: [FLINK-13428][Connectors /
FileSystem] allow part file names to be configurable
URL: https://github.com/apache/flink/pull/9228#discussion_r309676496
##########
File path:
flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkWriterTest.java
##########
@@ -62,44 +62,80 @@ public void testCustomBulkWriter() throws Exception {
new
DefaultBucketFactoryImpl<>())
) {
- testHarness.setup();
- testHarness.open();
-
- // this creates a new bucket "test1" and part-0-0
- testHarness.processElement(new
StreamRecord<>(Tuple2.of("test1", 1), 1L));
- TestUtils.checkLocalFs(outDir, 1, 0);
-
- // we take a checkpoint so we roll.
- testHarness.snapshot(1L, 1L);
-
- // these will close part-0-0 and open part-0-1
- testHarness.processElement(new
StreamRecord<>(Tuple2.of("test1", 2), 2L));
- testHarness.processElement(new
StreamRecord<>(Tuple2.of("test1", 3), 3L));
-
- // we take a checkpoint so we roll again.
- testHarness.snapshot(2L, 2L);
-
- TestUtils.checkLocalFs(outDir, 2, 0);
-
- Map<File, String> contents =
TestUtils.getFileContentByPath(outDir);
- int fileCounter = 0;
- for (Map.Entry<File, String> fileContents :
contents.entrySet()) {
- if
(fileContents.getKey().getName().contains(".part-0-0.inprogress")) {
- fileCounter++;
- Assert.assertEquals("test1@1\n",
fileContents.getValue());
- } else if
(fileContents.getKey().getName().contains(".part-0-1.inprogress")) {
- fileCounter++;
-
Assert.assertEquals("test1@2\ntest1@3\n", fileContents.getValue());
- }
+ testPartFiles(testHarness, outDir,
".part-0-0.inprogress", ".part-0-1.inprogress");
+
+ }
+ }
+
+ @Test
+ public void testCustomBulkWriterWithPartConfig() throws Exception {
+ final File outDir = TEMP_FOLDER.newFolder();
+
+ // we set the max bucket size to small so that we can know when
it rolls
+ try (
+ OneInputStreamOperatorTestHarness<Tuple2<String,
Integer>, Object> testHarness =
+ TestUtils.createTestSinkWithBulkEncoder(
+ outDir,
+ 1,
+ 0,
+ 10L,
+ new TestUtils.TupleToStringBucketer(),
+ new TestBulkWriterFactory(),
+ new DefaultBucketFactoryImpl<>(),
+ "prefix-",
+ ".ext")
+ ) {
+
Review comment:
Same here.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
