kl0u commented on a change in pull request #9228: [FLINK-13428][Connectors /
FileSystem] allow part file names to be configurable
URL: https://github.com/apache/flink/pull/9228#discussion_r309676818
##########
File path:
flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkWriterTest.java
##########
@@ -62,44 +62,80 @@ public void testCustomBulkWriter() throws Exception {
new
DefaultBucketFactoryImpl<>())
) {
- testHarness.setup();
- testHarness.open();
-
- // this creates a new bucket "test1" and part-0-0
- testHarness.processElement(new
StreamRecord<>(Tuple2.of("test1", 1), 1L));
- TestUtils.checkLocalFs(outDir, 1, 0);
-
- // we take a checkpoint so we roll.
- testHarness.snapshot(1L, 1L);
-
- // these will close part-0-0 and open part-0-1
- testHarness.processElement(new
StreamRecord<>(Tuple2.of("test1", 2), 2L));
- testHarness.processElement(new
StreamRecord<>(Tuple2.of("test1", 3), 3L));
-
- // we take a checkpoint so we roll again.
- testHarness.snapshot(2L, 2L);
-
- TestUtils.checkLocalFs(outDir, 2, 0);
-
- Map<File, String> contents =
TestUtils.getFileContentByPath(outDir);
- int fileCounter = 0;
- for (Map.Entry<File, String> fileContents :
contents.entrySet()) {
- if
(fileContents.getKey().getName().contains(".part-0-0.inprogress")) {
- fileCounter++;
- Assert.assertEquals("test1@1\n",
fileContents.getValue());
- } else if
(fileContents.getKey().getName().contains(".part-0-1.inprogress")) {
- fileCounter++;
-
Assert.assertEquals("test1@2\ntest1@3\n", fileContents.getValue());
- }
+ testPartFiles(testHarness, outDir,
".part-0-0.inprogress", ".part-0-1.inprogress");
+
+ }
+ }
+
+ @Test
+ public void testCustomBulkWriterWithPartConfig() throws Exception {
+ final File outDir = TEMP_FOLDER.newFolder();
+
+ // we set the max bucket size to small so that we can know when
it rolls
+ try (
+ OneInputStreamOperatorTestHarness<Tuple2<String,
Integer>, Object> testHarness =
+ TestUtils.createTestSinkWithBulkEncoder(
+ outDir,
+ 1,
+ 0,
+ 10L,
+ new TestUtils.TupleToStringBucketer(),
+ new TestBulkWriterFactory(),
+ new DefaultBucketFactoryImpl<>(),
+ "prefix-",
+ ".ext")
+ ) {
+
+ testPartFiles(testHarness, outDir,
".prefix-0-0.ext.inprogress", ".prefix-0-1.ext.inprogress");
+
+ }
+ }
+
+ private void testPartFiles(
+
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> testHarness,
Review comment:
No need for all these tabs. Two tabs indentation after the start of the
`private void testPartFiles(` line are enough.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
