[
https://issues.apache.org/jira/browse/FLINK-9750?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16538859#comment-16538859
]
ASF GitHub Bot commented on FLINK-9750:
---------------------------------------
Github user StephanEwen commented on a diff in the pull request:
https://github.com/apache/flink/pull/6281#discussion_r201401033
--- Diff:
flink-core/src/test/java/org/apache/flink/core/fs/AbstractResumableWriterTest.java
---
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.fs;
+
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.util.StringUtils;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import static org.junit.Assert.fail;
+
+public abstract class AbstractResumableWriterTest extends TestLogger {
+
+ private static final Random RND = new Random();
+
+ private static final String testData1 = "THIS IS A TEST 1.";
+ private static final String testData2 = "THIS IS A TEST 2.";
+ private static final String testData3 = "THIS IS A TEST 3.";
+
+ private Path basePathForTest;
+
+ private static FileSystem fileSystem;
+
+ public abstract Path getBasePath() throws Exception;
+
+ public abstract FileSystem initializeFileSystem();
+
+ public Path getBasePathForTest() {
+ return basePathForTest;
+ }
+
+ private FileSystem getFileSystem() {
+ if (fileSystem == null) {
+ fileSystem = initializeFileSystem();
+ }
+ return fileSystem;
+ }
+
+ private ResumableWriter getNewFileSystemWriter() throws IOException {
+ return getFileSystem().createRecoverableWriter();
+ }
+
+ @Before
+ public void prepare() throws Exception {
+ basePathForTest = new Path(getBasePath(), randomName());
+ getFileSystem().mkdirs(basePathForTest);
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ getFileSystem().delete(basePathForTest, true);
+ }
+
+ @Test
+ public void testCloseWithNoData() throws Exception {
+ final ResumableWriter writer = getNewFileSystemWriter();
+
+ final Path testDir = getBasePathForTest();
+
+ final Path path = new Path(testDir + File.separator + "part-0");
+
+ final RecoverableFsDataOutputStream stream = writer.open(path);
+ for (Map.Entry<Path, String> fileContents :
getFileContentByPath(testDir).entrySet()) {
+
Assert.assertTrue(fileContents.getKey().getName().startsWith(".part-0.inprogress."));
+ Assert.assertTrue(fileContents.getValue().isEmpty());
+ }
+
+ stream.closeForCommit().commit();
+
+ for (Map.Entry<Path, String> fileContents :
getFileContentByPath(testDir).entrySet()) {
+ Assert.assertEquals("part-0",
fileContents.getKey().getName());
+ Assert.assertTrue(fileContents.getValue().isEmpty());
+ }
+ }
+
+ @Test
+ public void testCommitAfterNormalClose() throws Exception {
+ final ResumableWriter writer = getNewFileSystemWriter();
+
+ final Path testDir = getBasePathForTest();
+
+ final Path path = new Path(testDir.getPath() + File.separator +
"part-0");
+
+ try (final RecoverableFsDataOutputStream stream =
writer.open(path)) {
+
stream.write(testData1.getBytes(Charset.forName("UTF-8")));
--- End diff --
Use `StandardCharsets.UTF_8` instead of "UTF-8".
> Create new StreamingFileSink that works on Flink's FileSystem abstraction
> -------------------------------------------------------------------------
>
> Key: FLINK-9750
> URL: https://issues.apache.org/jira/browse/FLINK-9750
> Project: Flink
> Issue Type: Sub-task
> Components: Streaming Connectors
> Reporter: Stephan Ewen
> Assignee: Kostas Kloudas
> Priority: Major
> Labels: pull-request-available
> Fix For: 1.6.0
>
>
> Using Flink's own file system abstraction means that we can add additional
> streaming/checkpointing related behavior.
> In addition, the new StreamingFileSink should only rely on internal
> checkpointed state what files are possibly in progress or need to roll over,
> never assume enumeration of files in the file system.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
