This is an automated email from the ASF dual-hosted git repository. zehnder pushed a commit to branch STREAMPIPES-379 in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit abc44137dea78964ddd3663116493c9f8726e6fd Author: Philipp Zehnder <[email protected]> AuthorDate: Tue Jun 8 09:33:59 2021 +0200 Remove bom in csv files --- .../streampipes/manager/file/FileManager.java | 11 ++++ .../streampipes/manager/file/TestFileManager.java | 64 ++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java index af1e7a6..382ee3b 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java @@ -17,6 +17,7 @@ */ package org.apache.streampipes.manager.file; +import org.apache.commons.io.input.BOMInputStream; import org.apache.streampipes.model.client.file.FileMetadata; import org.apache.streampipes.storage.api.IFileMetadataStorage; import org.apache.streampipes.storage.management.StorageDispatcher; @@ -33,6 +34,11 @@ public class FileManager { InputStream fileInputStream) throws IOException { String filetype = filename.substring(filename.lastIndexOf(".") + 1); + + if ("csv".equals(filetype)) { + fileInputStream = removeBom(fileInputStream); + } + String internalFilename = makeInternalFilename(filetype); FileMetadata fileMetadata = makeFileMetadata(user, filename, internalFilename, filetype); new FileHandler().storeFile(internalFilename, fileInputStream); @@ -79,4 +85,9 @@ public class FileManager { private static String makeInternalFilename(String filetype) { return UUID.randomUUID().toString() + "." + filetype; } + + public static InputStream removeBom(InputStream stream) { + return new BOMInputStream(stream); +// return stream; + } } diff --git a/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/file/TestFileManager.java b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/file/TestFileManager.java new file mode 100644 index 0000000..14b449e --- /dev/null +++ b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/file/TestFileManager.java @@ -0,0 +1,64 @@ +package org.apache.streampipes.manager.file; + +import org.apache.commons.io.ByteOrderMark; +import org.apache.commons.io.IOUtils; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.*; + +/* +Copyright 2021 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +public class TestFileManager { + + @Test + public void testRemoveBomWithNoBom() throws IOException { + String expected = "test"; + InputStream inputStream = IOUtils.toInputStream(expected, StandardCharsets.UTF_8); + InputStream resultStream = FileManager.removeBom(inputStream); + String resultString = IOUtils.toString(resultStream, StandardCharsets.UTF_8); + + assertEquals(expected, resultString); + } + + @Test + public void testRemoveBomWithBom() throws IOException { + String expected = "test"; + String UTF8_BOM = "\uFEFF"; + String inputString = UTF8_BOM + expected; + InputStream inputStream = IOUtils.toInputStream(inputString, StandardCharsets.UTF_8); + InputStream resultStream = FileManager.removeBom(inputStream); + String resultString = IOUtils.toString(resultStream, StandardCharsets.UTF_8); + + assertEquals(expected, resultString); + } + + @Test + public void testRemoveBomWithBomAndUmlauts() throws IOException { + String expected = "testäüö"; + String UTF8_BOM = "\uFEFF"; + String inputString = UTF8_BOM + expected; + InputStream inputStream = IOUtils.toInputStream(inputString, StandardCharsets.UTF_8); + InputStream resultStream = FileManager.removeBom(inputStream); + String resultString = IOUtils.toString(resultStream, StandardCharsets.ISO_8859_1); + + assertEquals(expected, resultString); + } +} \ No newline at end of file
