This is an automated email from the ASF dual-hosted git repository. joewitt pushed a commit to branch support/nifi-1.16 in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 2b6af1a2cb26752ad3c0a4ae7898352299f043de Author: Paul Grey <[email protected]> AuthorDate: Mon Apr 11 11:35:37 2022 -0400 NIFI-9902 - Improve handling of charsets when performing string/byte conversions This closes #5954 Signed-off-by: Mike Thomsen <[email protected]> --- nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java | 2 +- .../apache/nifi/jasn1/TestJASN1RecordReaderWithSimpleTypes.java | 3 ++- .../org/apache/nifi/serialization/record/MockRecordWriter.java | 9 +++++---- .../apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java | 5 +++-- .../apache/nifi/processors/slack/PostSlackTextMessageTest.java | 4 +++- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java index 7ee30a7c00..36c95652eb 100644 --- a/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java +++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java @@ -272,7 +272,7 @@ public class MockFlowFile implements FlowFileRecord { } public void assertContentEquals(final String data) { - assertContentEquals(data, "UTF-8"); + assertContentEquals(data, StandardCharsets.UTF_8); } public void assertContentEquals(final String data, final String charset) { diff --git a/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/test/java/org/apache/nifi/jasn1/TestJASN1RecordReaderWithSimpleTypes.java b/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/test/java/org/apache/nifi/jasn1/TestJASN1RecordReaderWithSimpleTypes.java index c6cb13a638..67cf5c6419 100644 --- a/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/test/java/org/apache/nifi/jasn1/TestJASN1RecordReaderWithSimpleTypes.java +++ b/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/test/java/org/apache/nifi/jasn1/TestJASN1RecordReaderWithSimpleTypes.java @@ -46,6 +46,7 @@ import org.apache.nifi.serialization.record.RecordSchema; import org.junit.jupiter.api.Test; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -156,7 +157,7 @@ public class TestJASN1RecordReaderWithSimpleTypes implements JASN1ReadRecordTest String dataFile = "target/bmpstring_wrapper.dat"; BMPStringWrapper berValue = new BMPStringWrapper(); - berValue.setValue(new BerBMPString("Some UTF-8 String. こんにちは世界。".getBytes())); + berValue.setValue(new BerBMPString("Some UTF-8 String. こんにちは世界。".getBytes(StandardCharsets.UTF_8))); Map<String, Object> expectedValues = new HashMap<String, Object>() {{ put("value", "Some UTF-8 String. こんにちは世界。"); diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java index 523510900c..fe7bde523f 100644 --- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java @@ -28,6 +28,7 @@ import org.apache.nifi.serialization.WriteResult; import java.io.BufferedOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; @@ -85,7 +86,7 @@ public class MockRecordWriter extends AbstractControllerService implements Recor @Override public WriteResult write(final RecordSet rs) throws IOException { if (header != null && !headerWritten) { - out.write(header.getBytes()); + out.write(header.getBytes(StandardCharsets.UTF_8)); out.write("\n".getBytes()); headerWritten = true; } @@ -105,10 +106,10 @@ public class MockRecordWriter extends AbstractControllerService implements Recor if (val != null) { if (quoteValues) { out.write("\"".getBytes()); - out.write(val.getBytes()); + out.write(val.getBytes(StandardCharsets.UTF_8)); out.write("\"".getBytes()); } else { - out.write(val.getBytes()); + out.write(val.getBytes(StandardCharsets.UTF_8)); } } @@ -134,7 +135,7 @@ public class MockRecordWriter extends AbstractControllerService implements Recor } if (header != null && !headerWritten) { - out.write(header.getBytes()); + out.write(header.getBytes(StandardCharsets.UTF_8)); out.write("\n".getBytes()); headerWritten = true; } diff --git a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java index 2070c781de..9e9183f444 100644 --- a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java +++ b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java @@ -21,6 +21,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -316,7 +317,7 @@ public class ConvertExcelToCSVProcessor ff = session.write(ff, new OutputStreamCallback() { @Override public void process(OutputStream out) throws IOException { - PrintStream outPrint = new PrintStream(out); + PrintStream outPrint = new PrintStream(out, false, StandardCharsets.UTF_8.name()); sheetHandler.setOutput(outPrint); try { @@ -386,7 +387,7 @@ public class ConvertExcelToCSVProcessor } public void setOutput(PrintStream output){ - final OutputStreamWriter streamWriter = new OutputStreamWriter(output); + final OutputStreamWriter streamWriter = new OutputStreamWriter(output, StandardCharsets.UTF_8); try { printer = new CSVPrinter(streamWriter, csvFormat); diff --git a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PostSlackTextMessageTest.java b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PostSlackTextMessageTest.java index c21f2e6b9d..dc5f379b75 100644 --- a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PostSlackTextMessageTest.java +++ b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PostSlackTextMessageTest.java @@ -28,6 +28,7 @@ import javax.json.Json; import javax.json.JsonObject; import javax.json.JsonReader; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -282,7 +283,8 @@ public class PostSlackTextMessageTest { private JsonObject getRequestBodyJson() { try { final RecordedRequest recordedRequest = mockWebServer.takeRequest(); - try (final JsonReader reader = Json.createReader(new InputStreamReader(recordedRequest.getBody().inputStream()))) { + try (final JsonReader reader = Json.createReader(new InputStreamReader( + recordedRequest.getBody().inputStream(), StandardCharsets.UTF_8))) { return reader.readObject(); } } catch (final InterruptedException e) {
