This is an automated email from the ASF dual-hosted git repository. chibenwa pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-mime4j.git
commit b0c40752f2fe61515983cfce705476686bca0324 Author: Benoit TELLIER <[email protected]> AuthorDate: Thu Nov 5 12:30:32 2026 +0100 [FIX] CharsetUtil#forEncoding: text its charset cannot encode takes UTF-8 Since 5a754cfb the builders rejected a text its charset cannot encode. But the charset and the text can both come from a received part: decoding replaces an undefined byte with U+FFFD, which US-ASCII, ISO-8859-x or windows-125x cannot encode. So any sender could make code that rebuilds a part in its own charset (footer, quote, forward) fail with an IllegalArgumentException, with one 8-bit byte in a part labelled US-ASCII. Such a text now takes UTF-8 and the part is labelled with it, as already done for a decode-only charset or when no charset is given: the text is neither altered nor rejected. Only a lone surrogate, which no charset encodes, is still rejected. StorageBodyFactory, which still wrote "?", now does the same. --- .../org/apache/james/mime4j/util/CharsetUtil.java | 32 +++++++------- .../james/mime4j/message/MultipartBuilderTest.java | 50 +++++++++++++++++----- .../james/mime4j/storage/StorageBodyFactory.java | 4 +- .../mime4j/storage/StorageBodyFactoryTest.java | 13 ++++++ 4 files changed, 70 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/org/apache/james/mime4j/util/CharsetUtil.java b/core/src/main/java/org/apache/james/mime4j/util/CharsetUtil.java index 3b10e144..ddf36cf7 100644 --- a/core/src/main/java/org/apache/james/mime4j/util/CharsetUtil.java +++ b/core/src/main/java/org/apache/james/mime4j/util/CharsetUtil.java @@ -216,29 +216,29 @@ public class CharsetUtil { } /** - * Returns the charset to encode a text with, so that the text is not silently altered. - * A charset chosen by the caller has to encode the whole text. Without a charset, the text - * stays in the default one (<code>null</code> is returned) or else takes UTF-8. + * Returns the charset to encode a text with and to label it with, so that the text is not + * silently altered. Text that the given charset cannot encode takes UTF-8: that charset may + * come from a received part, whose text can hold what decoding replaced with U+FFFD. Without + * a charset, the text stays in the default one (<code>null</code> is returned) or takes UTF-8. * - * @param charset the charset the caller chose, may be <code>null</code>. + * @param charset the charset wanted, may be <code>null</code>. * @param text the text to encode. - * @return the charset to use and to label the text with, <code>null</code> when the text fits - * the default charset and the caller chose none. - * @throws IllegalArgumentException if the charset cannot encode the text. A lone surrogate - * cannot be encoded by any charset. + * @return the charset to use, which callers have to label the text with: it may not be the + * given one. <code>null</code> when the text fits the default charset and none was given. + * @throws IllegalArgumentException if the text holds a lone surrogate, which no charset encodes. */ public static Charset forEncoding(final Charset charset, final CharSequence text) { Charset cs = forEncoding(charset); - if (cs == null) { - if (Charsets.DEFAULT_CHARSET.newEncoder().canEncode(text)) { - return null; - } - cs = Charsets.UTF_8; + if (cs == null && Charsets.DEFAULT_CHARSET.newEncoder().canEncode(text)) { + return null; } - if (!cs.newEncoder().canEncode(text)) { - throw new IllegalArgumentException("Charset " + cs.name() + " cannot encode the text"); + if (cs != null && cs.newEncoder().canEncode(text)) { + return cs; } - return cs; + if (!Charsets.UTF_8.newEncoder().canEncode(text)) { + throw new IllegalArgumentException("The text holds a lone surrogate: no charset can encode it"); + } + return Charsets.UTF_8; } } diff --git a/dom/src/test/java/org/apache/james/mime4j/message/MultipartBuilderTest.java b/dom/src/test/java/org/apache/james/mime4j/message/MultipartBuilderTest.java index 4456dcba..624ad2a1 100644 --- a/dom/src/test/java/org/apache/james/mime4j/message/MultipartBuilderTest.java +++ b/dom/src/test/java/org/apache/james/mime4j/message/MultipartBuilderTest.java @@ -295,17 +295,45 @@ public class MultipartBuilderTest { } @Test - public void textBodyInACharsetThatCannotEncodeItShouldBeRejected() throws Exception { - assertThatThrownBy(() -> Message.Builder.of().setBody("caf\u00e9", "plain", StandardCharsets.US_ASCII)) - .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> BodyPartBuilder.create().setBody("caf\u00e9", "plain", StandardCharsets.US_ASCII)) - .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> MultipartBuilder.create("mixed").addTextPart("\u20ac", StandardCharsets.ISO_8859_1)) - .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> BasicBodyFactory.INSTANCE.textBody("caf\u00e9", StandardCharsets.US_ASCII)) - .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> SingleBodyBuilder.create().setText("caf\u00e9").setCharset(StandardCharsets.US_ASCII).buildText()) - .isInstanceOf(IllegalArgumentException.class); + public void textThatItsCharsetCannotEncodeShouldTakeUtf8AndItsLabel() throws Exception { + Message built = Message.Builder.of().setBody("caf\u00e9", "plain", StandardCharsets.US_ASCII).build(); + assertThat(((ContentTypeField) built.getHeader().getField("Content-Type")).getCharset()).isEqualTo("UTF-8"); + assertThat(readText(writeAndParse(built))).isEqualTo("caf\u00e9"); + + Message part = Message.Builder.of() + .setBody(MultipartBuilder.create("mixed").addTextPart("\u20ac", StandardCharsets.ISO_8859_1).build()) + .build(); + Entity parsedPart = ((Multipart) writeAndParse(part).getBody()).getBodyParts().get(0); + assertThat(((ContentTypeField) parsedPart.getHeader().getField("Content-Type")).getCharset()).isEqualTo("UTF-8"); + assertThat(readText(parsedPart)).isEqualTo("\u20ac"); + + assertThat(BodyPartBuilder.create().setBody("caf\u00e9", "plain", StandardCharsets.US_ASCII) + .build().getCharset()).isEqualTo("UTF-8"); + assertThat(BasicBodyFactory.INSTANCE.textBody("caf\u00e9", StandardCharsets.US_ASCII).getMimeCharset()).isEqualTo("UTF-8"); + assertThat(BasicBodyFactory.INSTANCE.textBody("caf\u00e9", "US-ASCII").getMimeCharset()).isEqualTo("UTF-8"); + assertThat(SingleBodyBuilder.create().setText("caf\u00e9").setCharset(StandardCharsets.US_ASCII) + .buildText().getMimeCharset()).isEqualTo("UTF-8"); + } + + @Test + public void textThatItsCharsetCanEncodeShouldKeepThatCharset() throws Exception { + assertThat(BasicBodyFactory.INSTANCE.textBody("caf\u00e9", StandardCharsets.ISO_8859_1).getMimeCharset()).isEqualTo("ISO-8859-1"); + assertThat(BasicBodyFactory.INSTANCE.textBody("cafe", StandardCharsets.US_ASCII).getMimeCharset()).isEqualTo("US-ASCII"); + } + + @Test + public void textOfAReceivedPartShouldBeReusableWithTheCharsetOfThatPart() throws Exception { + byte[] raw = ("Content-Type: text/plain; charset=US-ASCII\r\n" + + "Content-Transfer-Encoding: 8bit\r\n\r\ncaf\u00e9").getBytes(StandardCharsets.ISO_8859_1); + Message message = new DefaultMessageBuilder().parseMessage(new ByteArrayInputStream(raw)); + TextBody received = (TextBody) message.getBody(); + String text = readText(message); + + TextBody rebuilt = SingleBodyBuilder.create().setText(text).setCharset(received.getCharset()).buildText(); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + rebuilt.writeTo(out); + assertThat(new String(out.toByteArray(), rebuilt.getCharset())).isEqualTo(text); } @Test diff --git a/storage/src/main/java/org/apache/james/mime4j/storage/StorageBodyFactory.java b/storage/src/main/java/org/apache/james/mime4j/storage/StorageBodyFactory.java index 465ebc27..af429090 100644 --- a/storage/src/main/java/org/apache/james/mime4j/storage/StorageBodyFactory.java +++ b/storage/src/main/java/org/apache/james/mime4j/storage/StorageBodyFactory.java @@ -249,7 +249,7 @@ public class StorageBodyFactory implements BodyFactory { if (text == null) throw new IllegalArgumentException(); - return new StringTextBody(text, Charsets.DEFAULT_CHARSET); + return new StringTextBody(text, CharsetUtil.forEncoding(Charsets.DEFAULT_CHARSET, text)); } /** @@ -275,7 +275,7 @@ public class StorageBodyFactory implements BodyFactory { throw new IllegalArgumentException(); Charset charset = toJavaCharset(mimeCharset, true, monitor); - return new StringTextBody(text, charset); + return new StringTextBody(text, CharsetUtil.forEncoding(charset, text)); } private static Charset toJavaCharset( diff --git a/storage/src/test/java/org/apache/james/mime4j/storage/StorageBodyFactoryTest.java b/storage/src/test/java/org/apache/james/mime4j/storage/StorageBodyFactoryTest.java index ff44fef1..239f6d0f 100644 --- a/storage/src/test/java/org/apache/james/mime4j/storage/StorageBodyFactoryTest.java +++ b/storage/src/test/java/org/apache/james/mime4j/storage/StorageBodyFactoryTest.java @@ -76,4 +76,17 @@ public class StorageBodyFactoryTest { .textBody("okay", "ISO-8859-1"); Assert.assertEquals("ISO-8859-1", body.getMimeCharset()); } + + @Test + public void textThatItsCharsetCannotEncodeShouldTakeUtf8AndItsLabel() throws Exception { + TextBody body = new StorageBodyFactory(new MemoryStorageProvider(), DecodeMonitor.SILENT) + .textBody("caf\u00e9", "US-ASCII"); + + Assert.assertEquals("UTF-8", body.getMimeCharset()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + body.writeTo(out); + Assert.assertEquals("caf\u00e9", new String(out.toByteArray(), StandardCharsets.UTF_8)); + Assert.assertEquals("UTF-8", new StorageBodyFactory(new MemoryStorageProvider(), DecodeMonitor.SILENT) + .textBody("caf\u00e9").getMimeCharset()); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
