This is an automated email from the ASF dual-hosted git repository. apupier pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel.git
commit 5279c082a9b4649e0f40a2ed5e562ac09993783a Author: smjain <[email protected]> AuthorDate: Tue Sep 29 09:18:27 2026 +0530 CAMEL-25126: camel-syslog - decode non-ASCII text with the exchange charset, not ISO-8859-1 SyslogConverter.parseMessage turned every byte into one char with (char) (b & 0xff), an ISO-8859-1 decoding, and SyslogDataFormat.unmarshal fed it body.getBytes() of a String it had just decoded with the exchange charset. On a JVM with a UTF-8 default charset, any text outside US-ASCII (the UTF-8 that RFC 5424 specifies for MSG and structured data values) came out as two to four wrong chars, the encoding option of the netty and mina endpoints did not help, and marshal/unmarshal was not a round trip. - New parseMessage(byte[], Charset) decodes the bytes of each field with the charset; parseMessage(byte[]) and toSyslogMessage(String) use UTF-8. - A MSG that starts with the UTF-8 BOM (MSG-UTF8) is decoded as UTF-8 and the BOM is not part of the log message. - The data format parses the received bytes and decodes them with ExchangeHelper.getCharset(exchange), and marshals with the same charset. US-ASCII messages are parsed as before. Adds a note to the 4.23 upgrade guide. Co-Authored-By: Claude Opus 5.5 <[email protected]> --- .../camel/component/syslog/SyslogConverter.java | 46 ++++- .../camel/component/syslog/SyslogDataFormat.java | 7 +- .../camel/component/syslog/SyslogCharsetTest.java | 225 +++++++++++++++++++++ .../ROOT/pages/camel-4x-upgrade-guide-4_23.adoc | 13 ++ 4 files changed, 281 insertions(+), 10 deletions(-) diff --git a/components/camel-syslog/src/main/java/org/apache/camel/component/syslog/SyslogConverter.java b/components/camel-syslog/src/main/java/org/apache/camel/component/syslog/SyslogConverter.java index cdae8135c0e0..d3585d7e7ec1 100644 --- a/components/camel-syslog/src/main/java/org/apache/camel/component/syslog/SyslogConverter.java +++ b/components/camel-syslog/src/main/java/org/apache/camel/component/syslog/SyslogConverter.java @@ -19,6 +19,8 @@ package org.apache.camel.component.syslog; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Calendar; import java.util.GregorianCalendar; import java.util.HashMap; @@ -52,6 +54,9 @@ public final class SyslogConverter { dec } + // the UTF-8 byte order mark that starts a MSG-UTF8 (RFC 5424), as the chars that the parse loops read for its bytes + private static final String UTF8_BOM_AS_BYTES = "\u00EF\u00BB\u00BF"; + private static Map<String, MONTHS> monthValueMap = new HashMap<String, MONTHS>() { private static final long serialVersionUID = 1L; @@ -142,10 +147,25 @@ public final class SyslogConverter { @Converter public static SyslogMessage toSyslogMessage(String body) { - return parseMessage(body.getBytes()); + return parseMessage(body.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8); } + /** + * Parses a syslog message whose text is encoded in UTF-8. + */ public static SyslogMessage parseMessage(byte[] bytes) { + return parseMessage(bytes, StandardCharsets.UTF_8); + } + + /** + * Parses a syslog message. + * + * @param bytes the message + * @param charset the charset of the text fields. A MSG that starts with the UTF-8 byte order mark is always + * decoded as UTF-8, without the byte order mark (MSG-UTF8 of RFC 5424). + * @return the parsed message + */ + public static SyslogMessage parseMessage(byte[] bytes, Charset charset) { ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length); byteBuffer.put(bytes); cast(byteBuffer).rewind(); @@ -219,7 +239,7 @@ public final class SyslogConverter { host.append(charFound); } - syslogMessage.setHostname(host.toString()); + syslogMessage.setHostname(decode(host, charset)); if (isRfc5424) { Rfc5424SyslogMessage rfc5424SyslogMessage = (Rfc5424SyslogMessage) syslogMessage; @@ -227,19 +247,19 @@ public final class SyslogConverter { while ((charFound = (char) (byteBuffer.get() & 0xff)) != ' ') { appName.append(charFound); } - rfc5424SyslogMessage.setAppName(appName.toString()); + rfc5424SyslogMessage.setAppName(decode(appName, charset)); StringBuilder procId = new StringBuilder(); while ((charFound = (char) (byteBuffer.get() & 0xff)) != ' ') { procId.append(charFound); } - rfc5424SyslogMessage.setProcId(procId.toString()); + rfc5424SyslogMessage.setProcId(decode(procId, charset)); StringBuilder msgId = new StringBuilder(); while ((charFound = (char) (byteBuffer.get() & 0xff)) != ' ') { msgId.append(charFound); } - rfc5424SyslogMessage.setMsgId(msgId.toString()); + rfc5424SyslogMessage.setMsgId(decode(msgId, charset)); StringBuilder structuredData = new StringBuilder(); boolean inblock = false; @@ -252,7 +272,7 @@ public final class SyslogConverter { } structuredData.append(charFound); } - rfc5424SyslogMessage.setStructuredData(structuredData.toString()); + rfc5424SyslogMessage.setStructuredData(decode(structuredData, charset)); } StringBuilder msg = new StringBuilder(); @@ -261,12 +281,24 @@ public final class SyslogConverter { msg.append(charFound); } - syslogMessage.setLogMessage(msg.toString()); + if (msg.toString().startsWith(UTF8_BOM_AS_BYTES)) { + syslogMessage.setLogMessage(decode(msg.substring(UTF8_BOM_AS_BYTES.length()), StandardCharsets.UTF_8)); + } else { + syslogMessage.setLogMessage(decode(msg, charset)); + } LOG.trace("Syslog message : {}", syslogMessage); return syslogMessage; } + /** + * The parse loops read one char per byte ({@code (char) (b & 0xff)}), so each char of a field is one of its bytes. + * Turns the chars back into those bytes, and decodes the bytes with the charset of the message. + */ + private static String decode(CharSequence bytesAsChars, Charset charset) { + return new String(bytesAsChars.toString().getBytes(StandardCharsets.ISO_8859_1), charset); + } + private static void addRfc3164TimeStamp(StringBuilder sbr, SyslogMessage message) { // SDF isn't going to help much here. diff --git a/components/camel-syslog/src/main/java/org/apache/camel/component/syslog/SyslogDataFormat.java b/components/camel-syslog/src/main/java/org/apache/camel/component/syslog/SyslogDataFormat.java index 0af4a879e813..8782238b18ec 100644 --- a/components/camel-syslog/src/main/java/org/apache/camel/component/syslog/SyslogDataFormat.java +++ b/components/camel-syslog/src/main/java/org/apache/camel/component/syslog/SyslogDataFormat.java @@ -38,13 +38,14 @@ public class SyslogDataFormat extends ServiceSupport implements DataFormat, Data @Override public void marshal(Exchange exchange, Object body, OutputStream stream) throws Exception { SyslogMessage message = ExchangeHelper.convertToMandatoryType(exchange, SyslogMessage.class, body); - stream.write(SyslogConverter.toString(message).getBytes()); + stream.write(SyslogConverter.toString(message).getBytes(ExchangeHelper.getCharset(exchange))); } @Override public Object unmarshal(Exchange exchange, InputStream inputStream) throws Exception { - String body = ExchangeHelper.convertToMandatoryType(exchange, String.class, inputStream); - SyslogMessage message = SyslogConverter.parseMessage(body.getBytes()); + // parse the bytes as received, and decode the text with the charset of the exchange (UTF-8 by default) + byte[] body = ExchangeHelper.convertToMandatoryType(exchange, byte[].class, inputStream); + SyslogMessage message = SyslogConverter.parseMessage(body, ExchangeHelper.getCharset(exchange)); exchange.getMessage().setHeader(SyslogConstants.SYSLOG_FACILITY, message.getFacility()); exchange.getMessage().setHeader(SyslogConstants.SYSLOG_SEVERITY, message.getSeverity()); diff --git a/components/camel-syslog/src/test/java/org/apache/camel/component/syslog/SyslogCharsetTest.java b/components/camel-syslog/src/test/java/org/apache/camel/component/syslog/SyslogCharsetTest.java new file mode 100644 index 000000000000..629832208aee --- /dev/null +++ b/components/camel-syslog/src/test/java/org/apache/camel/component/syslog/SyslogCharsetTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.syslog; + +import java.io.ByteArrayOutputStream; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.TimeZone; + +import org.apache.camel.Exchange; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.test.AvailablePortFinder; +import org.apache.camel.test.junit6.CamelTestSupport; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * Text outside US-ASCII in a syslog message (RFC 5424 MSG and structured data values are UTF-8). + */ +public class SyslogCharsetTest extends CamelTestSupport { + + // text in the Latin-1 range and in a CJK script + private static final String TEXT = "caf\u00e9 Gr\u00fc\u00dfe \u65e5\u672c"; + // a name with an umlaut + private static final String NAME = "J\u00fcrgen"; + private static final String RFC5424_HEADER = "<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 "; + private static final byte[] BOM = { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF }; + + @RegisterExtension + AvailablePortFinder.Port serverPort = AvailablePortFinder.find(); + + @Test + public void testConverterRfc3164() { + SyslogMessage message = SyslogConverter.toSyslogMessage("<165>Aug 4 05:34:00 mymachine " + TEXT); + + assertEquals("mymachine", message.getHostname()); + assertEquals(TEXT, message.getLogMessage()); + } + + @Test + public void testConverterRfc5424() { + SyslogMessage message + = SyslogConverter.toSyslogMessage(RFC5424_HEADER + "[exampleSDID@32473 user=\"" + NAME + "\"] " + TEXT); + + Rfc5424SyslogMessage rfc5424 = assertInstanceOf(Rfc5424SyslogMessage.class, message); + assertEquals("[exampleSDID@32473 user=\"" + NAME + "\"]", rfc5424.getStructuredData()); + assertEquals(TEXT, rfc5424.getLogMessage()); + } + + @Test + public void testParseUtf8Bytes() { + SyslogMessage message = SyslogConverter.parseMessage((RFC5424_HEADER + "- " + TEXT).getBytes(StandardCharsets.UTF_8)); + + assertEquals(TEXT, message.getLogMessage()); + } + + @Test + public void testParseMsgUtf8WithBom() throws Exception { + byte[] bytes = concat(RFC5424_HEADER + "- ", BOM, TEXT.getBytes(StandardCharsets.UTF_8)); + + // RFC 5424: the BOM marks MSG as UTF-8, it is not part of the text + assertEquals(TEXT, SyslogConverter.parseMessage(bytes).getLogMessage()); + assertEquals(TEXT, SyslogConverter.parseMessage(bytes, StandardCharsets.ISO_8859_1).getLogMessage()); + } + + @Test + public void testParseWithCharset() { + String text = "caf\u00e9 Gr\u00fc\u00dfe"; + byte[] bytes = ("<165>Aug 4 05:34:00 mymachine " + text).getBytes(StandardCharsets.ISO_8859_1); + + assertEquals(text, SyslogConverter.parseMessage(bytes, StandardCharsets.ISO_8859_1).getLogMessage()); + } + + @Test + public void testUnmarshalUtf8Bytes() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:unmarshal"); + mock.expectedMessageCount(1); + + byte[] bytes = concat(RFC5424_HEADER + "[exampleSDID@32473 user=\"" + NAME + "\"] ", BOM, + TEXT.getBytes(StandardCharsets.UTF_8)); + template.sendBody("direct:unmarshal", bytes); + + MockEndpoint.assertIsSatisfied(context); + Rfc5424SyslogMessage message + = assertInstanceOf(Rfc5424SyslogMessage.class, mock.getReceivedExchanges().get(0).getIn().getBody()); + assertEquals("[exampleSDID@32473 user=\"" + NAME + "\"]", message.getStructuredData()); + assertEquals(TEXT, message.getLogMessage()); + } + + @Test + public void testUnmarshalUsesExchangeCharset() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:unmarshal"); + mock.expectedMessageCount(1); + + String text = "caf\u00e9 Gr\u00fc\u00dfe"; + byte[] bytes = ("<165>Aug 4 05:34:00 mymachine " + text).getBytes(StandardCharsets.ISO_8859_1); + template.sendBodyAndHeader("direct:unmarshal", bytes, Exchange.CHARSET_NAME, "ISO-8859-1"); + + MockEndpoint.assertIsSatisfied(context); + SyslogMessage message = mock.getReceivedExchanges().get(0).getIn().getBody(SyslogMessage.class); + assertEquals(text, message.getLogMessage()); + } + + @Test + public void testMarshalUnmarshalRoundTripRfc3164() throws Exception { + SyslogMessage message = new SyslogMessage(); + message.setFacility(SyslogFacility.LOCAL4); + message.setSeverity(SyslogSeverity.NOTICE); + message.setHostname("host1"); + message.setTimestamp(timestamp()); + message.setLogMessage(TEXT); + + SyslogMessage back = roundTrip(message); + + assertEquals(SyslogFacility.LOCAL4, back.getFacility()); + assertEquals(SyslogSeverity.NOTICE, back.getSeverity()); + assertEquals("host1", back.getHostname()); + assertEquals(TEXT, back.getLogMessage()); + } + + @Test + public void testMarshalUnmarshalRoundTripRfc5424() throws Exception { + Rfc5424SyslogMessage message = new Rfc5424SyslogMessage(); + message.setFacility(SyslogFacility.LOCAL4); + message.setSeverity(SyslogSeverity.NOTICE); + message.setHostname("host1"); + message.setTimestamp(timestamp()); + message.setAppName("app"); + message.setProcId("42"); + message.setMsgId("ID1"); + message.setStructuredData("[exampleSDID@32473 user=\"" + NAME + "\"]"); + message.setLogMessage(TEXT); + + Rfc5424SyslogMessage back = assertInstanceOf(Rfc5424SyslogMessage.class, roundTrip(message)); + + assertEquals("host1", back.getHostname()); + assertEquals("app", back.getAppName()); + assertEquals("42", back.getProcId()); + assertEquals("ID1", back.getMsgId()); + assertEquals("[exampleSDID@32473 user=\"" + NAME + "\"]", back.getStructuredData()); + assertEquals(TEXT, back.getLogMessage()); + } + + @Test + public void testNettyUdpUtf8() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:netty"); + mock.expectedMessageCount(1); + + byte[] data = concat(RFC5424_HEADER + "[exampleSDID@32473 user=\"" + NAME + "\"] ", BOM, + TEXT.getBytes(StandardCharsets.UTF_8)); + try (DatagramSocket socket = new DatagramSocket()) { + socket.send(new DatagramPacket(data, data.length, InetAddress.getByName("127.0.0.1"), serverPort.getPort())); + } + + MockEndpoint.assertIsSatisfied(context); + Rfc5424SyslogMessage message + = assertInstanceOf(Rfc5424SyslogMessage.class, mock.getReceivedExchanges().get(0).getIn().getBody()); + assertEquals("[exampleSDID@32473 user=\"" + NAME + "\"]", message.getStructuredData()); + assertEquals(TEXT, message.getLogMessage()); + } + + private SyslogMessage roundTrip(SyslogMessage message) throws Exception { + MockEndpoint mock = getMockEndpoint("mock:roundTrip"); + mock.reset(); + mock.expectedMessageCount(1); + + template.sendBody("direct:roundTrip", message); + + MockEndpoint.assertIsSatisfied(context); + return mock.getReceivedExchanges().get(0).getIn().getBody(SyslogMessage.class); + } + + private static Calendar timestamp() { + Calendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + calendar.set(2026, Calendar.SEPTEMBER, 29, 10, 11, 12); + calendar.set(Calendar.MILLISECOND, 0); + return calendar; + } + + private static byte[] concat(String head, byte[]... parts) throws Exception { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + bos.write(head.getBytes(StandardCharsets.UTF_8)); + for (byte[] part : parts) { + bos.write(part); + } + return bos.toByteArray(); + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:unmarshal").unmarshal().syslog().to("mock:unmarshal"); + + from("direct:roundTrip").marshal().syslog().unmarshal().syslog().to("mock:roundTrip"); + + from("netty:udp://127.0.0.1:" + serverPort.getPort() + "?sync=false&allowDefaultCodec=false") + .unmarshal().syslog().to("mock:netty"); + } + }; + } +} diff --git a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc index 04d6027f2ded..7e4a2cd8edcb 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc @@ -544,6 +544,19 @@ When the flag is `false` (the default), any remaining `CamelExecCommand*`, and a WARN is logged once per exec endpoint. Those headers never overrode the URI without the flag; they were just silent before. +=== camel-syslog - text outside US-ASCII + +The syslog data format and the `SyslogMessage` type converter decoded every byte of a message as one +ISO-8859-1 character, so any text outside US-ASCII (for example the UTF-8 that RFC 5424 specifies for MSG +and for structured data values) came out as two to four wrong characters. The text is now decoded with +the charset of the exchange (the `CamelCharsetName` header or exchange property, which the `encoding` +option of camel-netty and camel-mina sets), and UTF-8 when none is set. A MSG that starts with the UTF-8 +byte order mark is decoded as UTF-8 and the byte order mark is no longer part of the log message. The +data format also writes the message with the charset of the exchange when it marshals. + +US-ASCII messages are parsed as before. A route that repaired the wrongly decoded text itself must stop +doing so. + === Components and Language removal ==== camel-csimple, camel-csimple-joor and csimple-maven-plugin
