[
https://issues.apache.org/jira/browse/CAMEL-10053?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Arno Noordover closed CAMEL-10053.
----------------------------------
Resolution: Not A Bug
See previous comments.
The developer can remove the characterset header and/or property to suppress
the conversion being done twice.
> Bindy does not seem to support Cp922 character encoding
> -------------------------------------------------------
>
> Key: CAMEL-10053
> URL: https://issues.apache.org/jira/browse/CAMEL-10053
> Project: Camel
> Issue Type: Bug
> Components: camel-bindy
> Affects Versions: 2.16.2
> Reporter: David Cornforth
>
> If I use the {{charset}} option when consuming from a non UTF8 encoded file
> and then unmarshal the result using a bindy object the result is either an
> exception or incorrect data. Below are some unit tests that illustrate the
> issue. The UTF8 ones work, but all of the Cp922 ones fail for a variety of
> reasons, both using {{BindyFixedLengthDataFormat}} and
> {{BindyCsvDataFormat}}. I have only tested this with Cp922, but it may be
> the case that other non-standard encodings also exhibit the same behaviour.
> {code}
> import org.apache.camel.EndpointInject;
> import org.apache.camel.builder.RouteBuilder;
> import org.apache.camel.component.mock.MockEndpoint;
> import org.apache.camel.dataformat.bindy.annotation.CsvRecord;
> import org.apache.camel.dataformat.bindy.annotation.DataField;
> import org.apache.camel.dataformat.bindy.annotation.FixedLengthRecord;
> import org.apache.camel.dataformat.bindy.csv.BindyCsvDataFormat;
> import org.apache.camel.dataformat.bindy.fixed.BindyFixedLengthDataFormat;
> import org.apache.camel.test.junit4.CamelTestSupport;
> import org.junit.After;
> import org.junit.Test;
> import java.io.IOException;
> import java.nio.file.Files;
> import java.nio.file.Paths;
> import java.util.List;
> public class BindyEncodingTest extends CamelTestSupport {
> private static final String RESULT_FIXED_LENGTH_UTF8 =
> "mock:result_fixed_length_utf8";
> private static final String RESULT_FIXED_LENGTH_CP922 =
> "mock:result_fixed_length_cp922";
> private static final String RESULT_CSV_UTF8 = "mock:result_csv_utf8";
> private static final String RESULT_CSV_CP922 = "mock:result_csv_cp922";
> private static final String TMP = "C:/tmp";
> private static final String FIXED_LENGTH_UTF8 =
> "BindyEncodingTest.FixedLength.utf8.txt";
> private static final String FIXED_LENGTH_CP922 =
> "BindyEncodingTest.FixedLength.cp922.txt";
> private static final String CSV_UTF8 = "BindyEncodingTest.csv.utf8.txt";
> private static final String CSV_CP922 = "BindyEncodingTest.csv.cp922.txt";
> @EndpointInject(uri = RESULT_FIXED_LENGTH_UTF8)
> protected MockEndpoint resultEndpointFixedLengthUtf8;
> @EndpointInject(uri = RESULT_FIXED_LENGTH_CP922)
> protected MockEndpoint resultEndpointFixedLengthCp922;
> @EndpointInject(uri = RESULT_CSV_UTF8)
> protected MockEndpoint resultEndpointCsvUtf8;
> @EndpointInject(uri = RESULT_CSV_CP922)
> protected MockEndpoint resultEndpointCsvCp922;
> @Override
> protected RouteBuilder createRouteBuilder() throws Exception {
> return new RouteBuilder() {
> @Override
> public void configure() throws Exception {
> from("file://"+TMP+"?fileName="+ FIXED_LENGTH_UTF8
> +"&charset=UTF8")
> .unmarshal(new
> BindyFixedLengthDataFormat(FixedLengthBindy.class))
> .to(RESULT_FIXED_LENGTH_UTF8);
> from("file://"+TMP+"?fileName="+ FIXED_LENGTH_CP922
> +"&charset=Cp922")
> .unmarshal(new
> BindyFixedLengthDataFormat(FixedLengthBindy.class))
> .to(RESULT_FIXED_LENGTH_CP922);
> from("file://"+TMP+"?fileName="+ CSV_UTF8 +"&charset=UTF8")
> .unmarshal(new BindyCsvDataFormat(CsvBindy.class))
> .to(RESULT_CSV_UTF8);
> from("file://"+TMP+"?fileName="+ CSV_CP922 +"&charset=Cp922")
> .unmarshal(new BindyCsvDataFormat(CsvBindy.class))
> .to(RESULT_CSV_CP922);
> }
> };
> }
> @After
> public void after() throws InterruptedException {
> assertMockEndpointsSatisfied();
> }
> /* works */
> @Test
> public void fixedLengthUtf8() throws IOException, InterruptedException {
> byte[] input = new byte[]{
> (byte)0xC3,(byte)0x85, // Å
> (byte)0xC3,(byte)0x98, // Ø
> (byte)0xC3,(byte)0x86, // Æ
>
> (byte)0x33,(byte)0x34,(byte)0x35,(byte)0x36,(byte)0x37,(byte)0x38,(byte)0x39
> // 3456789
> };
> writeFile(TMP+"/"+ FIXED_LENGTH_UTF8, input);
> resultEndpointFixedLengthUtf8.expectedMessageCount(1);
> Thread.sleep(5000);
> Object body =
> resultEndpointFixedLengthUtf8.getExchanges().get(0).getIn().getBody();
> assertTrue("The message body is null", body != null);
> assertTrue(body instanceof FixedLengthBindy);
> FixedLengthBindy fixedLengthBindy = (FixedLengthBindy) body;
> assertEquals("First is incorrect","Å",fixedLengthBindy.getFirst());
> assertEquals("Second is incorrect","ØÆ",fixedLengthBindy.getSecond());
> assertEquals("Third is
> incorrect","345678",fixedLengthBindy.getThird());
> assertEquals("Fourth is incorrect","9",fixedLengthBindy.getFourth());
> System.out.println(body);
> }
> /* fails with java.lang.StringIndexOutOfBoundsException */
> @Test
> public void fixedLengthCp922() throws IOException, InterruptedException {
> //
> ftp://ftp.software.ibm.com/software/globalization/gcoc/attachments/CP00922.pdf
> byte[] input = new byte[]{
> (byte)0xC5, // Å
> (byte)0xD8, // Ø
> (byte)0xC6, // Æ
>
> (byte)0x33,(byte)0x34,(byte)0x35,(byte)0x36,(byte)0x37,(byte)0x38,(byte)0x39
> // 3456789
> };
> writeFile(TMP + "/" + FIXED_LENGTH_CP922, input);
> resultEndpointFixedLengthCp922.expectedMessageCount(1);
> Thread.sleep(5000);
> Object body =
> resultEndpointFixedLengthCp922.getExchanges().get(0).getIn().getBody();
> assertTrue("The message body is null", body != null);
> assertTrue(body instanceof FixedLengthBindy);
> FixedLengthBindy fixedLengthBindy = (FixedLengthBindy) body;
> assertEquals("First is incorrect","Å",fixedLengthBindy.getFirst());
> assertEquals("Second is incorrect","ØÆ",fixedLengthBindy.getSecond());
> assertEquals("Third is
> incorrect","345678",fixedLengthBindy.getThird());
> assertEquals("Fourth is incorrect","9",fixedLengthBindy.getFourth());
> System.out.println(body);
> }
> /* fails as Third is incorrect */
> @Test
> public void fixedLengthCp922a() throws IOException, InterruptedException {
> //
> ftp://ftp.software.ibm.com/software/globalization/gcoc/attachments/CP00922.pdf
> byte[] input = new byte[]{
>
> (byte)0x31,(byte)0x32,(byte)0x33,(byte)0x34,(byte)0x35,(byte)0x36,(byte)0x37,
> // 1234567
> (byte)0xD8, // Ø
> (byte)0xC6, // Æ
> (byte)0xC5 // Å
> };
> writeFile(TMP + "/" + FIXED_LENGTH_CP922, input);
> resultEndpointFixedLengthCp922.expectedMessageCount(1);
> Thread.sleep(5000);
> Object body =
> resultEndpointFixedLengthCp922.getExchanges().get(0).getIn().getBody();
> assertTrue("The message body is null", body != null);
> assertTrue(body instanceof FixedLengthBindy);
> FixedLengthBindy fixedLengthBindy = (FixedLengthBindy) body;
> assertEquals("First is incorrect","1",fixedLengthBindy.getFirst());
> assertEquals("Second is incorrect","23",fixedLengthBindy.getSecond());
> assertEquals("Third is
> incorrect","4567ØÆ",fixedLengthBindy.getThird());
> assertEquals("Fourth is incorrect","Å",fixedLengthBindy.getFourth());
> System.out.println(body);
> }
> /* works */
> @Test
> public void csvUtf8() throws IOException, InterruptedException {
> byte[] input = new byte[]{
> (byte)0xC3,(byte)0x85, // Å
> (byte)0x2C, // ,
> (byte)0xC3,(byte)0x98, // Ø
> (byte)0xC3,(byte)0x86, // Æ
> (byte)0x2C, // ,
>
> (byte)0x33,(byte)0x34,(byte)0x35,(byte)0x36,(byte)0x37,(byte)0x38,
> (byte)0x2C, // ,
> (byte)0x39 // 3456789
> };
> writeFile(TMP+"/"+ CSV_UTF8, input);
> resultEndpointCsvUtf8.expectedMessageCount(1);
> Thread.sleep(5000);
> Object body =
> resultEndpointCsvUtf8.getExchanges().get(0).getIn().getBody();
> assertTrue("The message body is null", body != null);
> assertTrue(body instanceof CsvBindy);
> CsvBindy csvBindy = (CsvBindy) body;
> assertEquals("First is incorrect","Å",csvBindy.getFirst());
> assertEquals("First is incorrect","ØÆ",csvBindy.getSecond());
> assertEquals("First is incorrect","345678",csvBindy.getThird());
> assertEquals("First is incorrect","9",csvBindy.getFourth());
> System.out.println(body);
> }
> /* fails as First is incorrect */
> @Test
> public void csvCp922() throws IOException, InterruptedException {
> //
> ftp://ftp.software.ibm.com/software/globalization/gcoc/attachments/CP00922.pdf
> byte[] input = new byte[]{
> (byte)0xC5, // Å
> (byte)0x2C, // ,
> (byte)0xD8, // Ø
> (byte)0xC6, // Æ
> (byte)0x2C, // ,
>
> (byte)0x33,(byte)0x34,(byte)0x35,(byte)0x36,(byte)0x37,(byte)0x38, // 345678
> (byte)0x2C, // ,
> (byte)0x39 // 9
> };
> writeFile(TMP+"/"+ CSV_CP922, input);
> resultEndpointCsvCp922.expectedMessageCount(1);
> Thread.sleep(5000);
> Object body =
> resultEndpointCsvCp922.getExchanges().get(0).getIn().getBody();
> assertTrue("The message body is null", body != null);
> assertTrue("body is not List it is "+body.getClass(), body instanceof
> List);
> List<?> list = (List<?>)body;
> assertEquals("incorrect list size", 2, list.size());
> CsvBindy csvBindy = (CsvBindy) list.get(0);
> assertEquals("First is incorrect","Å",csvBindy.getFirst());
> assertEquals("First is incorrect","ØÆ",csvBindy.getSecond());
> assertEquals("First is incorrect","345678",csvBindy.getThird());
> assertEquals("First is incorrect","9",csvBindy.getFourth());
> System.out.println(body);
> }
> private void writeFile(String path, byte[] content) throws IOException {
> Files.write(Paths.get(path),content);
> }
> @FixedLengthRecord(ignoreTrailingChars = true)
> public static class FixedLengthBindy {
> @DataField(pos = 1, length = 1) // 1
> private String first;
> @DataField(pos = 2, length = 2) // 2 - 3
> private String second;
> @DataField(pos = 3, length = 6) // 4 - 9
> private String third;
> @DataField(pos = 4, length = 1) // 10
> private String fourth;
> public String getFirst() {
> return first;
> }
> public void setFirst(String first) {
> this.first = first;
> }
> public String getSecond() {
> return second;
> }
> public void setSecond(String second) {
> this.second = second;
> }
> public String getThird() {
> return third;
> }
> public void setThird(String third) {
> this.third = third;
> }
> public String getFourth() {
> return fourth;
> }
> public void setFourth(String fourth) {
> this.fourth = fourth;
> }
> @Override
> public String toString() {
> return "FixedLengthBindy{" +
> "first='" + first + '\'' +
> ", second='" + second + '\'' +
> ", third='" + third + '\'' +
> ", fourth='" + fourth + '\'' +
> '}';
> }
> }
> @CsvRecord(separator = ",", skipFirstLine = false)
> public static class CsvBindy{
> @DataField(pos = 1)
> private String first;
> @DataField(pos = 2)
> private String second;
> @DataField(pos = 3)
> private String third;
> @DataField(pos = 4)
> private String fourth;
> public String getFirst() {
> return first;
> }
> public void setFirst(String first) {
> this.first = first;
> }
> public String getSecond() {
> return second;
> }
> public void setSecond(String second) {
> this.second = second;
> }
> public String getThird() {
> return third;
> }
> public void setThird(String third) {
> this.third = third;
> }
> public String getFourth() {
> return fourth;
> }
> public void setFourth(String fourth) {
> this.fourth = fourth;
> }
> @Override
> public String toString() {
> return "CsvBindy{" +
> "first='" + first + '\'' +
> ", second='" + second + '\'' +
> ", third='" + third + '\'' +
> ", fourth='" + fourth + '\'' +
> '}';
> }
> }
> }
> {code}
> The relevant parts of the pom:
> {code:xml}
> <properties>
> <camel.version>2.16.2</camel.version>
> </properties>
> ...
> <build>
> <plugins>
> <plugin>
> <groupId>org.apache.maven.plugins</groupId>
> <artifactId>maven-compiler-plugin</artifactId>
> <version>2.5.1</version>
> <configuration>
> <source>1.7</source>
> <target>1.7</target>
> </configuration>
> </plugin>
> </plugins>
> </build>
> <dependencies>
> <dependency>
> <groupId>org.apache.camel</groupId>
> <artifactId>camel-core</artifactId>
> <version>${camel.version}</version>
> </dependency>
> <dependency>
> <groupId>org.apache.camel</groupId>
> <artifactId>camel-bindy</artifactId>
> <version>${camel.version}</version>
> </dependency>
> <dependency>
> <groupId>org.apache.camel</groupId>
> <artifactId>camel-jackson</artifactId>
> <version>${camel.version}</version>
> </dependency>
> <dependency>
> <groupId>org.apache.camel</groupId>
> <artifactId>camel-test</artifactId>
> <version>${camel.version}</version>
> <scope>test</scope>
> </dependency>
> <dependency>
> <groupId>commons-lang</groupId>
> <artifactId>commons-lang</artifactId>
> <version>2.6</version>
> </dependency>
> <dependency>
> <groupId>commons-io</groupId>
> <artifactId>commons-io</artifactId>
> <version>2.4</version>
> </dependency>
> <dependency>
> <groupId>org.slf4j</groupId>
> <artifactId>slf4j-jdk14</artifactId>
> <version>1.7.21</version>
> </dependency>
> </dependencies>
> {code}
> Note that the tests write to the filesystem, defaulting to {{C:/tmp}}.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
