CAMEL-7519 - new option quotingEscaped added for marshalling/unmarshalling with escape char and fix some typo
Project: http://git-wip-us.apache.org/repos/asf/camel/repo Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/95f1cf2a Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/95f1cf2a Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/95f1cf2a Branch: refs/heads/master Commit: 95f1cf2a4ccc2f5a5e1ba2baa593adae6409bb09 Parents: d6088eb Author: onders86 <[email protected]> Authored: Wed Apr 19 14:34:05 2017 +0300 Committer: Claus Ibsen <[email protected]> Committed: Thu Apr 20 09:47:44 2017 +0200 ---------------------------------------------------------------------- .../camel/dataformat/bindy/BindyCsvFactory.java | 22 ++- .../dataformat/bindy/annotation/CsvRecord.java | 5 + ...ecordFieldStartingWithSeperatorCharTest.java | 2 +- ...ContainingMultiQuoteCharEscapeFalseTest.java | 196 ++++++++++++++++++ ...vContainingMultiQuoteCharEscapeTrueTest.java | 198 +++++++++++++++++++ 5 files changed, 418 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/camel/blob/95f1cf2a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyCsvFactory.java ---------------------------------------------------------------------- diff --git a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyCsvFactory.java b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyCsvFactory.java index a055855..cb5dd3a 100755 --- a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyCsvFactory.java +++ b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyCsvFactory.java @@ -68,6 +68,7 @@ public class BindyCsvFactory extends BindyAbstractFactory implements BindyFactor private boolean quoting; private boolean autospanLine; private boolean allowEmptyStream; + private boolean quotingEscaped; public BindyCsvFactory(Class<?> type) throws Exception { super(type); @@ -207,7 +208,11 @@ public class BindyCsvFactory extends BindyAbstractFactory implements BindyFactor if (!data.equals("")) { try { - value = format.parse(data); + if (quoting && quote != null && (data.contains("\\" + quote) || data.contains(quote)) && quotingEscaped) { + value = format.parse(data.replaceAll("\\\\" + quote, "\\" + quote)); + } else { + value = format.parse(data); + } } catch (FormatException ie) { throw new IllegalArgumentException(ie.getMessage() + ", position: " + pos + ", line: " + line, ie); } catch (Exception e) { @@ -311,7 +316,12 @@ public class BindyCsvFactory extends BindyAbstractFactory implements BindyFactor if (quoting && quote != null) { buffer.append(quote); } - buffer.append(res); + // CAMEL-7519 - improvoment escape the token itself by prepending escape char + if (quoting && quote != null && (res.contains("\\" + quote) || res.contains(quote)) && quotingEscaped) { + buffer.append(res.replaceAll("\\" + quote, "\\\\" + quote)); + } else { + buffer.append(res); + } if (quoting && quote != null) { buffer.append(quote); } @@ -577,9 +587,13 @@ public class BindyCsvFactory extends BindyAbstractFactory implements BindyFactor autospanLine = record.autospanLine(); LOG.debug("Autospan line in last record: {}", autospanLine); - // Get skipFirstLine parameter + // Get allowEmptyStream parameter allowEmptyStream = record.allowEmptyStream(); - LOG.debug("Allo empty stream parameter of the CSV: {}" + allowEmptyStream); + LOG.debug("Allow empty stream parameter of the CSV: {}" + allowEmptyStream); + + // Get quotingEscaped parameter + quotingEscaped = record.quotingEscaped(); + LOG.debug("Escape quote character flag of the CSV: {}" + quotingEscaped); } if (section != null) { http://git-wip-us.apache.org/repos/asf/camel/blob/95f1cf2a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/annotation/CsvRecord.java ---------------------------------------------------------------------- diff --git a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/annotation/CsvRecord.java b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/annotation/CsvRecord.java index 6a1094d..63f1d11 100755 --- a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/annotation/CsvRecord.java +++ b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/annotation/CsvRecord.java @@ -79,6 +79,11 @@ public @interface CsvRecord { * Indicate if the values must be quoted when marshaling (optional) */ boolean quoting() default false; + + /** + * Indicate if the values must be escaped when quoting (optional) + */ + boolean quotingEscaped() default false; /** * Last record spans rest of line (optional) http://git-wip-us.apache.org/repos/asf/camel/blob/95f1cf2a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindyRecordFieldStartingWithSeperatorCharTest.java ---------------------------------------------------------------------- diff --git a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindyRecordFieldStartingWithSeperatorCharTest.java b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindyRecordFieldStartingWithSeperatorCharTest.java index d2e1b76..bee2c7b 100755 --- a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindyRecordFieldStartingWithSeperatorCharTest.java +++ b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindyRecordFieldStartingWithSeperatorCharTest.java @@ -72,7 +72,7 @@ public class BindyRecordFieldStartingWithSeperatorCharTest extends CamelTestSupp @Override public void configure() throws Exception { BindyCsvDataFormat camelDataFormat = - new BindyCsvDataFormat(BindyCsvRowFormat.class); + new BindyCsvDataFormat(BindyCsvRowFormat.class); from("direct:start").unmarshal(camelDataFormat).to("mock:result"); } }; http://git-wip-us.apache.org/repos/asf/camel/blob/95f1cf2a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindySimpleCsvContainingMultiQuoteCharEscapeFalseTest.java ---------------------------------------------------------------------- diff --git a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindySimpleCsvContainingMultiQuoteCharEscapeFalseTest.java b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindySimpleCsvContainingMultiQuoteCharEscapeFalseTest.java new file mode 100644 index 0000000..1e7f991 --- /dev/null +++ b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindySimpleCsvContainingMultiQuoteCharEscapeFalseTest.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dataformat.bindy.csv; + +import java.io.Serializable; +import java.math.BigDecimal; + +import org.apache.camel.EndpointInject; +import org.apache.camel.RoutesBuilder; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.dataformat.bindy.annotation.CsvRecord; +import org.apache.camel.dataformat.bindy.annotation.DataField; +import org.apache.camel.dataformat.bindy.util.ConverterUtils; +import org.apache.camel.test.junit4.CamelTestSupport; +import org.junit.Test; + +public class BindySimpleCsvContainingMultiQuoteCharEscapeFalseTest extends CamelTestSupport { + + @EndpointInject(uri = "mock:resultMarshal1") + private MockEndpoint mockEndPointMarshal1; + + @EndpointInject(uri = "mock:resultUnMarshal1") + private MockEndpoint mockEndPointUnMarshal1; + + @EndpointInject(uri = "mock:resultMarshal2") + private MockEndpoint mockEndPointMarshal2; + + @EndpointInject(uri = "mock:resultUnMarshal2") + private MockEndpoint mockEndPointUnMarshal2; + + @Test + public void testMarshallCsvRecordFieldContainingMultiEscapedQuoteChar() throws Exception { + + mockEndPointMarshal1.expectedMessageCount(1); + mockEndPointMarshal1.expectedBodiesReceived("\"123\",\"\"\"foo\"\"\",\"10\"" + ConverterUtils.getStringCarriageReturn("WINDOWS")); + + BindyCsvRowFormat75191 body = new BindyCsvRowFormat75191(); + body.setFirstField("123"); + body.setSecondField("\"\"foo\"\""); + body.setNumber(new BigDecimal(10)); + template.sendBody("direct:startMarshal1", body); + + assertMockEndpointsSatisfied(); + + BindyCsvRowFormat75191 model = mockEndPointUnMarshal1.getReceivedExchanges().get(0).getIn().getBody(BindyCsvRowFormat75191.class); + + assertEquals("123", model.getFirstField()); + assertEquals("\"\"foo\"\"", model.getSecondField()); + assertEquals(new BigDecimal(10), model.getNumber()); + } + + @Test + public void testMarshallCsvRecordFieldContainingMultiNonEscapedQuoteChar() throws Exception { + + mockEndPointMarshal2.expectedMessageCount(1); + mockEndPointMarshal2.expectedBodiesReceived("'123','''foo''','10'" + ConverterUtils.getStringCarriageReturn("WINDOWS")); + + BindyCsvRowFormat75192 body = new BindyCsvRowFormat75192(); + body.setFirstField("123"); + body.setSecondField("''foo''"); + body.setNumber(new BigDecimal(10)); + template.sendBody("direct:startMarshal2", body); + + assertMockEndpointsSatisfied(); + + BindyCsvRowFormat75192 model = mockEndPointUnMarshal2.getReceivedExchanges().get(0).getIn().getBody(BindyCsvRowFormat75192.class); + + assertEquals("123", model.getFirstField()); + assertEquals("''foo''", model.getSecondField()); + assertEquals(new BigDecimal(10), model.getNumber()); + } + + @Override + protected RoutesBuilder createRouteBuilder() throws Exception { + return new RouteBuilder() { + @Override + public void configure() throws Exception { + BindyCsvDataFormat camelDataFormat1 = + new BindyCsvDataFormat(BindyCsvRowFormat75191.class); + + from("direct:startMarshal1") + .marshal(camelDataFormat1) + .to("mock:resultMarshal1") + .to("direct:middle1"); + + from("direct:middle1") + .unmarshal(camelDataFormat1) + .to("mock:resultUnMarshal1"); + + BindyCsvDataFormat camelDataFormat2 = + new BindyCsvDataFormat(BindyCsvRowFormat75192.class); + + from("direct:startMarshal2") + .marshal(camelDataFormat2) + .to("mock:resultMarshal2") + .to("direct:middle2"); + + from("direct:middle2") + .unmarshal(camelDataFormat2) + .to("mock:resultUnMarshal2"); + } + }; + } + + //from https://issues.apache.org/jira/browse/CAMEL-7519 + @CsvRecord(separator = ",", quote = "\"", quoting = true, quotingEscaped = false) + public static class BindyCsvRowFormat75191 implements Serializable { + + @DataField(pos = 1) + private String firstField; + + @DataField(pos = 2) + private String secondField; + + @DataField(pos = 3, pattern = "########.##") + private BigDecimal number; + + public String getFirstField() { + return firstField; + } + + public void setFirstField(String firstField) { + this.firstField = firstField; + } + + public String getSecondField() { + return secondField; + } + + public void setSecondField(String secondField) { + this.secondField = secondField; + } + + public BigDecimal getNumber() { + return number; + } + + public void setNumber(BigDecimal number) { + this.number = number; + } + } + + @CsvRecord(separator = ",", quote = "'", quoting = true, quotingEscaped = false) + public static class BindyCsvRowFormat75192 implements Serializable { + + @DataField(pos = 1) + private String firstField; + + @DataField(pos = 2) + private String secondField; + + @DataField(pos = 3, pattern = "########.##") + private BigDecimal number; + + public String getFirstField() { + return firstField; + } + + public void setFirstField(String firstField) { + this.firstField = firstField; + } + + public String getSecondField() { + return secondField; + } + + public void setSecondField(String secondField) { + this.secondField = secondField; + } + + public BigDecimal getNumber() { + return number; + } + + public void setNumber(BigDecimal number) { + this.number = number; + } + } + + +} http://git-wip-us.apache.org/repos/asf/camel/blob/95f1cf2a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindySimpleCsvContainingMultiQuoteCharEscapeTrueTest.java ---------------------------------------------------------------------- diff --git a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindySimpleCsvContainingMultiQuoteCharEscapeTrueTest.java b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindySimpleCsvContainingMultiQuoteCharEscapeTrueTest.java new file mode 100644 index 0000000..56d5d94 --- /dev/null +++ b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindySimpleCsvContainingMultiQuoteCharEscapeTrueTest.java @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dataformat.bindy.csv; + +import java.io.Serializable; +import java.math.BigDecimal; + +import org.apache.camel.EndpointInject; +import org.apache.camel.RoutesBuilder; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.dataformat.bindy.annotation.CsvRecord; +import org.apache.camel.dataformat.bindy.annotation.DataField; +import org.apache.camel.dataformat.bindy.csv.BindySimpleCsvContainingMultiQuoteCharEscapeFalseTest.BindyCsvRowFormat75191; +import org.apache.camel.dataformat.bindy.csv.BindySimpleCsvContainingMultiQuoteCharEscapeFalseTest.BindyCsvRowFormat75192; +import org.apache.camel.dataformat.bindy.util.ConverterUtils; +import org.apache.camel.test.junit4.CamelTestSupport; +import org.junit.Test; + +public class BindySimpleCsvContainingMultiQuoteCharEscapeTrueTest extends CamelTestSupport { + + @EndpointInject(uri = "mock:resultMarshal1") + private MockEndpoint mockEndPointMarshal1; + + @EndpointInject(uri = "mock:resultUnMarshal1") + private MockEndpoint mockEndPointUnMarshal1; + + @EndpointInject(uri = "mock:resultMarshal2") + private MockEndpoint mockEndPointMarshal2; + + @EndpointInject(uri = "mock:resultUnMarshal2") + private MockEndpoint mockEndPointUnMarshal2; + + @Test + public void testMarshallCsvRecordFieldContainingMultiEscapedQuoteChar() throws Exception { + + mockEndPointMarshal1.expectedMessageCount(1); + mockEndPointMarshal1.expectedBodiesReceived("\"123\",\"\\\"\\\"foo\\\"\\\"\",\"10\"" + ConverterUtils.getStringCarriageReturn("WINDOWS")); + + BindyCsvRowFormat75191 body = new BindyCsvRowFormat75191(); + body.setFirstField("123"); + body.setSecondField("\"\"foo\"\""); + body.setNumber(new BigDecimal(10)); + template.sendBody("direct:startMarshal1", body); + + assertMockEndpointsSatisfied(); + + BindyCsvRowFormat75191 model = mockEndPointUnMarshal1.getReceivedExchanges().get(0).getIn().getBody(BindyCsvRowFormat75191.class); + + assertEquals("123", model.getFirstField()); + assertEquals("\"\"foo\"\"", model.getSecondField()); + assertEquals(new BigDecimal(10), model.getNumber()); + } + + @Test + public void testMarshallCsvRecordFieldContainingMultiNonEscapedQuoteChar() throws Exception { + + mockEndPointMarshal2.expectedMessageCount(1); + mockEndPointMarshal2.expectedBodiesReceived("'123','\\'\\'foo\\'\\'','10'" + ConverterUtils.getStringCarriageReturn("WINDOWS")); + + BindyCsvRowFormat75192 body = new BindyCsvRowFormat75192(); + body.setFirstField("123"); + body.setSecondField("''foo''"); + body.setNumber(new BigDecimal(10)); + template.sendBody("direct:startMarshal2", body); + + assertMockEndpointsSatisfied(); + + BindyCsvRowFormat75192 model = mockEndPointUnMarshal2.getReceivedExchanges().get(0).getIn().getBody(BindyCsvRowFormat75192.class); + + assertEquals("123", model.getFirstField()); + assertEquals("''foo''", model.getSecondField()); + assertEquals(new BigDecimal(10), model.getNumber()); + } + + @Override + protected RoutesBuilder createRouteBuilder() throws Exception { + return new RouteBuilder() { + @Override + public void configure() throws Exception { + BindyCsvDataFormat camelDataFormat1 = + new BindyCsvDataFormat(BindyCsvRowFormat75191.class); + + from("direct:startMarshal1") + .marshal(camelDataFormat1) + .to("mock:resultMarshal1") + .to("direct:middle1"); + + from("direct:middle1") + .unmarshal(camelDataFormat1) + .to("mock:resultUnMarshal1"); + + BindyCsvDataFormat camelDataFormat2 = + new BindyCsvDataFormat(BindyCsvRowFormat75192.class); + + from("direct:startMarshal2") + .marshal(camelDataFormat2) + .to("mock:resultMarshal2") + .to("direct:middle2"); + + from("direct:middle2") + .unmarshal(camelDataFormat2) + .to("mock:resultUnMarshal2"); + } + }; + } + + //from https://issues.apache.org/jira/browse/CAMEL-7519 + @CsvRecord(separator = ",", quote = "\"", quoting = true, quotingEscaped = true) + public static class BindyCsvRowFormat75191 implements Serializable { + + @DataField(pos = 1) + private String firstField; + + @DataField(pos = 2) + private String secondField; + + @DataField(pos = 3, pattern = "########.##") + private BigDecimal number; + + public String getFirstField() { + return firstField; + } + + public void setFirstField(String firstField) { + this.firstField = firstField; + } + + public String getSecondField() { + return secondField; + } + + public void setSecondField(String secondField) { + this.secondField = secondField; + } + + public BigDecimal getNumber() { + return number; + } + + public void setNumber(BigDecimal number) { + this.number = number; + } + } + + @CsvRecord(separator = ",", quote = "'", quoting = true, quotingEscaped = true) + public static class BindyCsvRowFormat75192 implements Serializable { + + @DataField(pos = 1) + private String firstField; + + @DataField(pos = 2) + private String secondField; + + @DataField(pos = 3, pattern = "########.##") + private BigDecimal number; + + public String getFirstField() { + return firstField; + } + + public void setFirstField(String firstField) { + this.firstField = firstField; + } + + public String getSecondField() { + return secondField; + } + + public void setSecondField(String secondField) { + this.secondField = secondField; + } + + public BigDecimal getNumber() { + return number; + } + + public void setNumber(BigDecimal number) { + this.number = number; + } + } + + +}
