[
https://issues.apache.org/jira/browse/CAMEL-12192?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16344912#comment-16344912
]
ASF GitHub Bot commented on CAMEL-12192:
----------------------------------------
neoxu999 closed pull request #2199: CAMEL-12192 support csv bindy skip fields
URL: https://github.com/apache/camel/pull/2199
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyCsvFactory.java
b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyCsvFactory.java
index 9906402ef6b..001594746bf 100755
---
a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyCsvFactory.java
+++
b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyCsvFactory.java
@@ -75,12 +75,20 @@
private boolean allowEmptyStream;
private boolean quotingEscaped;
private boolean endWithLineBreak;
+
+ private boolean isSkipField;
public BindyCsvFactory(Class<?> type) throws Exception {
+ this(type, false);
+ }
+
+ public BindyCsvFactory(Class<?> type, boolean isSkipField) throws
Exception {
super(type);
// initialize specific parameters of the csv model
initCsvModel();
+
+ this.isSkipField = isSkipField;
}
/**
@@ -174,105 +182,118 @@ public void bind(CamelContext camelContext,
List<String> tokens, Map<String, Obj
// Get DataField from model
DataField dataField = dataFields.get(pos);
- ObjectHelper.notNull(dataField, "No position " + pos + " defined
for the field: " + data + ", line: " + line);
-
- if (dataField.trim()) {
- data = data.trim();
+
+ // If a DataField can be skipped, it needs to check whether it is
in dataFields keyset
+ if (isSkipField()) {
+ if (dataFields.keySet().contains(pos)) {
+ counterMandatoryFields = setDataFieldValue(camelContext,
model, line, pos, counterMandatoryFields, data, dataField);
+ }
+ } else {
+ counterMandatoryFields = setDataFieldValue(camelContext,
model, line, pos, counterMandatoryFields, data, dataField);
}
+
+ ++pos;
- if (dataField.required()) {
- // Increment counter of mandatory fields
- ++counterMandatoryFields;
+ }
- // Check if content of the field is empty
- // This is not possible for mandatory fields
- if (data.equals("")) {
- throw new IllegalArgumentException("The mandatory field
defined at the position " + pos + " is empty for the line: " + line);
- }
- }
+ LOG.debug("Counter mandatory fields: {}", counterMandatoryFields);
- // Get Field to be setted
- Field field = annotatedFields.get(pos);
- field.setAccessible(true);
+ if (counterMandatoryFields < numberMandatoryFields) {
+ throw new IllegalArgumentException("Some mandatory fields are
missing, line: " + line);
+ }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Pos: {}, Data: {}, Field type: {}", new
Object[]{pos, data, field.getType()});
+ if (pos < totalFields) {
+ setDefaultValuesForFields(model);
+ }
+
+ }
+
+ private int setDataFieldValue(CamelContext camelContext, Map<String,
Object> model, int line, int pos, int counterMandatoryFields, String data,
DataField dataField) throws Exception {
+ ObjectHelper.notNull(dataField, "No position " + pos + " defined for
the field: " + data + ", line: " + line);
+
+ if (dataField.trim()) {
+ data = data.trim();
+ }
+
+ if (dataField.required()) {
+ // Increment counter of mandatory fields
+ ++counterMandatoryFields;
+
+ // Check if content of the field is empty
+ // This is not possible for mandatory fields
+ if (data.equals("")) {
+ throw new IllegalArgumentException("The mandatory field
defined at the position " + pos + " is empty for the line: " + line);
}
+ }
- // Create format object to format the field
- FormattingOptions formattingOptions =
ConverterUtils.convert(dataField,
- field.getType(),
- field.getAnnotation(BindyConverter.class),
- getLocale());
- Format<?> format = formatFactory.getFormat(formattingOptions);
+ // Get Field to be setted
+ Field field = annotatedFields.get(pos);
+ field.setAccessible(true);
- // field object to be set
- Object modelField = model.get(field.getDeclaringClass().getName());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Pos: {}, Data: {}, Field type: {}", new Object[]{pos,
data, field.getType()});
+ }
- // format the data received
- Object value = null;
+ // Create format object to format the field
+ FormattingOptions formattingOptions = ConverterUtils.convert(dataField,
+ field.getType(),
+ field.getAnnotation(BindyConverter.class),
+ getLocale());
+ Format<?> format = formatFactory.getFormat(formattingOptions);
- if (!data.equals("")) {
- try {
- if (quoting && quote != null && (data.contains("\\" +
quote) || data.contains(quote)) && quotingEscaped) {
- value = format.parse(data.replaceAll("\\\\" + quote,
"\\" + quote));
- } else {
- value = format.parse(data);
- }
- } catch (FormatException ie) {
- throw new IllegalArgumentException(ie.getMessage() + ",
position: " + pos + ", line: " + line, ie);
- } catch (Exception e) {
- throw new IllegalArgumentException("Parsing error detected
for field defined at the position: " + pos + ", line: " + line, e);
- }
- } else {
- if (!dataField.defaultValue().isEmpty()) {
- value = format.parse(dataField.defaultValue());
+ // field object to be set
+ Object modelField = model.get(field.getDeclaringClass().getName());
+
+ // format the data received
+ Object value = null;
+
+ if (!data.equals("")) {
+ try {
+ if (quoting && quote != null && (data.contains("\\" + quote)
|| data.contains(quote)) && quotingEscaped) {
+ value = format.parse(data.replaceAll("\\\\" + quote, "\\"
+ quote));
} else {
- value = getDefaultValueForPrimitive(field.getType());
+ value = format.parse(data);
}
+ } catch (FormatException ie) {
+ throw new IllegalArgumentException(ie.getMessage() + ",
position: " + pos + ", line: " + line, ie);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Parsing error detected for
field defined at the position: " + pos + ", line: " + line, e);
}
-
- if (value != null && !dataField.method().isEmpty()) {
- Class<?> clazz;
- if (dataField.method().contains(".")) {
- clazz =
camelContext.getClassResolver().resolveMandatoryClass(dataField.method().substring(0,
dataField.method().lastIndexOf(".")));
- } else {
- clazz = field.getType();
- }
-
- String methodName =
dataField.method().substring(dataField.method().lastIndexOf(".") + 1,
-
dataField.method().length());
-
- Method m = ReflectionHelper.findMethod(clazz, methodName,
field.getType());
- if (m != null) {
- // this method must be static and return type
- // must be the same as the datafield and
- // must receive only the datafield value
- // as the method argument
- value = ObjectHelper.invokeMethod(m, null, value);
- } else {
- // fallback to method without parameter, that is on the
value itself
- m = ReflectionHelper.findMethod(clazz, methodName);
- value = ObjectHelper.invokeMethod(m, value);
- }
+ } else {
+ if (!dataField.defaultValue().isEmpty()) {
+ value = format.parse(dataField.defaultValue());
+ } else {
+ value = getDefaultValueForPrimitive(field.getType());
}
-
- field.set(modelField, value);
-
- ++pos;
-
}
- LOG.debug("Counter mandatory fields: {}", counterMandatoryFields);
+ if (value != null && !dataField.method().isEmpty()) {
+ Class<?> clazz;
+ if (dataField.method().contains(".")) {
+ clazz =
camelContext.getClassResolver().resolveMandatoryClass(dataField.method().substring(0,
dataField.method().lastIndexOf(".")));
+ } else {
+ clazz = field.getType();
+ }
- if (counterMandatoryFields < numberMandatoryFields) {
- throw new IllegalArgumentException("Some mandatory fields are
missing, line: " + line);
- }
+ String methodName =
dataField.method().substring(dataField.method().lastIndexOf(".") + 1,
+ dataField.method().length());
- if (pos < totalFields) {
- setDefaultValuesForFields(model);
+ Method m = ReflectionHelper.findMethod(clazz, methodName,
field.getType());
+ if (m != null) {
+ // this method must be static and return type
+ // must be the same as the datafield and
+ // must receive only the datafield value
+ // as the method argument
+ value = ObjectHelper.invokeMethod(m, null, value);
+ } else {
+ // fallback to method without parameter, that is on the value
itself
+ m = ReflectionHelper.findMethod(clazz, methodName);
+ value = ObjectHelper.invokeMethod(m, value);
+ }
}
+ field.set(modelField, value);
+ return counterMandatoryFields;
}
@Override
@@ -719,4 +740,13 @@ public boolean isAllowEmptyStream() {
public boolean isEndWithLineBreak() {
return endWithLineBreak;
}
+
+ /**
+ * Indicate if DataField can be ignored
+ *
+ * @return boolean
+ */
+ public boolean isSkipField() {
+ return this.isSkipField;
+ }
}
diff --git
a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/csv/BindyCsvDataFormat.java
b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/csv/BindyCsvDataFormat.java
index 00ea80900bd..d076198bc94 100755
---
a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/csv/BindyCsvDataFormat.java
+++
b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/csv/BindyCsvDataFormat.java
@@ -47,11 +47,21 @@
public class BindyCsvDataFormat extends BindyAbstractDataFormat {
private static final Logger LOG =
LoggerFactory.getLogger(BindyCsvDataFormat.class);
+ /**
+ * If isSkipField = true, a CSV file doesn't need to declare all the
fields, otherwise, all the fields are mandatory.
+ */
+ private boolean isSkipField;
+
public BindyCsvDataFormat() {
}
public BindyCsvDataFormat(Class<?> type) {
+ this(type, false);
+ }
+
+ public BindyCsvDataFormat(Class<?> type, boolean isSkipField) {
super(type);
+ this.isSkipField = isSkipField;
}
@Override
@@ -310,8 +320,12 @@ public Object unmarshal(Exchange exchange, InputStream
inputStream) throws Excep
@Override
protected BindyAbstractFactory createModelFactory(FormatFactory
formatFactory) throws Exception {
- BindyCsvFactory bindyCsvFactory = new BindyCsvFactory(getClassType());
+ BindyCsvFactory bindyCsvFactory = new BindyCsvFactory(getClassType(),
isSkipField());
bindyCsvFactory.setFormatFactory(formatFactory);
return bindyCsvFactory;
}
+
+ private boolean isSkipField() {
+ return this.isSkipField;
+ }
}
diff --git
a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindyCsvSkipFieldTest.java
b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindyCsvSkipFieldTest.java
new file mode 100644
index 00000000000..f9907b584d1
--- /dev/null
+++
b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/csv/BindyCsvSkipFieldTest.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dataformat.bindy.csv;
+
+import org.apache.camel.EndpointInject;
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.Produce;
+import org.apache.camel.ProducerTemplate;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.apache.camel.dataformat.bindy.annotation.CsvRecord;
+import org.apache.camel.dataformat.bindy.annotation.DataField;
+import org.junit.Test;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.ContextConfiguration;
+import
org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
+
+
+@ContextConfiguration
+public class BindyCsvSkipFieldTest extends AbstractJUnit4SpringContextTests {
+
+ private static final String URI_MOCK_RESULT = "mock:result";
+ private static final String URI_DIRECT_START = "direct:start";
+
+ private static String input = "VOA,12 abc street,Skip
Street,Melbourne,VIC,3000,Australia,Skip dummy1,end of record";
+
+ @Produce(uri = URI_DIRECT_START)
+ private ProducerTemplate template;
+
+ @EndpointInject(uri = URI_MOCK_RESULT)
+ private MockEndpoint result;
+
+ private String expected;
+
+ @Test
+ @DirtiesContext
+ public void testUnMarshalAndMarshal() throws Exception {
+
+ template.sendBody(input);
+ result.expectedMessageCount(1);
+ result.assertIsSatisfied();
+ }
+
+ public static class ContextConfig extends RouteBuilder {
+ BindyCsvDataFormat camelDataFormat = new
BindyCsvDataFormat(CsvSkipField.class, true);
+
+ public void configure() {
+ from(URI_DIRECT_START).unmarshal(camelDataFormat)
+ .process(new Processor() {
+ @Override
+ public void process(Exchange exchange) throws
Exception {
+ CsvSkipField csvSkipField = (CsvSkipField)
exchange.getIn().getBody();
+ assert csvSkipField.getAttention().equals("VOA");
+ assert csvSkipField.getAddressLine1().equals("12
abc street");
+ assert csvSkipField.getCity().equals("Melbourne");
+ assert csvSkipField.getState().equals("VIC");
+ assert csvSkipField.getZip().equals("3000");
+ assert csvSkipField.getCountry()
.equals("Australia");
+ assert csvSkipField.getDummy2().equals("end of
record");
+ }
+ })
+
+ .marshal(camelDataFormat)
+ .convertBodyTo(String.class)
+ .to(URI_MOCK_RESULT);
+ }
+
+ }
+
+ @CsvRecord(separator = ",")
+ public static class CsvSkipField {
+ @DataField(pos = 1)
+ private String attention;
+
+ @DataField(pos = 2)
+ private String addressLine1;
+
+ @DataField(pos = 4)
+ private String city;
+
+ @DataField(pos = 5)
+ private String state;
+
+ @DataField(pos = 6)
+ private String zip;
+
+ @DataField(pos = 7)
+ private String country;
+
+ @DataField(pos = 9)
+ private String dummy2;
+
+ public String getAttention() {
+ return attention;
+ }
+
+ public void setAttention(String attention) {
+ this.attention = attention;
+ }
+
+ public String getAddressLine1() {
+ return addressLine1;
+ }
+
+ public void setAddressLine1(String addressLine1) {
+ this.addressLine1 = addressLine1;
+ }
+
+ public String getCity() {
+ return city;
+ }
+
+ public void setCity(String city) {
+ this.city = city;
+ }
+
+ public String getState() {
+ return state;
+ }
+
+ public void setState(String state) {
+ this.state = state;
+ }
+
+ public String getZip() {
+ return zip;
+ }
+
+ public void setZip(String zip) {
+ this.zip = zip;
+ }
+
+ public String getCountry() {
+ return country;
+ }
+
+ public void setCountry(String country) {
+ this.country = country;
+ }
+
+ public String getDummy2() {
+ return dummy2;
+ }
+
+ public void setDummy2(String dummy2) {
+ this.dummy2 = dummy2;
+ }
+
+ @Override
+ public String toString() {
+ return "Record [attention=" + getAttention() + ", addressLine1=" +
getAddressLine1() + ", " + "city=" + getCity() + ", state=" + getState() + ",
zip=" + getZip() + ", country="
+ + getCountry() + ", dummy2=" + getDummy2() + "]";
+ }
+ }
+}
diff --git
a/components/camel-bindy/src/test/resources/org/apache/camel/dataformat/bindy/csv/BindyCsvSkipFieldTest-context.xml
b/components/camel-bindy/src/test/resources/org/apache/camel/dataformat/bindy/csv/BindyCsvSkipFieldTest-context.xml
new file mode 100644
index 00000000000..c1c6ebd1380
--- /dev/null
+++
b/components/camel-bindy/src/test/resources/org/apache/camel/dataformat/bindy/csv/BindyCsvSkipFieldTest-context.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<beans xmlns="http://www.springframework.org/schema/beans";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd
+ http://camel.apache.org/schema/spring
+ http://camel.apache.org/schema/spring/camel-spring.xsd";>
+
+ <camelContext xmlns="http://camel.apache.org/schema/spring";>
+ <routeBuilder ref="myBuilder" />
+ </camelContext>
+
+ <bean id="myBuilder"
class="org.apache.camel.dataformat.bindy.csv.BindyCsvSkipFieldTest$ContextConfig"/>
+
+</beans>
\ No newline at end of file
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> BindyCsvDataFormat does not support skip fields
> ------------------------------------------------
>
> Key: CAMEL-12192
> URL: https://issues.apache.org/jira/browse/CAMEL-12192
> Project: Camel
> Issue Type: Improvement
> Components: camel-bindy
> Affects Versions: 2.15.1, 2.20.1
> Reporter: Neo Xu
> Priority: Major
> Original Estimate: 48h
> Remaining Estimate: 48h
>
> Currently: BindyCsvDataFormat has to map all the fields in a CSV file.
> If there are 200 fields in a CSV file, it has to create 200 fields in a
> class. But sometimes, it is only interested in 20 fields. So it needs to skip
> some positions if they are not required. But BindyCsvDataFormat will throw an
> exception in this case.
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
