Repository: metron Updated Branches: refs/heads/master c99df8ff1 -> 7b9f20c93
METRON-681 CSVConverter should trim values (basvdl via ottobackwards) closes apache/metron#674 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7b9f20c9 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7b9f20c9 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7b9f20c9 Branch: refs/heads/master Commit: 7b9f20c937c471632faae789520f15f63b3617e1 Parents: c99df8f Author: basvdl <[email protected]> Authored: Tue Aug 1 08:41:57 2017 -0400 Committer: otto <[email protected]> Committed: Tue Aug 1 08:41:57 2017 -0400 ---------------------------------------------------------------------- .../apache/metron/common/csv/CSVConverter.java | 9 ++++++++- .../metron/parsers/csv/CSVParserTest.java | 21 ++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/7b9f20c9/metron-platform/metron-common/src/main/java/org/apache/metron/common/csv/CSVConverter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/csv/CSVConverter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/csv/CSVConverter.java index ce23deb..f200d84 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/csv/CSVConverter.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/csv/CSVConverter.java @@ -32,6 +32,13 @@ import java.util.Map; import static org.apache.commons.lang3.StringUtils.isEmpty; +/** + * Based on the parser config field `COLUMNS_KEY` the log line is + * transformed into a intermediate result using the CSV parser. + * + * All keys in `COLUMNS_KEY` and values extracted from the log line + * will be trimmed before adding to `values`. + */ public class CSVConverter implements Serializable { public static final String COLUMNS_KEY="columns"; public static final String SEPARATOR_KEY="separator"; @@ -53,7 +60,7 @@ public class CSVConverter implements Serializable { String[] tokens = parser.parseLine(line); Map<String, String> values = new HashMap<>(); for(Map.Entry<String, Integer> kv : columnMap.entrySet()) { - values.put(kv.getKey(), tokens[kv.getValue()]); + values.put(kv.getKey().trim(), tokens[kv.getValue()].trim()); } return values; } http://git-wip-us.apache.org/repos/asf/metron/blob/7b9f20c9/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java index c4b7ff5..4ac60d0 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java @@ -40,8 +40,8 @@ public class CSVParserTest { { "columns" : { "col1" : 0 - ,"col2" : 1 - ,"col3" : 2 + ," col2" : 1 + ,"col3 " : 2 } } } @@ -94,8 +94,21 @@ public class CSVParserTest { Assert.assertTrue(parser.validate(o)); Assert.assertEquals(5, o.size()); Assert.assertEquals("foo", o.get("col1")); - Assert.assertEquals(" bar", o.get("col2")); - Assert.assertEquals(" grok", o.get("col3")); + Assert.assertEquals("bar", o.get("col2")); + Assert.assertEquals("grok", o.get("col3")); + } + { + String line = " foo , bar , grok "; + List<JSONObject> results = parser.parse(Bytes.toBytes(line)); + Assert.assertEquals(1, results.size()); + JSONObject o = results.get(0); + Assert.assertTrue(parser.validate(o)); + Assert.assertEquals(5, o.size()); + Assert.assertEquals("foo", o.get("col1")); + Assert.assertEquals("bar", o.get("col2")); + Assert.assertEquals("grok", o.get("col3")); + Assert.assertEquals(null, o.get(" col2")); + Assert.assertEquals(null, o.get("col3 ")); } { UnitTestHelper.setLog4jLevel(CSVParser.class, Level.FATAL);
