Repository: nifi Updated Branches: refs/heads/master eb37f300c -> a298f1a70
NIFI-3466 - Address non English locales in ParseCEF Signed-off-by: Pierre Villard <[email protected]> This closes #1614. Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/a298f1a7 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/a298f1a7 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/a298f1a7 Branch: refs/heads/master Commit: a298f1a7076a2a306a38e71bff1eb45da9614517 Parents: eb37f30 Author: Andre F de Miranda <[email protected]> Authored: Wed Mar 22 23:20:18 2017 +1100 Committer: Pierre Villard <[email protected]> Committed: Thu Mar 23 09:04:16 2017 +0100 ---------------------------------------------------------------------- .../nifi-standard-processors/pom.xml | 2 +- .../nifi/processors/standard/ParseCEF.java | 48 ++++++++++++- .../nifi/processors/standard/TestParseCEF.java | 73 ++++++++++++++++++++ 3 files changed, 120 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/a298f1a7/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index 3be700a..f2e71a4 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -258,7 +258,7 @@ language governing permissions and limitations under the License. --> <dependency> <groupId>com.fluenda</groupId> <artifactId>ParCEFone</artifactId> - <version>1.2.0</version> + <version>1.2.2</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> http://git-wip-us.apache.org/repos/asf/nifi/blob/a298f1a7/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java index 3417c0d..4107810 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java @@ -43,6 +43,9 @@ import org.apache.nifi.annotation.documentation.SeeAlso; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.Validator; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processor.AbstractProcessor; @@ -69,6 +72,7 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TimeZone; @@ -146,6 +150,17 @@ public class ParseCEF extends AbstractProcessor { .defaultValue(LOCAL_TZ) .build(); + public static final PropertyDescriptor DATETIME_REPRESENTATION = new PropertyDescriptor.Builder() + .name("DATETIME_REPRESENTATION") + .displayName("DateTime Locale") + .description("The IETF BCP 47 representation of the Locale to be used when parsing date " + + "fields with long or short month names (e.g. may <en-US> vs. mai. <fr-FR>. The default" + + "value is generally safe. Only change if having issues parsing CEF messages") + .required(true) + .addValidator(new ValidateLocale()) + .defaultValue("en-US") + .build(); + static final Relationship REL_FAILURE = new Relationship.Builder() .name("failure") .description("Any FlowFile that could not be parsed as a CEF message will be transferred to this Relationship without any attributes being added") @@ -157,10 +172,11 @@ public class ParseCEF extends AbstractProcessor { @Override public List<PropertyDescriptor> getSupportedPropertyDescriptors() { - final List<PropertyDescriptor>properties =new ArrayList<>(); + final List<PropertyDescriptor>properties = new ArrayList<>(); properties.add(FIELDS_DESTINATION); properties.add(APPEND_RAW_MESSAGE_TO_JSON); properties.add(TIME_REPRESENTATION); + properties.add(DATETIME_REPRESENTATION); return properties; } @@ -193,6 +209,7 @@ public class ParseCEF extends AbstractProcessor { tzId = UTC; break; } + } @Override @@ -214,7 +231,11 @@ public class ParseCEF extends AbstractProcessor { CommonEvent event; try { - event = parser.parse(buffer, true); + // parcefoneLocale defaults to en_US, so this should not fail. But we force failure in case the custom + // validator failed to identify an invalid Locale + final Locale parcefoneLocale = Locale.forLanguageTag(context.getProperty(DATETIME_REPRESENTATION).getValue()); + event = parser.parse(buffer, true, parcefoneLocale); + } catch (Exception e) { // This should never trigger but adding in here as a fencing mechanism to // address possible ParCEFone bugs. @@ -322,6 +343,29 @@ public class ParseCEF extends AbstractProcessor { throws IOException, JsonProcessingException { jsonGenerator.writeObject(macAddress.toString()); } + } + + protected static class ValidateLocale implements Validator { + @Override + public ValidationResult validate(String subject, String input, ValidationContext context) { + if (null == input || input.isEmpty()) { + return new ValidationResult.Builder().subject(subject).input(input).valid(false) + .explanation(subject + " cannot be empty").build(); + } + final Locale testLocale = Locale.forLanguageTag(input); + final Locale[] availableLocales = Locale.getAvailableLocales(); + + // Check if the provided Locale is valid by checking against the first value of the array (i.e. "null" locale) + if (availableLocales[0].equals(testLocale)) { + // Locale matches the "null" locale so it is treated as invalid + return new ValidationResult.Builder().subject(subject).input(input).valid(false) + .explanation(input + " is not a valid locale format.").build(); + } else { + return new ValidationResult.Builder().subject(subject).input(input).valid(true).build(); + + } + + } } } http://git-wip-us.apache.org/repos/asf/nifi/blob/a298f1a7/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java index 0da4d2b..5406613 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java @@ -28,6 +28,7 @@ import org.junit.Test; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.Locale; import java.util.TimeZone; @@ -239,6 +240,78 @@ public class TestParseCEF { Assert.assertEquals(prettyEvent, extension.get("deviceCustomDate1").asText()); } + @Test + public void testNonEnglishDate() { + final TestRunner runner = TestRunners.newTestRunner(new ParseCEF()); + runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_ATTRIBUTES); + runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, "fr-FR"); + runner.assertValid(); + + String sample = sample1.replace("Feb", "févr."); + runner.enqueue(sample); + runner.run(); + + runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 1); + MockFlowFile mff1 = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0); + + mff1.assertAttributeEquals("cef.extension.rt", sdf.format(new Date(1423441663000L))); + + runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_ATTRIBUTES); + runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, "et-EE"); + runner.assertValid(); + + sample = sample1.replace("Feb", "veebr"); + runner.enqueue(sample); + runner.run(); + + runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 2); + MockFlowFile mff2 = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0); + + mff2.assertAttributeEquals("cef.extension.rt", sdf.format(new Date(1423441663000L))); + + runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_ATTRIBUTES); + runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, "ja-JP"); + runner.assertValid(); + + SimpleDateFormat dateFormat = new SimpleDateFormat("LLLL", Locale.forLanguageTag("ja-JP") ); + String jpFeb = dateFormat.format(1423441663000L); + + sample = sample1.replace("Feb", jpFeb ); + runner.enqueue(sample); + runner.run(); + + runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 3); + MockFlowFile mff3 = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0); + + mff3.assertAttributeEquals("cef.extension.rt", sdf.format(new Date(1423441663000L))); + + + } + + + @Test + public void testCustomValidator() { + final TestRunner runner = TestRunners.newTestRunner(new ParseCEF()); + runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_CONTENT); + runner.setProperty(ParseCEF.TIME_REPRESENTATION, ParseCEF.UTC); + + + runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, "SPANGLISH"); + runner.assertNotValid(); + runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, "en-US"); + runner.assertValid(); + + + Locale availableLocales[] = Locale.getAvailableLocales(); + + for (Locale listedLocale : availableLocales ) { + if (!listedLocale.toString().isEmpty()) { + String input = listedLocale.toLanguageTag(); + runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, input ); + runner.assertValid(); + } + } + } }
