NIFI-5337 Syslog 5424 Record Reader and nifi-syslog-utils - Create nifi-syslog-utils to move syslog parsing functionalty to a central location shared by the processors and serialization/record system. - Refactor Processors to use these utils - Update 5424 syslog classes using simple-syslog-5424 to pick up new changes to support this work, as well as keep dependencies/types from bleeding out to the processors or readers - Refactor Syslog5424Event and Parser - Create Syslog5424RecordReader - per review, handle blank message differently from eof - name schema per review
This closes #2816. Signed-off-by: Bryan Bende <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b1022043 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b1022043 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b1022043 Branch: refs/heads/master Commit: b10220439cfddd876e01f1a2279b33552e47c7ac Parents: ec7f131 Author: Otto Fowler <[email protected]> Authored: Fri Jun 22 15:44:38 2018 -0400 Committer: Bryan Bende <[email protected]> Committed: Mon Jul 16 11:50:55 2018 -0400 ---------------------------------------------------------------------- .../nifi-syslog-utils/pom.xml | 48 ++++ .../syslog/attributes/Syslog5424Attributes.java | 50 ++++ .../syslog/attributes/SyslogAttributes.java | 60 +++++ .../nifi/syslog/events/Syslog5424Event.java | 116 +++++++++ .../apache/nifi/syslog/events/SyslogEvent.java | 180 +++++++++++++ .../syslog/keyproviders/SimpleKeyProvider.java | 105 ++++++++ .../keyproviders/SyslogPrefixedKeyProvider.java | 105 ++++++++ .../syslog/parsers/StrictSyslog5424Parser.java | 130 ++++++++++ .../nifi/syslog/parsers/SyslogParser.java | 175 +++++++++++++ .../syslog/utils/NifiStructuredDataPolicy.java | 35 +++ .../nifi/syslog/utils/NilHandlingPolicy.java | 36 +++ .../syslog/BaseStrictSyslog5424ParserTest.java | 206 +++++++++++++++ .../StrictSyslog5424ParserDashPolicyTest.java | 26 ++ .../StrictSyslog5424ParserNullPolicyTest.java | 26 ++ .../StrictSyslog5424ParserOmitPolicyTest.java | 26 ++ .../apache/nifi/syslog/TestSyslogParser.java | 255 +++++++++++++++++++ nifi-nar-bundles/nifi-extension-utils/pom.xml | 1 + .../nifi-standard-processors/pom.xml | 9 +- .../nifi/processors/standard/ListenSyslog.java | 30 +-- .../nifi/processors/standard/ParseSyslog.java | 39 +-- .../processors/standard/ParseSyslog5424.java | 32 ++- .../nifi/processors/standard/PutSyslog.java | 2 +- .../standard/syslog/StrictSyslog5424Parser.java | 206 --------------- .../standard/syslog/Syslog5424Attributes.java | 43 ---- .../standard/syslog/Syslog5424Event.java | 103 -------- .../standard/syslog/SyslogAttributes.java | 48 ---- .../processors/standard/syslog/SyslogEvent.java | 180 ------------- .../standard/syslog/SyslogParser.java | 173 ------------- .../standard/ITListenSyslogGroovy.groovy | 2 +- .../standard/ParseSyslogGroovyTest.groovy | 2 +- .../processors/standard/ITListenSyslog.java | 22 +- .../processors/standard/TestListenSyslog.java | 12 +- .../processors/standard/TestParseSyslog.java | 38 +-- .../standard/TestParseSyslog5424.java | 8 +- .../util/BaseStrictSyslog5424ParserTest.java | 205 --------------- .../StrictSyslog5424ParserDashPolicyTest.java | 25 -- .../StrictSyslog5424ParserNullPolicyTest.java | 25 -- .../StrictSyslog5424ParserOmitPolicyTest.java | 25 -- .../standard/util/TestSyslogParser.java | 255 ------------------- nifi-nar-bundles/nifi-standard-bundle/pom.xml | 5 - .../src/main/resources/META-INF/NOTICE | 8 + .../nifi-record-serialization-services/pom.xml | 9 + .../apache/nifi/syslog/Syslog5424Reader.java | 155 +++++++++++ .../nifi/syslog/Syslog5424RecordReader.java | 121 +++++++++ ...org.apache.nifi.controller.ControllerService | 2 +- .../additionalDetails.html | 91 +++++++ .../nifi/syslog/TestSyslog5424RecordReader.java | 223 ++++++++++++++++ .../test/resources/syslog/syslog5424/log.txt | 1 + .../resources/syslog/syslog5424/log_all.txt | 1 + .../resources/syslog/syslog5424/log_mix.txt | 3 + .../syslog/syslog5424/log_mix_in_error.txt | 4 + 51 files changed, 2302 insertions(+), 1385 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/pom.xml new file mode 100644 index 0000000..dd94370 --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/pom.xml @@ -0,0 +1,48 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-extension-utils</artifactId> + <version>1.8.0-SNAPSHOT</version> + </parent> + <artifactId>nifi-syslog-utils</artifactId> + <packaging>jar</packaging> + <dependencies> + <dependency> + <groupId>com.github.palindromicity</groupId> + <artifactId>simple-syslog-5424</artifactId> + <version>0.0.7</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-utils</artifactId> + <version>1.8.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-mock</artifactId> + <version>1.8.0-SNAPSHOT</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/Syslog5424Attributes.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/Syslog5424Attributes.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/Syslog5424Attributes.java new file mode 100644 index 0000000..346bd7a --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/Syslog5424Attributes.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog.attributes; + +import org.apache.nifi.flowfile.attributes.FlowFileAttributeKey; + +/** + * FlowFile Attributes for each Syslog message. + */ +public enum Syslog5424Attributes implements FlowFileAttributeKey { + + SYSLOG_APP_NAME("syslog.appName"), + SYSLOG_PROCID("syslog.procid"), + SYSLOG_MESSAGEID("syslog.messageid"), + SYSLOG_STRUCTURED_BASE("syslog.structuredData"), + SYSLOG_STRUCTURED_ELEMENT_ID_FMT("syslog.structuredData.%s"), + SYSLOG_STRUCTURED_ELEMENT_ID_PNAME_FMT("syslog.structuredData.%s.%s"), + SYSLOG_STRUCTURED_ELEMENT_ID_PNAME_PATTERN("syslog.structuredData\\.(.*)\\.(.*)$"), + APP_NAME("appName"), + PROCID("procid"), + MESSAGEID("messageid"), + STRUCTURED_BASE("structuredData"), + STRUCTURED_ELEMENT_ID_FMT("structuredData.%s"), + STRUCTURED_ELEMENT_ID_PNAME_FMT("structuredData.%s.%s"), + STRUCTURED_ELEMENT_ID_PNAME_PATTERN("structuredData\\.(.*)\\.(.*)$"); + private String key; + + Syslog5424Attributes(String key) { + this.key = key; + } + + @Override + public String key() { + return key; + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/SyslogAttributes.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/SyslogAttributes.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/SyslogAttributes.java new file mode 100644 index 0000000..1eb2dc8 --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/SyslogAttributes.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog.attributes; + +import org.apache.nifi.flowfile.attributes.FlowFileAttributeKey; + +/** + * FlowFile Attributes for each Syslog message. + */ +public enum SyslogAttributes implements FlowFileAttributeKey { + + SYSLOG_PRIORITY("syslog.priority"), + SYSLOG_SEVERITY("syslog.severity"), + SYSLOG_FACILITY("syslog.facility"), + SYSLOG_VERSION("syslog.version"), + SYSLOG_TIMESTAMP("syslog.timestamp"), + SYSLOG_HOSTNAME("syslog.hostname"), + SYSLOG_SENDER("syslog.sender"), + SYSLOG_BODY("syslog.body"), + SYSLOG_VALID("syslog.valid"), + SYSLOG_PROTOCOL("syslog.protocol"), + SYSLOG_PORT("syslog.port"), + + PRIORITY("priority"), + SEVERITY("severity"), + FACILITY("facility"), + VERSION("version"), + TIMESTAMP("timestamp"), + HOSTNAME("hostname"), + SENDER("sender"), + BODY("body"), + VALID("valid"), + PROTOCOL("protocol"), + PORT("port"); + + private String key; + + SyslogAttributes(String key) { + this.key = key; + } + + @Override + public String key() { + return key; + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/Syslog5424Event.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/Syslog5424Event.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/Syslog5424Event.java new file mode 100644 index 0000000..7fb00bb --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/Syslog5424Event.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog.events; + +import java.util.Map; + +/** + * Encapsulates the parsed information for a single Syslog 5424 event. + */ +public class Syslog5424Event { + private final Map<String, Object> fieldMap; + private final String fullMessage; + private final byte[] rawMessage; + private final String sender; + private final boolean valid; + private final Exception exception; + + private Syslog5424Event(final Builder builder) { + this.fieldMap = builder.fieldMap; + this.fullMessage = builder.fullMessage; + this.rawMessage = builder.rawMessage; + this.sender = builder.sender; + this.valid = builder.valid; + this.exception = builder.exception; + } + + public Map<String, Object> getFieldMap() { + return fieldMap; + } + + public Exception getException() { + return exception; + } + + public String getFullMessage() { + return fullMessage; + } + + public byte[] getRawMessage() { + return rawMessage; + } + + public String getSender() { + return sender; + } + + public boolean isValid() { + return valid; + } + + public static final class Builder { + private Exception exception; + private String fullMessage; + private String sender; + private Map<String, Object> fieldMap; + private byte[] rawMessage; + private boolean valid; + + public void reset() { + this.fieldMap = null; + this.sender = null; + this.fullMessage = null; + this.valid = false; + this.exception = null; + } + + public Builder sender(String sender) { + this.sender = sender; + return this; + } + + public Builder exception(Exception exception) { + this.exception = exception; + return this; + } + + public Builder fieldMap(Map<String, Object> fieldMap) { + this.fieldMap = fieldMap; + return this; + } + + public Builder fullMessage(String fullMessage) { + this.fullMessage = fullMessage; + return this; + } + + public Builder rawMessage(byte[] rawMessage) { + this.rawMessage = rawMessage; + return this; + } + + public Builder valid(boolean valid) { + this.valid = valid; + return this; + } + + public Syslog5424Event build() { + return new Syslog5424Event(this); + } + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/SyslogEvent.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/SyslogEvent.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/SyslogEvent.java new file mode 100644 index 0000000..a6f711c --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/SyslogEvent.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog.events; + +/** + * Encapsulates the parsed information for a single Syslog event. + */ +public class SyslogEvent { + + private final String priority; + private final String severity; + private final String facility; + private final String version; + private final String timeStamp; + private final String hostName; + private final String sender; + private final String msgBody; + private final String fullMessage; + private final byte[] rawMessage; + private final boolean valid; + + private SyslogEvent(final Builder builder) { + this.priority = builder.priority; + this.severity = builder.severity; + this.facility = builder.facility; + this.version = builder.version; + this.timeStamp = builder.timeStamp; + this.hostName = builder.hostName; + this.sender = builder.sender; + this.msgBody = builder.msgBody; + this.fullMessage = builder.fullMessage; + this.rawMessage = builder.rawMessage; + this.valid = builder.valid; + } + + public String getPriority() { + return priority; + } + + public String getSeverity() { + return severity; + } + + public String getFacility() { + return facility; + } + + public String getVersion() { + return version; + } + + public String getTimeStamp() { + return timeStamp; + } + + public String getHostName() { + return hostName; + } + + public String getSender() { + return sender; + } + + public String getMsgBody() { + return msgBody; + } + + public String getFullMessage() { + return fullMessage; + } + + public byte[] getRawMessage() { + return rawMessage; + } + + public boolean isValid() { + return valid; + } + + public static final class Builder { + private String priority; + private String severity; + private String facility; + private String version; + private String timeStamp; + private String hostName; + private String sender; + private String msgBody; + private String fullMessage; + private byte[] rawMessage; + private boolean valid; + + public void reset() { + this.priority = null; + this.severity = null; + this.facility = null; + this.version = null; + this.timeStamp = null; + this.hostName = null; + this.sender = null; + this.msgBody = null; + this.fullMessage = null; + this.valid = false; + } + + public Builder priority(String priority) { + this.priority = priority; + return this; + } + + public Builder severity(String severity) { + this.severity = severity; + return this; + } + + public Builder facility(String facility) { + this.facility = facility; + return this; + } + + public Builder version(String version) { + this.version = version; + return this; + } + + public Builder timestamp(String timestamp) { + this.timeStamp = timestamp; + return this; + } + + public Builder hostname(String hostName) { + this.hostName = hostName; + return this; + } + + public Builder sender(String sender) { + this.sender = sender; + return this; + } + + public Builder msgBody(String msgBody) { + this.msgBody = msgBody; + return this; + } + + public Builder fullMessage(String fullMessage) { + this.fullMessage = fullMessage; + return this; + } + + public Builder rawMessage(byte[] rawMessage) { + this.rawMessage = rawMessage; + return this; + } + + public Builder valid(boolean valid) { + this.valid = valid; + return this; + } + + public SyslogEvent build() { + return new SyslogEvent(this); + } + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SimpleKeyProvider.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SimpleKeyProvider.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SimpleKeyProvider.java new file mode 100644 index 0000000..76959d7 --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SimpleKeyProvider.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.syslog.keyproviders; + +import com.github.palindromicity.syslog.KeyProvider; +import org.apache.nifi.syslog.attributes.Syslog5424Attributes; +import org.apache.nifi.syslog.attributes.SyslogAttributes; + +import java.util.regex.Pattern; + +public class SimpleKeyProvider implements KeyProvider { + private Pattern pattern; + + public SimpleKeyProvider() { + } + + @Override + public String getMessage() { + return SyslogAttributes.BODY.key(); + } + + @Override + public String getHeaderAppName(){ + return Syslog5424Attributes.APP_NAME.key(); + } + + @Override + public String getHeaderHostName() { + return SyslogAttributes.HOSTNAME.key(); + } + + @Override + public String getHeaderPriority() { + return SyslogAttributes.PRIORITY.key(); + } + + @Override + public String getHeaderFacility() { + return SyslogAttributes.FACILITY.key(); + } + + @Override + public String getHeaderSeverity() { + return SyslogAttributes.SEVERITY.key(); + } + + + @Override + public String getHeaderProcessId() { + return Syslog5424Attributes.PROCID.key(); + } + + @Override + public String getHeaderTimeStamp() { + return SyslogAttributes.TIMESTAMP.key(); + } + + @Override + public String getHeaderMessageId() { + return Syslog5424Attributes.MESSAGEID.key(); + } + + @Override + public String getHeaderVersion() { + return SyslogAttributes.VERSION.key(); + } + + @Override + public String getStructuredBase() { + return Syslog5424Attributes.STRUCTURED_BASE.key(); + } + + @Override + public String getStructuredElementIdFormat() { + return Syslog5424Attributes.STRUCTURED_ELEMENT_ID_FMT.key(); + } + + @Override + public String getStructuredElementIdParamNameFormat() { + return Syslog5424Attributes.STRUCTURED_ELEMENT_ID_PNAME_FMT.key(); + } + + @Override + public Pattern getStructuredElementIdParamNamePattern() { + if (pattern == null) { + pattern = Pattern.compile(Syslog5424Attributes.STRUCTURED_ELEMENT_ID_PNAME_PATTERN.key()); + } + return pattern; + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SyslogPrefixedKeyProvider.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SyslogPrefixedKeyProvider.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SyslogPrefixedKeyProvider.java new file mode 100644 index 0000000..c23aab5 --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SyslogPrefixedKeyProvider.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.syslog.keyproviders; + +import com.github.palindromicity.syslog.KeyProvider; +import org.apache.nifi.syslog.attributes.Syslog5424Attributes; +import org.apache.nifi.syslog.attributes.SyslogAttributes; + +import java.util.regex.Pattern; + +public class SyslogPrefixedKeyProvider implements KeyProvider { + private Pattern pattern; + + public SyslogPrefixedKeyProvider() { + } + + @Override + public String getMessage() { + return SyslogAttributes.SYSLOG_BODY.key(); + } + + @Override + public String getHeaderAppName() { + return Syslog5424Attributes.SYSLOG_APP_NAME.key(); + } + + @Override + public String getHeaderHostName() { + return SyslogAttributes.SYSLOG_HOSTNAME.key(); + } + + @Override + public String getHeaderPriority() { + return SyslogAttributes.SYSLOG_PRIORITY.key(); + } + + @Override + public String getHeaderFacility() { + return SyslogAttributes.SYSLOG_FACILITY.key(); + } + + @Override + public String getHeaderSeverity() { + return SyslogAttributes.SYSLOG_SEVERITY.key(); + } + + + @Override + public String getHeaderProcessId() { + return Syslog5424Attributes.SYSLOG_PROCID.key(); + } + + @Override + public String getHeaderTimeStamp() { + return SyslogAttributes.SYSLOG_TIMESTAMP.key(); + } + + @Override + public String getHeaderMessageId() { + return Syslog5424Attributes.SYSLOG_MESSAGEID.key(); + } + + @Override + public String getHeaderVersion() { + return SyslogAttributes.SYSLOG_VERSION.key(); + } + + @Override + public String getStructuredBase() { + return Syslog5424Attributes.SYSLOG_STRUCTURED_BASE.key(); + } + + @Override + public String getStructuredElementIdFormat() { + return Syslog5424Attributes.SYSLOG_STRUCTURED_ELEMENT_ID_FMT.key(); + } + + @Override + public String getStructuredElementIdParamNameFormat() { + return Syslog5424Attributes.SYSLOG_STRUCTURED_ELEMENT_ID_PNAME_FMT.key(); + } + + @Override + public Pattern getStructuredElementIdParamNamePattern() { + if (pattern == null) { + pattern = Pattern.compile(Syslog5424Attributes.SYSLOG_STRUCTURED_ELEMENT_ID_PNAME_PATTERN.key()); + } + return pattern; + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java new file mode 100644 index 0000000..ed36c9d --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog.parsers; + +import com.github.palindromicity.syslog.KeyProvider; +import com.github.palindromicity.syslog.NilPolicy; +import com.github.palindromicity.syslog.StructuredDataPolicy; +import com.github.palindromicity.syslog.SyslogParserBuilder; +import org.apache.nifi.syslog.events.Syslog5424Event; +import org.apache.nifi.syslog.keyproviders.SyslogPrefixedKeyProvider; +import org.apache.nifi.syslog.utils.NifiStructuredDataPolicy; +import org.apache.nifi.syslog.utils.NilHandlingPolicy; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * Parses a Syslog message from a ByteBuffer into a Syslog5424Event instance. + * For 5424 we use simple-syslog-5424 since it parsers out structured data. + */ +public class StrictSyslog5424Parser { + private Charset charset; + private com.github.palindromicity.syslog.SyslogParser parser; + + public StrictSyslog5424Parser() { + this(StandardCharsets.UTF_8, NilHandlingPolicy.NULL, NifiStructuredDataPolicy.FLATTEN, new SyslogPrefixedKeyProvider()); + } + + public StrictSyslog5424Parser(final Charset charset, final NilHandlingPolicy nilPolicy, + NifiStructuredDataPolicy structuredDataPolicy, KeyProvider keyProvider) { + this.charset = charset; + parser = new SyslogParserBuilder() + .withNilPolicy(NilPolicy.valueOf(nilPolicy.name())) + .withStructuredDataPolicy(StructuredDataPolicy.valueOf(structuredDataPolicy.name())) + .withKeyProvider(keyProvider) + .build(); + } + + /** + * Parses a Syslog5424Event from a {@code ByteBuffer}. + * + * @param buffer a {@code ByteBuffer} containing a syslog message + * @return a Syslog5424Event parsed from the {@code {@code byte array}} + */ + public Syslog5424Event parseEvent(final ByteBuffer buffer) { + return parseEvent(buffer, null); + } + + /** + * Parses a Syslog5424Event from a {@code ByteBuffer}. + * + * @param buffer a {@code ByteBuffer} containing a syslog message + * @param sender the hostname of the syslog server that sent the message + * @return a Syslog5424Event parsed from the {@code byte array} + */ + public Syslog5424Event parseEvent(final ByteBuffer buffer, final String sender) { + if (buffer == null) { + return null; + } + return parseEvent(bufferToBytes(buffer), sender); + } + + /** + * Parses a Syslog5424Event from a {@code byte array}. + * + * @param bytes a {@code byte array} containing a syslog message + * @param sender the hostname of the syslog server that sent the message + * @return a Syslog5424Event parsed from the {@code byte array} + */ + public Syslog5424Event parseEvent(final byte[] bytes, final String sender) { + if (bytes == null || bytes.length == 0) { + return null; + } + + // remove trailing new line before parsing + int length = bytes.length; + if (bytes[length - 1] == '\n') { + length = length - 1; + } + + final String message = new String(bytes, 0, length, charset); + + final Syslog5424Event.Builder builder = new Syslog5424Event.Builder() + .valid(false).fullMessage(message).rawMessage(bytes).sender(sender); + + try { + parser.parseLine(message, builder::fieldMap); + builder.valid(true); + } catch (Exception e) { + // this is not a valid 5424 message + builder.valid(false); + builder.exception(e); + } + + // either invalid w/original msg, or fully parsed event + return builder.build(); + } + + public String getCharsetName() { + return charset == null ? StandardCharsets.UTF_8.name() : charset.name(); + } + + + private byte[] bufferToBytes(ByteBuffer buffer) { + if (buffer == null) { + return null; + } + if (buffer.position() != 0) { + buffer.flip(); + } + byte bytes[] = new byte[buffer.limit()]; + buffer.get(bytes, 0, buffer.limit()); + return bytes; + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/SyslogParser.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/SyslogParser.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/SyslogParser.java new file mode 100644 index 0000000..165dc39 --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/SyslogParser.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog.parsers; + +import org.apache.nifi.syslog.events.SyslogEvent; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.regex.MatchResult; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parses a Syslog message from a ByteBuffer into a SyslogEvent instance. + * + * The Syslog regular expressions below were adapted from the Apache Flume project. + */ +public class SyslogParser { + + public static final String SYSLOG_MSG_RFC5424_0 = + "(?:\\<(\\d{1,3})\\>)" + // priority + "(?:(\\d)?\\s?)" + // version + /* yyyy-MM-dd'T'HH:mm:ss.SZ or yyyy-MM-dd'T'HH:mm:ss.S+hh:mm or - (null stamp) */ + "(?:" + + "(\\d{4}[-]\\d{2}[-]\\d{2}[T]\\d{2}[:]\\d{2}[:]\\d{2}" + + "(?:\\.\\d{1,6})?(?:[+-]\\d{2}[:]\\d{2}|Z)?)|-)" + // stamp + "\\s" + // separator + "(?:([\\w][\\w\\d\\.@\\-]*)|-)" + // host name or - (null) + "\\s" + // separator + "(.*)$"; // body + + public static final String SYSLOG_MSG_RFC3164_0 = + "(?:\\<(\\d{1,3})\\>)" + + "(?:(\\d)?\\s?)" + // version + // stamp MMM d HH:mm:ss, single digit date has two spaces + "([A-Z][a-z][a-z]\\s{1,2}\\d{1,2}\\s\\d{2}[:]\\d{2}[:]\\d{2})" + + "\\s" + // separator + "([\\w][\\w\\d(\\.|\\:)@-]*)" + // host + "\\s(.*)$"; // body + + public static final Collection<Pattern> MESSAGE_PATTERNS; + static { + List<Pattern> patterns = new ArrayList<>(); + patterns.add(Pattern.compile(SYSLOG_MSG_RFC5424_0)); + patterns.add(Pattern.compile(SYSLOG_MSG_RFC3164_0)); + MESSAGE_PATTERNS = Collections.unmodifiableList(patterns); + } + + // capture group positions from the above message patterns + public static final int SYSLOG_PRIORITY_POS = 1; + public static final int SYSLOG_VERSION_POS = 2; + public static final int SYSLOG_TIMESTAMP_POS = 3; + public static final int SYSLOG_HOSTNAME_POS = 4; + public static final int SYSLOG_BODY_POS = 5; + + private Charset charset; + + public SyslogParser() { + this(StandardCharsets.UTF_8); + } + + public SyslogParser(final Charset charset) { + this.charset = charset; + } + + /** + * Parses a SyslogEvent from a byte buffer. + * + * @param buffer a byte buffer containing a syslog message + * @return a SyslogEvent parsed from the byte array + */ + public SyslogEvent parseEvent(final ByteBuffer buffer) { + return parseEvent(buffer, null); + } + + /** + * Parses a SyslogEvent from a byte buffer. + * + * @param buffer a byte buffer containing a syslog message + * @param sender the hostname of the syslog server that sent the message + * @return a SyslogEvent parsed from the byte array + */ + public SyslogEvent parseEvent(final ByteBuffer buffer, final String sender) { + if (buffer == null) { + return null; + } + if (buffer.position() != 0) { + buffer.flip(); + } + byte bytes[] = new byte[buffer.limit()]; + buffer.get(bytes, 0, buffer.limit()); + return parseEvent(bytes, sender); + } + + /** + * Parses a SyslogEvent from a byte array. + * + * @param bytes a byte array containing a syslog message + * @param sender the hostname of the syslog server that sent the message + * @return a SyslogEvent parsed from the byte array + */ + public SyslogEvent parseEvent(final byte[] bytes, final String sender) { + if (bytes == null || bytes.length == 0) { + return null; + } + + // remove trailing new line before parsing + int length = bytes.length; + if (bytes[length - 1] == '\n') { + length = length - 1; + } + + final String message = new String(bytes, 0, length, charset); + + final SyslogEvent.Builder builder = new SyslogEvent.Builder() + .valid(false).fullMessage(message).rawMessage(bytes).sender(sender); + + for (Pattern pattern : MESSAGE_PATTERNS) { + final Matcher matcher = pattern.matcher(message); + if (!matcher.matches()) { + continue; + } + + final MatchResult res = matcher.toMatchResult(); + for (int grp = 1; grp <= res.groupCount(); grp++) { + String value = res.group(grp); + if (grp == SYSLOG_TIMESTAMP_POS) { + builder.timestamp(value); + } else if (grp == SYSLOG_HOSTNAME_POS) { + builder.hostname(value); + } else if (grp == SYSLOG_PRIORITY_POS) { + int pri = Integer.parseInt(value); + int sev = pri % 8; + int facility = pri / 8; + builder.priority(value); + builder.severity(String.valueOf(sev)); + builder.facility(String.valueOf(facility)); + } else if (grp == SYSLOG_VERSION_POS) { + builder.version(value); + } else if (grp == SYSLOG_BODY_POS) { + builder.msgBody(value); + } + } + + builder.valid(true); + break; + } + + // either invalid w/original msg, or fully parsed event + return builder.build(); + } + + public String getCharsetName() { + return charset == null ? StandardCharsets.UTF_8.name() : charset.name(); + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NifiStructuredDataPolicy.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NifiStructuredDataPolicy.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NifiStructuredDataPolicy.java new file mode 100644 index 0000000..5af2191 --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NifiStructuredDataPolicy.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.syslog.utils; + +/** + * Policy for handling Structured Data + * must match the simple-syslog-5424 StructuredDataPolicy + */ +public enum NifiStructuredDataPolicy { + /** + * The Structured Data will be flattened per the KeyProvider provided values. + */ + FLATTEN, + /** + * The Structued Data will be returned as a Map field named structuredData. + * Each map entry will have the value of the Structured Data ID, and a value + * of a map of each element param name and value + */ + MAP_OF_MAPS +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NilHandlingPolicy.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NilHandlingPolicy.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NilHandlingPolicy.java new file mode 100644 index 0000000..d2c3d3c --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NilHandlingPolicy.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog.utils; + +/** + * Policies available for handling NIL '-' values. + * Must match the simple-syslog-5424 NilPolicy + */ +public enum NilHandlingPolicy { + /** + * a nil value will result msg part being omitted from the map. + */ + OMIT, + /** + * a nil value will result in a null value in the map. + */ + NULL, + /** + * a nil value will result in a '-' symbol in the map. + */ + DASH +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/BaseStrictSyslog5424ParserTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/BaseStrictSyslog5424ParserTest.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/BaseStrictSyslog5424ParserTest.java new file mode 100644 index 0000000..082fbbd --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/BaseStrictSyslog5424ParserTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog; + +import org.apache.nifi.syslog.attributes.Syslog5424Attributes; +import org.apache.nifi.syslog.attributes.SyslogAttributes; +import org.apache.nifi.syslog.events.Syslog5424Event; +import org.apache.nifi.syslog.keyproviders.SyslogPrefixedKeyProvider; +import org.apache.nifi.syslog.parsers.StrictSyslog5424Parser; +import org.apache.nifi.syslog.utils.NifiStructuredDataPolicy; +import org.apache.nifi.syslog.utils.NilHandlingPolicy; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +public abstract class BaseStrictSyslog5424ParserTest { + + private static final Charset CHARSET = Charset.forName("UTF-8"); + private static final String NIL_VALUE = "-"; + private StrictSyslog5424Parser parser; + + protected abstract NilHandlingPolicy getPolicy(); + + protected void validateForPolicy(String expected, Object actual) { + switch (getPolicy()) { + case DASH: + Assert.assertEquals(actual, NIL_VALUE); + break; + case OMIT: + case NULL: + Assert.assertNull(actual); + + } + } + + @Before + public void setup() { + parser = new StrictSyslog5424Parser(CHARSET, getPolicy(), NifiStructuredDataPolicy.FLATTEN, new SyslogPrefixedKeyProvider()); + } + + @Test + public void testRFC5424WithVersion() { + final String pri = "34"; + final String version = "1"; + final String stamp = "2003-10-11T22:14:15.003Z"; + final String host = "mymachine.example.com"; + final String appName = "su"; + final String procId = "-"; + final String msgId = "ID17"; + final String structuredData = "-"; + final String body = "BOM'su root' failed for lonvick on /dev/pts/8"; + + final String message = "<" + pri + ">" + version + " " + stamp + " " + host + " " + + appName + " " + procId + " " + msgId + " " + "-" + " " + body; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final Syslog5424Event event = parser.parseEvent(buffer); + Assert.assertNotNull(event); + Assert.assertTrue(event.isValid()); + Assert.assertFalse(event.getFieldMap().isEmpty()); + Map<String, Object> fieldMap = event.getFieldMap(); + Assert.assertEquals(pri, fieldMap.get(SyslogAttributes.SYSLOG_PRIORITY.key())); + Assert.assertEquals("2", fieldMap.get(SyslogAttributes.SYSLOG_SEVERITY.key())); + Assert.assertEquals("4", fieldMap.get(SyslogAttributes.SYSLOG_FACILITY.key())); + Assert.assertEquals(version, fieldMap.get(SyslogAttributes.SYSLOG_VERSION.key())); + Assert.assertEquals(stamp, fieldMap.get(SyslogAttributes.SYSLOG_TIMESTAMP.key())); + Assert.assertEquals(host, fieldMap.get(SyslogAttributes.SYSLOG_HOSTNAME.key())); + Assert.assertEquals(appName, fieldMap.get(Syslog5424Attributes.SYSLOG_APP_NAME.key())); + validateForPolicy(procId, fieldMap.get(Syslog5424Attributes.SYSLOG_PROCID.key())); + Assert.assertEquals(msgId, fieldMap.get(Syslog5424Attributes.SYSLOG_MESSAGEID.key())); + + Pattern structuredPattern = new SyslogPrefixedKeyProvider().getStructuredElementIdParamNamePattern(); + fieldMap.forEach((key, value) -> { + if (value != null) { + Assert.assertFalse(structuredPattern.matcher(key).matches()); + } + }); + + Assert.assertEquals(body, fieldMap.get(SyslogAttributes.SYSLOG_BODY.key())); + Assert.assertEquals(message, event.getFullMessage()); + Assert.assertNull(event.getSender()); + } + + @Test + public void testRFC5424WithoutVersion() { + final String pri = "34"; + final String version = "-"; + final String stamp = "2003-10-11T22:14:15.003Z"; + final String host = "mymachine.example.com"; + final String appName = "su"; + final String procId = "-"; + final String msgId = "ID17"; + final String structuredData = "-"; + final String body = "BOM'su root' failed for lonvick on /dev/pts/8"; + + final String message = "<" + pri + ">" + version + " " + stamp + " " + host + " " + + appName + " " + procId + " " + msgId + " " + "-" + " " + body; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final Syslog5424Event event = parser.parseEvent(buffer); + Assert.assertFalse(event.isValid()); + } + + @Test + public void testTrailingNewLine() { + final String message = "<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - " + + "ID47 - BOM'su root' failed for lonvick on /dev/pts/8\n"; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final Syslog5424Event event = parser.parseEvent(buffer); + Assert.assertNotNull(event); + Assert.assertTrue(event.isValid()); + } + + @Test + public void testVariety() { + final List<String> messages = new ArrayList<>(); + + // supported examples from RFC 5424 + messages.add("<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - " + + "ID47 - BOM'su root' failed for lonvick on /dev/pts/8"); + messages.add("<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc " + + "8710 - - %% It's time to make the do-nuts."); + messages.add("<14>1 2014-06-20T09:14:07+00:00 loggregator" + + " d0602076-b14a-4c55-852a-981e7afeed38 DEA MSG-01" + + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]" + + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance"); + + for (final String message : messages) { + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final Syslog5424Event event = parser.parseEvent(buffer); + Assert.assertTrue(event.isValid()); + } + } + + @Test + public void testInvalidPriority() { + final String message = "10 Oct 13 14:14:43 localhost some body of the message"; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final Syslog5424Event event = parser.parseEvent(buffer); + Assert.assertNotNull(event); + Assert.assertFalse(event.isValid()); + Assert.assertEquals(message, event.getFullMessage()); + } + + @Test + public void testParseWithSender() { + final String sender = "127.0.0.1"; + final String message = "<14>1 2014-06-20T09:14:07+00:00 loggregator" + + " d0602076-b14a-4c55-852a-981e7afeed38 DEA MSG-01" + + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]" + + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance"; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final Syslog5424Event event = parser.parseEvent(buffer, sender); + Assert.assertNotNull(event); + Assert.assertTrue(event.isValid()); + Assert.assertEquals(sender, event.getSender()); + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserDashPolicyTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserDashPolicyTest.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserDashPolicyTest.java new file mode 100644 index 0000000..c330551 --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserDashPolicyTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog; + + +import org.apache.nifi.syslog.utils.NilHandlingPolicy; + +public class StrictSyslog5424ParserDashPolicyTest extends BaseStrictSyslog5424ParserTest { + protected NilHandlingPolicy getPolicy() { + return NilHandlingPolicy.DASH; + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserNullPolicyTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserNullPolicyTest.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserNullPolicyTest.java new file mode 100644 index 0000000..921d9bf --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserNullPolicyTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog; + + +import org.apache.nifi.syslog.utils.NilHandlingPolicy; + +public class StrictSyslog5424ParserNullPolicyTest extends BaseStrictSyslog5424ParserTest { + protected NilHandlingPolicy getPolicy() { + return NilHandlingPolicy.NULL; + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserOmitPolicyTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserOmitPolicyTest.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserOmitPolicyTest.java new file mode 100644 index 0000000..04bc151 --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserOmitPolicyTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog; + + +import org.apache.nifi.syslog.utils.NilHandlingPolicy; + +public class StrictSyslog5424ParserOmitPolicyTest extends BaseStrictSyslog5424ParserTest { + protected NilHandlingPolicy getPolicy() { + return NilHandlingPolicy.OMIT; + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/TestSyslogParser.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/TestSyslogParser.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/TestSyslogParser.java new file mode 100644 index 0000000..054d1eb --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/TestSyslogParser.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.syslog; + +import org.apache.nifi.syslog.events.SyslogEvent; +import org.apache.nifi.syslog.parsers.SyslogParser; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; + +public class TestSyslogParser { + + static final Charset CHARSET = Charset.forName("UTF-8"); + + private SyslogParser parser; + + @Before + public void setup() { + parser = new SyslogParser(CHARSET); + } + + @Test + public void testRFC3164SingleDigitDay() { + final String pri = "10"; + final String stamp = "Oct 1 13:14:04"; + final String host = "my.host.com"; + final String body = "some body message"; + final String message = "<" + pri + ">" + stamp + " " + host + " " + body; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final SyslogEvent event = parser.parseEvent(buffer); + Assert.assertNotNull(event); + Assert.assertEquals(pri, event.getPriority()); + Assert.assertEquals("2", event.getSeverity()); + Assert.assertEquals("1", event.getFacility()); + Assert.assertNull(event.getVersion()); + Assert.assertEquals(stamp, event.getTimeStamp()); + Assert.assertEquals(host, event.getHostName()); + Assert.assertEquals(body, event.getMsgBody()); + Assert.assertEquals(message, event.getFullMessage()); + Assert.assertTrue(event.isValid()); + } + + @Test + public void testRFC3164DoubleDigitDay() { + final String pri = "31"; + final String stamp = "Oct 13 14:14:43"; + final String host = "localhost"; + final String body = "AppleCameraAssistant[470]: DeviceMessageNotificationCallback: kIOPMMessageSystemPowerEventOccurred: 0x00000000"; + final String message = "<" + pri + ">" + stamp + " " + host + " " + body; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final SyslogEvent event = parser.parseEvent(buffer); + Assert.assertNotNull(event); + Assert.assertEquals(pri, event.getPriority()); + Assert.assertEquals("7", event.getSeverity()); + Assert.assertEquals("3", event.getFacility()); + Assert.assertNull(event.getVersion()); + Assert.assertEquals(stamp, event.getTimeStamp()); + Assert.assertEquals(host, event.getHostName()); + Assert.assertEquals(body, event.getMsgBody()); + Assert.assertEquals(message, event.getFullMessage()); + Assert.assertTrue(event.isValid()); + } + + @Test + public void testRFC3164WithVersion() { + final String pri = "31"; + final String version = "1"; + final String stamp = "Oct 13 14:14:43"; + final String host = "localhost"; + final String body = "AppleCameraAssistant[470]: DeviceMessageNotificationCallback: kIOPMMessageSystemPowerEventOccurred: 0x00000000"; + final String message = "<" + pri + ">" + version + " " + stamp + " " + host + " " + body; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final SyslogEvent event = parser.parseEvent(buffer); + Assert.assertNotNull(event); + Assert.assertEquals(pri, event.getPriority()); + Assert.assertEquals("7", event.getSeverity()); + Assert.assertEquals("3", event.getFacility()); + Assert.assertEquals(version, event.getVersion()); + Assert.assertEquals(stamp, event.getTimeStamp()); + Assert.assertEquals(host, event.getHostName()); + Assert.assertEquals(body, event.getMsgBody()); + Assert.assertEquals(message, event.getFullMessage()); + Assert.assertTrue(event.isValid()); + } + + @Test + public void testRFC5424WithVersion() { + final String pri = "34"; + final String version = "1"; + final String stamp = "2003-10-11T22:14:15.003Z"; + final String host = "mymachine.example.com"; + final String body = "su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8"; + + final String message = "<" + pri + ">" + version + " " + stamp + " " + host + " " + body; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final SyslogEvent event = parser.parseEvent(buffer); + Assert.assertNotNull(event); + Assert.assertEquals(pri, event.getPriority()); + Assert.assertEquals("2", event.getSeverity()); + Assert.assertEquals("4", event.getFacility()); + Assert.assertEquals(version, event.getVersion()); + Assert.assertEquals(stamp, event.getTimeStamp()); + Assert.assertEquals(host, event.getHostName()); + Assert.assertEquals(body, event.getMsgBody()); + Assert.assertEquals(message, event.getFullMessage()); + Assert.assertTrue(event.isValid()); + } + + @Test + public void testRFC5424WithoutVersion() { + final String pri = "34"; + final String stamp = "2003-10-11T22:14:15.003Z"; + final String host = "mymachine.example.com"; + final String body = "su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8"; + + final String message = "<" + pri + ">" + stamp + " " + host + " " + body; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final SyslogEvent event = parser.parseEvent(buffer); + Assert.assertNotNull(event); + Assert.assertEquals(pri, event.getPriority()); + Assert.assertEquals("2", event.getSeverity()); + Assert.assertEquals("4", event.getFacility()); + Assert.assertNull(event.getVersion()); + Assert.assertEquals(stamp, event.getTimeStamp()); + Assert.assertEquals(host, event.getHostName()); + Assert.assertEquals(body, event.getMsgBody()); + Assert.assertEquals(message, event.getFullMessage()); + Assert.assertTrue(event.isValid()); + } + + @Test + public void testTrailingNewLine() { + final String message = "<31>Oct 13 15:43:23 localhost.home some message\n"; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final SyslogEvent event = parser.parseEvent(buffer); + Assert.assertNotNull(event); + Assert.assertTrue(event.isValid()); + } + + @Test + public void testVariety() { + final List<String> messages = new ArrayList<>(); + + // supported examples from RFC 3164 + messages.add("<34>Oct 11 22:14:15 mymachine su: 'su root' failed for " + + "lonvick on /dev/pts/8"); + messages.add("<13>Feb 5 17:32:18 10.0.0.99 Use the BFG!"); + messages.add("<165>Aug 24 05:34:00 CST 1987 mymachine myproc[10]: %% " + + "It's time to make the do-nuts. %% Ingredients: Mix=OK, Jelly=OK # " + + "Devices: Mixer=OK, Jelly_Injector=OK, Frier=OK # Transport: " + + "Conveyer1=OK, Conveyer2=OK # %%"); + messages.add("<0>Oct 22 10:52:12 scapegoat 1990 Oct 22 10:52:01 TZ-6 " + + "scapegoat.dmz.example.org 10.1.2.3 sched[0]: That's All Folks!"); + + // supported examples from RFC 5424 + messages.add("<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - " + + "ID47 - BOM'su root' failed for lonvick on /dev/pts/8"); + messages.add("<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc " + + "8710 - - %% It's time to make the do-nuts."); + + // non-standard (but common) messages (RFC3339 dates, no version digit) + messages.add("<13>2003-08-24T05:14:15Z localhost snarf?"); + messages.add("<13>2012-08-16T14:34:03-08:00 127.0.0.1 test shnap!"); + + for (final String message : messages) { + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final SyslogEvent event = parser.parseEvent(buffer); + Assert.assertTrue(event.isValid()); + } + } + + @Test + public void testInvalidPriority() { + final String message = "10 Oct 13 14:14:43 localhost some body of the message"; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final SyslogEvent event = parser.parseEvent(buffer); + Assert.assertNotNull(event); + Assert.assertFalse(event.isValid()); + Assert.assertEquals(message, event.getFullMessage()); + } + + @Test + public void testParseWithSender() { + final String sender = "127.0.0.1"; + final String message = "<31>Oct 13 15:43:23 localhost.home some message\n"; + + final byte[] bytes = message.getBytes(CHARSET); + final ByteBuffer buffer = ByteBuffer.allocate(bytes.length); + buffer.clear(); + buffer.put(bytes); + + final SyslogEvent event = parser.parseEvent(buffer, sender); + Assert.assertNotNull(event); + Assert.assertTrue(event.isValid()); + Assert.assertEquals(sender, event.getSender()); + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-extension-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/pom.xml index 289cb55..85f2a4e 100644 --- a/nifi-nar-bundles/nifi-extension-utils/pom.xml +++ b/nifi-nar-bundles/nifi-extension-utils/pom.xml @@ -31,6 +31,7 @@ <module>nifi-hadoop-utils</module> <module>nifi-processor-utils</module> <module>nifi-reporting-utils</module> + <module>nifi-syslog-utils</module> </modules> </project> http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index a3fe1f6..39d7a6d 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -31,6 +31,11 @@ </dependency> <dependency> <groupId>org.apache.nifi</groupId> + <artifactId>nifi-syslog-utils</artifactId> + <version>1.8.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> <artifactId>nifi-utils</artifactId> <version>1.8.0-SNAPSHOT</version> </dependency> @@ -327,10 +332,6 @@ <version>1.8.0-SNAPSHOT</version> <scope>test</scope> </dependency> - <dependency> - <groupId>com.github.palindromicity</groupId> - <artifactId>simple-syslog-5424</artifactId> - </dependency> </dependencies> <build> <plugins> http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java index fb239ee..0509e0c 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java @@ -48,12 +48,12 @@ import org.apache.nifi.processor.util.listen.event.EventFactory; import org.apache.nifi.processor.util.listen.handler.ChannelHandlerFactory; import org.apache.nifi.processor.util.listen.handler.socket.SocketChannelHandlerFactory; import org.apache.nifi.processor.util.listen.response.ChannelResponder; -import org.apache.nifi.processors.standard.syslog.SyslogAttributes; -import org.apache.nifi.processors.standard.syslog.SyslogEvent; -import org.apache.nifi.processors.standard.syslog.SyslogParser; import org.apache.nifi.security.util.SslContextFactory; import org.apache.nifi.ssl.RestrictedSSLContextService; import org.apache.nifi.ssl.SSLContextService; +import org.apache.nifi.syslog.attributes.SyslogAttributes; +import org.apache.nifi.syslog.events.SyslogEvent; +import org.apache.nifi.syslog.parsers.SyslogParser; import javax.net.ssl.SSLContext; import java.io.IOException; @@ -419,8 +419,8 @@ public class ListenSyslog extends AbstractSyslogProcessor { final String protocol = context.getProperty(PROTOCOL).getValue(); final Map<String, String> defaultAttributes = new HashMap<>(4); - defaultAttributes.put(SyslogAttributes.PROTOCOL.key(), protocol); - defaultAttributes.put(SyslogAttributes.PORT.key(), port); + defaultAttributes.put(SyslogAttributes.SYSLOG_PROTOCOL.key(), protocol); + defaultAttributes.put(SyslogAttributes.SYSLOG_PORT.key(), port); defaultAttributes.put(CoreAttributes.MIME_TYPE.key(), "text/plain"); @@ -461,7 +461,7 @@ public class ListenSyslog extends AbstractSyslogProcessor { FlowFile invalidFlowFile = session.create(); invalidFlowFile = session.putAllAttributes(invalidFlowFile, defaultAttributes); if (sender != null) { - invalidFlowFile = session.putAttribute(invalidFlowFile, SyslogAttributes.SENDER.key(), sender); + invalidFlowFile = session.putAttribute(invalidFlowFile, SyslogAttributes.SYSLOG_SENDER.key(), sender); } try { @@ -486,14 +486,14 @@ public class ListenSyslog extends AbstractSyslogProcessor { getLogger().trace(event.getFullMessage()); final Map<String, String> attributes = new HashMap<>(numAttributes); - attributes.put(SyslogAttributes.PRIORITY.key(), event.getPriority()); - attributes.put(SyslogAttributes.SEVERITY.key(), event.getSeverity()); - attributes.put(SyslogAttributes.FACILITY.key(), event.getFacility()); - attributes.put(SyslogAttributes.VERSION.key(), event.getVersion()); - attributes.put(SyslogAttributes.TIMESTAMP.key(), event.getTimeStamp()); - attributes.put(SyslogAttributes.HOSTNAME.key(), event.getHostName()); - attributes.put(SyslogAttributes.BODY.key(), event.getMsgBody()); - attributes.put(SyslogAttributes.VALID.key(), String.valueOf(event.isValid())); + attributes.put(SyslogAttributes.SYSLOG_PRIORITY.key(), event.getPriority()); + attributes.put(SyslogAttributes.SYSLOG_SEVERITY.key(), event.getSeverity()); + attributes.put(SyslogAttributes.SYSLOG_FACILITY.key(), event.getFacility()); + attributes.put(SyslogAttributes.SYSLOG_VERSION.key(), event.getVersion()); + attributes.put(SyslogAttributes.SYSLOG_TIMESTAMP.key(), event.getTimeStamp()); + attributes.put(SyslogAttributes.SYSLOG_HOSTNAME.key(), event.getHostName()); + attributes.put(SyslogAttributes.SYSLOG_BODY.key(), event.getMsgBody()); + attributes.put(SyslogAttributes.SYSLOG_VALID.key(), String.valueOf(event.isValid())); flowFile = session.putAllAttributes(flowFile, attributes); } @@ -536,7 +536,7 @@ public class ListenSyslog extends AbstractSyslogProcessor { final Map<String, String> newAttributes = new HashMap<>(defaultAttributes.size() + 1); newAttributes.putAll(defaultAttributes); - newAttributes.put(SyslogAttributes.SENDER.key(), sender); + newAttributes.put(SyslogAttributes.SYSLOG_SENDER.key(), sender); flowFile = session.putAllAttributes(flowFile, newAttributes); getLogger().debug("Transferring {} to success", new Object[] {flowFile}); http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java index 31691eb..d8f8b8b 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java @@ -17,15 +17,6 @@ package org.apache.nifi.processors.standard; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; import org.apache.nifi.annotation.behavior.EventDriven; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; @@ -45,10 +36,20 @@ import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.io.InputStreamCallback; import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.processors.standard.syslog.SyslogAttributes; -import org.apache.nifi.processors.standard.syslog.SyslogEvent; -import org.apache.nifi.processors.standard.syslog.SyslogParser; import org.apache.nifi.stream.io.StreamUtils; +import org.apache.nifi.syslog.attributes.SyslogAttributes; +import org.apache.nifi.syslog.events.SyslogEvent; +import org.apache.nifi.syslog.parsers.SyslogParser; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; @EventDriven @@ -145,13 +146,13 @@ public class ParseSyslog extends AbstractProcessor { } final Map<String, String> attributes = new HashMap<>(8); - attributes.put(SyslogAttributes.PRIORITY.key(), event.getPriority()); - attributes.put(SyslogAttributes.SEVERITY.key(), event.getSeverity()); - attributes.put(SyslogAttributes.FACILITY.key(), event.getFacility()); - attributes.put(SyslogAttributes.VERSION.key(), event.getVersion()); - attributes.put(SyslogAttributes.TIMESTAMP.key(), event.getTimeStamp()); - attributes.put(SyslogAttributes.HOSTNAME.key(), event.getHostName()); - attributes.put(SyslogAttributes.BODY.key(), event.getMsgBody()); + attributes.put(SyslogAttributes.SYSLOG_PRIORITY.key(), event.getPriority()); + attributes.put(SyslogAttributes.SYSLOG_SEVERITY.key(), event.getSeverity()); + attributes.put(SyslogAttributes.SYSLOG_FACILITY.key(), event.getFacility()); + attributes.put(SyslogAttributes.SYSLOG_VERSION.key(), event.getVersion()); + attributes.put(SyslogAttributes.SYSLOG_TIMESTAMP.key(), event.getTimeStamp()); + attributes.put(SyslogAttributes.SYSLOG_HOSTNAME.key(), event.getHostName()); + attributes.put(SyslogAttributes.SYSLOG_BODY.key(), event.getMsgBody()); flowFile = session.putAllAttributes(flowFile, attributes); session.transfer(flowFile, REL_SUCCESS);
