Github user ehashman commented on a diff in the pull request:
https://github.com/apache/incubator-metron/pull/11#discussion_r50460904
--- Diff:
metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicBroParser.java
---
@@ -22,130 +22,122 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.metron.tldextractor.BasicTldExtractor;
-
@SuppressWarnings("serial")
public class BasicBroParser extends AbstractParser {
- protected static final Logger _LOG = LoggerFactory
- .getLogger(BasicBroParser.class);
- private JSONCleaner cleaner = new JSONCleaner();
- private BasicTldExtractor tldex = new BasicTldExtractor();
-
- @SuppressWarnings("unchecked")
- public JSONObject parse(byte[] msg) {
-
- _LOG.trace("[Metron] Starting to parse incoming message");
-
- String raw_message = null;
-
- try {
-
- raw_message = new String(msg, "UTF-8");
- _LOG.trace("[Metron] Received message: " + raw_message);
-
- JSONObject cleaned_message = cleaner.Clean(raw_message);
- _LOG.debug("[Metron] Cleaned message: " + raw_message);
-
- if (cleaned_message == null ||
cleaned_message.isEmpty())
- throw new Exception("Unable to clean message: "
+ raw_message);
-
- String key =
cleaned_message.keySet().iterator().next().toString();
-
- if (key == null)
- throw new Exception("Unable to retrieve key for
message: "
- + raw_message);
-
- JSONObject payload = (JSONObject)
cleaned_message.get(key);
-
- String originalString = " |";
- for (Object k : payload.keySet()) {
- originalString = originalString + " " +
k.toString() + ":"
- + payload.get(k).toString();
- }
- originalString = key.toUpperCase() + originalString;
- payload.put("original_string", originalString);
-
- if (payload == null)
- throw new Exception("Unable to retrieve payload
for message: "
- + raw_message);
-
- if (payload.containsKey("ts")) {
- String ts = payload.remove("ts").toString();
- payload.put("timestamp", ts);
- _LOG.trace("[Metron] Added ts to: " + payload);
- }
-
- if (payload.containsKey("id.orig_h")) {
- String source_ip =
payload.remove("id.orig_h").toString();
- payload.put("ip_src_addr", source_ip);
- _LOG.trace("[Metron] Added ip_src_addr to: " +
payload);
- } else if (payload.containsKey("tx_hosts")) {
- JSONArray txHosts = (JSONArray)
payload.remove("tx_hosts");
- if (txHosts != null && !txHosts.isEmpty()) {
- payload.put("ip_src_addr",
txHosts.get(0));
- _LOG.trace("[Metron] Added ip_src_addr
to: " + payload);
- }
- }
-
- if (payload.containsKey("id.resp_h")) {
- String source_ip =
payload.remove("id.resp_h").toString();
- payload.put("ip_dst_addr", source_ip);
- _LOG.trace("[Metron] Added ip_dst_addr to: " +
payload);
- } else if (payload.containsKey("rx_hosts")) {
- JSONArray rxHosts = (JSONArray)
payload.remove("rx_hosts");
- if (rxHosts != null && !rxHosts.isEmpty()) {
- payload.put("ip_dst_addr",
rxHosts.get(0));
- _LOG.trace("[Metron] Added ip_dst_addr
to: " + payload);
- }
- }
-
- if (payload.containsKey("id.orig_p")) {
- String source_port =
payload.remove("id.orig_p").toString();
- payload.put("ip_src_port", source_port);
- _LOG.trace("[Metron] Added ip_src_port to: " +
payload);
- }
- if (payload.containsKey("id.resp_p")) {
- String dest_port =
payload.remove("id.resp_p").toString();
- payload.put("ip_dst_port", dest_port);
- _LOG.trace("[Metron] Added ip_dst_port to: " +
payload);
- }
-
-// if (payload.containsKey("host")) {
-//
-// String host =
payload.get("host").toString().trim();
-// String tld = tldex.extractTLD(host);
-//
-// payload.put("tld", tld);
-// _LOG.trace("[Metron] Added tld to: " + payload);
-//
-// }
-// if (payload.containsKey("query")) {
-// String host = payload.get("query").toString();
-// String[] parts = host.split("\\.");
-// int length = parts.length;
-// if (length >= 2) {
-// payload.put("tld", parts[length - 2] +
"."
-// + parts[length - 1]);
-// _LOG.trace("[Metron] Added tld to: " +
payload);
-// }
-// }
-
- _LOG.trace("[Metron] Inner message: " + payload);
-
- payload.put("protocol", key);
- _LOG.debug("[Metron] Returning parsed message: " +
payload);
-
- return payload;
-
- } catch (Exception e) {
-
- _LOG.error("Unable to Parse Message: " + raw_message);
- e.printStackTrace();
- return null;
- }
-
- }
-
-
+ protected static final Logger _LOG = LoggerFactory
+ .getLogger(BasicBroParser.class);
+ private JSONCleaner cleaner = new JSONCleaner();
+
+ @SuppressWarnings("unchecked")
+ public JSONObject parse(byte[] msg) {
+
+ _LOG.trace("[Metron] Starting to parse incoming message");
+
+ String rawMessage = null;
+
+ try {
+ rawMessage = new String(msg, "UTF-8");
+ _LOG.trace("[Metron] Received message: " + rawMessage);
+
+ JSONObject cleanedMessage = cleaner.Clean(rawMessage);
+ _LOG.debug("[Metron] Cleaned message: " + cleanedMessage);
+
+ if (cleanedMessage == null || cleanedMessage.isEmpty()) {
+ throw new Exception("Unable to clean message: " +
rawMessage);
+ }
+
+ String key;
+ JSONObject payload;
+ if (cleanedMessage.containsKey("type")) {
+ key = cleanedMessage.get("type").toString();
+ payload = cleanedMessage;
+ } else {
+ key = cleanedMessage.keySet().iterator().next().toString();
+
+ if (key == null) {
+ throw new Exception("Unable to retrieve key for
message: "
+ + rawMessage);
+ }
+
+ payload = (JSONObject) cleanedMessage.get(key);
+ }
+
+ if (payload == null) {
+ throw new Exception("Unable to retrieve payload for
message: "
+ + rawMessage);
+ }
+
+ String originalString = key.toUpperCase() + " |";
+ for (Object k : payload.keySet()) {
+ String value = payload.get(k).toString();
+ originalString += " " + k.toString() + ":" + value;
+ }
+ payload.put("original_string", originalString);
+
+ replaceKey(payload, "timestamp", new String[]{ "ts" });
+
+ if (payload.containsKey("timestamp")) {
+ try {
+ long timestamp =
Long.parseLong(payload.get("timestamp").toString());
+ payload.put("timestamp", timestamp);
+ } catch (NumberFormatException nfe) {
+ _LOG.error(String.format("[Metron] tiemstamp is
invalid: %s", payload.get("timestamp")));
--- End diff --
Typo: `tiemstamp -> timestamp`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
