Repository: metron
Updated Branches:
  refs/heads/master 1d95b8316 -> 9fb0d06e6


METRON-1463: Adjust the groupings and shuffles in enrichment to be more 
efficient closes apache/metron#944


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9fb0d06e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9fb0d06e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9fb0d06e

Branch: refs/heads/master
Commit: 9fb0d06e644597032da58c5df020a841b93d19c0
Parents: 1d95b83
Author: cstella <ceste...@gmail.com>
Authored: Wed Mar 7 09:21:52 2018 -0500
Committer: cstella <ceste...@gmail.com>
Committed: Wed Mar 7 09:21:52 2018 -0500

----------------------------------------------------------------------
 .../src/main/flux/enrichment/remote.yaml        | 42 +++++++++-----------
 .../enrichment/bolt/EnrichmentJoinBolt.java     |  4 +-
 .../enrichment/bolt/EnrichmentSplitterBolt.java |  4 +-
 .../enrichment/bolt/GenericEnrichmentBolt.java  |  2 +-
 .../enrichment/bolt/ThreatIntelJoinBolt.java    |  4 +-
 .../bolt/ThreatIntelSplitterBolt.java           |  4 +-
 6 files changed, 28 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml 
b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
index 7ee1b2e..fd7ceff 100644
--- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
@@ -411,7 +411,7 @@ streams:
         from: "kafkaSpout"
         to: "enrichmentSplitBolt"
         grouping:
-            type: SHUFFLE
+            type: LOCAL_OR_SHUFFLE
 
 #enrichment
     -   name: "enrichmentSplit -> host"
@@ -420,14 +420,15 @@ streams:
         grouping:
             streamId: "host"
             type: FIELDS
-            args: ["key"]
+            args: ["message"]
+
     -   name: "enrichmentSplit -> geo"
         from: "enrichmentSplitBolt"
         to: "geoEnrichmentBolt"
         grouping:
             streamId: "geo"
             type: FIELDS
-            args: ["key"]
+            args: ["message"]
 
     -   name: "enrichmentSplit -> stellar"
         from: "enrichmentSplitBolt"
@@ -435,7 +436,7 @@ streams:
         grouping:
             streamId: "stellar"
             type: FIELDS
-            args: ["key"]
+            args: ["message"]
 
 
     -   name: "enrichmentSplit -> simpleHBaseEnrichmentBolt"
@@ -444,7 +445,7 @@ streams:
         grouping:
             streamId: "hbaseEnrichment"
             type: FIELDS
-            args: ["key"]
+            args: ["message"]
 
     -   name: "splitter -> join"
         from: "enrichmentSplitBolt"
@@ -453,6 +454,7 @@ streams:
             streamId: "message"
             type: FIELDS
             args: ["key"]
+
     -   name: "geo -> join"
         from: "geoEnrichmentBolt"
         to: "enrichmentJoinBolt"
@@ -460,6 +462,7 @@ streams:
             streamId: "geo"
             type: FIELDS
             args: ["key"]
+
     -   name: "stellar -> join"
         from: "stellarEnrichmentBolt"
         to: "enrichmentJoinBolt"
@@ -468,8 +471,6 @@ streams:
             type: FIELDS
             args: ["key"]
 
-
-
     -   name: "simpleHBaseEnrichmentBolt -> join"
         from: "simpleHBaseEnrichmentBolt"
         to: "enrichmentJoinBolt"
@@ -477,6 +478,7 @@ streams:
             streamId: "hbaseEnrichment"
             type: FIELDS
             args: ["key"]
+
     -   name: "host -> join"
         from: "hostEnrichmentBolt"
         to: "enrichmentJoinBolt"
@@ -491,32 +493,28 @@ streams:
         to: "enrichmentErrorOutputBolt"
         grouping:
             streamId: "error"
-            type: FIELDS
-            args: ["message"]
+            type: LOCAL_OR_SHUFFLE
 
     -   name: "stellarEnrichmentBolt -> enrichmentErrorOutputBolt"
         from: "stellarEnrichmentBolt"
         to: "enrichmentErrorOutputBolt"
         grouping:
             streamId: "error"
-            type: FIELDS
-            args: ["message"]
+            type: LOCAL_OR_SHUFFLE
 
     -   name: "hostEnrichmentBolt -> enrichmentErrorOutputBolt"
         from: "hostEnrichmentBolt"
         to: "enrichmentErrorOutputBolt"
         grouping:
             streamId: "error"
-            type: FIELDS
-            args: ["message"]
+            type: LOCAL_OR_SHUFFLE
 
     -   name: "simpleHBaseEnrichmentBolt -> enrichmentErrorOutputBolt"
         from: "simpleHBaseEnrichmentBolt"
         to: "enrichmentErrorOutputBolt"
         grouping:
             streamId: "error"
-            type: FIELDS
-            args: ["message"]
+            type: LOCAL_OR_SHUFFLE
 
 #threat intel
     -   name: "enrichmentJoin -> threatSplit"
@@ -533,14 +531,15 @@ streams:
         grouping:
             streamId: "hbaseThreatIntel"
             type: FIELDS
-            args: ["key"]
+            args: ["message"]
+
     -   name: "threatSplit -> stellarThreatIntel"
         from: "threatIntelSplitBolt"
         to: "stellarThreatIntelBolt"
         grouping:
             streamId: "stellar"
             type: FIELDS
-            args: ["key"]
+            args: ["message"]
 
 
     -   name: "simpleHBaseThreatIntel -> join"
@@ -572,8 +571,7 @@ streams:
         to: "outputBolt"
         grouping:
             streamId: "message"
-            type: FIELDS
-            args: ["key"]
+            type: LOCAL_OR_SHUFFLE
 
     # Error output
     -   name: "simpleHBaseThreatIntelBolt -> threatIntelErrorOutputBolt"
@@ -581,14 +579,12 @@ streams:
         to: "threatIntelErrorOutputBolt"
         grouping:
             streamId: "error"
-            type: FIELDS
-            args: ["message"]
+            type: LOCAL_OR_SHUFFLE
 
     -   name: "stellarThreatIntelBolt -> threatIntelErrorOutputBolt"
         from: "stellarThreatIntelBolt"
         to: "threatIntelErrorOutputBolt"
         grouping:
             streamId: "error"
-            type: FIELDS
-            args: ["message"]
+            type: LOCAL_OR_SHUFFLE
 

http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
index 3f25479..fd5f874 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
@@ -96,7 +96,7 @@ public class EnrichmentJoinBolt extends JoinBolt<JSONObject> {
       if (config != null) {
         return config.getEnrichment().getEnrichmentConfigs();
       } else {
-        LOG.info("Unable to retrieve a sensor enrichment config of {}", 
sensorType);
+        LOG.debug("Unable to retrieve a sensor enrichment config of {}", 
sensorType);
       }
     } else {
       LOG.error("Trying to retrieve a field map with sensor type of null");
@@ -111,7 +111,7 @@ public class EnrichmentJoinBolt extends 
JoinBolt<JSONObject> {
         return config.getEnrichment().getFieldMap();
       }
       else {
-        LOG.info("Unable to retrieve a sensor enrichment config of {}", 
sourceType);
+        LOG.debug("Unable to retrieve a sensor enrichment config of {}", 
sourceType);
       }
     }
     else {

http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
index ae75508..3298c76 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
@@ -147,7 +147,7 @@ public class EnrichmentSplitterBolt extends 
SplitBolt<JSONObject> {
       if (config != null) {
         return config.getEnrichment().getEnrichmentConfigs();
       } else {
-        LOG.info("Unable to retrieve a sensor enrichment config of {}", 
sensorType);
+        LOG.debug("Unable to retrieve a sensor enrichment config of {}", 
sensorType);
       }
     } else {
       LOG.error("Trying to retrieve a field map with sensor type of null");
@@ -160,7 +160,7 @@ public class EnrichmentSplitterBolt extends 
SplitBolt<JSONObject> {
       if (config != null) {
         return config.getEnrichment().getFieldMap();
       } else {
-        LOG.info("Unable to retrieve a sensor enrichment config of {}", 
sensorType);
+        LOG.debug("Unable to retrieve a sensor enrichment config of {}", 
sensorType);
       }
     } else {
       LOG.error("Trying to retrieve a field map with sensor type of null");

http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
index dbbb7b6..7d67d2d 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
@@ -212,7 +212,7 @@ public class GenericEnrichmentBolt extends 
ConfiguredEnrichmentBolt {
           if (value != null) {
             SensorEnrichmentConfig config = 
getConfigurations().getSensorEnrichmentConfig(sourceType);
             if(config == null) {
-              LOG.error("Unable to find SensorEnrichmentConfig for sourceType: 
{}", sourceType);
+              LOG.debug("Unable to find SensorEnrichmentConfig for sourceType: 
{}", sourceType);
               MetronError metronError = new MetronError()
                       .withErrorType(Constants.ErrorType.ENRICHMENT_ERROR)
                       .withMessage("Unable to find SensorEnrichmentConfig for 
sourceType: " + sourceType)

http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
index 1ce0b16..cfa101d 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
@@ -68,7 +68,7 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
       if (config != null) {
         return config.getThreatIntel().getEnrichmentConfigs();
       } else {
-        LOG.info("Unable to retrieve a sensor enrichment config of {}", 
sensorType);
+        LOG.debug("Unable to retrieve a sensor enrichment config of {}", 
sensorType);
       }
     } else {
       LOG.error("Trying to retrieve a field map with sensor type of null");
@@ -100,7 +100,7 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt 
{
       return config.getThreatIntel().getFieldMap();
     }
     else {
-      LOG.info("Unable to retrieve sensor config: {}", sourceType);
+      LOG.debug("Unable to retrieve sensor config: {}", sourceType);
       return null;
     }
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
index bd408e0..76c65c6 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
@@ -37,7 +37,7 @@ public class ThreatIntelSplitterBolt extends 
EnrichmentSplitterBolt {
       if (config != null) {
         return config.getThreatIntel().getEnrichmentConfigs();
       } else {
-        LOG.info("Unable to retrieve a sensor config of {}", sensorType);
+        LOG.debug("Unable to retrieve a sensor config of {}", sensorType);
       }
     } else {
       LOG.error("Trying to retrieve a field map with sensor type of null");
@@ -52,7 +52,7 @@ public class ThreatIntelSplitterBolt extends 
EnrichmentSplitterBolt {
       if (config != null) {
         return config.getThreatIntel().getFieldMap();
       } else {
-        LOG.info("Unable to retrieve sensor config: {}", sensorType);
+        LOG.debug("Unable to retrieve sensor config: {}", sensorType);
       }
     } else {
       LOG.error("Trying to retrieve a field map with sensor type of null");

Reply via email to