Repository: metron Updated Branches: refs/heads/master 1d95b8316 -> 9fb0d06e6
METRON-1463: Adjust the groupings and shuffles in enrichment to be more efficient closes apache/metron#944 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9fb0d06e Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9fb0d06e Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9fb0d06e Branch: refs/heads/master Commit: 9fb0d06e644597032da58c5df020a841b93d19c0 Parents: 1d95b83 Author: cstella <[email protected]> Authored: Wed Mar 7 09:21:52 2018 -0500 Committer: cstella <[email protected]> Committed: Wed Mar 7 09:21:52 2018 -0500 ---------------------------------------------------------------------- .../src/main/flux/enrichment/remote.yaml | 42 +++++++++----------- .../enrichment/bolt/EnrichmentJoinBolt.java | 4 +- .../enrichment/bolt/EnrichmentSplitterBolt.java | 4 +- .../enrichment/bolt/GenericEnrichmentBolt.java | 2 +- .../enrichment/bolt/ThreatIntelJoinBolt.java | 4 +- .../bolt/ThreatIntelSplitterBolt.java | 4 +- 6 files changed, 28 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml index 7ee1b2e..fd7ceff 100644 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml @@ -411,7 +411,7 @@ streams: from: "kafkaSpout" to: "enrichmentSplitBolt" grouping: - type: SHUFFLE + type: LOCAL_OR_SHUFFLE #enrichment - name: "enrichmentSplit -> host" @@ -420,14 +420,15 @@ streams: grouping: streamId: "host" type: FIELDS - args: ["key"] + args: ["message"] + - name: "enrichmentSplit -> geo" from: "enrichmentSplitBolt" to: "geoEnrichmentBolt" grouping: streamId: "geo" type: FIELDS - args: ["key"] + args: ["message"] - name: "enrichmentSplit -> stellar" from: "enrichmentSplitBolt" @@ -435,7 +436,7 @@ streams: grouping: streamId: "stellar" type: FIELDS - args: ["key"] + args: ["message"] - name: "enrichmentSplit -> simpleHBaseEnrichmentBolt" @@ -444,7 +445,7 @@ streams: grouping: streamId: "hbaseEnrichment" type: FIELDS - args: ["key"] + args: ["message"] - name: "splitter -> join" from: "enrichmentSplitBolt" @@ -453,6 +454,7 @@ streams: streamId: "message" type: FIELDS args: ["key"] + - name: "geo -> join" from: "geoEnrichmentBolt" to: "enrichmentJoinBolt" @@ -460,6 +462,7 @@ streams: streamId: "geo" type: FIELDS args: ["key"] + - name: "stellar -> join" from: "stellarEnrichmentBolt" to: "enrichmentJoinBolt" @@ -468,8 +471,6 @@ streams: type: FIELDS args: ["key"] - - - name: "simpleHBaseEnrichmentBolt -> join" from: "simpleHBaseEnrichmentBolt" to: "enrichmentJoinBolt" @@ -477,6 +478,7 @@ streams: streamId: "hbaseEnrichment" type: FIELDS args: ["key"] + - name: "host -> join" from: "hostEnrichmentBolt" to: "enrichmentJoinBolt" @@ -491,32 +493,28 @@ streams: to: "enrichmentErrorOutputBolt" grouping: streamId: "error" - type: FIELDS - args: ["message"] + type: LOCAL_OR_SHUFFLE - name: "stellarEnrichmentBolt -> enrichmentErrorOutputBolt" from: "stellarEnrichmentBolt" to: "enrichmentErrorOutputBolt" grouping: streamId: "error" - type: FIELDS - args: ["message"] + type: LOCAL_OR_SHUFFLE - name: "hostEnrichmentBolt -> enrichmentErrorOutputBolt" from: "hostEnrichmentBolt" to: "enrichmentErrorOutputBolt" grouping: streamId: "error" - type: FIELDS - args: ["message"] + type: LOCAL_OR_SHUFFLE - name: "simpleHBaseEnrichmentBolt -> enrichmentErrorOutputBolt" from: "simpleHBaseEnrichmentBolt" to: "enrichmentErrorOutputBolt" grouping: streamId: "error" - type: FIELDS - args: ["message"] + type: LOCAL_OR_SHUFFLE #threat intel - name: "enrichmentJoin -> threatSplit" @@ -533,14 +531,15 @@ streams: grouping: streamId: "hbaseThreatIntel" type: FIELDS - args: ["key"] + args: ["message"] + - name: "threatSplit -> stellarThreatIntel" from: "threatIntelSplitBolt" to: "stellarThreatIntelBolt" grouping: streamId: "stellar" type: FIELDS - args: ["key"] + args: ["message"] - name: "simpleHBaseThreatIntel -> join" @@ -572,8 +571,7 @@ streams: to: "outputBolt" grouping: streamId: "message" - type: FIELDS - args: ["key"] + type: LOCAL_OR_SHUFFLE # Error output - name: "simpleHBaseThreatIntelBolt -> threatIntelErrorOutputBolt" @@ -581,14 +579,12 @@ streams: to: "threatIntelErrorOutputBolt" grouping: streamId: "error" - type: FIELDS - args: ["message"] + type: LOCAL_OR_SHUFFLE - name: "stellarThreatIntelBolt -> threatIntelErrorOutputBolt" from: "stellarThreatIntelBolt" to: "threatIntelErrorOutputBolt" grouping: streamId: "error" - type: FIELDS - args: ["message"] + type: LOCAL_OR_SHUFFLE http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java index 3f25479..fd5f874 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java @@ -96,7 +96,7 @@ public class EnrichmentJoinBolt extends JoinBolt<JSONObject> { if (config != null) { return config.getEnrichment().getEnrichmentConfigs(); } else { - LOG.info("Unable to retrieve a sensor enrichment config of {}", sensorType); + LOG.debug("Unable to retrieve a sensor enrichment config of {}", sensorType); } } else { LOG.error("Trying to retrieve a field map with sensor type of null"); @@ -111,7 +111,7 @@ public class EnrichmentJoinBolt extends JoinBolt<JSONObject> { return config.getEnrichment().getFieldMap(); } else { - LOG.info("Unable to retrieve a sensor enrichment config of {}", sourceType); + LOG.debug("Unable to retrieve a sensor enrichment config of {}", sourceType); } } else { http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java index ae75508..3298c76 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java @@ -147,7 +147,7 @@ public class EnrichmentSplitterBolt extends SplitBolt<JSONObject> { if (config != null) { return config.getEnrichment().getEnrichmentConfigs(); } else { - LOG.info("Unable to retrieve a sensor enrichment config of {}", sensorType); + LOG.debug("Unable to retrieve a sensor enrichment config of {}", sensorType); } } else { LOG.error("Trying to retrieve a field map with sensor type of null"); @@ -160,7 +160,7 @@ public class EnrichmentSplitterBolt extends SplitBolt<JSONObject> { if (config != null) { return config.getEnrichment().getFieldMap(); } else { - LOG.info("Unable to retrieve a sensor enrichment config of {}", sensorType); + LOG.debug("Unable to retrieve a sensor enrichment config of {}", sensorType); } } else { LOG.error("Trying to retrieve a field map with sensor type of null"); http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java index dbbb7b6..7d67d2d 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java @@ -212,7 +212,7 @@ public class GenericEnrichmentBolt extends ConfiguredEnrichmentBolt { if (value != null) { SensorEnrichmentConfig config = getConfigurations().getSensorEnrichmentConfig(sourceType); if(config == null) { - LOG.error("Unable to find SensorEnrichmentConfig for sourceType: {}", sourceType); + LOG.debug("Unable to find SensorEnrichmentConfig for sourceType: {}", sourceType); MetronError metronError = new MetronError() .withErrorType(Constants.ErrorType.ENRICHMENT_ERROR) .withMessage("Unable to find SensorEnrichmentConfig for sourceType: " + sourceType) http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java index 1ce0b16..cfa101d 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java @@ -68,7 +68,7 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt { if (config != null) { return config.getThreatIntel().getEnrichmentConfigs(); } else { - LOG.info("Unable to retrieve a sensor enrichment config of {}", sensorType); + LOG.debug("Unable to retrieve a sensor enrichment config of {}", sensorType); } } else { LOG.error("Trying to retrieve a field map with sensor type of null"); @@ -100,7 +100,7 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt { return config.getThreatIntel().getFieldMap(); } else { - LOG.info("Unable to retrieve sensor config: {}", sourceType); + LOG.debug("Unable to retrieve sensor config: {}", sourceType); return null; } } http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java index bd408e0..76c65c6 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java @@ -37,7 +37,7 @@ public class ThreatIntelSplitterBolt extends EnrichmentSplitterBolt { if (config != null) { return config.getThreatIntel().getEnrichmentConfigs(); } else { - LOG.info("Unable to retrieve a sensor config of {}", sensorType); + LOG.debug("Unable to retrieve a sensor config of {}", sensorType); } } else { LOG.error("Trying to retrieve a field map with sensor type of null"); @@ -52,7 +52,7 @@ public class ThreatIntelSplitterBolt extends EnrichmentSplitterBolt { if (config != null) { return config.getThreatIntel().getFieldMap(); } else { - LOG.info("Unable to retrieve sensor config: {}", sensorType); + LOG.debug("Unable to retrieve sensor config: {}", sensorType); } } else { LOG.error("Trying to retrieve a field map with sensor type of null");
