Repository: metron Updated Branches: refs/heads/master 4a074becc -> e3900c4db
METRON-1156 Simulate Triage Rules in the Stellar REPL (nickwallen) closes apache/metron#733 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e3900c4d Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e3900c4d Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e3900c4d Branch: refs/heads/master Commit: e3900c4dbd2dea183d6b7048ffda421d79d5daa9 Parents: 4a074be Author: nickwallen <[email protected]> Authored: Mon Sep 25 16:16:14 2017 -0400 Committer: nickallen <[email protected]> Committed: Mon Sep 25 16:16:14 2017 -0400 ---------------------------------------------------------------------- .../triage/ThreatTriageProcessor.java | 21 +- metron-platform/metron-management/README.md | 147 ++++++- .../management/ThreatTriageFunctions.java | 434 ++++++++++++++----- .../management/ThreatTriageFunctionsTest.java | 302 ++++++++++++- 4 files changed, 758 insertions(+), 146 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/e3900c4d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java index cb11a27..4f24b68 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java @@ -26,13 +26,13 @@ import org.apache.metron.common.configuration.enrichment.threatintel.RuleScore; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatScore; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig; +import org.apache.metron.stellar.common.StellarPredicateProcessor; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.MapVariableResolver; import org.apache.metron.stellar.dsl.VariableResolver; import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.stellar.common.StellarPredicateProcessor; -import org.apache.metron.stellar.common.StellarProcessor; -import org.apache.metron.stellar.common.utils.ConversionUtils; import javax.annotation.Nullable; import java.util.List; @@ -96,8 +96,8 @@ public class ThreatTriageProcessor implements Function<Map, ThreatScore> { Aggregators aggregators = threatTriageConfig.getAggregator(); List<Number> allScores = threatScore.getRuleScores().stream().map(score -> score.getRule().getScore()).collect(Collectors.toList()); Double aggregateScore = aggregators.aggregate(allScores, threatTriageConfig.getAggregationConfig()); - // return the overall threat score threatScore.setScore(aggregateScore); + return threatScore; } @@ -105,4 +105,17 @@ public class ThreatTriageProcessor implements Function<Map, ThreatScore> { Object result = processor.parse(expression, resolver, functionResolver, context); return ConversionUtils.convert(result, clazz); } + + public List<RiskLevelRule> getRiskLevelRules() { + return threatTriageConfig.getRiskLevelRules(); + } + + public SensorEnrichmentConfig getSensorConfig() { + return sensorConfig; + } + + @Override + public String toString() { + return String.format("ThreatTriage{%d rule(s)}", threatTriageConfig.getRiskLevelRules().size()); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/e3900c4d/metron-platform/metron-management/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/README.md b/metron-platform/metron-management/README.md index 07c6908..07a5b1a 100644 --- a/metron-platform/metron-management/README.md +++ b/metron-platform/metron-management/README.md @@ -15,7 +15,23 @@ Additionally, some shell functions have been added to This functionality is exposed as a pack of Stellar functions in this project. -## Function Details +* [Functions](#functions) + * [Grok Functions](#grok-functions) + * [File Functions](#file-functions) + * [Shell Functions](#shell-functions) + * [Configuration Functions](#configuration-functions) + * [Parser Functions](#parser-functions) + * [Indexing Functions](#indexing-functions) + * [Enrichment Functions](#enrichment-functions) + * [Threat Triage Functions](#threat-triage-functions) +* [Examples](#examples) + * [Iterate to Find a Valid Grok Pattern](#iterate-to-find-a-valid-grok-pattern) + * [Manage Stellar Field Transformations](#manage-stellar-field-transformations) + * [Manage Stellar Enrichments](#manage-stellar-enrichments) + * [Manage Threat Triage Rules](#manage-threat-triage-rules) + * [Simulate Threat Triage Rules](#simulate-threat-triage-rules) + +## Functions The functions are split roughly into a few sections: * Shell functions - Functions surrounding interacting with the shell in either a nicer way or a more functional way. @@ -153,7 +169,7 @@ The functions are split roughly into a few sections: * sensorConfig - Sensor config to add transformation to. * stellarTransforms - A Map associating fields to stellar expressions * Returns: The String representation of the config in zookeeper -* `PARSER_STELLAR_TRANSFORM_PRINT` +* `PARSER-STELLAR_TRANSFORM_PRINT` * Description: Retrieve stellar field transformations. * Input: * sensorConfig - Sensor config to add transformation to. @@ -218,6 +234,22 @@ The functions are split roughly into a few sections: ### Threat Triage Functions +* `THREAT_TRIAGE_INIT` + * Description: Create a threat triage engine. + * Input: + * config - the threat triage configuration (optional) + * Returns: A threat triage engine. +* `THREAT_TRIAGE_CONFIG` + * Description: Export the configuration used by a threat triage engine. + * Input: + * engine - threat triage engine returned by THREAT_TRIAGE_INIT. + * Returns: The configuration used by the threat triage engine. +* `THREAT_TRIAGE_SCORE` + * Description: Scores a message using a set of triage rules. + * Inputs: + * message - a string containing the message to score. + * engine - threat triage engine returned by THREAT_TRIAGE_INIT. + * Returns: A threat triage engine. * `THREAT_TRIAGE_ADD` * Description: Add a threat triage rule. * Input: @@ -225,17 +257,17 @@ The functions are split roughly into a few sections: * stellarTransforms - A Map associating stellar rules to scores * triageRules - Map (or list of Maps) representing a triage rule. It must contain 'rule' and 'score' keys, the stellar expression for the rule and triage score respectively. It may contain 'name' and 'comment', the name of the rule and comment associated with the rule respectively." * Returns: The String representation of the threat triage rules -* `THREAT_TRIAGE_PRINT` - * Description: Retrieve stellar enrichment transformations. - * Input: - * sensorConfig - Sensor config to add transformation to. - * Returns: The String representation of the threat triage rules * `THREAT_TRIAGE_REMOVE` * Description: Remove stellar threat triage rule(s). * Input: * sensorConfig - Sensor config to add transformation to. * rules - A list of stellar rules or rule names to remove * Returns: The String representation of the enrichment config +* `THREAT_TRIAGE_PRINT` + * Description: Retrieve stellar enrichment transformations. + * Input: + * sensorConfig - Sensor config to add transformation to. + * Returns: The String representation of the threat triage rules * `THREAT_TRIAGE_SET_AGGREGATOR` * Description: Set the threat triage aggregator. * Input: @@ -256,7 +288,7 @@ The functions are split roughly into a few sections: Included for description and education purposes are a couple example Stellar REPL transcripts with helpful comments to illustrate some common operations. -### Iterate in finding a valid Grok pattern +### Iterate to Find a Valid Grok pattern ``` Stellar, Go! Please note that functions are loading lazily in the background and will be unavailable until loaded fully. @@ -957,3 +989,102 @@ SION('is_both') ] ) } [Stellar]>>> ``` + +### Simulate Threat Triage Rules + +1. Create a threat triage engine. + + ``` + [Stellar]>>> t := THREAT_TRIAGE_INIT() + [Stellar]>>> t + ThreatTriage{0 rule(s)} + ``` + +1. Add a few triage rules. + + ``` + [Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule1", "rule":"value>10", + ``` + ``` + [Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule2", "rule":"value>20", "score":20}) + ``` + ``` + [Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule3", "rule":"value>30", "score":30}) + ``` + +1. Review the rules that you have created. + ``` + [Stellar]>>> THREAT_TRIAGE_PRINT(t) + âââââââââ¤ââââââââââ¤ââââââââââââââ¤ââââââââ¤âââââââââ + â Name â Comment â Triage Rule â Score â Reason â + â ââââââââªââââââââââªââââââââââââââªââââââââªââââââââ⣠+ â rule1 â â value>10 â 10 â â + âââââââââ¼ââââââââââ¼ââââââââââââââ¼ââââââââ¼ââââââââ⢠+ â rule2 â â value>20 â 20 â â + âââââââââ¼ââââââââââ¼ââââââââââââââ¼ââââââââ¼ââââââââ⢠+ â rule3 â â value>30 â 30 â â + âââââââââ§ââââââââââ§ââââââââââââââ§ââââââââ§âââââââââ + ``` + +1. Create a few test messages to simulate your telemetry. + ``` + [Stellar]>>> msg1 := "{ \"value\":22 }" + [Stellar]>>> msg1 + { "value":22 } + ``` + ``` + [Stellar]>>> msg2 := "{ \"value\":44 }" + [Stellar]>>> msg2 + { "value":44 } + ``` + +1. Score a message based on the rules that have been defined. The result allows you to see the total score, the aggregator, along with details about each rule that fired. + + ``` + [Stellar]>>> THREAT_TRIAGE_SCORE( msg1, t) + {score=20.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value>10}, {score=20.0, name=rule2, rule=value>20}]} + ``` + ``` + [Stellar]>>> THREAT_TRIAGE_SCORE( msg2, t) + {score=30.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value>10}, {score=20.0, name=rule2, rule=value>20}, {score=30.0, name=rule3, rule=value>30}]} + ``` + +1. From here you can iterate on your rule set until it does exactly what you need it to do. Once you have a working rule set, extract the configuration and push it into your live, Metron cluster. + + ``` + [Stellar]>>> conf := THREAT_TRIAGE_CONFIG( t) + [Stellar]>>> conf + { + "enrichment" : { + "fieldMap" : { }, + "fieldToTypeMap" : { }, + "config" : { } + }, + "threatIntel" : { + "fieldMap" : { }, + "fieldToTypeMap" : { }, + "config" : { }, + "triageConfig" : { + "riskLevelRules" : [ { + "name" : "rule1", + "rule" : "value>10", + "score" : 10.0 + }, { + "name" : "rule2", + "rule" : "value>20", + "score" : 20.0 + }, { + "name" : "rule3", + "rule" : "value>30", + "score" : 30.0 + }], + "aggregator" : "MAX", + "aggregationConfig" : { } + } + }, + "configuration" : { } + } + ``` + ``` + [Stellar]>>> CONFIG_PUT("ENRICHMENT", conf, "bro") + ``` http://git-wip-us.apache.org/repos/asf/metron/blob/e3900c4d/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java index 94d35dc..e142b23 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java @@ -17,67 +17,245 @@ */ package org.apache.metron.management; -import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; -import static org.apache.metron.management.EnrichmentConfigFunctions.getConfig; - import com.fasterxml.jackson.core.JsonProcessingException; import com.jakewharton.fliptables.FlipTable; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.ClassUtils; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.RiskLevelRule; +import org.apache.metron.common.configuration.enrichment.threatintel.RuleScore; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig; -import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig; +import org.apache.metron.common.configuration.enrichment.threatintel.ThreatScore; import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.profiler.client.stellar.Util; import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver; +import org.apache.metron.threatintel.triage.ThreatTriageProcessor; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static java.lang.String.format; +import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; +import static org.apache.metron.management.EnrichmentConfigFunctions.getConfig; + +/** + * Stellar functions related to Threat Triage. + */ public class ThreatTriageFunctions { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected static final String SCORE_KEY = "score"; + protected static final String RULES_KEY = "rules"; + protected static final String AGG_KEY = "aggregator"; + protected static final String RULE_NAME_KEY = "name"; + protected static final String RULE_EXPR_KEY = "rule"; + protected static final String RULE_SCORE_KEY = "score"; + protected static final String RULE_REASON_KEY = "reason"; + protected static final String RULE_COMMENT_KEY = "comment"; + + @Stellar( + namespace = "THREAT_TRIAGE" + ,name = "INIT" + ,description = "Create a threat triage engine to execute triage rules." + ,params = {"config - the threat triage configuration (optional)" } + ,returns = "A threat triage engine." + ) + public static class ThreatTriageInit implements StellarFunction { + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + ThreatTriageProcessor processor; + SensorEnrichmentConfig config = new SensorEnrichmentConfig(); + + // the user can provide an initial config + if(args.size() > 0) { + String json = Util.getArg(0, String.class, args); + if (json != null) { + config = (SensorEnrichmentConfig) ENRICHMENT.deserialize(json); + + } else { + throw new IllegalArgumentException(format("Invalid configuration: unable to deserialize '%s'", json)); + } + } + + processor = new ThreatTriageProcessor(config, new ClasspathFunctionResolver(), context); + return processor; + } + + @Override + public void initialize(Context context) { + // nothing to do + } + + @Override + public boolean isInitialized() { + return true; + } + } + + @Stellar( + namespace = "THREAT_TRIAGE" + ,name = "SCORE" + ,description = "Scores a message using a set of triage rules." + ,params = { + "message - a string containing the message to score.", + "engine - threat triage engine returned by THREAT_TRIAGE_INIT."} + ,returns = "A threat triage engine." + ) + public static class ThreatTriageScore implements StellarFunction { + + private JSONParser parser; + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + + // the user must provide the message as a string + String arg0 = Util.getArg(0, String.class, args); + if(arg0 == null) { + throw new IllegalArgumentException(format("expected string, got null")); + } + + // parse the message + JSONObject message; + try { + message = (JSONObject) parser.parse(arg0); + } catch(org.json.simple.parser.ParseException e) { + throw new IllegalArgumentException("invalid message", e); + } + + // the user must provide the threat triage processor + ThreatTriageProcessor processor = Util.getArg(1, ThreatTriageProcessor.class, args); + if(processor == null) { + throw new IllegalArgumentException(format("expected threat triage engine; got null")); + } + + ThreatScore score = processor.apply(message); + return transform(score, processor.getSensorConfig()); + } + + /** + * Transforms a ThreatScore into a Map that can be more easily manipulated + * in the REPL. + * @param score The ThreatScore to transform. + * @return The transformed ThreatScore. + */ + private Map<String, Object> transform(ThreatScore score, SensorEnrichmentConfig config) { + List<Map<String, Object>> scores = new ArrayList<>(); + for(RuleScore ruleScore: score.getRuleScores()) { + + // transform the score from each rule + Map<String, Object> map = new HashMap<>(); + if(ruleScore.getRule().getName() != null) { + map.put(RULE_NAME_KEY, ruleScore.getRule().getName()); + } + if(ruleScore.getRule().getRule() != null) { + map.put(RULE_EXPR_KEY, ruleScore.getRule().getRule()); + } + if(ruleScore.getRule().getScore() != null) { + map.put(RULE_SCORE_KEY, ruleScore.getRule().getScore()); + } + if(ruleScore.getReason() != null) { + map.put(RULE_REASON_KEY, ruleScore.getReason()); + } + if(ruleScore.getRule().getComment() != null) { + map.put(RULE_COMMENT_KEY, ruleScore.getRule().getComment()); + } + scores.add(map); + } + + // contains the total score and details on the score from each rule + Map<String, Object> result = new HashMap<>(); + result.put(SCORE_KEY, score.getScore()); + result.put(RULES_KEY, scores); + result.put(AGG_KEY, config.getThreatIntel().getTriageConfig().getAggregator().toString()); + return result; + } + + @Override + public void initialize(Context context) { + parser = new JSONParser(); + } + + @Override + public boolean isInitialized() { + return parser != null; + } + } + + @Stellar( + namespace = "THREAT_TRIAGE" + ,name = "CONFIG" + ,description = "Export the configuration used by a threat triage engine." + ,params = { "engine - threat triage engine returned by THREAT_TRIAGE_INIT." } + ,returns = "The configuration used by the threat triage engine." + ) + public static class ThreatTriageConfig implements StellarFunction { + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + + // the user must provide the threat triage processor + ThreatTriageProcessor processor = Util.getArg(0, ThreatTriageProcessor.class, args); + if(processor == null) { + throw new IllegalArgumentException(format("expected threat triage engine; got null")); + } + + // serialize the configuration to JSON + SensorEnrichmentConfig config = processor.getSensorConfig(); + return toJSON(config); + } + + @Override + public void initialize(Context context) { + // do nothing + } + + @Override + public boolean isInitialized() { + return true; + } + } @Stellar( namespace = "THREAT_TRIAGE" ,name = "PRINT" ,description = "Retrieve stellar enrichment transformations." - ,params = {"sensorConfig - Sensor config to add transformation to." - } + ,params = { "config (or engine) - JSON configuration as a string (or Threat Triage engine returned by THREAT_TRIAGE_INIT)" } ,returns = "The String representation of the threat triage rules" ) public static class GetStellarTransformation implements StellarFunction { @Override public Object apply(List<Object> args, Context context) throws ParseException { - String config = (String) args.get(0); - SensorEnrichmentConfig configObj; - if(config == null || config.isEmpty()) { - configObj = new SensorEnrichmentConfig(); - } - else { - configObj = (SensorEnrichmentConfig) ENRICHMENT.deserialize(config); - } - ThreatIntelConfig tiConfig = (ThreatIntelConfig) getConfig(configObj, EnrichmentConfigFunctions.Type.THREAT_INTEL); + SensorEnrichmentConfig config = getSensorEnrichmentConfig(args, 0); + + ThreatIntelConfig tiConfig = (ThreatIntelConfig) getConfig(config, EnrichmentConfigFunctions.Type.THREAT_INTEL); if(tiConfig == null) { return ""; } - ThreatTriageConfig triageConfig = tiConfig.getTriageConfig(); + org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig triageConfig = tiConfig.getTriageConfig(); if(triageConfig == null) { return ""; } - List<RiskLevelRule> triageRules = triageConfig.getRiskLevelRules(); - if(triageRules == null) { - triageRules = new ArrayList<>(); - } + + // print each rule + List<RiskLevelRule> triageRules = ListUtils.emptyIfNull(triageConfig.getRiskLevelRules()); String[] headers = new String[] {"Name", "Comment", "Triage Rule", "Score", "Reason"}; String[][] data = new String[triageRules.size()][5]; int i = 0; @@ -90,9 +268,9 @@ public class ThreatTriageFunctions { data[i++] = new String[] {name, comment, rule.getRule(), score, reason}; } String ret = FlipTable.of(headers, data); - if(!triageRules.isEmpty()) { - ret += "\n\n"; + // print the aggregation + if(!triageRules.isEmpty()) { ret += "Aggregation: " + triageConfig.getAggregator().name(); } return ret; @@ -100,7 +278,7 @@ public class ThreatTriageFunctions { @Override public void initialize(Context context) { - + // nothing to do } @Override @@ -124,69 +302,70 @@ public class ThreatTriageFunctions { @Override public Object apply(List<Object> args, Context context) throws ParseException { - String config = (String) args.get(0); - SensorEnrichmentConfig configObj; - if(config == null || config.isEmpty()) { - configObj = new SensorEnrichmentConfig(); - } - else { - configObj = (SensorEnrichmentConfig) ENRICHMENT.deserialize(config); - } - ThreatIntelConfig tiConfig = (ThreatIntelConfig) getConfig(configObj, EnrichmentConfigFunctions.Type.THREAT_INTEL); + SensorEnrichmentConfig config = getSensorEnrichmentConfig(args, 0); + + ThreatIntelConfig tiConfig = (ThreatIntelConfig) getConfig(config, EnrichmentConfigFunctions.Type.THREAT_INTEL); if(tiConfig == null) { tiConfig = new ThreatIntelConfig(); - configObj.setThreatIntel(tiConfig); + config.setThreatIntel(tiConfig); } - ThreatTriageConfig triageConfig = tiConfig.getTriageConfig(); + + org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig triageConfig = tiConfig.getTriageConfig(); if(triageConfig == null) { - triageConfig = new ThreatTriageConfig(); + triageConfig = new org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig(); tiConfig.setTriageConfig(triageConfig); } - List<RiskLevelRule> triageRules = triageConfig.getRiskLevelRules(); - if(triageRules == null) { - triageRules = new ArrayList<>(); - } - Object newRuleObj = args.get(1); - List<Map<String, Object>> newRules = new ArrayList<>(); - if(newRuleObj != null && newRuleObj instanceof List) { - newRules = (List<Map<String, Object>>) newRuleObj; - } - else if(newRuleObj != null && newRuleObj instanceof Map) { - newRules.add((Map<String, Object>) newRuleObj); - } - else if(newRuleObj != null) { - throw new IllegalStateException("triageRule must be either a Map representing a single rule or a List of rules."); - } - for(Map<String, Object> newRule : newRules) { - if(!(newRule == null || !newRule.containsKey("rule") || !newRule.containsKey("score"))) { + + // build the new rules + List<RiskLevelRule> newRules = new ArrayList<>(); + for(Map<String, Object> newRule : getNewRuleDefinitions(args)) { + + if(newRule != null && newRule.containsKey("rule") && newRule.containsKey("score")) { + + // create the rule RiskLevelRule ruleToAdd = new RiskLevelRule(); - ruleToAdd.setRule((String) newRule.get("rule")); - ruleToAdd.setScore(ConversionUtils.convert(newRule.get("score"), Double.class)); - if (newRule.containsKey("name")) { - ruleToAdd.setName((String) newRule.get("name")); + ruleToAdd.setRule((String) newRule.get(RULE_EXPR_KEY)); + ruleToAdd.setScore(ConversionUtils.convert(newRule.get(RULE_SCORE_KEY), Double.class)); + + // add optional rule fields + if (newRule.containsKey(RULE_NAME_KEY)) { + ruleToAdd.setName((String) newRule.get(RULE_NAME_KEY)); } - if (newRule.containsKey("comment")) { - ruleToAdd.setComment((String) newRule.get("comment")); + if (newRule.containsKey(RULE_COMMENT_KEY)) { + ruleToAdd.setComment((String) newRule.get(RULE_COMMENT_KEY)); } - if (newRule.containsKey("reason")) { - ruleToAdd.setReason((String) newRule.get("reason")); + if (newRule.containsKey(RULE_REASON_KEY)) { + ruleToAdd.setReason((String) newRule.get(RULE_REASON_KEY)); } - triageRules.add(ruleToAdd); + newRules.add(ruleToAdd); } } - triageConfig.setRiskLevelRules(triageRules); - try { - return JSONUtils.INSTANCE.toJSON(configObj, true); - } catch (JsonProcessingException e) { - LOG.error("Unable to convert object to JSON: {}", configObj, e); - return config; - } + // combine the new and existing rules + List<RiskLevelRule> allRules = ListUtils.union(triageConfig.getRiskLevelRules(), newRules); + triageConfig.setRiskLevelRules(allRules); + + return toJSON(config); + } + + private List<Map<String, Object>> getNewRuleDefinitions(List<Object> args) { + List<Map<String, Object>> newRules = new ArrayList<>(); + Object arg1 = Util.getArg(1, Object.class, args); + if(arg1 instanceof Map) { + newRules.add((Map<String, Object>) arg1); + + } else if(arg1 instanceof List) { + newRules.addAll((List<Map<String, Object>>) arg1); + + } else { + throw new IllegalArgumentException(String.format("triage rule expected to be map or list, got %s", + ClassUtils.getShortClassName(arg1, "null"))); + } return newRules; } @Override public void initialize(Context context) { - + // nothing to do } @Override @@ -208,22 +387,16 @@ public class ThreatTriageFunctions { @Override public Object apply(List<Object> args, Context context) throws ParseException { - String config = (String) args.get(0); - SensorEnrichmentConfig configObj; - if(config == null || config.isEmpty()) { - configObj = new SensorEnrichmentConfig(); - } - else { - configObj = (SensorEnrichmentConfig) ENRICHMENT.deserialize(config); - } - ThreatIntelConfig tiConfig = (ThreatIntelConfig) getConfig(configObj, EnrichmentConfigFunctions.Type.THREAT_INTEL); + SensorEnrichmentConfig config = getSensorEnrichmentConfig(args, 0); + + ThreatIntelConfig tiConfig = (ThreatIntelConfig) getConfig(config, EnrichmentConfigFunctions.Type.THREAT_INTEL); if(tiConfig == null) { tiConfig = new ThreatIntelConfig(); - configObj.setThreatIntel(tiConfig); + config.setThreatIntel(tiConfig); } - ThreatTriageConfig triageConfig = tiConfig.getTriageConfig(); + org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig triageConfig = tiConfig.getTriageConfig(); if(triageConfig == null) { - triageConfig = new ThreatTriageConfig(); + triageConfig = new org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig(); tiConfig.setTriageConfig(triageConfig); } List<RiskLevelRule> triageRules = triageConfig.getRiskLevelRules(); @@ -243,18 +416,13 @@ public class ThreatTriageFunctions { it.remove(); } } - try { - return JSONUtils.INSTANCE.toJSON(configObj, true); - } catch (JsonProcessingException e) { - LOG.error("Unable to convert object to JSON: {}", configObj, e); - return config; - } + return toJSON(config); } @Override public void initialize(Context context) { - + // nothing to do } @Override @@ -277,24 +445,16 @@ public class ThreatTriageFunctions { @Override public Object apply(List<Object> args, Context context) throws ParseException { - String config = (String) args.get(0); + SensorEnrichmentConfig config = getSensorEnrichmentConfig(args, 0); - SensorEnrichmentConfig configObj; - if(config == null || config.isEmpty()) { - configObj = new SensorEnrichmentConfig(); - } - else { - configObj = (SensorEnrichmentConfig) ENRICHMENT.deserialize(config); - } - - ThreatIntelConfig tiConfig = (ThreatIntelConfig) getConfig(configObj, EnrichmentConfigFunctions.Type.THREAT_INTEL); + ThreatIntelConfig tiConfig = (ThreatIntelConfig) getConfig(config, EnrichmentConfigFunctions.Type.THREAT_INTEL); if(tiConfig == null) { tiConfig = new ThreatIntelConfig(); - configObj.setThreatIntel(tiConfig); + config.setThreatIntel(tiConfig); } - ThreatTriageConfig triageConfig = tiConfig.getTriageConfig(); + org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig triageConfig = tiConfig.getTriageConfig(); if(triageConfig == null) { - triageConfig = new ThreatTriageConfig(); + triageConfig = new org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig(); tiConfig.setTriageConfig(triageConfig); } List<RiskLevelRule> triageRules = triageConfig.getRiskLevelRules(); @@ -308,18 +468,13 @@ public class ThreatTriageFunctions { Map<String, Object> aggConfig = (Map<String, Object>) args.get(2); triageConfig.setAggregationConfig(aggConfig); } - try { - return JSONUtils.INSTANCE.toJSON(configObj, true); - } catch (JsonProcessingException e) { - LOG.error("Unable to convert object to JSON: {}", configObj, e); - return config; - } + return toJSON(config); } @Override public void initialize(Context context) { - + // nothing to do } @Override @@ -327,4 +482,51 @@ public class ThreatTriageFunctions { return true; } } + + /** + * + * Serializes the Enrichment configuration to JSON. + * @param enrichmentConfig The Enrichment configuration to serialize to JSON. + * @return The Enrichment configuration as JSON. + */ + private static String toJSON(SensorEnrichmentConfig enrichmentConfig) { + try { + return JSONUtils.INSTANCE.toJSON(enrichmentConfig, true); + + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Unable to serialize enrichment config to JSON", e); + } + } + + /** + * Retrieves the sensor enrichment configuration from the function arguments. The manner + * of retrieving the configuration can differ based on what the user passes in. + * @param args The function arguments. + * @param position The position from which the configuration will be extracted. + * @return The sensor enrichment configuration. + */ + private static SensorEnrichmentConfig getSensorEnrichmentConfig(List<Object> args, int position) { + Object arg0 = Util.getArg(position, Object.class, args); + SensorEnrichmentConfig config = new SensorEnrichmentConfig(); + if(arg0 instanceof String) { + + // deserialize the configuration from json + String json = Util.getArg(0, String.class, args); + if(json != null) { + config = (SensorEnrichmentConfig) ENRICHMENT.deserialize(json); + } + } else if(arg0 instanceof ThreatTriageProcessor) { + + // extract the configuration from the engine + ThreatTriageProcessor engine = Util.getArg(0, ThreatTriageProcessor.class, args); + config = engine.getSensorConfig(); + + } else { + + // unexpected type + throw new IllegalArgumentException(String.format("Unexpected type: got '%s'", ClassUtils.getShortClassName(arg0, "null"))); + } + + return config; + } } http://git-wip-us.apache.org/repos/asf/metron/blob/e3900c4d/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java index bb8d64b..2b154d8 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java @@ -21,22 +21,25 @@ import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.RiskLevelRule; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.shell.StellarExecutor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.DefaultVariableResolver; +import org.apache.metron.stellar.dsl.MapVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; -import org.apache.metron.stellar.common.StellarProcessor; -import org.apache.metron.stellar.common.shell.StellarExecutor; +import org.apache.metron.threatintel.triage.ThreatTriageProcessor; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; import static org.apache.metron.management.EnrichmentConfigFunctionsTest.emptyTransformationsConfig; import static org.apache.metron.management.EnrichmentConfigFunctionsTest.toMap; -import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; public class ThreatTriageFunctionsTest { @@ -66,17 +69,50 @@ public class ThreatTriageFunctionsTest { return processor.parse(rule, new DefaultVariableResolver(x -> variables.get(x),x -> variables.containsKey(x)), StellarFunctions.FUNCTION_RESOLVER(), context); } + private Object run(String rule) { + StellarProcessor processor = new StellarProcessor(); + return processor.parse(rule, new MapVariableResolver(Collections.emptyMap()), StellarFunctions.FUNCTION_RESOLVER(), context); + } + + /** + * Sequentially runs a set of expressions and returns the result of the last expression. + * @param expressions The set of expressions to execute. + * @return The result of running the last expression. + */ + private Object run(String... expressions) { + Object result = null; + + for(String expression: expressions) { + result = run(expression); + } + + return result; + } + @Test public void testSetAggregation() { + String newConfig = (String) run("THREAT_TRIAGE_SET_AGGREGATOR(config, 'MIN' )", toMap("config", configStr)); + SensorEnrichmentConfig sensorConfig = (SensorEnrichmentConfig) ENRICHMENT.deserialize(newConfig); + Assert.assertEquals("MIN", sensorConfig.getThreatIntel().getTriageConfig().getAggregator().toString()); + } - String newConfig = (String) run( - "THREAT_TRIAGE_SET_AGGREGATOR(config, 'MIN' )" - , toMap("config", configStr - ) - ); + @Test + public void testSetAggregationWithEngine() { + // init the engine + ThreatTriageProcessor engine = (ThreatTriageProcessor) run("THREAT_TRIAGE_INIT()"); + Map<String, Object> vars = new HashMap<>(); + vars.put("engine", engine); + + // set the aggregator + String newConfig = (String) run("THREAT_TRIAGE_SET_AGGREGATOR(engine, 'MIN')", vars); + + // validate the return configuration SensorEnrichmentConfig sensorConfig = (SensorEnrichmentConfig) ENRICHMENT.deserialize(newConfig); Assert.assertEquals("MIN", sensorConfig.getThreatIntel().getTriageConfig().getAggregator().toString()); + + // validate that the engine was updated + Assert.assertEquals("MIN", engine.getSensorConfig().getThreatIntel().getTriageConfig().getAggregator().toString()); } @Test @@ -96,6 +132,22 @@ public class ThreatTriageFunctionsTest { } @Test + public void testAddEmptyWithEngine() { + // init the engine + ThreatTriageProcessor engine = (ThreatTriageProcessor) run("THREAT_TRIAGE_INIT()"); + Map<String, Object> vars = new HashMap<>(); + vars.put("engine", engine); + String newConfig = (String) run("THREAT_TRIAGE_ADD(engine, {'rule' : SHELL_GET_EXPRESSION('less'), 'score' : 10 } )", vars); + + // validate the returned configuration + List<RiskLevelRule> triageRules = getTriageRules(newConfig); + Assert.assertEquals(1, triageRules.size()); + + // validate that the engine was updated + Assert.assertEquals(1, engine.getSensorConfig().getThreatIntel().getTriageConfig().getRiskLevelRules().size()); + } + + @Test public void testAddHasExisting() { String newConfig = (String) run( @@ -153,6 +205,41 @@ public class ThreatTriageFunctionsTest { } @Test + public void testAddMultiple() { + + // add a new rule + String newConfig = (String) run( + "THREAT_TRIAGE_ADD(config, { 'name':'rule1', 'rule':'value < 2', 'score':10 } )", + toMap("config", configStr)); + + // add another rule + newConfig = (String) run( + "THREAT_TRIAGE_ADD(config, { 'name':'rule2', 'rule':'value < 4', 'score':10 } )", + toMap("config", newConfig)); + + List<RiskLevelRule> triageRules = getTriageRules(newConfig); + Assert.assertEquals(2, triageRules.size()); + } + + @Test + public void testAddMultipleWithEngine() { + + // init the engine + ThreatTriageProcessor engine = (ThreatTriageProcessor) run("THREAT_TRIAGE_INIT()"); + Map<String, Object> vars = new HashMap<>(); + vars.put("engine", engine); + + // add a new rule + run("THREAT_TRIAGE_ADD(engine, { 'name':'rule1', 'rule':'value < 2', 'score':10 } )", vars); + + // add another rule + run("THREAT_TRIAGE_ADD(engine, { 'name':'rule2', 'rule':'value < 4', 'score':10 } )", vars); + + List<RiskLevelRule> triageRules = engine.getRiskLevelRules(); + Assert.assertEquals(2, triageRules.size()); + } + + @Test public void testRemove() { String newConfig = (String) run( "THREAT_TRIAGE_ADD(config, [ { 'rule' : SHELL_GET_EXPRESSION('less'), 'score' : 10 }, { 'rule' : SHELL_GET_EXPRESSION('greater'), 'score' : 20 } ] )" @@ -174,6 +261,31 @@ public class ThreatTriageFunctionsTest { } @Test + public void testRemoveWithEngine() { + // init the engine + ThreatTriageProcessor engine = (ThreatTriageProcessor) run("THREAT_TRIAGE_INIT()"); + + // set the aggregator + Map<String, Object> vars = new HashMap<>(); + vars.put("engine", engine); + + // add 2 rules + String newConfig = (String) run("THREAT_TRIAGE_ADD(engine, [" + + "{ 'rule' : SHELL_GET_EXPRESSION('less'), 'score' : 10 }, " + + "{ 'rule' : SHELL_GET_EXPRESSION('greater'), 'score' : 20 } ] )", vars); + + // remove 1 rule + newConfig = (String) run("THREAT_TRIAGE_REMOVE(engine, [ " + + "SHELL_GET_EXPRESSION('greater')] )", vars); + + List<RiskLevelRule> triageRules = engine.getRiskLevelRules(); + Assert.assertEquals(1, triageRules.size()); + RiskLevelRule rule = triageRules.get(0); + Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() ); + Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 ); + } + + @Test public void testRemoveMultiple() { String newConfig = (String) run( "THREAT_TRIAGE_ADD(config, [ { 'rule' : SHELL_GET_EXPRESSION('less'), 'score' : 10 }, { 'rule' : SHELL_GET_EXPRESSION('greater'), 'score' : 20 } ] )" @@ -224,17 +336,16 @@ public class ThreatTriageFunctionsTest { ââââââââ¼ââââââââââ¼ââââââââââââââ¼ââââââââ¼ââââââââ⢠â â â 1 > 2 â 20 â â ââââââââ§ââââââââââ§ââââââââââââââ§ââââââââ§âââââââââ - - Aggregation: MAX*/ @Multiline static String testPrintExpected; @Test public void testPrint() { - String newConfig = (String) run( - "THREAT_TRIAGE_ADD(config, [ { 'rule' : SHELL_GET_EXPRESSION('less'), 'score' : 10, 'reason' : '2 + 2' }, { 'rule' : SHELL_GET_EXPRESSION('greater'), 'score' : 20 } ] )" + "THREAT_TRIAGE_ADD(config, [ " + + "{ 'rule' : SHELL_GET_EXPRESSION('less'), 'score' : 10, 'reason' : '2 + 2' }, " + + "{ 'rule' : SHELL_GET_EXPRESSION('greater'), 'score' : 20 } ] )" , toMap("config", configStr ) ); @@ -247,6 +358,26 @@ Aggregation: MAX*/ Assert.assertEquals(testPrintExpected, out); } + @Test + public void testPrintWithEngine() { + + // init the engine + ThreatTriageProcessor engine = (ThreatTriageProcessor) run("THREAT_TRIAGE_INIT()"); + Map<String, Object> vars = new HashMap<>(); + vars.put("engine", engine); + + // add 2 rules + run("THREAT_TRIAGE_ADD(engine, [ " + + "{ 'rule' : SHELL_GET_EXPRESSION('less'), 'score' : 10, 'reason' : '2 + 2' }, " + + "{ 'rule' : SHELL_GET_EXPRESSION('greater'), 'score' : 20 } ] )" + , vars); + + // print + String out = (String) run("THREAT_TRIAGE_PRINT(engine)", vars); + Assert.assertEquals(testPrintExpected, out); + } + + /** ââââââââ¤ââââââââââ¤ââââââââââââââ¤ââââââââ¤âââââââââ â Name â Comment â Triage Rule â Score â Reason â @@ -267,17 +398,152 @@ Aggregation: MAX*/ Assert.assertEquals(testPrintEmptyExpected, out); } - @Test + @Test(expected = IllegalArgumentException.class) public void testPrintNull() { Map<String,Object> variables = new HashMap<String,Object>(){{ - put("config",null); + put("config", null); }}; - String out = (String) run( - "THREAT_TRIAGE_PRINT(config)" - , variables - ); + String out = (String) run("THREAT_TRIAGE_PRINT(config)", variables); Assert.assertEquals(out, testPrintEmptyExpected); } + /** + * { + * "timestamp": 1504548045948, + * "source.type": "example", + * "value": 22 + * } + */ + @Multiline + private String message; + + @Test + public void testTriageInitNoArg() { + Object result = run("THREAT_TRIAGE_INIT()"); + Assert.assertNotNull(result); + Assert.assertTrue(result instanceof ThreatTriageProcessor); + + // there should be no triage rules defined + ThreatTriageProcessor engine = (ThreatTriageProcessor) result; + Assert.assertEquals(0, engine.getRiskLevelRules().size()); + } + + @Test + public void testTriageInitWithArg() { + + // add a triage rule + String confWithRule = (String) run("THREAT_TRIAGE_ADD(conf, [{ 'rule': 'value > 0', 'score' : 10 } ])", + toMap("conf", configStr)); + + // initialize the engine + Object result = run("THREAT_TRIAGE_INIT(confWithRule)", + toMap("confWithRule", confWithRule)); + + Assert.assertNotNull(result); + Assert.assertTrue(result instanceof ThreatTriageProcessor); + + // validate that there is 1 triage rule + ThreatTriageProcessor engine = (ThreatTriageProcessor) result; + Assert.assertEquals(1, engine.getRiskLevelRules().size()); + } + + @Test(expected = IllegalArgumentException.class) + public void testTriageInitWithBadArg() { + run("THREAT_TRIAGE_INIT(missing)"); + } + + @Test + public void testTriageScoreWithNoRules() { + + // init the engine + Object engine = run("THREAT_TRIAGE_INIT()"); + + Map<String, Object> vars = new HashMap<>(); + vars.put("engine", engine); + vars.put("msg", message); + + // score the message + Object result = run("THREAT_TRIAGE_SCORE(msg, engine)", vars); + Assert.assertNotNull(result); + Assert.assertTrue(result instanceof Map); + + // validate the rules that were scored + Map<String, Object> score = (Map) result; + Assert.assertEquals(0, ((List) score.get(ThreatTriageFunctions.RULES_KEY)).size()); + + // validate the total score + Object totalScore = score.get(ThreatTriageFunctions.SCORE_KEY); + Assert.assertTrue(totalScore instanceof Double); + Assert.assertEquals(0.0, (Double) totalScore, 0.001); + } + + @Test + public void testTriageScoreWithRules() { + + // add a triage rule + String confWithRule = (String) run("THREAT_TRIAGE_ADD(conf, [{ 'rule': 'value > 0', 'score' : 10 }])", + toMap("conf", configStr)); + + // initialize the engine + Object engine = run("THREAT_TRIAGE_INIT(confWithRule)", + toMap("confWithRule", confWithRule)); + + Map<String, Object> vars = new HashMap<>(); + vars.put("engine", engine); + vars.put("msg", message); + + // score the message + Object result = run("THREAT_TRIAGE_SCORE(msg, engine)", vars); + Assert.assertNotNull(result); + Assert.assertTrue(result instanceof Map); + + // validate the rules that were scored + Map<String, Object> score = (Map) result; + Assert.assertEquals(1, ((List) score.get(ThreatTriageFunctions.RULES_KEY)).size()); + + // validate the total score + Object totalScore = score.get(ThreatTriageFunctions.SCORE_KEY); + Assert.assertTrue(totalScore instanceof Double); + Assert.assertEquals(10.0, (Double) totalScore, 0.001); + + // validate the aggregator + Assert.assertEquals("MAX", score.get(ThreatTriageFunctions.AGG_KEY)); + } + + @Test(expected = Exception.class) + public void testTriageScoreWithNoMessage() { + // add a triage rule + String confWithRule = (String) run("THREAT_TRIAGE_ADD(conf, [{ 'rule': 'value > 0', 'score' : 10 }])", + toMap("conf", configStr)); + + // initialize the engine + Object engine = run("THREAT_TRIAGE_INIT(confWithRule)", + toMap("confWithRule", confWithRule)); + + Map<String, Object> vars = new HashMap<>(); + vars.put("engine", engine); + + // score the message + run("THREAT_TRIAGE_SCORE(11, engine)", vars); + } + + @Test + public void testTriageConfig() { + + // init the engine + Object engine = run("THREAT_TRIAGE_INIT()"); + + Map<String, Object> vars = new HashMap<>(); + vars.put("engine", engine); + + // score the message + Object result = run("THREAT_TRIAGE_CONFIG(engine)", vars); + Assert.assertNotNull(result); + Assert.assertTrue(result instanceof String); + + // validate the configuration + String json = (String) result; + Assert.assertEquals(emptyTransformationsConfig(), json); + } }
