This is an automated email from the ASF dual-hosted git repository.
nickallen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 39fa0f1 METRON-685 Scores in Threat Triage should be a Stellar
Statement (nickwallen) closes apache/metron#1311
39fa0f1 is described below
commit 39fa0f134fbf4d16da2001792971135d9f24cf4c
Author: nickwallen <[email protected]>
AuthorDate: Mon Jan 7 13:53:35 2019 -0500
METRON-685 Scores in Threat Triage should be a Stellar Statement
(nickwallen) closes apache/metron#1311
---
.../enrichment/threatintel/RiskLevelRule.java | 63 ++++++-----
.../enrichment/threatintel/RuleScore.java | 37 +++++--
.../enrichment/threatintel/ThreatTriageConfig.java | 2 +-
.../threatintel/ThreatTriageConfigTest.java | 115 +++++++++++++++++++++
metron-platform/metron-enrichment/README.md | 58 +++++------
.../metron/enrichment/utils/ThreatIntelUtils.java | 2 +-
.../threatintel/triage/ThreatTriageProcessor.java | 24 +++--
.../threatintel/triage/ThreatTriageTest.java | 55 ++++++++++
.../metron/management/ThreatTriageFunctions.java | 10 +-
.../management/ThreatTriageFunctionsTest.java | 55 ++++++++--
10 files changed, 326 insertions(+), 95 deletions(-)
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java
index 94ab0c8..0443c27 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RiskLevelRule.java
@@ -17,6 +17,10 @@
*/
package org.apache.metron.common.configuration.enrichment.threatintel;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Objects;
+
/**
* This class represents a rule that is used to triage threats.
*
@@ -34,24 +38,24 @@ public class RiskLevelRule {
/**
* The name of the rule. This field is optional.
*/
- String name;
+ private String name;
/**
* A description of the rule. This field is optional.
*/
- String comment;
+ private String comment;
/**
* A predicate, in the form of a Stellar expression, that determines whether
* the rule is applied to an alert or not. This field is required.
*/
- String rule;
+ private String rule;
/**
- * A numeric value that represents the score that is applied to the alert.
This
- * field is required.
+ * A Stellar expression that when evaluated results in a numeric score. The
expression
+ * can refer to fields within the message undergoing triage.
*/
- Number score;
+ private String scoreExpression;
/**
* Allows a rule author to provide contextual information when a rule is
applied
@@ -60,7 +64,7 @@ public class RiskLevelRule {
* This is expected to be a valid Stellar expression and can refer to any of
the
* fields within the message itself.
*/
- String reason;
+ private String reason;
public String getName() {
return name;
@@ -86,12 +90,26 @@ public class RiskLevelRule {
this.rule = rule;
}
- public Number getScore() {
- return score;
+ @JsonProperty("score")
+ public String getScoreExpression() {
+ return scoreExpression;
}
- public void setScore(Number score) {
- this.score = score;
+ @JsonProperty("score")
+ public void setScoreExpression(Object scoreExpression) {
+ if(scoreExpression instanceof Number) {
+ // a numeric value was provided
+ scoreExpression = Number.class.cast(scoreExpression).toString();
+
+ } else if (scoreExpression instanceof String) {
+ // a stellar expression was provided
+ scoreExpression = String.class.cast(scoreExpression);
+
+ } else {
+ throw new IllegalArgumentException(String.format("Expected 'score' to be
number or string, but got '%s'", scoreExpression));
+ }
+
+ this.scoreExpression = scoreExpression.toString();
}
public String getReason() {
@@ -105,25 +123,18 @@ public class RiskLevelRule {
@Override
public boolean equals(Object o) {
if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
+ if (!(o instanceof RiskLevelRule)) return false;
RiskLevelRule that = (RiskLevelRule) o;
-
- if (name != null ? !name.equals(that.name) : that.name != null) return
false;
- if (comment != null ? !comment.equals(that.comment) : that.comment !=
null) return false;
- if (rule != null ? !rule.equals(that.rule) : that.rule != null) return
false;
- if (score != null ? !score.equals(that.score) : that.score != null) return
false;
- return reason != null ? reason.equals(that.reason) : that.reason == null;
+ return Objects.equals(name, that.name) &&
+ Objects.equals(comment, that.comment) &&
+ Objects.equals(rule, that.rule) &&
+ Objects.equals(scoreExpression, that.scoreExpression) &&
+ Objects.equals(reason, that.reason);
}
@Override
public int hashCode() {
- int result = name != null ? name.hashCode() : 0;
- result = 31 * result + (comment != null ? comment.hashCode() : 0);
- result = 31 * result + (rule != null ? rule.hashCode() : 0);
- result = 31 * result + (score != null ? score.hashCode() : 0);
- result = 31 * result + (reason != null ? reason.hashCode() : 0);
- return result;
+ return Objects.hash(name, comment, rule, scoreExpression, reason);
}
@Override
@@ -132,7 +143,7 @@ public class RiskLevelRule {
"name='" + name + '\'' +
", comment='" + comment + '\'' +
", rule='" + rule + '\'' +
- ", score=" + score +
+ ", scoreExpression='" + scoreExpression + '\'' +
", reason='" + reason + '\'' +
'}';
}
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RuleScore.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RuleScore.java
index 71bd13b..7889fac 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RuleScore.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/RuleScore.java
@@ -17,6 +17,8 @@
*/
package org.apache.metron.common.configuration.enrichment.threatintel;
+import java.util.Objects;
+
/**
* This class represents the score resulting from applying a RiskLevelRule
* to a message.
@@ -44,12 +46,19 @@ public class RuleScore {
private String reason;
/**
+ * The numeric score which is the result of executing the {@link
RiskLevelRule} score Stellar expression.
+ */
+ private Number score;
+
+ /**
* @param rule The threat triage rule that when applied resulted in this
score.
* @param reason The result of executing the rule's 'reason' expression.
Provides context to why a rule was applied.
+ * @param score The result of executing the rule's 'score' expression.
*/
- public RuleScore(RiskLevelRule rule, String reason) {
+ public RuleScore(RiskLevelRule rule, String reason, Number score) {
this.rule = rule;
this.reason = reason;
+ this.score = score;
}
public String getReason() {
@@ -60,22 +69,27 @@ public class RuleScore {
return rule;
}
+ public Number getScore() {
+ return score;
+ }
+
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- RuleScore that = (RuleScore) o;
-
- if (rule != null ? !rule.equals(that.rule) : that.rule != null) return
false;
- return reason != null ? reason.equals(that.reason) : that.reason == null;
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof RuleScore)) {
+ return false;
+ }
+ RuleScore ruleScore = (RuleScore) o;
+ return Objects.equals(rule, ruleScore.rule) &&
+ Objects.equals(reason, ruleScore.reason) &&
+ Objects.equals(score, ruleScore.score);
}
@Override
public int hashCode() {
- int result = rule != null ? rule.hashCode() : 0;
- result = 31 * result + (reason != null ? reason.hashCode() : 0);
- return result;
+ return Objects.hash(rule, reason, score);
}
@Override
@@ -83,6 +97,7 @@ public class RuleScore {
return "RuleScore{" +
"rule=" + rule +
", reason='" + reason + '\'' +
+ ", score=" + score +
'}';
}
}
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java
index 961f9d5..13effc4 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfig.java
@@ -47,7 +47,7 @@ public class ThreatTriageConfig {
StellarProcessor processor = new StellarProcessor();
for(RiskLevelRule rule : riskLevelRules) {
- if(rule.getRule() == null || rule.getScore() == null) {
+ if(rule.getRule() == null || rule.getScoreExpression() == null) {
throw new IllegalStateException("Risk level rules must contain both a
rule and a score.");
}
if(ruleIndex.contains(rule.getRule())) {
diff --git
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfigTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfigTest.java
new file mode 100644
index 0000000..9b54430
--- /dev/null
+++
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/enrichment/threatintel/ThreatTriageConfigTest.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.configuration.enrichment.threatintel;
+
+import org.adrianwalker.multilinestring.Multiline;
+import
org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.junit.Test;
+
+import java.util.List;
+
+import static
org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+
+public class ThreatTriageConfigTest {
+
+ /**
+ * {
+ * "enrichment": {
+ * },
+ * "threatIntel": {
+ * "triageConfig": {
+ * "riskLevelRules": [
+ * {
+ * "name": "Rule Name",
+ * "comment": "Rule Comment",
+ * "reason": "'Rule Reason'",
+ * "rule": "ip_src_addr == '10.0.2.3'",
+ * "score": 10
+ * }
+ * ]
+ * }
+ * }
+ * }
+ */
+ @Multiline
+ private String triageRuleWithNumericScore;
+
+ @Test
+ public void shouldAllowNumericRuleScore() throws Exception {
+ // deserialize
+ SensorEnrichmentConfig enrichment = (SensorEnrichmentConfig)
ENRICHMENT.deserialize(triageRuleWithNumericScore);
+
+ ThreatTriageConfig threatTriage =
enrichment.getThreatIntel().getTriageConfig();
+ assertNotNull(threatTriage);
+
+ List<RiskLevelRule> rules = threatTriage.getRiskLevelRules();
+ assertEquals(1, rules.size());
+
+ RiskLevelRule rule = rules.get(0);
+ assertEquals("Rule Name", rule.getName());
+ assertEquals("Rule Comment", rule.getComment());
+ assertEquals("ip_src_addr == '10.0.2.3'", rule.getRule());
+ assertEquals("'Rule Reason'", rule.getReason());
+ assertEquals("10", rule.getScoreExpression());
+ }
+
+ /**
+ * {
+ * "enrichment": {
+ * },
+ * "threatIntel": {
+ * "triageConfig": {
+ * "riskLevelRules": [
+ * {
+ * "name": "Rule Name",
+ * "comment": "Rule Comment",
+ * "reason": "'Rule Reason'",
+ * "rule": "ip_src_addr == '10.0.2.3'",
+ * "score": "10 + 10"
+ * }
+ * ]
+ * }
+ * }
+ * }
+ */
+ @Multiline
+ private String triageRuleWithScoreExpression;
+
+ @Test
+ public void shouldAllowScoreAsStellarExpression() throws Exception {
+ // deserialize the enrichment configuration
+ SensorEnrichmentConfig enrichment = (SensorEnrichmentConfig)
ENRICHMENT.deserialize(triageRuleWithScoreExpression);
+
+ ThreatTriageConfig threatTriage =
enrichment.getThreatIntel().getTriageConfig();
+ assertNotNull(threatTriage);
+
+ List<RiskLevelRule> rules = threatTriage.getRiskLevelRules();
+ assertEquals(1, rules.size());
+
+ RiskLevelRule rule = rules.get(0);
+ assertEquals("Rule Name", rule.getName());
+ assertEquals("Rule Comment", rule.getComment());
+ assertEquals("'Rule Reason'", rule.getReason());
+ assertEquals("ip_src_addr == '10.0.2.3'", rule.getRule());
+ assertEquals("10 + 10", rule.getScoreExpression());
+ }
+}
diff --git a/metron-platform/metron-enrichment/README.md
b/metron-platform/metron-enrichment/README.md
index c72970f..9e35f48 100644
--- a/metron-platform/metron-enrichment/README.md
+++ b/metron-platform/metron-enrichment/README.md
@@ -22,7 +22,7 @@ limitations under the License.
The `enrichment` topology is a topology dedicated to taking the data
from the parsing topologies that have been normalized into the Metron
data format (e.g. a JSON Map structure with `original_message` and
-`timestamp`) and
+`timestamp`) and
* Enriching messages with external data from data stores (e.g. hbase) by
adding new fields based on existing fields in the messages.
* Marking messages as threats based on data in external data stores
@@ -38,7 +38,7 @@ data format (e.g. a JSON Map structure with
`original_message` and
The unified enrichment topology uses data parallelism as opposed to the
deprecated
split/join topology's task parallelism. This architecture uses a worker pool
to fully
enrich any message within a worker. This results in
-* Fewer bolts in the topology
+* Fewer bolts in the topology
* Each bolt fully operates on a message.
* Fewer network hops
@@ -84,7 +84,7 @@ defined by JSON documents stored in zookeeper.
There are two types of configurations at the moment, `global` and
`sensor` specific.
-## Global Configuration
+## Global Configuration
There are a few enrichments which have independent configurations, such
as from the global config. You can also configure the enrichment topology's
@@ -104,7 +104,7 @@ The location on HDFS of the GeoLite2 database file to use
for GeoIP
lookups. This file will be localized on the storm supervisors running
the topology and used from there. This is lazy, so if this property
changes in a running topology, the file will be localized from HDFS upon first
-time the file is used via the geo enrichment.
+time the file is used via the geo enrichment.
### Writer Batching
@@ -115,7 +115,7 @@ The size of the batch that is written to Kafka at once.
Defaults to `15` (size o
#### `enrichment.writer.batchTimeout`
The timeout after which a batch will be flushed even if batchSize has not been
met. Optional.
-If unspecified, or set to `0`, it defaults to a system-determined duration
which is a fraction of the Storm
+If unspecified, or set to `0`, it defaults to a system-determined duration
which is a fraction of the Storm
parameter `topology.message.timeout.secs`. Ignored if batchSize is `1`, since
this disables batching.
## Sensor Enrichment Configuration
@@ -161,16 +161,16 @@ The simplest, by far, is just providing a simple list as
in
]
}
```
-Based on this sample config, both `ip_src_addr` and `ip_dst_addr` will go to
the `geo`, `host`, and
-`hbaseEnrichment` adapter bolts.
-
+Based on this sample config, both `ip_src_addr` and `ip_dst_addr` will go to
the `geo`, `host`, and
+`hbaseEnrichment` adapter bolts.
+
#### Stellar Enrichment Configuration
-For the `geo`, `host` and `hbaseEnrichment`, this is sufficient. However, more
complex enrichments
+For the `geo`, `host` and `hbaseEnrichment`, this is sufficient. However, more
complex enrichments
may contain their own configuration. Currently, the `stellar` enrichment is
more adaptable and thus
requires a more nuanced configuration.
At its most basic, we want to take a message and apply a couple of
enrichments, such as converting the
-`hostname` field to lowercase. We do this by specifying the transformation
inside of the
+`hostname` field to lowercase. We do this by specifying the transformation
inside of the
`config` for the `stellar` fieldMap. There are two syntaxes that are
supported, specifying the transformations
as a map with the key as the field and the value the stellar expression:
```
@@ -199,7 +199,7 @@ in the Stellar REPL, such as:
Sometimes arbitrary stellar enrichments may take enough time that you would
prefer to split some of them
into groups and execute the groups of stellar enrichments in parallel. Take,
for instance, if you wanted
-to do an HBase enrichment and a profiler call which were independent of one
another. This usecase is
+to do an HBase enrichment and a profiler call which were independent of one
another. This usecase is
supported by splitting the enrichments up as groups.
Consider the following example:
@@ -212,13 +212,13 @@ Consider the following example:
"is_bad_domain" : "ENRICHMENT_EXISTS('malicious_domains',
ip_dst_addr, 'enrichments', 'cf')"
},
"login_profile" : [
- "profile_window := PROFILE_WINDOW('from 6 months ago')",
+ "profile_window := PROFILE_WINDOW('from 6 months ago')",
"global_login_profile := PROFILE_GET('distinct_login_attempts',
'global', profile_window)",
"stats := STATS_MERGE(global_login_profile)",
- "auth_attempts_median := STATS_PERCENTILE(stats, 0.5)",
+ "auth_attempts_median := STATS_PERCENTILE(stats, 0.5)",
"auth_attempts_sd := STATS_SD(stats)",
- "profile_window := null",
- "global_login_profile := null",
+ "profile_window := null",
+ "global_login_profile := null",
"stats := null"
]
}
@@ -275,20 +275,20 @@ A risk level rule is of the following format:
* `name` : The name of the threat triage rule
* `comment` : A comment describing the rule
* `rule` : The rule, represented as a Stellar statement
-* `score` : Associated threat triage score for the rule
+* `score` : The score attributed to the rule. Can be either numeric or a
Stellar expression. The expression has access to all fields with the message
being triaged.
* `reason` : Reason the rule tripped. Can be represented as a Stellar statement
An example of a rule is as follows:
```
- "riskLevelRules" : [
- {
- "name" : "is internal"
- , "comment" : "determines if the destination is internal."
- , "rule" : "IN_SUBNET(ip_dst_addr, '192.168.0.0/24')"
- , "score" : 10
- , "reason" : "FORMAT('%s is internal', ip_dst_addr)"
+ "riskLevelRules" : [
+ {
+ "name" : "is internal",
+ "comment" : "determines if the destination is internal.",
+ "rule" : "IN_SUBNET(ip_dst_addr, '192.168.0.0/24')",
+ "score" : 10,
+ "reason" : "FORMAT('%s is internal', ip_dst_addr)"
}
- ]
+ ]
```
The supported aggregation functions are:
@@ -343,7 +343,7 @@ An example configuration for the YAF sensor is as follows:
]
},
"triageConfig" : {
- "riskLevelRules" : [
+ "riskLevelRules" : [
{
"rule" : "ip_src_addr == '10.0.2.3' or ip_dst_addr == '10.0.2.3'",
"score" : 10
@@ -372,7 +372,7 @@ Start Squid via `service squid start`
## Adjust Enrichment Configurations for Squid to Call Stellar
Let's adjust the configurations for the Squid topology to annotate the
messages using some Stellar functions.
-* Edit the squid enrichment configuration at
`$METRON_HOME/config/zookeeper/enrichments/squid.json` (this file will not
exist, so create a new one) to add some new fields based on stellar queries:
+* Edit the squid enrichment configuration at
`$METRON_HOME/config/zookeeper/enrichments/squid.json` (this file will not
exist, so create a new one) to add some new fields based on stellar queries:
```
{
@@ -394,7 +394,7 @@ Let's adjust the configurations for the Squid topology to
annotate the messages
"config" : {
"bar" : "TO_UPPER(source.type)"
}
- }
+ }
},
"triageConfig" : {
}
@@ -403,7 +403,7 @@ Let's adjust the configurations for the Squid topology to
annotate the messages
```
We have added the following fields as part of the enrichment phase of the
enrichment topology:
* `foo` == 2
-* `ALL_CAPS` == SQUID
+* `ALL_CAPS` == SQUID
We have added the following as part of the threat intel:
* ` bar` == SQUID
@@ -424,5 +424,3 @@ Now we need to start the topologies and send some data:
* Ensure that the documents have new fields `foo`, `bar` and `ALL_CAPS` with
values as described above.
Note that we could have used any Stellar statements here, including calling
out to HBase via `ENRICHMENT_GET` and `ENRICHMENT_EXISTS` or even calling a
machine learning model via [Model as a
Service](../../metron-analytics/metron-maas-service).
-
-
diff --git
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
index 870d709..646547e 100644
---
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
+++
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
@@ -152,7 +152,7 @@ public static JSONObject triage(JSONObject ret,
SensorEnrichmentConfig config, F
for(RuleScore score: threatScore.getRuleScores()) {
message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i,
THREAT_TRIAGE_RULE_NAME), score.getRule().getName());
message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i,
THREAT_TRIAGE_RULE_COMMENT), score.getRule().getComment());
- message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i,
THREAT_TRIAGE_RULE_SCORE), score.getRule().getScore());
+ message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i,
THREAT_TRIAGE_RULE_SCORE), score.getRule().getScoreExpression());
message.put(joiner.join(THREAT_TRIAGE_RULES_KEY, i++,
THREAT_TRIAGE_RULE_REASON), score.getReason());
}
}
diff --git
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
index 4f24b68..64a2ae6 100644
---
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
+++
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
@@ -35,9 +35,9 @@ import org.apache.metron.stellar.dsl.VariableResolver;
import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver;
import javax.annotation.Nullable;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.stream.Collectors;
/**
* Applies the threat triage rules to an alert and produces a threat score
that is
@@ -72,30 +72,36 @@ public class ThreatTriageProcessor implements Function<Map,
ThreatScore> {
this.context = context;
}
+ /**
+ * @param message The message being triaged.
+ */
@Nullable
@Override
- public ThreatScore apply(@Nullable Map input) {
+ public ThreatScore apply(@Nullable Map message) {
ThreatScore threatScore = new ThreatScore();
StellarPredicateProcessor predicateProcessor = new
StellarPredicateProcessor();
StellarProcessor processor = new StellarProcessor();
- VariableResolver resolver = new MapVariableResolver(input,
sensorConfig.getConfiguration(), threatIntelConfig.getConfig());
+ VariableResolver variableResolver = new MapVariableResolver(message,
sensorConfig.getConfiguration(), threatIntelConfig.getConfig());
// attempt to apply each rule to the threat
for(RiskLevelRule rule : threatTriageConfig.getRiskLevelRules()) {
- if(predicateProcessor.parse(rule.getRule(), resolver, functionResolver,
context)) {
+ if(predicateProcessor.parse(rule.getRule(), variableResolver,
functionResolver, context)) {
// add the rule's score to the overall threat score
- String reason = execute(rule.getReason(), processor, resolver,
String.class);
- RuleScore score = new RuleScore(rule, reason);
- threatScore.addRuleScore(score);
+ String reason = execute(rule.getReason(), processor, variableResolver,
String.class);
+ Double score = execute(rule.getScoreExpression(), processor,
variableResolver, Double.class);
+ threatScore.addRuleScore(new RuleScore(rule, reason, score));
}
}
// calculate the aggregate threat score
+ List<Number> ruleScores = new ArrayList<>();
+ for(RuleScore ruleScore: threatScore.getRuleScores()) {
+ ruleScores.add(ruleScore.getScore());
+ }
Aggregators aggregators = threatTriageConfig.getAggregator();
- List<Number> allScores = threatScore.getRuleScores().stream().map(score ->
score.getRule().getScore()).collect(Collectors.toList());
- Double aggregateScore = aggregators.aggregate(allScores,
threatTriageConfig.getAggregationConfig());
+ Double aggregateScore = aggregators.aggregate(ruleScores,
threatTriageConfig.getAggregationConfig());
threatScore.setScore(aggregateScore);
return threatScore;
diff --git
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
index 63f720b..da79fae 100644
---
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
+++
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java
@@ -30,6 +30,7 @@ import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -353,6 +354,60 @@ public class ThreatTriageTest {
}
}
+ /**
+ * {
+ * "threatIntel" : {
+ * "triageConfig": {
+ * "riskLevelRules": [
+ * {
+ * "rule" : "true",
+ * "score" : 10
+ * }
+ * ],
+ * "aggregator" : "MAX"
+ * }
+ * }
+ * }
+ */
+ @Multiline
+ private static String shouldAllowNumericRuleScore;
+
+ @Test
+ public void shouldAllowNumericRuleScore() throws Exception {
+ Map<String, Object> message = new HashMap<>();
+ ThreatTriageProcessor threatTriageProcessor =
getProcessor(shouldAllowNumericRuleScore);
+ Assert.assertEquals(10d, threatTriageProcessor.apply(message).getScore(),
1e-10);
+ }
+
+ /**
+ * {
+ * "threatIntel" : {
+ * "triageConfig": {
+ * "riskLevelRules": [
+ * {
+ * "rule" : "true",
+ * "score" : "priority * 10.1"
+ * }
+ * ],
+ * "aggregator" : "MAX"
+ * }
+ * }
+ * }
+ */
+ @Multiline
+ private static String shouldAllowScoreAsStellarExpression;
+
+ @Test
+ public void shouldAllowScoreAsStellarExpression() throws Exception {
+ // the message being triaged has a field 'priority' that is referenced in
the score expression
+ Map<Object, Object> message = new HashMap<Object, Object>() {{
+ put("priority", 100);
+ }};
+
+ ThreatTriageProcessor threatTriageProcessor =
getProcessor(shouldAllowScoreAsStellarExpression);
+ Assert.assertEquals(1010.0d,
threatTriageProcessor.apply(message).getScore(), 1e-10);
+ }
+
private static ThreatTriageProcessor getProcessor(String config) throws
IOException {
SensorEnrichmentConfig c = JSONUtils.INSTANCE.load(config,
SensorEnrichmentConfig.class);
return new ThreatTriageProcessor(c, StellarFunctions.FUNCTION_RESOLVER(),
Context.EMPTY_CONTEXT());
diff --git
a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java
b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java
index e142b23..9c1589f 100644
---
a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java
+++
b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java
@@ -28,7 +28,6 @@ import
org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntel
import
org.apache.metron.common.configuration.enrichment.threatintel.ThreatScore;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.profiler.client.stellar.Util;
-import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.ParseException;
import org.apache.metron.stellar.dsl.Stellar;
@@ -167,8 +166,8 @@ public class ThreatTriageFunctions {
if(ruleScore.getRule().getRule() != null) {
map.put(RULE_EXPR_KEY, ruleScore.getRule().getRule());
}
- if(ruleScore.getRule().getScore() != null) {
- map.put(RULE_SCORE_KEY, ruleScore.getRule().getScore());
+ if(ruleScore.getRule().getScoreExpression() != null) {
+ map.put(RULE_SCORE_KEY, ruleScore.getRule().getScoreExpression());
}
if(ruleScore.getReason() != null) {
map.put(RULE_REASON_KEY, ruleScore.getReason());
@@ -260,8 +259,7 @@ public class ThreatTriageFunctions {
String[][] data = new String[triageRules.size()][5];
int i = 0;
for(RiskLevelRule rule : triageRules) {
- double d = rule.getScore().doubleValue();
- String score = d == (long)d ? String.format("%d", (long)d) :
String.format("%s", d);
+ String score = rule.getScoreExpression();
String name = Optional.ofNullable(rule.getName()).orElse("");
String comment = Optional.ofNullable(rule.getComment()).orElse("");
String reason = Optional.ofNullable(rule.getReason()).orElse("");
@@ -325,7 +323,7 @@ public class ThreatTriageFunctions {
// create the rule
RiskLevelRule ruleToAdd = new RiskLevelRule();
ruleToAdd.setRule((String) newRule.get(RULE_EXPR_KEY));
-
ruleToAdd.setScore(ConversionUtils.convert(newRule.get(RULE_SCORE_KEY),
Double.class));
+ ruleToAdd.setScoreExpression(newRule.get(RULE_SCORE_KEY));
// add optional rule fields
if (newRule.containsKey(RULE_NAME_KEY)) {
diff --git
a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
index 0809dcf..fc7558e 100644
---
a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
+++
b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java
@@ -129,7 +129,7 @@ public class ThreatTriageFunctionsTest {
Assert.assertEquals(1, triageRules.size());
RiskLevelRule rule = triageRules.get(0);
Assert.assertEquals(variables.get("less").getExpression().get(),
rule.getRule() );
- Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 );
+ Assert.assertEquals("10", rule.getScoreExpression());
}
@Test
@@ -159,19 +159,19 @@ public class ThreatTriageFunctionsTest {
newConfig = (String) run(
"THREAT_TRIAGE_ADD(config, { 'rule' :
SHELL_GET_EXPRESSION('greater'), 'score' : 20 } )"
- , toMap("config",newConfig
+ , toMap("config", newConfig
)
);
List<RiskLevelRule> triageRules = getTriageRules(newConfig);
Assert.assertEquals(2, triageRules.size());
RiskLevelRule less = triageRules.get(0);
- Assert.assertEquals(variables.get("less").getExpression().get(),
less.getRule() );
- Assert.assertEquals(10.0, less.getScore().doubleValue(), 1e-6 );
+ Assert.assertEquals(variables.get("less").getExpression().get(),
less.getRule());
+ Assert.assertEquals("10", less.getScoreExpression());
RiskLevelRule greater = triageRules.get(1);
- Assert.assertEquals(variables.get("greater").getExpression().get(),
greater.getRule() );
- Assert.assertEquals(20.0, greater.getScore().doubleValue(), 1e-6 );
+ Assert.assertEquals(variables.get("greater").getExpression().get(),
greater.getRule());
+ Assert.assertEquals("20", greater.getScoreExpression());
}
@Test(expected=ParseException.class)
@@ -202,7 +202,7 @@ public class ThreatTriageFunctionsTest {
Assert.assertEquals(1, triageRules.size());
RiskLevelRule rule = triageRules.get(0);
Assert.assertEquals(variables.get("less").getExpression().get(),
rule.getRule() );
- Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 );
+ Assert.assertEquals("10", rule.getScoreExpression());
}
@Test
@@ -258,7 +258,7 @@ public class ThreatTriageFunctionsTest {
Assert.assertEquals(1, triageRules.size());
RiskLevelRule rule = triageRules.get(0);
Assert.assertEquals(variables.get("less").getExpression().get(),
rule.getRule() );
- Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 );
+ Assert.assertEquals("10", rule.getScoreExpression());
}
@Test
@@ -283,7 +283,7 @@ public class ThreatTriageFunctionsTest {
Assert.assertEquals(1, triageRules.size());
RiskLevelRule rule = triageRules.get(0);
Assert.assertEquals(variables.get("less").getExpression().get(),
rule.getRule() );
- Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 );
+ Assert.assertEquals("10", rule.getScoreExpression());
}
@Test
@@ -322,11 +322,11 @@ public class ThreatTriageFunctionsTest {
Assert.assertEquals(2, triageRules.size());
RiskLevelRule less = triageRules.get(0);
Assert.assertEquals(variables.get("less").getExpression().get(),
less.getRule() );
- Assert.assertEquals(10.0, less.getScore().doubleValue(), 1e-6 );
+ Assert.assertEquals("10", less.getScoreExpression());
RiskLevelRule greater = triageRules.get(1);
Assert.assertEquals(variables.get("greater").getExpression().get(),
greater.getRule() );
- Assert.assertEquals(20.0, greater.getScore().doubleValue(), 1e-6 );
+ Assert.assertEquals("20", greater.getScoreExpression());
}
/**
@@ -511,6 +511,39 @@ Aggregation: MAX*/
Assert.assertEquals("MAX", score.get(ThreatTriageFunctions.AGG_KEY));
}
+ @Test
+ public void testTriageWithScoreExpression() {
+
+ // add a triage rule that uses an expression for the score
+ String confWithRule = (String) run("THREAT_TRIAGE_ADD(conf, [{ 'rule':
'value > 0', 'score' : 'value * 10' }])",
+ toMap("conf", configStr));
+
+ // initialize the engine
+ Object engine = run("THREAT_TRIAGE_INIT(confWithRule)",
+ toMap("confWithRule", confWithRule));
+
+ Map<String, Object> vars = new HashMap<>();
+ vars.put("engine", engine);
+ vars.put("msg", message);
+
+ // score the message
+ Object result = run("THREAT_TRIAGE_SCORE(msg, engine)", vars);
+ Assert.assertNotNull(result);
+ Assert.assertTrue(result instanceof Map);
+
+ // validate the rules that were scored
+ Map<String, Object> score = (Map) result;
+ Assert.assertEquals(1, ((List)
score.get(ThreatTriageFunctions.RULES_KEY)).size());
+
+ // validate the total score
+ Object totalScore = score.get(ThreatTriageFunctions.SCORE_KEY);
+ Assert.assertTrue(totalScore instanceof Double);
+ Assert.assertEquals(220.0, (Double) totalScore, 0.001);
+
+ // validate the aggregator
+ Assert.assertEquals("MAX", score.get(ThreatTriageFunctions.AGG_KEY));
+ }
+
@Test(expected = Exception.class)
public void testTriageScoreWithNoMessage() {
