METRON-1620: Fixes for forensic clustering use case example (mmiklavc via mmiklavc) closes apache/metron#1065
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0c20fd1a Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0c20fd1a Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0c20fd1a Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 0c20fd1a3a809a8ad2e8cc280a2a5382f3d5a7e8 Parents: a4bec95 Author: mmiklavc <michael.miklav...@gmail.com> Authored: Mon Jul 16 12:54:53 2018 -0600 Committer: Michael Miklavcic <michael.miklav...@gmail.com> Committed: Mon Jul 16 12:54:53 2018 -0600 ---------------------------------------------------------------------- use-cases/forensic_clustering/README.md | 228 +++++++++++++++++++-------- use-cases/typosquat_detection/README.md | 9 +- 2 files changed, 166 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/0c20fd1a/use-cases/forensic_clustering/README.md ---------------------------------------------------------------------- diff --git a/use-cases/forensic_clustering/README.md b/use-cases/forensic_clustering/README.md index dac116c..fd631c1 100644 --- a/use-cases/forensic_clustering/README.md +++ b/use-cases/forensic_clustering/README.md @@ -157,28 +157,138 @@ we have valid data. ``` -Before we start, we will want to install ES mappings so ES knows how to interpret our fields: +Before we start, we will want to install ES template mappings so ES knows how to interpret our fields: ``` -curl -XPUT 'http://$ES_HOST/cowrie*/_mapping/cowrie_doc' -d ' +curl -XPUT $ES_HOST'/_template/cowrie_index' -d ' { + "template": "cowrie_index*", + "mappings": { + "cowrie_doc": { + "dynamic_templates": [ + { + "geo_location_point": { + "match": "enrichments:geo:*:location_point", + "match_mapping_type": "*", + "mapping": { + "type": "geo_point" + } + } + }, + { + "geo_country": { + "match": "enrichments:geo:*:country", + "match_mapping_type": "*", + "mapping": { + "type": "keyword" + } + } + }, + { + "geo_city": { + "match": "enrichments:geo:*:city", + "match_mapping_type": "*", + "mapping": { + "type": "keyword" + } + } + }, + { + "geo_location_id": { + "match": "enrichments:geo:*:locID", + "match_mapping_type": "*", + "mapping": { + "type": "keyword" + } + } + }, + { + "geo_dma_code": { + "match": "enrichments:geo:*:dmaCode", + "match_mapping_type": "*", + "mapping": { + "type": "keyword" + } + } + }, + { + "geo_postal_code": { + "match": "enrichments:geo:*:postalCode", + "match_mapping_type": "*", + "mapping": { + "type": "keyword" + } + } + }, + { + "geo_latitude": { + "match": "enrichments:geo:*:latitude", + "match_mapping_type": "*", + "mapping": { + "type": "float" + } + } + }, + { + "geo_longitude": { + "match": "enrichments:geo:*:longitude", + "match_mapping_type": "*", + "mapping": { + "type": "float" + } + } + }, + { + "timestamps": { + "match": "*:ts", + "match_mapping_type": "*", + "mapping": { + "type": "date", + "format": "epoch_millis" + } + } + }, + { + "threat_triage_score": { + "mapping": { + "type": "float" + }, + "match": "threat:triage:*score", + "match_mapping_type": "*" + } + }, + { + "threat_triage_reason": { + "mapping": { + "type": "text", + "fielddata": "true" + }, + "match": "threat:triage:rules:*:reason", + "match_mapping_type": "*" + } + }, + { + "threat_triage_name": { + "mapping": { + "type": "text", + "fielddata": "true" + }, + "match": "threat:triage:rules:*:name", + "match_mapping_type": "*" + } + } + ], "properties" : { - "adapter:stellaradapter:begin:ts" : { - "type" : "string" - }, - "adapter:stellaradapter:end:ts" : { - "type" : "string" - }, "blacklisted" : { "type" : "boolean" }, "compCS" : { - "type" : "string" + "type" : "keyword" }, "data" : { - "type" : "string" + "type" : "keyword" }, "dst_ip" : { - "type" : "string" + "type" : "keyword" }, "dst_port" : { "type" : "long" @@ -187,117 +297,87 @@ curl -XPUT 'http://$ES_HOST/cowrie*/_mapping/cowrie_doc' -d ' "type" : "double" }, "encCS" : { - "type" : "string" - }, - "enrichmentjoinbolt:joiner:ts" : { - "type" : "string" - }, - "enrichmentsplitterbolt:splitter:begin:ts" : { - "type" : "string" - }, - "enrichmentsplitterbolt:splitter:end:ts" : { - "type" : "string" + "type" : "keyword" }, "eventid" : { - "type" : "string" + "type" : "keyword" }, "guid" : { - "type" : "string" + "type" : "keyword" }, "input" : { - "type" : "string" + "type" : "keyword" }, "isError" : { "type" : "long" }, "is_alert" : { - "type" : "string" + "type" : "keyword" }, "kexAlgs" : { - "type" : "string" + "type" : "keyword" }, "keyAlgs" : { - "type" : "string" + "type" : "keyword" }, "macCS" : { - "type" : "string" + "type" : "keyword" }, "message" : { - "type" : "string" + "type" : "keyword" }, - "original_string" : { - "type" : "string" + "original_keyword" : { + "type" : "keyword" }, "password" : { - "type" : "string" + "type" : "keyword" }, "sensor" : { - "type" : "string" + "type" : "keyword" }, "session" : { - "type" : "string" + "type" : "keyword" }, "similarity_bin" : { - "type" : "string" + "type" : "keyword" }, "size" : { "type" : "long" }, "source:type" : { - "type" : "string" + "type" : "keyword" }, "src_ip" : { - "type" : "string" + "type" : "keyword" }, "src_port" : { "type" : "long" }, "system" : { - "type" : "string" - }, - "threat:triage:rules:0:comment" : { - "type" : "string" - }, - "threat:triage:rules:0:name" : { - "type" : "string" + "type" : "keyword" }, - "threat:triage:rules:0:reason" : { - "type" : "string" - }, - "threat:triage:rules:0:score" : { - "type" : "long" - }, - "threat:triage:score" : { - "type" : "double" - }, - "threatinteljoinbolt:joiner:ts" : { - "type" : "string" - }, - "threatintelsplitterbolt:splitter:begin:ts" : { - "type" : "string" - }, - "threatintelsplitterbolt:splitter:end:ts" : { - "type" : "string" - }, - "timestamp" : { - "type" : "long" + "timestamp": { + "type": "date", + "format": "epoch_millis" }, "tlsh" : { - "type" : "string" + "type" : "keyword" }, "ttylog" : { - "type" : "string" + "type" : "keyword" }, "username" : { - "type" : "string" + "type" : "keyword" }, "version" : { - "type" : "string" + "type" : "keyword" }, - "alert" : { + "metron_alert" : { "type" : "nested" } } + } + } } ' ``` @@ -408,7 +488,7 @@ We want to pull a snapshot of the cowrie logs, so create `~/load_data.sh` with t COWRIE_HOME=~/cowrie for i in cowrie.1626302-1636522.json cowrie.16879981-16892488.json cowrie.21312194-21331475.json cowrie.698260-710913.json cowrie.762933-772239.json cowrie.929866-939552.json cowrie.1246880-1248235.json cowrie.19285959-19295444.json cowrie.16542668-16581213.json cowrie.5849832-5871517.json cowrie.6607473-6609163.json;do echo $i - cat $COWRIE_HOME/$i | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic cowrie + cat $COWRIE_HOME/$i | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic cowrie sleep 2 done ``` @@ -448,3 +528,11 @@ As you can see, we have found a few more malicious actors: * 94.78.80.45 Now we can look at *other* things that they're doing to build and refine our definition of what an alert is without resorting to hard-coding of rules. Note that nothing in our enrichments actually used the string `busybox`, so this is a more general purpose way of navigating similar things. + +### Version Info + +Verified against: + +- METRON_VERSION=0.5.0 +- ELASTIC_VERSION=5.6.2 + http://git-wip-us.apache.org/repos/asf/metron/blob/0c20fd1a/use-cases/typosquat_detection/README.md ---------------------------------------------------------------------- diff --git a/use-cases/typosquat_detection/README.md b/use-cases/typosquat_detection/README.md index 63624c7..12a770d 100644 --- a/use-cases/typosquat_detection/README.md +++ b/use-cases/typosquat_detection/README.md @@ -417,7 +417,7 @@ curl -XPOST "http://$ES_HOST/_template/squid_index"; -d '{ "url" : { "type" : "text","fielddata" : true }, - "alert" : { + "metron_alert" : { "type" : "nested" } } @@ -448,3 +448,10 @@ From there you should see the following data from squid with one as an alert and Now, if you drill down into the alert, you can see our fields and the reasons specified  + +### Version Info + +Verified against: + +- METRON_VERSION=0.5.0 +- ELASTIC_VERSION=5.6.2
