METRON-943 Create traffic connections report in zeppelin (justinleet) closes apache/incubator-metron#573
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b3e7222f Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b3e7222f Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b3e7222f Branch: refs/heads/Metron_0.4.0 Commit: b3e7222f1c4fb527ff3b18e8bb47383528d2eb57 Parents: e375936 Author: justinleet <[email protected]> Authored: Fri May 12 08:49:28 2017 -0400 Committer: leet <[email protected]> Committed: Fri May 12 08:49:28 2017 -0400 ---------------------------------------------------------------------- metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec | 3 +++ .../config/zeppelin/metron/metron-connection-volume-report.json | 1 + 2 files changed, 4 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/b3e7222f/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index e851c7f..b49e0e6 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -271,6 +271,7 @@ This package installs the Metron Indexing files %{metron_home}/config/zeppelin/metron/metron-yaf-telemetry.json %{metron_home}/config/zeppelin/metron/metron-connection-report.json %{metron_home}/config/zeppelin/metron/metron-ip-report.json +%{metron_home}/config/zeppelin/metron/metron-connection-volume-report.json # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -400,6 +401,8 @@ This package installs the Metron Management UI %{metron_home} # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ %changelog +* Tue May 9 2017 Apache Metron <[email protected]> - 0.4.0 +- Add Zeppelin Connection Volume Report Dashboard * Thu May 4 2017 Ryan Merriman <[email protected]> - 0.4.0 - Added REST * Tue May 2 2017 David Lyle <[email protected]> - 0.4.0 http://git-wip-us.apache.org/repos/asf/metron/blob/b3e7222f/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json b/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json new file mode 100644 index 0000000..22765d4 --- /dev/null +++ b/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json @@ -0,0 +1 @@ +{"paragraphs":[{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1494339150477_-816854736","id":"20170509-141230_1340330181","dateCreated":"2017-05-09T14:12:30+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:7251","text":"%spark.sql\n\n#\n# load the Yaf telemetry that has been archived by Metron\n#\ncreate temporary table yaf\n using org.apache.spark.sql.json\n options (path \"hdfs:///apps/metron/indexing/indexed/yaf\")","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:11+0000","dateStarted":"2017-05-09T14:20:10+0000","result":{"code":"SUCCESS","type":"TEXT","msg":""}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_14943391 84335_-2067041830","id":"20170509-141304_1479312597","dateCreated":"2017-05-09T14:13:04+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7322","text":"%md\n\n### Connection Volume (Source) - Yaf\n\nThe volume of connections made from source IPs\n\nThis IPs retrieved are given by an IPv4 CIDR block.","dateUpdated":"2017-05-09T14:22:02+0000","dateFinished":"2017-05-09T14:22:02+0000","dateStarted":"2017-05-09T14:22:02+0000","result":{"code":"SUCCESS","type":"HTML","msg":"<h3>Connection Volume (Source) - Yaf</h3>\n<p>The volume of connections made from source IPs</p>\n<p>This IPs retrieved are given by an IPv4 CIDR block.</p>\n"}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{"CIDR":"192.0.0.0/8"},"forms":{"CIDR":{"name":"CIDR","displayName":"CIDR","type":"input","defaultValue":"","hidden":false}}},"jobName":"paragraph_14943391 91894_-766575224","id":"20170509-141311_2132481247","dateCreated":"2017-05-09T14:13:11+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7398","text":"%spark\nimport org.apache.spark.sql.Row\nimport scala.concurrent.duration._\nimport java.util.concurrent.TimeUnit\nimport org.apache.commons.net.util.SubnetUtils\n\nval cidr = z.input(\"CIDR\").toString\n@transient val utils = new SubnetUtils(cidr)\nutils.setInclusiveHostCount(true)\n@transient val info = utils.getInfo\n\nval ipToLong = (ipAddress: String) => {\n val ipAddressInArray = ipAddress.split(\"\\\\.\")\n var result = 0L\n for (i <- 0 to ipAddressInArray.length-1) {\n\t val power = 3 - i\n\t val ip = Integer.parseInt(ipAddressInArray(i))\n\t result += ip * Math.pow(256.0, power.toDouble).toLong\n }\n result\n}\n\nval bcLow = sc.broadcast(ipToLong(info.getLowAddress))\nval bcHigh = sc.broadcast(ipToLong(info.getHighAddress))\nval bcIpToLong = sc.broadcast(ipToLong )\n\nval results = sqlContext.sql(\ns\"\"\"SELECT\n ip_src_addr,\n COUNT(*) AS count\nFROM\n yaf\nGROUP BY ip_src_addr\nORDER BY ip_src_addr\n\"\"\").flatMap {\n case Row(ip_src_addr: String, count: Long) => {\n val longSrc = bcIpToLong.value(ip_src_addr)\n if(bcLow.value <= longSrc && longSrc <= bcHigh.value) {\n\t\t List(ip_src_addr + \"\\t\" + count)\n } else {\n List.empty[String]\n }\n }\n }.collect()\n\nprint(\"%table ip_src_addr\\tcount\\n\" + results.mkString(\"\\n\"))","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:21+0000","dateStarted":"2017-05-09T14:20:10+0000","result":{"code":"SUCCESS","type":"TABLE","msg":"ip_src_addr\tcount\n192.168.138.158\t111\n192.168.138.2\t1\n192.168.66.1\t89\n192.168.66.121\t69","comment":"","msgTable":[[{"key":"count","value":"192.168.138.158"},{"key":"count","value":"111"}],[{"value":"192.168.138.2"},{"value":"1"}],[{"value":"192.168.66.1"},{"value":"89"}],[{"val ue":"192.168.66.121"},{"value":"69"}]],"columnNames":[{"name":"ip_src_addr","index":0,"aggr":"sum"},{"name":"count","index":1,"aggr":"sum"}],"rows":[["192.168.138.158","111"],["192.168.138.2","1"],["192.168.66.1","89"],["192.168.66.121","69"]]}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1494339578537_-194906756","id":"20170509-141938_486503393","dateCreated":"2017-05-09T14:19:38+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7794","text":"%md\n\n### Connection Volume (Destination) - Yaf\n\nThe volume of connections made to destination IPs.\n\nThis IPs retrieved are given by an IPv4 CIDR block.","dateUpdated":"2017-05-09T14:22:06+0000","dateFinished":"2017-05-09T14:22:06+0000","dateStarted":"2017-05-09T14:22:06+0000","result":{"code":"SUCCESS","type":"HTM L","msg":"<h3>Connection Volume (Destination) - Yaf</h3>\n<p>The volume of connections made to destination IPs.</p>\n<p>This IPs retrieved are given by an IPv4 CIDR block.</p>\n"}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{"CIDR":"192.0.0.0/8"},"forms":{"CIDR":{"name":"CIDR","displayName":"CIDR","type":"input","defaultValue":"","hidden":false}}},"jobName":"paragraph_1494339202329_1284921236","id":"20170509-141322_1098639923","dateCreated":"2017-05-09T14:13:22+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7474","text":"%spark\nimport org.apache.spark.sql.Row\nimport scala.concurrent.duration._\nimport java.util.concurrent.TimeUnit\nimport org.apache.commons.net.util.SubnetUtils\n\nval cidr = z.input(\"CIDR\").toString\n@transient val utils = new SubnetUtils(cidr)\nutils.setInclusiveHostCo unt(true)\n@transient val info = utils.getInfo\n\nval ipToLong = (ipAddress: String) => {\n val ipAddressInArray = ipAddress.split(\"\\\\.\")\n var result = 0L\n for (i <- 0 to ipAddressInArray.length-1) {\n\t val power = 3 - i\n\t val ip = Integer.parseInt(ipAddressInArray(i))\n\t result += ip * Math.pow(256.0, power.toDouble).toLong\n }\n result\n}\n\nval bcLow = sc.broadcast(ipToLong(info.getLowAddress))\nval bcHigh = sc.broadcast(ipToLong(info.getHighAddress))\nval bcIpToLong = sc.broadcast(ipToLong)\n\nval results = sqlContext.sql(\ns\"\"\"SELECT\n ip_dst_addr,\n COUNT(*) AS count\nFROM\n yaf\nGROUP BY ip_dst_addr\nORDER BY ip_dst_addr\n\"\"\").flatMap {\n case Row(ip_dst_addr: String, count: Long) => {\n val longDst = bcIpToLong.value(ip_dst_addr)\n if(bcLow.value <= longDst && longDst <= bcHigh.value) {\n\t\t List(ip_dst_addr + \"\\t\" + count)\n } else {\n List.empty[String]\n }\n }\n }.collect()\n \nprint(\"%table ip_dst_addr\\tcount\\n\" + results.mkString(\"\\n\"))\n","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:31+0000","dateStarted":"2017-05-09T14:20:11+0000","result":{"code":"SUCCESS","type":"TABLE","msg":"ip_dst_addr\tcount\n192.168.138.158\t151\n192.168.138.2\t2\n192.168.66.1\t69\n192.168.66.121\t86","comment":"","msgTable":[[{"key":"count","value":"192.168.138.158"},{"key":"count","value":"151"}],[{"value":"192.168.138.2"},{"value":"2"}],[{"value":"192.168.66.1"},{"value":"69"}],[{"value":"192.168.66.121"},{"value":"86"}]],"columnNames":[{"name":"ip_dst_addr","index":0,"aggr":"sum"},{"name":"count","index":1,"aggr":"sum"}],"rows":[["192.168.138.158","151"],["192.168.138.2","2"],["192.168.66.1","69"],["192.168.66.121","86"]]}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1494339303735_1 424887757","id":"20170509-141503_898772342","dateCreated":"2017-05-09T14:15:03+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7557","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:31+0000","dateStarted":"2017-05-09T14:20:21+0000","result":{"code":"SUCCESS","type":"TEXT","msg":""}}],"name":"Metron - Connection Volume Report","id":"2CER9F199","angularObjects":{"2CET9UGAN:shared_process":[],"2CHC4B4TT:shared_process":[],"2CF24S5PD:shared_process":[],"2CF6W9QPU:shared_process":[],"2CJJ4RS82:shared_process":[],"2CGYCAYBG:shared_process":[]},"config":{"looknfeel":"simple"},"info":{}} \ No newline at end of file
