METRON-907: Zeppelin Dashboard to execute and download pcap queries closes apache/incubator-metron#559
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1d27a32d Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1d27a32d Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1d27a32d Branch: refs/heads/Metron_0.4.0 Commit: 1d27a32d5fe20ecb57a0cc143c1cb707f79ad20c Parents: 494643c Author: cstella <[email protected]> Authored: Wed May 3 10:14:41 2017 -0400 Committer: cstella <[email protected]> Committed: Wed May 3 10:14:41 2017 -0400 ---------------------------------------------------------------------- .../docker/rpm-docker/SPECS/metron.spec | 2 + .../config/zeppelin/metron/metron-pcap.json | 1 + .../org/apache/metron/pcap/query/CliConfig.java | 19 ++++- .../org/apache/metron/pcap/query/CliParser.java | 4 +- .../metron/pcap/query/FixedCliConfig.java | 3 +- .../metron/pcap/query/FixedCliParser.java | 9 +- .../org/apache/metron/pcap/query/PcapCli.java | 23 +++-- .../metron/pcap/query/QueryCliConfig.java | 4 + .../metron/pcap/query/QueryCliParser.java | 9 +- .../src/main/scripts/pcap_zeppelin_run.sh | 88 ++++++++++++++++++++ .../apache/metron/pcap/query/PcapCliTest.java | 27 +++--- 11 files changed, 158 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index b1ba637..c435c6c 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -288,7 +288,9 @@ This package installs the Metron PCAP files %{metron_home} %{metron_home}/bin/pcap_inspector.sh %{metron_home}/bin/pcap_query.sh %{metron_home}/bin/start_pcap_topology.sh +%{metron_home}/bin/pcap_zeppelin_run.sh %{metron_home}/flux/pcap/remote.yaml +%{metron_home}/config/zeppelin/metron/metron-pcap.json %attr(0644,root,root) %{metron_home}/lib/metron-pcap-backend-%{full_version}.jar # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json b/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json new file mode 100644 index 0000000..8c853a5 --- /dev/null +++ b/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json @@ -0,0 +1 @@ +{"paragraphs":[{"text":"%md\n# Execute Packet Capture Queries\n\nSpecify the following to filter the packet capture query:\n* *end time* - The ending time of the query in yyyyMMdd format (e.g. 20170428)\n* *start time* - The starting time of the query in yyyyMMdd format (e.g. 20170428)\n* *query* - The [Stellar](https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-language) query (i.e. a Stellar expression that returns `true` or `false`) to specify the packets.\n\nThe available fields to use in the queries are as follows:\n* `ip_src_addr` - The source IP address of the packets filtered\n* `ip_src_port` - The source port of the packets filtered\n* `ip_dst_addr` - The destination IP address of the packets filtered\n* `ip_dst_port` - The destination port of the packets filtered\n* `packet` - The raw packet (for use with the `BYTEARRAY_MATCHER` function)\n\nYou can use any [Stellar function](https://github.com/apache/incubator-metron/tree/maste r/metron-platform/metron-common#stellar-core-functions)\n\n## Simple Boolean Expressions\n\nFor example:\n* `ip_dst_port == 8080` would return all packets where the destination port is `8080`\n* `ip_dst_port in [ 8080, 80 ]` would return all packets where the destination port is either `8080` or `80`\n\n## Common Network Functions on Metadata\n\nFor example:\n* `IN_SUBNET(ip_dst_addr, '192.168.0.0/24')` would return all packets whose destination conforms to the CIDR `192.168.0.0/24`\n* `IN_SUBNET(ip_dst_addr, '192.168.0.0/24') && ip_dst_port == 8080` would return all packets matching the CIDR and whose destination port is `8080`\n \n## Filtering based on the Packet Contents\n\nWe use byteseek regular expressions to filter packets. The syntax for these is described [here](https://github.com/nishihatapalmer/byteseek/blob/master/sequencesyntax.md).\n* `BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)` would return all packets that contain the string `/api/v1` in them anywhere.\n* `ip_ds t_port==8080 && BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)` would return all packets that contain the string `/api/v1` and have a destination port of `8080`\n* `BYTEARRAY_MATCHER('ff(.){5}ff', packet)` would return all packets containing a binary regex with `0xff` followed by any 5 bytes and then `0xff`\n\n# The Output\nThe output will be a table of links to the various parts of the packet capture files. The files will be named in temporal order.","dateUpdated":"2017-05-01T17:36:27+0000","config":{"colWidth":12,"editorMode":"ace/mode/markdown","editorHide":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{"query":"ip_dst_port==8080 && BYTEARRAY_MATCHER('\\\\\\'/api/v1/\\\\\\'', packet)"},"forms":{}},"jobName":"paragraph_1493658709556_1010155640","id":"20170428-183346_777875025","result":{"code":"SUCCESS","type":"HTML","msg":"<h1>Execute Packet Capture Queries</h1>\n<p>Specify the fo llowing to filter the packet capture query:</p>\n<ul>\n<li><em>end time</em> - The ending time of the query in yyyyMMdd format (e.g. 20170428)</li>\n<li><em>start time</em> - The starting time of the query in yyyyMMdd format (e.g. 20170428)</li>\n<li><em>query</em> - The <a href=\"https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-language\";>Stellar</a> query (i.e. a Stellar expression that returns <code>true</code> or <code>false</code>) to specify the packets.</li>\n</ul>\n<p>The available fields to use in the queries are as follows:</p>\n<ul>\n<li><code>ip_src_addr</code> - The source IP address of the packets filtered</li>\n<li><code>ip_src_port</code> - The source port of the packets filtered</li>\n<li><code>ip_dst_addr</code> - The destination IP address of the packets filtered</li>\n<li><code>ip_dst_port</code> - The destination port of the packets filtered</li>\n<li><code>packet</code> - The raw packet (for use with the <code>BYTEARR AY_MATCHER</code> function)</li>\n</ul>\n<p>You can use any <a href=\"https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-core-functions\";>Stellar function</a></p>\n<h2>Simple Boolean Expressions</h2>\n<p>For example:</p>\n<ul>\n<li><code>ip_dst_port == 8080</code> would return all packets where the destination port is <code>8080</code></li>\n<li><code>ip_dst_port in [ 8080, 80 ]</code> would return all packets where the destination port is either <code>8080</code> or <code>80</code></li>\n</ul>\n<h2>Common Network Functions on Metadata</h2>\n<p>For example:</p>\n<ul>\n<li><code>IN_SUBNET(ip_dst_addr, '192.168.0.0/24')</code> would return all packets whose destination conforms to the CIDR <code>192.168.0.0/24</code></li>\n<li><code>IN_SUBNET(ip_dst_addr, '192.168.0.0/24') && ip_dst_port == 8080</code> would return all packets matching the CIDR and whose destination port is <code>8080</code></li>\n</ul>\n<h2>Filtering based on the Pack et Contents</h2>\n<p>We use byteseek regular expressions to filter packets. The syntax for these is described <a href=\"https://github.com/nishihatapalmer/byteseek/blob/master/sequencesyntax.md\";>here</a>.</p>\n<ul>\n<li><code>BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)</code> would return all packets that contain the string <code>/api/v1</code> in them anywhere.</li>\n<li><code>ip_dst_port==8080 && BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)</code> would return all packets that contain the string <code>/api/v1</code> and have a destination port of <code>8080</code></li>\n<li><code>BYTEARRAY_MATCHER('ff(.){5}ff', packet)</code> would return all packets containing a binary regex with <code>0xff</code> followed by any 5 bytes and then <code>0xff</code></li>\n</ul>\n<h1>The Output</h1>\n<p>The output will be a table of links to the various parts of the packet capture files. The files will be named in temporal order.</p>\n"},"dateCreated":"2017-05-01T17:11:49+0000","stat us":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1724","dateFinished":"2017-05-01T17:36:26+0000","dateStarted":"2017-05-01T17:36:26+0000","focus":true},{"text":"%sh\nexport PCAP_ZEPPELIN_RUN=$(find /usr -name pcap_zeppelin_run.sh)\nexport RECORDS_PER_FILE=10000\nexport NUMBER_OF_REDUCERS=10\nexport DATE_FORMAT=\"yyyyMMdd\"\nexport PCAP_DATA_PATH=\"/apps/metron/pcap\"\n\n$PCAP_ZEPPELIN_RUN \"${query}\" \"${start time}\" \"${optional end time}\"","dateUpdated":"2017-05-01T17:35:04+0000","config":{"colWidth":12,"editorMode":"ace/mode/sh","editorHide":false,"graph":{"mode":"table","height":164,"optionOpen":false,"keys":[{"name":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now","index":0,"aggr":"sum"}],"values":[],"groups":[],"scatter":{"xAxis":{"name":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now","index":0,"aggr":"sum"}}},"e nabled":true},"settings":{"params":{"start time":"20170428","end time":"","query":"BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet) && ip_dst_port == 8080","optional end time":""},"forms":{"optional end time":{"name":"optional end time","defaultValue":"","hidden":false},"query":{"name":"query","defaultValue":"","hidden":false},"start time":{"name":"start time","defaultValue":"","hidden":false}}},"jobName":"paragraph_1493658709562_1009386142","id":"20170428-181957_829993114","result":{"code":"SUCCESS","type":"TABLE","msg":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now\n%html <a href=\"http://node1:50070/webhdfs/v1/user/zeppelin/queries/pcap-data-20170501173442578/pcap-data-20170501173442578%2B0000.pcap?op=OPEN\";>pcap-data-20170501173442578+0000.pcap</a>\n","comment":"","msgTable":[[{"value":"%html <a href=\"http://node1:50070/webhdfs/v1/user/zeppelin/queries/pcap-data-20170501173442578/pcap-data-20170501173442578 %2B0000.pcap?op=OPEN\">pcap-data-20170501173442578+0000.pcap</a>"}]],"columnNames":[{"name":"Packets conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting at 20170428 ending now","index":0,"aggr":"sum"}],"rows":[["%html <a href=\"http://node1:50070/webhdfs/v1/user/zeppelin/queries/pcap-data-20170501173442578/pcap-data-20170501173442578%2B0000.pcap?op=OPEN\";>pcap-data-20170501173442578+0000.pcap</a>"]]},"dateCreated":"2017-05-01T17:11:49+0000","dateStarted":"2017-05-01T17:35:04+0000","dateFinished":"2017-05-01T17:34:55+0000","status":"RUNNING","progressUpdateIntervalMs":500,"$$hashKey":"object:1725","focus":true},{"dateUpdated":"2017-05-01T17:34:55+0000","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/markdown","editorHide":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1493658709563_1009001393","id":"20170428-184335_ 1604096389","dateCreated":"2017-05-01T17:11:49+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1726","dateFinished":"2017-05-01T17:34:54+0000","dateStarted":"2017-05-01T17:34:54+0000","result":{"code":"SUCCESS","type":"HTML","msg":"<h1>Troubleshooting</h1>\n<p>If you are having problems with the above form, the following may help.</p>\n<h2>I see <code>Terminated by SIGINTERRUPT</code> or something similar in the output!</h2>\n<p>PCAP filtering happens via a batch process and on busy systems, this can take some time. You very well may need to request this of your system administrator.\n<br />They can do this by changing the <code>shell.command.timeout.millisecs</code> property for the <code>sh</code> interpreter to a larger value, likely <code>100000</code>.</p>\n<h2>I do not see a table of URLs to pcap files in my output, what happened?</h2>\n<p>If an error happens, the log of the pcap querying utility will be displayed instead of an output. Please co ntact an administrator with this output to debug further.</p>\n"},"text":"%md\n# Troubleshooting\n\nIf you are having problems with the above form, the following may help.\n\n## I see `Terminated by SIGINTERRUPT` or something similar in the output!\n\nPCAP filtering happens via a batch process and on busy systems, this can take some time. You very well may need to request this of your system administrator.\nThey can do this by changing the `shell.command.timeout.millisecs` property for the `sh` interpreter to a larger value, likely `100000`.\n\n## I do not see a table of URLs to pcap files in my output, what happened?\n\nIf an error happens, the log of the pcap querying utility will be displayed instead of an output. Please contact an administrator with this output to debug further.","focus":true},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobN ame":"paragraph_1493659778398_1082698332","id":"20170501-172938_213921861","dateCreated":"2017-05-01T17:29:38+0000","status":"READY","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:1972"}],"name":"metron/pcap","id":"2CEEXVR1W","angularObjects":{"2CFNGE6TP:shared_process":[],"2CGUX5TSW:shared_process":[],"2CETVR2AB:shared_process":[],"2CF163WFX:shared_process":[],"2CG4YXKUV:shared_process":[],"2CG6QDFF7:shared_process":[]},"config":{"looknfeel":"default"},"info":{}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java index 294844f..1d8e3f3 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java @@ -18,12 +18,19 @@ package org.apache.metron.pcap.query; import org.apache.commons.lang3.StringUtils; +import org.apache.metron.common.system.Clock; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.Function; public class CliConfig { + public interface PrefixStrategy extends Function<Clock, String>{} + private boolean showHelp; + private String prefix; private String basePath; private String baseOutputPath; private long startTime; @@ -32,13 +39,23 @@ public class CliConfig { private int numRecordsPerFile; private DateFormat dateFormat; - public CliConfig() { + + public CliConfig(PrefixStrategy prefixStrategy) { showHelp = false; basePath = ""; baseOutputPath = ""; startTime = -1L; endTime = -1L; numReducers = 0; + prefix = prefixStrategy.apply(new Clock()); + } + + public String getPrefix() { + return prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; } public int getNumReducers() { http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java index 83e9fcf..d5976ae 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java @@ -29,8 +29,10 @@ public class CliParser { public static final int NUM_REDUCERS_DEFAULT = 10; public static final int NUM_RECORDS_PER_FILE_DEFAULT = 10000; private CommandLineParser parser; + protected CliConfig.PrefixStrategy prefixStrategy; - public CliParser() { + public CliParser(CliConfig.PrefixStrategy prefixStrategy) { + this.prefixStrategy = prefixStrategy; parser = new PosixParser(); } http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java index df653e1..03caed7 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java @@ -27,7 +27,8 @@ public class FixedCliConfig extends CliConfig { private Map<String, String> fixedFields; - public FixedCliConfig() { + public FixedCliConfig(PrefixStrategy prefixStrategy) { + super(prefixStrategy); this.fixedFields = new LinkedHashMap<>(); } http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java index fda8692..4e1bfcf 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java @@ -26,7 +26,8 @@ import org.apache.metron.pcap.PcapHelper; public class FixedCliParser extends CliParser { private Options fixedOptions; - public FixedCliParser() { + public FixedCliParser(CliConfig.PrefixStrategy prefixStrategy) { + super(prefixStrategy); fixedOptions = buildFixedOptions(); } @@ -38,6 +39,7 @@ public class FixedCliParser extends CliParser { options.addOption(newOption("dp", "ip_dst_port", true, "Destination port")); options.addOption(newOption("p", "protocol", true, "IP Protocol")); options.addOption(newOption("pf", "packet_filter", true, "Packet Filter regex")); + options.addOption(newOption("pre", "prefix", true, "Result file prefix to use")); options.addOption(newOption("ir", "include_reverse", false, "Indicates if filter should check swapped src/dest addresses and IPs")); return options; } @@ -51,7 +53,7 @@ public class FixedCliParser extends CliParser { */ public FixedCliConfig parse(String[] args) throws ParseException, java.text.ParseException { CommandLine commandLine = getParser().parse(fixedOptions, args); - FixedCliConfig config = new FixedCliConfig(); + FixedCliConfig config = new FixedCliConfig(prefixStrategy); super.parse(commandLine, config); config.putFixedField(Constants.Fields.SRC_ADDR.getName(), commandLine.getOptionValue("ip_src_addr")); config.putFixedField(Constants.Fields.DST_ADDR.getName(), commandLine.getOptionValue("ip_dst_addr")); @@ -60,6 +62,9 @@ public class FixedCliParser extends CliParser { config.putFixedField(Constants.Fields.PROTOCOL.getName(), commandLine.getOptionValue("protocol")); config.putFixedField(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), Boolean.toString(commandLine.hasOption("include_reverse"))); config.putFixedField(PcapHelper.PacketFields.PACKET_FILTER.getName(), commandLine.getOptionValue("packet_filter")); + if(commandLine.hasOption("prefix")) { + config.setPrefix(commandLine.getOptionValue("prefix")); + } return config; } http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java index d2e6807..b02e2e2 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java @@ -36,22 +36,28 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.UUID; public class PcapCli { private static final Logger LOGGER = LoggerFactory.getLogger(PcapCli.class); + public static final CliConfig.PrefixStrategy PREFIX_STRATEGY = clock -> { + String timestamp = new Clock().currentTimeFormatted("yyyyMMddHHmm"); + String uuid = UUID.randomUUID().toString().replaceAll("-", ""); + return String.format("%s-%s", timestamp, uuid); + }; private final PcapJob jobRunner; private final ResultsWriter resultsWriter; - private final Clock clock; + private final CliConfig.PrefixStrategy prefixStrategy; public static void main(String[] args) { - int status = new PcapCli(new PcapJob(), new ResultsWriter(), new Clock()).run(args); + int status = new PcapCli(new PcapJob(), new ResultsWriter(), PREFIX_STRATEGY).run(args); System.exit(status); } - public PcapCli(PcapJob jobRunner, ResultsWriter resultsWriter, Clock clock) { + public PcapCli(PcapJob jobRunner, ResultsWriter resultsWriter, CliConfig.PrefixStrategy prefixStrategy) { this.jobRunner = jobRunner; this.resultsWriter = resultsWriter; - this.clock = clock; + this.prefixStrategy = prefixStrategy; } public int run(String[] args) { @@ -72,7 +78,7 @@ public class PcapCli { } CliConfig commonConfig = null; if ("fixed".equals(jobType)) { - FixedCliParser fixedParser = new FixedCliParser(); + FixedCliParser fixedParser = new FixedCliParser(prefixStrategy); FixedCliConfig config = null; try { config = fixedParser.parse(otherArgs); @@ -110,7 +116,7 @@ public class PcapCli { return -1; } } else if ("query".equals(jobType)) { - QueryCliParser queryParser = new QueryCliParser(); + QueryCliParser queryParser = new QueryCliParser(prefixStrategy); QueryCliConfig config = null; try { config = queryParser.parse(otherArgs); @@ -151,11 +157,12 @@ public class PcapCli { return -1; } try { + Iterable<List<byte[]>> partitions = Iterables.partition(results, commonConfig.getNumRecordsPerFile()); + int part = 1; if (partitions.iterator().hasNext()) { for (List<byte[]> data : partitions) { - String timestamp = clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ"); - String outFileName = String.format("pcap-data-%s.pcap", timestamp); + String outFileName = String.format("pcap-data-%s+%04d.pcap", commonConfig.getPrefix(), part++); if(data.size() > 0) { resultsWriter.write(data, outFileName); } http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java index 3d06e1d..67f045f 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java @@ -20,6 +20,10 @@ package org.apache.metron.pcap.query; public class QueryCliConfig extends CliConfig { private String query; + public QueryCliConfig(PrefixStrategy prefixStrategy) { + super(prefixStrategy); + } + public String getQuery() { return query; } http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java index 72b5a95..d6e5cd1 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java @@ -24,13 +24,15 @@ import org.apache.commons.cli.ParseException; public class QueryCliParser extends CliParser { private Options queryOptions; - public QueryCliParser() { + public QueryCliParser(CliConfig.PrefixStrategy prefixStrategy) { + super(prefixStrategy); queryOptions = setupOptions(); } private Options setupOptions() { Options options = buildOptions(); options.addOption(newOption("q", "query", true, "Query string to use as a filter")); + options.addOption(newOption("pre", "prefix", true, "Result file prefix to use")); return options; } @@ -43,11 +45,14 @@ public class QueryCliParser extends CliParser { */ public QueryCliConfig parse(String[] args) throws ParseException, java.text.ParseException { CommandLine commandLine = getParser().parse(queryOptions, args); - QueryCliConfig config = new QueryCliConfig(); + QueryCliConfig config = new QueryCliConfig(prefixStrategy); super.parse(commandLine, config); if (commandLine.hasOption("query")) { config.setQuery(commandLine.getOptionValue("query")); } + if(commandLine.hasOption("prefix")) { + config.setPrefix(commandLine.getOptionValue("prefix")); + } return config; } http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh new file mode 100755 index 0000000..3561c9d --- /dev/null +++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +METRON_VERSION=0.4.0 +METRON_HOME=${METRON_HOME:-"/usr/metron/$METRON_VERSION"} +DATE_FORMAT=${DATE_FORMAT:-"yyyyMMdd"} +USER=$(whoami) +USER_HOMEDIR=${USER_HOMEDIR:-`hdfs getconf -confKey dfs.user.home.dir.prefix`/$USER} +QUERY_HOME=${QUERY_HOME:-"$USER_HOMEDIR/queries"} +WEBHDFS_HOSTNAME=${WEBHDFS_HOSTNAME:-`hdfs getconf -confKey dfs.namenode.http-address`} +QUERY=${QUERY:-$1} +START_TIME=${START_TIME:-$2} +END_TIME=${END_TIME:-$3} +RECORDS_PER_FILE=${RECORDS_PER_FILE:-10000} +NUMBER_OF_REDUCERS=${NUMBER_OF_REDUCERS:-10} +PCAP_DATA_PATH=${PCAP_DATA_PATH:-/apps/metron/pcap} +if [ -z "$QUERY" ]; then + echo "You must specify a query." + exit 1 +fi +if [ -z "$START_TIME" ]; then + echo "You must specify a start time." + exit 2 +fi +TIMESTAMP_EPOCH=$(date +%s%N | cut -b1-13) +if [ -f /proc/sys/kernel/random/uuid ];then + UUID=$(cat /proc/sys/kernel/random/uuid | sed 's/-//g') + PREFIX="$TIMESTAMP_EPOCH-$UUID" +else + QUERY_HASH=$(echo $QUERY | md5sum | awk '{print $1}') + PREFIX="$TIMESTAMP_EPOCH-$QUERY_HASH" +fi +if [ -z "$END_TIME" ]; then + CMD=$($METRON_HOME/bin/pcap_query.sh query --prefix "$PREFIX" --query "$QUERY" -st "$START_TIME" -df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp "$PCAP_DATA_PATH" \"2>&1) + SUMMARY="Packets conforming to $QUERY starting at $START_TIME ending now" +else + CMD=$($METRON_HOME/bin/pcap_query.sh query --prefix "$PREFIX" --query "$QUERY" -st "$START_TIME" -et "$END_TIME" -df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp "$PCAP_DATA_PATH" 2>&1) + SUMMARY="Packets conforming to $QUERY starting at $START_TIME ending at $END_TIME" +fi + +FAILED=$(echo $CMD | grep "Unable to complete query due to errors") +if [ -z "$FAILED" ];then + PATTERN="pcap-data-$PREFIX" + hadoop fs -mkdir -p $QUERY_HOME/$PATTERN && hadoop fs -put $PATTERN* $QUERY_HOME/$PATTERN + FAILED=$? + if [ $FAILED -eq 0 ];then + echo "%html" + echo "<h4>$SUMMARY</h4>" + echo "<ul>" + for i in $(ls $PATTERN*.pcap | sort -n);do + FILENAME=$(echo $i | sed 's/+/%2B/g') + SIZE=$(du -h $i | awk '{print $1}') + echo "<li><a href=\"http://$WEBHDFS_HOSTNAME/webhdfs/v1$QUERY_HOME/$PATTERN/$FILENAME?op=OPEN\";>$i</a> ($SIZE)</li>" + rm $i + done + echo "</ul>" + echo "<small>NOTE: There are $RECORDS_PER_FILE records per file</small>" + else + echo "Unable to create $QUERY_HOME/$PATTERN" + exit 3 + fi +else + echo "%html <pre>FAILED JOB:" + echo "QUERY: $QUERY" + echo "START_TIME: $START_TIME" + echo "DATE_FORMAT: $DATE_FORMAT" + echo "METRON_HOME: $METRON_HOME" + echo "DATE_FORMAT: $DATE_FORMAT" + echo "Output:" + echo "$CMD" + echo "</pre>" +fi http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java index 4f441f1..7202819 100644 --- a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java +++ b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java @@ -92,11 +92,10 @@ public class PcapCliTest { }}; when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(FixedPcapFilter.Configurator.class))).thenReturn(iterable); - when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000"); - PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock); + PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix"); assertThat("Expect no errors on run", cli.run(args), equalTo(0)); - Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap"); + Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap"); } @Test @@ -133,11 +132,10 @@ public class PcapCliTest { }}; when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(FixedPcapFilter.Configurator.class))).thenReturn(iterable); - when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000"); - PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock); + PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix"); assertThat("Expect no errors on run", cli.run(args), equalTo(0)); - Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap"); + Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap"); } @Test @@ -177,11 +175,10 @@ public class PcapCliTest { long startAsNanos = asNanos("2016-06-13-18:35.00", "yyyy-MM-dd-HH:mm.ss"); long endAsNanos = asNanos("2016-06-15-18:35.00", "yyyy-MM-dd-HH:mm.ss"); when(jobRunner.query(eq(base_path), eq(base_output_path), eq(startAsNanos), eq(endAsNanos), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(FixedPcapFilter.Configurator.class))).thenReturn(iterable); - when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000"); - PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock); + PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix"); assertThat("Expect no errors on run", cli.run(args), equalTo(0)); - Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap"); + Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap"); } private long asNanos(String inDate, String format) throws ParseException { @@ -211,11 +208,10 @@ public class PcapCliTest { String query = "some query string"; when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(QueryPcapFilter.Configurator.class))).thenReturn(iterable); - when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000"); - PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock); + PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix"); assertThat("Expect no errors on run", cli.run(args), equalTo(0)); - Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap"); + Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap"); } @Test @@ -240,11 +236,10 @@ public class PcapCliTest { String query = "some query string"; when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(QueryPcapFilter.Configurator.class))).thenReturn(iterable); - when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000"); - PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock); + PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix"); assertThat("Expect no errors on run", cli.run(args), equalTo(0)); - Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap"); + Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap"); } // INVALID OPTION CHECKS @@ -281,7 +276,7 @@ public class PcapCliTest { PrintStream errOutStream = new PrintStream(new BufferedOutputStream(ebos)); System.setErr(errOutStream); - PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock); + PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix"); assertThat("Expect errors on run", cli.run(args), equalTo(-1)); assertThat("Expect missing required option error: " + ebos.toString(), ebos.toString().contains(optMsg), equalTo(true)); assertThat("Expect usage to be printed: " + bos.toString(), bos.toString().contains("usage: " + type + " filter options"), equalTo(true));
