METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes apache/metron#1138
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b29833cc Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b29833cc Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b29833cc Branch: refs/heads/master Commit: b29833cc5db4efca2a6e9025042baf705fb347c6 Parents: d8a5922 Author: merrimanr <merrim...@gmail.com> Authored: Fri Aug 10 08:25:13 2018 -0500 Committer: rmerriman <merrim...@gmail.com> Committed: Fri Aug 10 08:25:13 2018 -0500 ---------------------------------------------------------------------- metron-platform/metron-pcap-backend/README.md | 2 ++ .../java/org/apache/metron/pcap/query/CliParser.java | 4 ++++ .../java/org/apache/metron/pcap/query/PcapCliTest.java | 12 +++++++++--- .../java/org/apache/metron/pcap/config/PcapConfig.java | 9 +++++++++ .../java/org/apache/metron/pcap/config/PcapOptions.java | 3 ++- .../main/java/org/apache/metron/pcap/mr/PcapJob.java | 11 +++++++++++ 6 files changed, 37 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/README.md b/metron-platform/metron-pcap-backend/README.md index e7960e3..2ff20d8 100644 --- a/metron-platform/metron-pcap-backend/README.md +++ b/metron-platform/metron-pcap-backend/README.md @@ -132,6 +132,7 @@ usage: Fixed filter options -nr,--num_reducers <arg> The number of reducers to use. Default is 10. -h,--help Display help + -ps,--print_status Print the status of the job as it runs -ir,--include_reverse Indicates if filter should check swapped src/dest addresses and IPs -p,--protocol <arg> IP Protocol @@ -154,6 +155,7 @@ usage: Query filter options -nr,--num_reducers <arg> The number of reducers to use. Default is 10. -h,--help Display help + -ps,--print_status Print the status of the job as it runs -q,--query <arg> Query string to use as a filter -st,--start_time <arg> (required) Packet start time range. ``` http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java index 2d15e8b..69c725c 100644 --- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java +++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java @@ -55,6 +55,7 @@ public class CliParser { options.addOption(newOption("rpf", "records_per_file", true, String.format("Number of records to include in each output pcap file (defaults to %s)", NUM_RECORDS_PER_FILE_DEFAULT))); options.addOption(newOption("et", "end_time", true, "Packet end time range. Default is current system time.")); options.addOption(newOption("df", "date_format", true, "Date format to use for parsing start_time and end_time. Default is to use time in millis since the epoch.")); + options.addOption(newOption("ps", "print_status", false, "Print the status of the job as it runs")); return options; } @@ -125,6 +126,9 @@ public class CliParser { //no-op } } + if (commandLine.hasOption("print_status")) { + config.setPrintJobStatus(true); + } } public void printHelp(String msg, Options opts) { http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java index 96ca354..7c75224 100644 --- a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java +++ b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java @@ -147,7 +147,8 @@ public class PcapCliTest { "-protocol", "6", "-include_reverse", "-num_reducers", "10", - "-records_per_file", "1000" + "-records_per_file", "1000", + "-ps" }; Map<String, String> query = new HashMap<String, String>() {{ put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1"); @@ -165,6 +166,7 @@ public class PcapCliTest { PcapOptions.START_TIME_MS.put(config, 500L); PcapOptions.END_TIME_MS.put(config, 1000L); PcapOptions.NUM_RECORDS_PER_FILE.put(config, 1000); + PcapOptions.PRINT_JOB_STATUS.put(config, true); when(jobRunner.submit(isA(Finalizer.class), argThat(mapContaining(config)))).thenReturn(jobRunner); @@ -189,7 +191,8 @@ public class PcapCliTest { "-protocol", "6", "-include_reverse", "-num_reducers", "10", - "-records_per_file", "1000" + "-records_per_file", "1000", + "-ps" }; Map<String, String> query = new HashMap<String, String>() {{ put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1"); @@ -211,6 +214,7 @@ public class PcapCliTest { PcapOptions.START_TIME_MS.put(config, startAsNanos / 1000000L); // needed bc defaults in config PcapOptions.END_TIME_MS.put(config, endAsNanos / 1000000L); // needed bc defaults in config PcapOptions.NUM_RECORDS_PER_FILE.put(config, 1000); + PcapOptions.PRINT_JOB_STATUS.put(config, true); when(jobRunner.submit(isA(Finalizer.class), argThat(mapContaining(config)))).thenReturn(jobRunner); @@ -262,7 +266,8 @@ public class PcapCliTest { "-base_path", "/base/path", "-base_output_path", "/base/output/path", "-query", "some query string", - "-records_per_file", "1000" + "-records_per_file", "1000", + "-ps" }; String query = "some query string"; @@ -274,6 +279,7 @@ public class PcapCliTest { PcapOptions.START_TIME_MS.put(config, 500L); // needed bc defaults in config PcapOptions.END_TIME_MS.put(config, 1000L); // needed bc defaults in config PcapOptions.NUM_RECORDS_PER_FILE.put(config, 1000); + PcapOptions.PRINT_JOB_STATUS.put(config, true); when(jobRunner.submit(isA(Finalizer.class), argThat(mapContaining(config)))).thenReturn(jobRunner); http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java index 26509be..cbb8170 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java @@ -40,6 +40,7 @@ public class PcapConfig extends AbstractMapDecorator<String, Object>{ public PcapConfig(PrefixStrategy prefixStrategy) { this(); setShowHelp(false); + setPrintJobStatus(false); setBasePath(""); setBaseInterimResultPath(""); setStartTimeMs(-1L); @@ -73,6 +74,14 @@ public class PcapConfig extends AbstractMapDecorator<String, Object>{ this.showHelp = showHelp; } + public boolean printJobStatus() { + return PcapOptions.PRINT_JOB_STATUS.get(this, Boolean.class); + } + + public void setPrintJobStatus(boolean printJobStatus) { + PcapOptions.PRINT_JOB_STATUS.put(this, printJobStatus); + } + public String getBasePath() { return PcapOptions.BASE_PATH.get(this, String.class); } http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java index 3d7c4f6..203c800 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java @@ -40,7 +40,8 @@ public enum PcapOptions implements ConfigOption { FIELDS("fields"), FILTER_IMPL("filterImpl"), HADOOP_CONF("hadoopConf"), - FILESYSTEM("fileSystem"); + FILESYSTEM("fileSystem"), + PRINT_JOB_STATUS("printJobStatus"); public static final BiFunction<String, Object, Path> STRING_TO_PATH = (s, o) -> o == null ? null : new Path(o.toString()); http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java index 23bd510..10f31b4 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java @@ -467,6 +467,13 @@ public class PcapJob<T> implements Statusable<Path> { */ @Override public Pageable<Path> get() throws JobException, InterruptedException { + if (PcapOptions.PRINT_JOB_STATUS.get(configuration, Boolean.class)) { + try { + mrJob.monitorAndPrintJob(); + } catch (IOException e) { + throw new JobException("Could not monitor job status", e); + } + } for (; ; ) { JobStatus status = getStatus(); if (status.getState() == State.SUCCEEDED @@ -478,6 +485,10 @@ public class PcapJob<T> implements Statusable<Path> { } } + public void monitorJob() throws IOException, InterruptedException { + mrJob.monitorAndPrintJob(); + } + private synchronized Pageable<Path> getFinalResults() { return new PcapPages(finalResults); }