Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1554-pcap-query-panel d8a5922b4 -> b29833cc5


METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes 
apache/metron#1138


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b29833cc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b29833cc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b29833cc

Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: b29833cc5db4efca2a6e9025042baf705fb347c6
Parents: d8a5922
Author: merrimanr <merrim...@gmail.com>
Authored: Fri Aug 10 08:25:13 2018 -0500
Committer: rmerriman <merrim...@gmail.com>
Committed: Fri Aug 10 08:25:13 2018 -0500

----------------------------------------------------------------------
 metron-platform/metron-pcap-backend/README.md           |  2 ++
 .../java/org/apache/metron/pcap/query/CliParser.java    |  4 ++++
 .../java/org/apache/metron/pcap/query/PcapCliTest.java  | 12 +++++++++---
 .../java/org/apache/metron/pcap/config/PcapConfig.java  |  9 +++++++++
 .../java/org/apache/metron/pcap/config/PcapOptions.java |  3 ++-
 .../main/java/org/apache/metron/pcap/mr/PcapJob.java    | 11 +++++++++++
 6 files changed, 37 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/README.md 
b/metron-platform/metron-pcap-backend/README.md
index e7960e3..2ff20d8 100644
--- a/metron-platform/metron-pcap-backend/README.md
+++ b/metron-platform/metron-pcap-backend/README.md
@@ -132,6 +132,7 @@ usage: Fixed filter options
  -nr,--num_reducers <arg>        The number of reducers to use.  Default
                                  is 10.
  -h,--help                       Display help
+ -ps,--print_status              Print the status of the job as it runs
  -ir,--include_reverse           Indicates if filter should check swapped
                                  src/dest addresses and IPs
  -p,--protocol <arg>             IP Protocol
@@ -154,6 +155,7 @@ usage: Query filter options
  -nr,--num_reducers <arg>        The number of reducers to use.  Default
                                  is 10.
  -h,--help                       Display help
+ -ps,--print_status              Print the status of the job as it runs
  -q,--query <arg>                Query string to use as a filter
  -st,--start_time <arg>          (required) Packet start time range.
 ```

http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
 
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 2d15e8b..69c725c 100644
--- 
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++ 
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -55,6 +55,7 @@ public class CliParser {
     options.addOption(newOption("rpf", "records_per_file", true, 
String.format("Number of records to include in each output pcap file (defaults 
to %s)", NUM_RECORDS_PER_FILE_DEFAULT)));
     options.addOption(newOption("et", "end_time", true, "Packet end time 
range. Default is current system time."));
     options.addOption(newOption("df", "date_format", true, "Date format to use 
for parsing start_time and end_time. Default is to use time in millis since the 
epoch."));
+    options.addOption(newOption("ps", "print_status", false, "Print the status 
of the job as it runs"));
     return options;
   }
 
@@ -125,6 +126,9 @@ public class CliParser {
         //no-op
       }
     }
+    if (commandLine.hasOption("print_status")) {
+      config.setPrintJobStatus(true);
+    }
   }
 
   public void printHelp(String msg, Options opts) {

http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
 
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index 96ca354..7c75224 100644
--- 
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++ 
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -147,7 +147,8 @@ public class PcapCliTest {
             "-protocol", "6",
             "-include_reverse",
             "-num_reducers", "10",
-            "-records_per_file", "1000"
+            "-records_per_file", "1000",
+            "-ps"
     };
     Map<String, String> query = new HashMap<String, String>() {{
       put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1");
@@ -165,6 +166,7 @@ public class PcapCliTest {
     PcapOptions.START_TIME_MS.put(config, 500L);
     PcapOptions.END_TIME_MS.put(config, 1000L);
     PcapOptions.NUM_RECORDS_PER_FILE.put(config, 1000);
+    PcapOptions.PRINT_JOB_STATUS.put(config, true);
 
     when(jobRunner.submit(isA(Finalizer.class), 
argThat(mapContaining(config)))).thenReturn(jobRunner);
 
@@ -189,7 +191,8 @@ public class PcapCliTest {
             "-protocol", "6",
             "-include_reverse",
             "-num_reducers", "10",
-            "-records_per_file", "1000"
+            "-records_per_file", "1000",
+            "-ps"
     };
     Map<String, String> query = new HashMap<String, String>() {{
       put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1");
@@ -211,6 +214,7 @@ public class PcapCliTest {
     PcapOptions.START_TIME_MS.put(config, startAsNanos / 1000000L); // needed 
bc defaults in config
     PcapOptions.END_TIME_MS.put(config, endAsNanos / 1000000L);  // needed bc 
defaults in config
     PcapOptions.NUM_RECORDS_PER_FILE.put(config, 1000);
+    PcapOptions.PRINT_JOB_STATUS.put(config, true);
 
     when(jobRunner.submit(isA(Finalizer.class), 
argThat(mapContaining(config)))).thenReturn(jobRunner);
 
@@ -262,7 +266,8 @@ public class PcapCliTest {
             "-base_path", "/base/path",
             "-base_output_path", "/base/output/path",
             "-query", "some query string",
-            "-records_per_file", "1000"
+            "-records_per_file", "1000",
+            "-ps"
     };
 
     String query = "some query string";
@@ -274,6 +279,7 @@ public class PcapCliTest {
     PcapOptions.START_TIME_MS.put(config, 500L); // needed bc defaults in 
config
     PcapOptions.END_TIME_MS.put(config, 1000L);  // needed bc defaults in 
config
     PcapOptions.NUM_RECORDS_PER_FILE.put(config, 1000);
+    PcapOptions.PRINT_JOB_STATUS.put(config, true);
 
     when(jobRunner.submit(isA(Finalizer.class), 
argThat(mapContaining(config)))).thenReturn(jobRunner);
 

http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
 
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
index 26509be..cbb8170 100644
--- 
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
+++ 
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
@@ -40,6 +40,7 @@ public class PcapConfig extends AbstractMapDecorator<String, 
Object>{
   public PcapConfig(PrefixStrategy prefixStrategy) {
     this();
     setShowHelp(false);
+    setPrintJobStatus(false);
     setBasePath("");
     setBaseInterimResultPath("");
     setStartTimeMs(-1L);
@@ -73,6 +74,14 @@ public class PcapConfig extends AbstractMapDecorator<String, 
Object>{
     this.showHelp = showHelp;
   }
 
+  public boolean printJobStatus() {
+    return PcapOptions.PRINT_JOB_STATUS.get(this, Boolean.class);
+  }
+
+  public void setPrintJobStatus(boolean printJobStatus) {
+    PcapOptions.PRINT_JOB_STATUS.put(this, printJobStatus);
+  }
+
   public String getBasePath() {
     return PcapOptions.BASE_PATH.get(this, String.class);
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java
 
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java
index 3d7c4f6..203c800 100644
--- 
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java
+++ 
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapOptions.java
@@ -40,7 +40,8 @@ public enum PcapOptions implements ConfigOption {
   FIELDS("fields"),
   FILTER_IMPL("filterImpl"),
   HADOOP_CONF("hadoopConf"),
-  FILESYSTEM("fileSystem");
+  FILESYSTEM("fileSystem"),
+  PRINT_JOB_STATUS("printJobStatus");
 
   public static final BiFunction<String, Object, Path> STRING_TO_PATH =
       (s, o) -> o == null ? null : new Path(o.toString());

http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
 
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
index 23bd510..10f31b4 100644
--- 
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
+++ 
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
@@ -467,6 +467,13 @@ public class PcapJob<T> implements Statusable<Path> {
    */
   @Override
   public Pageable<Path> get() throws JobException, InterruptedException {
+    if (PcapOptions.PRINT_JOB_STATUS.get(configuration, Boolean.class)) {
+      try {
+        mrJob.monitorAndPrintJob();
+      } catch (IOException e) {
+        throw new JobException("Could not monitor job status", e);
+      }
+    }
     for (; ; ) {
       JobStatus status = getStatus();
       if (status.getState() == State.SUCCEEDED
@@ -478,6 +485,10 @@ public class PcapJob<T> implements Statusable<Path> {
     }
   }
 
+  public void monitorJob() throws IOException, InterruptedException {
+    mrJob.monitorAndPrintJob();
+  }
+
   private synchronized Pageable<Path> getFinalResults() {
     return new PcapPages(finalResults);
   }

Reply via email to