Repository: metron
Updated Branches:
  refs/heads/master 3d3c43c71 -> c26abbbae


METRON-1438 STELLAR: Move shell functions to common from metron-management 
(ottobackwards) closes apache/metron#920


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c26abbba
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c26abbba
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c26abbba

Branch: refs/heads/master
Commit: c26abbbaeaeea3551e218cb9aa8ba97b32655958
Parents: 3d3c43c
Author: ottobackwards <ottobackwa...@gmail.com>
Authored: Fri Feb 2 09:58:59 2018 -0500
Committer: otto <o...@apache.org>
Committed: Fri Feb 2 09:58:59 2018 -0500

----------------------------------------------------------------------
 metron-platform/metron-management/README.md     |  32 +-
 .../metron/management/ShellFunctions.java       | 302 -------------------
 .../metron/management/ShellFunctionsTest.java   | 171 -----------
 metron-stellar/stellar-common/README.md         |  31 ++
 metron-stellar/stellar-common/pom.xml           |   5 +
 .../stellar/common/shell/cli/PausableInput.java |  23 +-
 .../stellar/dsl/functions/ShellFunctions.java   | 301 ++++++++++++++++++
 .../dsl/functions/ShellFunctionsTest.java       | 176 +++++++++++
 8 files changed, 529 insertions(+), 512 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-platform/metron-management/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/README.md 
b/metron-platform/metron-management/README.md
index 812583c..bf939c2 100644
--- a/metron-platform/metron-management/README.md
+++ b/metron-platform/metron-management/README.md
@@ -35,14 +35,13 @@ project.
 * [Functions](#functions)
     * [Grok Functions](#grok-functions)
     * [File Functions](#file-functions)
-    * [Shell Functions](#shell-functions)
     * [Configuration Functions](#configuration-functions)
     * [Parser Functions](#parser-functions)
     * [Indexing Functions](#indexing-functions)
     * [Enrichment Functions](#enrichment-functions)
     * [Threat Triage Functions](#threat-triage-functions)
 * [Examples](#examples)
-    *  [Iterate to Find a Valid Grok 
Pattern](#iterate-to-find-a-valid-grok-pattern)
+    * [Iterate to Find a Valid Grok 
Pattern](#iterate-to-find-a-valid-grok-pattern)
     * [Manage Stellar Field 
Transformations](#manage-stellar-field-transformations)
     * [Manage Stellar Enrichments](#manage-stellar-enrichments)
     * [Manage Threat Triage Rules](#manage-threat-triage-rules)
@@ -132,35 +131,6 @@ The functions are split roughly into a few sections:
       * path - The path of the file
     * Returns: true if the file was written and false otherwise.
 
-### Shell Functions 
-
-* `SHELL_EDIT`
-  * Description: Open an editor (optionally initialized with text) and return 
whatever is saved from the editor.  The editor to use is pulled from `EDITOR` 
or `VISUAL` environment variable.
-  * Input:
-    * string - (Optional) A string whose content is used to initialize the 
editor.
-  * Returns: The content that the editor saved after editor exit.
-* `SHELL_GET_EXPRESSION`
-  * Description: Get a stellar expression from a variable
-  * Input:
-    * variable - variable name
-  * Returns: The stellar expression associated with the variable.
-* `SHELL_LIST_VARS`
-  * Description: Return the variables in a tabular form
-  * Input:
-    * wrap : Length of string to wrap the columns
-  * Returns: A tabular representation of the variables.
-* `SHELL_MAP2TABLE`
-  * Description: Take a map and return a table
-  * Input:
-    * map - Map
-  * Returns: The map in table form
-* `SHELL_VARS2MAP`
-  * Description: Take a set of variables and return a map
-  * Input:
-    * variables* - variable names to use to create map 
-  * Returns: A map associating the variable name with the stellar expression.
-
-
 ### Configuration Functions
 
 * `CONFIG_GET`

http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
 
b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
deleted file mode 100644
index afac7f0..0000000
--- 
a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.management;
-
-import com.jakewharton.fliptables.FlipTable;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.text.WordUtils;
-import org.apache.metron.stellar.common.shell.VariableResult;
-import org.apache.metron.stellar.common.shell.cli.PausableInput;
-import org.apache.metron.stellar.common.utils.ConversionUtils;
-import org.apache.metron.stellar.dsl.BaseStellarFunction;
-import org.apache.metron.stellar.dsl.Context;
-import org.apache.metron.stellar.dsl.ParseException;
-import org.apache.metron.stellar.dsl.Stellar;
-import org.apache.metron.stellar.dsl.StellarFunction;
-import org.jboss.aesh.console.Console;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.lang.invoke.MethodHandles;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE;
-
-public class ShellFunctions {
-  private static final org.slf4j.Logger LOG = 
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
-  private static Map<String, VariableResult> getVariables(Context context) {
-    return (Map<String, VariableResult>) 
context.getCapability(Context.Capabilities.SHELL_VARIABLES).get();
-  }
-
-  @Stellar(
-           namespace = "SHELL"
-          ,name = "MAP2TABLE"
-          ,description = "Take a map and return a table"
-          ,params = {"map - Map"
-                    }
-          ,returns = "The map in table form"
-          )
-  public static class Map2Table extends BaseStellarFunction {
-
-    @Override
-    public Object apply(List<Object> args) {
-      if(args.size() < 1) {
-        return null;
-      }
-      Map<Object, Object> map = (Map<Object, Object>) args.get(0);
-      if(map == null) {
-        map = new HashMap<>();
-      }
-      String[] headers = {"KEY", "VALUE"};
-      String[][] data = new String[map.size()][2];
-      int i = 0;
-      for(Map.Entry<Object, Object> kv : map.entrySet()) {
-        data[i++] = new String[] {kv.getKey().toString(), 
kv.getValue().toString()};
-      }
-      return FlipTable.of(headers, data);
-    }
-  }
-
-  @Stellar(
-           namespace = "SHELL"
-          ,name = "LIST_VARS"
-          ,description = "Return the variables in a tabular form"
-          ,params = {
-             "wrap : Length of string to wrap the columns"
-                    }
-          ,returns = "A tabular representation of the variables."
-          )
-  public static class ListVars implements StellarFunction {
-
-    @Override
-    public Object apply(List<Object> args, Context context) throws 
ParseException {
-
-      Map<String, VariableResult> variables = getVariables(context);
-      String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"};
-      String[][] data = new String[variables.size()][3];
-      int wordWrap = -1;
-      if(args.size() > 0) {
-        wordWrap = ConversionUtils.convert(args.get(0), Integer.class);
-      }
-      int i = 0;
-      for(Map.Entry<String, VariableResult> kv : variables.entrySet()) {
-        VariableResult result = kv.getValue();
-        data[i++] = new String[] { toWrappedString(kv.getKey().toString(), 
wordWrap)
-                                 , toWrappedString(result.getResult(), 
wordWrap)
-                                 , 
toWrappedString(result.getExpression().get(), wordWrap)
-                                 };
-      }
-      return FlipTable.of(headers, data);
-    }
-
-    private static String toWrappedString(Object o, int wrap) {
-      String s = "" + o;
-      if(wrap <= 0) {
-        return s;
-      }
-      return WordUtils.wrap(s, wrap);
-    }
-
-    @Override
-    public void initialize(Context context) {
-
-    }
-
-    @Override
-    public boolean isInitialized() {
-      return true;
-    }
-  }
-
-  @Stellar(
-           namespace = "SHELL"
-          ,name = "VARS2MAP"
-          ,description = "Take a set of variables and return a map"
-          ,params = {"variables* - variable names to use to create map "
-                    }
-          ,returns = "A map associating the variable name with the stellar 
expression."
-          )
-  public static class Var2Map implements StellarFunction {
-
-    @Override
-    public Object apply(List<Object> args, Context context) throws 
ParseException {
-      Map<String, VariableResult> variables = getVariables(context);
-      LinkedHashMap<String, String> ret = new LinkedHashMap<>();
-      for(Object arg : args) {
-        if(arg == null) {
-          continue;
-        }
-        String variable = (String)arg;
-        VariableResult result = variables.get(variable);
-        if(result != null && result.getExpression().isPresent()) {
-          ret.put(variable, result.getExpression().orElseGet(() -> ""));
-        }
-      }
-      return ret;
-    }
-
-    @Override
-    public void initialize(Context context) {
-
-    }
-
-    @Override
-    public boolean isInitialized() {
-      return true;
-    }
-  }
-
-  @Stellar(
-           namespace = "SHELL"
-          ,name = "GET_EXPRESSION"
-          ,description = "Get a stellar expression from a variable"
-          ,params = {"variable - variable name"
-                    }
-          ,returns = "The stellar expression associated with the variable."
-          )
-  public static class GetExpression implements StellarFunction {
-
-    @Override
-    public Object apply(List<Object> args, Context context) throws 
ParseException {
-      Map<String, VariableResult> variables = getVariables(context);
-      if(args.size() == 0) {
-        return null;
-      }
-      String variable = (String) args.get(0);
-      if(variable == null) {
-        return null;
-      }
-      VariableResult result = variables.get(variable);
-      if(result != null && result.getExpression().isPresent()) {
-        return result.getExpression().get();
-      }
-      return null;
-    }
-
-    @Override
-    public void initialize(Context context) {
-
-    }
-
-    @Override
-    public boolean isInitialized() {
-      return true;
-    }
-  }
-
-  @Stellar(
-           namespace = "SHELL"
-          ,name = "EDIT"
-          ,description = "Open an editor (optionally initialized with text) 
and return " +
-                         "whatever is saved from the editor.  The editor to 
use is pulled " +
-                         "from `EDITOR` or `VISUAL` environment variable."
-          ,params = {   "string - (Optional) A string whose content is used to 
initialize the editor."
-                    }
-          ,returns = "The content that the editor saved after editor exit."
-          )
-  public static class Edit implements StellarFunction {
-
-    private String getEditor() {
-      // if we have editor in the system properties, it should
-      // override the env so we check that first
-      String editor = System.getProperty("EDITOR");
-      if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
-        editor = System.getenv().get("EDITOR");
-      }
-      if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
-        editor = System.getenv("VISUAL");
-      }
-      if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
-        editor = "/bin/vi";
-      }
-      return editor;
-    }
-
-    @Override
-    public Object apply(List<Object> args, Context context) throws 
ParseException {
-      File outFile = null;
-      String editor = getEditor();
-      try {
-        outFile = File.createTempFile("stellar_shell", "out");
-        if(args.size() > 0) {
-          String arg = (String)args.get(0);
-          try(PrintWriter pw = new PrintWriter(outFile)) {
-            IOUtils.write(arg, pw);
-          }
-        }
-      } catch (IOException e) {
-        String message = "Unable to create temp file: " + e.getMessage();
-        LOG.error(message, e);
-        throw new IllegalStateException(message, e);
-      }
-      Optional<Object> console =  context.getCapability(CONSOLE, false);
-      try {
-        PausableInput.INSTANCE.pause();
-        //shut down the IO for the console
-        ProcessBuilder processBuilder = new ProcessBuilder(editor, 
outFile.getAbsolutePath());
-        processBuilder.redirectInput(ProcessBuilder.Redirect.INHERIT);
-        processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
-        processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
-        try {
-          Process p = processBuilder.start();
-          // wait for termination.
-          p.waitFor();
-          try (BufferedReader br = new BufferedReader(new 
FileReader(outFile))) {
-            String ret = IOUtils.toString(br).trim();
-            return ret;
-          }
-        } catch (Exception e) {
-          String message = "Unable to read output: " + e.getMessage();
-          LOG.error(message, e);
-          return null;
-        }
-      } finally {
-        try {
-          PausableInput.INSTANCE.unpause();
-          if(console.isPresent()) {
-            ((Console)console.get()).pushToInputStream("\b\n");
-          }
-        } catch (IOException e) {
-          LOG.error("Unable to unpause: {}", e.getMessage(), e);
-        }
-        if(outFile.exists()) {
-          outFile.delete();
-        }
-      }
-    }
-
-    @Override
-    public void initialize(Context context) {
-
-    }
-
-    @Override
-    public boolean isInitialized() {
-      return true;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
----------------------------------------------------------------------
diff --git 
a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
 
b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
deleted file mode 100644
index 83c2bce..0000000
--- 
a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.management;
-
-import com.google.common.collect.ImmutableMap;
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.metron.stellar.common.shell.VariableResult;
-import org.apache.metron.stellar.dsl.Context;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
-
-public class ShellFunctionsTest {
-
-  Map<String, VariableResult> variables = ImmutableMap.of(
-          "var1" , VariableResult.withExpression("CASEY", "TO_UPPER('casey')"),
-          "var2" , VariableResult.withValue("foo"),
-          "var3" , VariableResult.withValue(null),
-          "var4" , VariableResult.withExpression(null, "blah")
-  );
-
-  Context context = new Context.Builder()
-            .with(Context.Capabilities.SHELL_VARIABLES , () -> variables)
-            .build();
-/**
-╔══════════╤═══════╤════════════╗
-║ VARIABLE │ VALUE │ EXPRESSION ║
-╠
══════════╪═══════╪════════════╣
-║ foo      │ 2.0   │ 1 + 1      ║
-╚══════════╧═══════╧════════════╝
- **/
-  @Multiline
-  static String expectedListWithFoo;
-
-  @Test
-  public void testListVarsWithVars() {
-    Map<String, VariableResult> variables = ImmutableMap.of(
-            "foo", VariableResult.withExpression(2.0, "1 + 1"));
-
-    Context context = new Context.Builder()
-            .with(Context.Capabilities.SHELL_VARIABLES , () -> variables)
-            .build();
-    Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
-    Assert.assertEquals(expectedListWithFoo, out);
-  }
-
-/**
-╔══════════╤═══════╤════════════╗
-║ VARIABLE │ VALUE │ EXPRESSION ║
-╠
══════════╧═══════╧════════════╣
-║ (empty)                       ║
-╚═══════════════════════════════╝
- **/
-  @Multiline
-  static String expectedEmptyList;
-
-  @Test
-  public void testListVarsWithoutVars() {
-    Context context = new Context.Builder()
-            .with(Context.Capabilities.SHELL_VARIABLES, () -> new HashMap<>())
-            .build();
-    Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
-    Assert.assertEquals(expectedEmptyList, out);
-  }
-/**
-╔════════╤═══════╗
-║ KEY    │ VALUE ║
-╠════════╪═══════╣
-║ field1 │ val1  ║
-╟────────┼───────╢
-║ field2 │ val2  ║
-╚════════╧═══════╝
- **/
-  @Multiline
-  static String expectedMap2Table;
-
-  @Test
-  public void testMap2Table() {
-    Map<String, Object> variables = ImmutableMap.of("map_field", 
ImmutableMap.of("field1", "val1", "field2", "val2"));
-    Context context = Context.EMPTY_CONTEXT();
-    Object out = run("SHELL_MAP2TABLE(map_field)", variables, context);
-    Assert.assertEquals(expectedMap2Table, out);
-  }
- /**
-╔═════╤═══════╗
-║ KEY │ VALUE ║
-╠═════╧═══════╣
-║ (empty)     ║
-╚═════════════╝
- **/
-  @Multiline
-  static String expectedMap2TableNullInput;
-
-  @Test
-  public void testMap2TableNullInput() {
-    Map<String,Object> variables = new HashMap<String,Object>(){{
-      put("map_field",null);
-    }};
-    Context context = Context.EMPTY_CONTEXT();
-    Object out = run("SHELL_MAP2TABLE(map_field)", variables, context);
-    Assert.assertEquals(expectedMap2TableNullInput, out);
-  }
-
-  @Test
-  public void testMap2TableInsufficientArgs() {
-    Map<String, Object> variables = new HashMap<>();
-    Context context = Context.EMPTY_CONTEXT();
-    Object out = run("SHELL_MAP2TABLE()", variables, context);
-    Assert.assertNull(out);
-  }
-
-  @Test
-  public void testVars2Map() {
-    Object out = run("SHELL_VARS2MAP('var1', 'var2')", new HashMap<>(), 
context);
-    Assert.assertTrue(out instanceof Map);
-    Map<String, String> mapOut = (Map<String, String>)out;
-    //second one is null, so we don't want it there.
-    Assert.assertEquals(1, mapOut.size());
-    Assert.assertEquals("TO_UPPER('casey')", mapOut.get("var1"));
-  }
-
-  @Test
-  public void testVars2MapEmpty() {
-    Object out = run("SHELL_VARS2MAP()", new HashMap<>(), context);
-    Map<String, String> mapOut = (Map<String, String>)out;
-    Assert.assertEquals(0, mapOut.size());
-  }
-
-  @Test
-  public void testGetExpression() {
-    Object out = run("SHELL_GET_EXPRESSION('var1')", new HashMap<>(), context);
-    Assert.assertTrue(out instanceof String);
-    String expression = (String)out;
-    //second one is null, so we don't want it there.
-    Assert.assertEquals("TO_UPPER('casey')", expression);
-  }
-
-  @Test
-  public void testGetExpressionEmpty() {
-    Object out = run("SHELL_GET_EXPRESSION()", new HashMap<>(), context);
-    Assert.assertNull(out );
-  }
-
-  @Test
-  public void testEdit() throws Exception {
-    System.getProperties().put("EDITOR", "/bin/cat");
-    Object out = run("TO_UPPER(SHELL_EDIT(foo))", ImmutableMap.of("foo", 
"foo"), context);
-    Assert.assertEquals("FOO", out);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md 
b/metron-stellar/stellar-common/README.md
index a2f1242..078799e 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -243,6 +243,11 @@ Where:
 | [ `SET_MERGE`](#set_merge)                                                   
                      |
 | [ `SET_REMOVE`](#set_remove)                                                 
                      |
 | [ `SIN`](#sin)                                                               
                      |
+| [ `SHELL_EDIT`](#shell_edit)                                                 
                      |
+| [ `SHELL_GET_EXPRESSION`](#shell_get_expression)                             
                      |
+| [ `SHELL_LIST_VARS`](#shell_list_vars)                                       
                      |
+| [ `SHELL_MAP2TABLE`](#shell_map2table)                                       
                      |
+| [ `SHELL_VARS2MAP`](#shell_vars2map)                                         
                      |
 | [ `SPLIT`](#split)                                                           
                      |
 | [ `SQRT`](#sqrt)                                                             
                      |
 | [ `STARTS_WITH`](#starts_with)                                               
                      |
@@ -920,6 +925,32 @@ Where:
     * o - object to add to set
   * Returns: A Set
 
+### `SHELL_EDIT`
+  * Description: Open an editor (optionally initialized with text) and return 
whatever is saved from the editor.  The editor to use is pulled from `EDITOR` 
or `VISUAL` environment variable.
+  * Input:
+    * string - (Optional) A string whose content is used to initialize the 
editor.
+  * Returns: The content that the editor saved after editor exit.
+### `SHELL_GET_EXPRESSION`
+  * Description: Get a stellar expression from a variable
+  * Input:
+    * variable - variable name
+  * Returns: The stellar expression associated with the variable.
+### `SHELL_LIST_VARS`
+  * Description: Return the variables in a tabular form
+  * Input:
+    * wrap : Length of string to wrap the columns
+  * Returns: A tabular representation of the variables.
+### `SHELL_MAP2TABLE`
+  * Description: Take a map and return a table
+  * Input:
+    * map - Map
+  * Returns: The map in table form
+### `SHELL_VARS2MAP`
+  * Description: Take a set of variables and return a map
+  * Input:
+    * variables* - variable names to use to create map 
+  * Returns: A map associating the variable name with the stellar expression.
+
 ### `SIN`
   * Description: Returns the sine of a number.
   * Input:

http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/pom.xml 
b/metron-stellar/stellar-common/pom.xml
index b43fcc1..6b07e68 100644
--- a/metron-stellar/stellar-common/pom.xml
+++ b/metron-stellar/stellar-common/pom.xml
@@ -203,6 +203,11 @@
             <version>${global_kryo_serializers_version}</version>
         </dependency>
         <dependency>
+            <groupId>com.jakewharton.fliptables</groupId>
+            <artifactId>fliptables</artifactId>
+            <version>1.0.2</version>
+        </dependency>
+        <dependency>
             <!-- junit dependency added with default scope to allow inclusion 
of StellarProcessorUtils in main jar.
                  It is excluded from the uber-jar. -->
             <groupId>junit</groupId>

http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java
----------------------------------------------------------------------
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java
index fad0115..c72d66f 100644
--- 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java
@@ -21,6 +21,7 @@ package org.apache.metron.stellar.common.shell.cli;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * An input stream which mirrors System.in, but allows you to 'pause' and 
'unpause' it.
@@ -36,8 +37,8 @@ import java.io.InputStream;
  *
  */
 public class PausableInput extends InputStream {
-  InputStream in = System.in;
-  boolean paused = false;
+  private InputStream in = System.in;
+  private AtomicBoolean paused = new AtomicBoolean(false);
   private PausableInput() {
     super();
   }
@@ -46,7 +47,7 @@ public class PausableInput extends InputStream {
    * Stop mirroring stdin
    */
   public void pause() {
-    paused = true;
+    paused.set(true);
   }
 
   /**
@@ -54,8 +55,7 @@ public class PausableInput extends InputStream {
    * @throws IOException
    */
   public void unpause() throws IOException {
-    in.read(new byte[in.available()]);
-    paused = false;
+    paused.set(false);
   }
 
   public final static PausableInput INSTANCE = new PausableInput();
@@ -76,7 +76,14 @@ public class PausableInput extends InputStream {
    */
   @Override
   public int read() throws IOException {
-
+    if(paused.get()) {
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+      return 0;
+    }
     return in.read();
   }
 
@@ -116,7 +123,7 @@ public class PausableInput extends InputStream {
   @Override
   public int read(byte[] b) throws IOException {
 
-    if(paused) {
+    if(paused.get()) {
       try {
         Thread.sleep(1000);
       } catch (InterruptedException e) {
@@ -187,7 +194,7 @@ public class PausableInput extends InputStream {
    */
   @Override
   public int read(byte[] b, int off, int len) throws IOException {
-    if(paused) {
+    if(paused.get()) {
       try {
         Thread.sleep(1000);
       } catch (InterruptedException e) {

http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java
----------------------------------------------------------------------
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java
new file mode 100644
index 0000000..1df4a51
--- /dev/null
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.dsl.functions;
+
+import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE;
+
+import com.jakewharton.fliptables.FlipTable;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.text.WordUtils;
+import org.apache.metron.stellar.common.shell.VariableResult;
+import org.apache.metron.stellar.common.shell.cli.PausableInput;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.BaseStellarFunction;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.jboss.aesh.console.Console;
+import org.slf4j.LoggerFactory;
+
+public class ShellFunctions {
+  private static final org.slf4j.Logger LOG = 
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  @SuppressWarnings("unchecked")
+  private static Map<String, VariableResult> getVariables(Context context) {
+    return (Map<String, VariableResult>) 
context.getCapability(Context.Capabilities.SHELL_VARIABLES).get();
+  }
+
+  @Stellar(
+           namespace = "SHELL"
+          ,name = "MAP2TABLE"
+          ,description = "Take a map and return a table"
+          ,params = {"map - Map"
+                    }
+          ,returns = "The map in table form"
+          )
+  public static class Map2Table extends BaseStellarFunction {
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public Object apply(List<Object> args) {
+      if (args.size() < 1) {
+        return null;
+      }
+      Map<Object, Object> map = (Map<Object, Object>) args.get(0);
+      if (map == null) {
+        map = new HashMap<>();
+      }
+      String[] headers = {"KEY", "VALUE"};
+      String[][] data = new String[map.size()][2];
+      int i = 0;
+      for (Map.Entry<Object, Object> kv : map.entrySet()) {
+        data[i++] = new String[]{kv.getKey().toString(), 
kv.getValue().toString()};
+      }
+      return FlipTable.of(headers, data);
+    }
+  }
+
+  @Stellar(
+           namespace = "SHELL"
+          ,name = "LIST_VARS"
+          ,description = "Return the variables in a tabular form"
+          ,params = {
+             "wrap : Length of string to wrap the columns"
+                    }
+          ,returns = "A tabular representation of the variables."
+          )
+  public static class ListVars implements StellarFunction {
+
+    @Override
+    public Object apply(List<Object> args, Context context) throws 
ParseException {
+      Map<String, VariableResult> variables = getVariables(context);
+      String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"};
+      String[][] data = new String[variables.size()][3];
+      int wordWrap = -1;
+      if (args.size() > 0) {
+        wordWrap = ConversionUtils.convert(args.get(0), Integer.class);
+      }
+      int i = 0;
+      for (Map.Entry<String, VariableResult> kv : variables.entrySet()) {
+        VariableResult result = kv.getValue();
+        data[i++] = new String[]{toWrappedString(kv.getKey(), wordWrap),
+            toWrappedString(result.getResult(), wordWrap),
+            toWrappedString(result.getExpression().get(), wordWrap)};
+      }
+      return FlipTable.of(headers, data);
+    }
+
+    private static String toWrappedString(Object o, int wrap) {
+      String s = "" + o;
+      if(wrap <= 0) {
+        return s;
+      }
+      return WordUtils.wrap(s, wrap);
+    }
+
+    @Override
+    public void initialize(Context context) {
+
+    }
+
+    @Override
+    public boolean isInitialized() {
+      return true;
+    }
+  }
+
+  @Stellar(
+           namespace = "SHELL"
+          ,name = "VARS2MAP"
+          ,description = "Take a set of variables and return a map"
+          ,params = {"variables* - variable names to use to create map "
+                    }
+          ,returns = "A map associating the variable name with the stellar 
expression."
+          )
+  public static class Var2Map implements StellarFunction {
+
+    @Override
+    public Object apply(List<Object> args, Context context) throws 
ParseException {
+      Map<String, VariableResult> variables = getVariables(context);
+      LinkedHashMap<String, String> ret = new LinkedHashMap<>();
+      for (Object arg : args) {
+        if (arg == null) {
+          continue;
+        }
+        String variable = (String) arg;
+        VariableResult result = variables.get(variable);
+        if (result != null && result.getExpression().isPresent()) {
+          ret.put(variable, result.getExpression().orElseGet(() -> ""));
+        }
+      }
+      return ret;
+    }
+
+    @Override
+    public void initialize(Context context) {
+
+    }
+
+    @Override
+    public boolean isInitialized() {
+      return true;
+    }
+  }
+
+  @Stellar(
+           namespace = "SHELL"
+          ,name = "GET_EXPRESSION"
+          ,description = "Get a stellar expression from a variable"
+          ,params = {"variable - variable name"
+                    }
+          ,returns = "The stellar expression associated with the variable."
+          )
+  public static class GetExpression implements StellarFunction {
+
+    @Override
+    public Object apply(List<Object> args, Context context) throws 
ParseException {
+      Map<String, VariableResult> variables = getVariables(context);
+      if (args.size() == 0) {
+        return null;
+      }
+      String variable = (String) args.get(0);
+      if (variable == null) {
+        return null;
+      }
+      VariableResult result = variables.get(variable);
+      if (result != null && result.getExpression().isPresent()) {
+        return result.getExpression().get();
+      }
+      return null;
+    }
+
+    @Override
+    public void initialize(Context context) {
+
+    }
+
+    @Override
+    public boolean isInitialized() {
+      return true;
+    }
+  }
+
+  @Stellar(
+           namespace = "SHELL"
+          ,name = "EDIT"
+          ,description = "Open an editor (optionally initialized with text) 
and return " +
+                         "whatever is saved from the editor.  The editor to 
use is pulled " +
+                         "from `EDITOR` or `VISUAL` environment variable."
+          ,params = {   "string - (Optional) A string whose content is used to 
initialize the editor."
+                    }
+          ,returns = "The content that the editor saved after editor exit."
+          )
+  public static class Edit implements StellarFunction {
+
+    private String getEditor() {
+      // if we have editor in the system properties, it should
+      // override the env so we check that first
+      String editor = System.getProperty("EDITOR");
+      if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
+        editor = System.getenv().get("EDITOR");
+      }
+      if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
+        editor = System.getenv("VISUAL");
+      }
+      if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
+        editor = "/bin/vi";
+      }
+      return editor;
+    }
+
+    @Override
+    public Object apply(List<Object> args, Context context) throws 
ParseException {
+      File outFile = null;
+      String editor = getEditor();
+      try {
+        outFile = File.createTempFile("stellar_shell", "out");
+        if (args.size() > 0) {
+          String arg = (String) args.get(0);
+          try (PrintWriter pw = new PrintWriter(outFile)) {
+            IOUtils.write(arg, pw);
+          }
+        }
+      } catch (IOException e) {
+        String message = "Unable to create temp file: " + e.getMessage();
+        LOG.error(message, e);
+        throw new IllegalStateException(message, e);
+      }
+      Optional<Object> console = context.getCapability(CONSOLE, false);
+      try {
+        PausableInput.INSTANCE.pause();
+        //shut down the IO for the console
+        ProcessBuilder processBuilder = new ProcessBuilder(editor, 
outFile.getAbsolutePath());
+        processBuilder.redirectInput(ProcessBuilder.Redirect.INHERIT);
+        processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
+        processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
+        try {
+          Process p = processBuilder.start();
+          // wait for termination.
+          p.waitFor();
+          try (BufferedReader br = new BufferedReader(new 
FileReader(outFile))) {
+            String ret = IOUtils.toString(br).trim();
+            return ret;
+          }
+        } catch (Exception e) {
+          String message = "Unable to read output: " + e.getMessage();
+          LOG.error(message, e);
+          return null;
+        }
+      } finally {
+        try {
+          PausableInput.INSTANCE.unpause();
+          if (console.isPresent()) {
+            ((Console) console.get()).pushToInputStream("\b\n");
+          }
+        } catch (IOException e) {
+          LOG.error("Unable to unpause: {}", e.getMessage(), e);
+        }
+        if (outFile.exists()) {
+          outFile.delete();
+        }
+      }
+    }
+
+    @Override
+    public void initialize(Context context) {
+
+    }
+
+    @Override
+    public boolean isInitialized() {
+      return true;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java
----------------------------------------------------------------------
diff --git 
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java
 
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java
new file mode 100644
index 0000000..354e0c3
--- /dev/null
+++ 
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.dsl.functions;
+
+import com.google.common.collect.ImmutableMap;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.stellar.common.shell.VariableResult;
+import org.apache.metron.stellar.common.shell.cli.PausableInput;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.Context.Capabilities;
+import org.jboss.aesh.console.Console;
+import org.jboss.aesh.console.settings.Settings;
+import org.jboss.aesh.console.settings.SettingsBuilder;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
+
+public class ShellFunctionsTest {
+
+  Map<String, VariableResult> variables = ImmutableMap.of(
+          "var1" , VariableResult.withExpression("CASEY", "TO_UPPER('casey')"),
+          "var2" , VariableResult.withValue("foo"),
+          "var3" , VariableResult.withValue(null),
+          "var4" , VariableResult.withExpression(null, "blah")
+  );
+
+  Context context = new Context.Builder()
+            .with(Context.Capabilities.SHELL_VARIABLES , () -> 
variables).build();
+
+/**
+╔══════════╤═══════╤════════════╗
+║ VARIABLE │ VALUE │ EXPRESSION ║
+╠
══════════╪═══════╪════════════╣
+║ foo      │ 2.0   │ 1 + 1      ║
+╚══════════╧═══════╧════════════╝
+ **/
+  @Multiline
+  static String expectedListWithFoo;
+
+  @Test
+  public void testListVarsWithVars() {
+    Map<String, VariableResult> variables = ImmutableMap.of(
+            "foo", VariableResult.withExpression(2.0, "1 + 1"));
+
+    Context context = new Context.Builder()
+            .with(Context.Capabilities.SHELL_VARIABLES , () -> variables)
+            .build();
+    Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
+    Assert.assertEquals(expectedListWithFoo, out);
+  }
+
+/**
+╔══════════╤═══════╤════════════╗
+║ VARIABLE │ VALUE │ EXPRESSION ║
+╠
══════════╧═══════╧════════════╣
+║ (empty)                       ║
+╚═══════════════════════════════╝
+ **/
+  @Multiline
+  static String expectedEmptyList;
+
+  @Test
+  public void testListVarsWithoutVars() {
+    Context context = new Context.Builder()
+            .with(Context.Capabilities.SHELL_VARIABLES, () -> new HashMap<>())
+            .build();
+    Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
+    Assert.assertEquals(expectedEmptyList, out);
+  }
+/**
+╔════════╤═══════╗
+║ KEY    │ VALUE ║
+╠════════╪═══════╣
+║ field1 │ val1  ║
+╟────────┼───────╢
+║ field2 │ val2  ║
+╚════════╧═══════╝
+ **/
+  @Multiline
+  static String expectedMap2Table;
+
+  @Test
+  public void testMap2Table() {
+    Map<String, Object> variables = ImmutableMap.of("map_field", 
ImmutableMap.of("field1", "val1", "field2", "val2"));
+    Context context = Context.EMPTY_CONTEXT();
+    Object out = run("SHELL_MAP2TABLE(map_field)", variables, context);
+    Assert.assertEquals(expectedMap2Table, out);
+  }
+ /**
+╔═════╤═══════╗
+║ KEY │ VALUE ║
+╠═════╧═══════╣
+║ (empty)     ║
+╚═════════════╝
+ **/
+  @Multiline
+  static String expectedMap2TableNullInput;
+
+  @Test
+  public void testMap2TableNullInput() {
+    Map<String,Object> variables = new HashMap<String,Object>(){{
+      put("map_field",null);
+    }};
+    Context context = Context.EMPTY_CONTEXT();
+    Object out = run("SHELL_MAP2TABLE(map_field)", variables, context);
+    Assert.assertEquals(expectedMap2TableNullInput, out);
+  }
+
+  @Test
+  public void testMap2TableInsufficientArgs() {
+    Map<String, Object> variables = new HashMap<>();
+    Context context = Context.EMPTY_CONTEXT();
+    Object out = run("SHELL_MAP2TABLE()", variables, context);
+    Assert.assertNull(out);
+  }
+
+  @Test
+  public void testVars2Map() {
+    Object out = run("SHELL_VARS2MAP('var1', 'var2')", new HashMap<>(), 
context);
+    Assert.assertTrue(out instanceof Map);
+    Map<String, String> mapOut = (Map<String, String>)out;
+    //second one is null, so we don't want it there.
+    Assert.assertEquals(1, mapOut.size());
+    Assert.assertEquals("TO_UPPER('casey')", mapOut.get("var1"));
+  }
+
+  @Test
+  public void testVars2MapEmpty() {
+    Object out = run("SHELL_VARS2MAP()", new HashMap<>(), context);
+    Map<String, String> mapOut = (Map<String, String>)out;
+    Assert.assertEquals(0, mapOut.size());
+  }
+
+  @Test
+  public void testGetExpression() {
+    Object out = run("SHELL_GET_EXPRESSION('var1')", new HashMap<>(), context);
+    Assert.assertTrue(out instanceof String);
+    String expression = (String)out;
+    //second one is null, so we don't want it there.
+    Assert.assertEquals("TO_UPPER('casey')", expression);
+  }
+
+  @Test
+  public void testGetExpressionEmpty() {
+    Object out = run("SHELL_GET_EXPRESSION()", new HashMap<>(), context);
+    Assert.assertNull(out );
+  }
+
+  @Test
+  public void testEdit() throws Exception {
+    System.getProperties().put("EDITOR", "/bin/cat");
+    Object out = run("TO_UPPER(SHELL_EDIT(foo))", ImmutableMap.of("foo", 
"foo"), context);
+    Assert.assertEquals("FOO", out);
+  }
+
+}

Reply via email to