METRON-1382 Run Stellar in a Zeppelin Notebook (nickwallen) closes apache/metron#884
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2dd01b17 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2dd01b17 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2dd01b17 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 2dd01b17782af7756776b8e07afe3661ac19bb0a Parents: 55a7379 Author: nickwallen <[email protected]> Authored: Fri Jan 12 13:08:32 2018 -0500 Committer: nickallen <[email protected]> Committed: Fri Jan 12 13:08:32 2018 -0500 ---------------------------------------------------------------------- dependencies_with_url.csv | 16 + .../metron-common/src/main/scripts/stellar | 2 +- .../metron/management/ShellFunctions.java | 57 +- .../management/ConfigurationFunctionsTest.java | 1 - .../EnrichmentConfigFunctionsTest.java | 24 +- .../management/IndexingConfigFunctionsTest.java | 10 +- .../management/ParserConfigFunctionsTest.java | 10 +- .../metron/management/ShellFunctionsTest.java | 22 +- .../management/ThreatTriageFunctionsTest.java | 26 +- metron-stellar/pom.xml | 1 + metron-stellar/stellar-common/README.md | 35 +- metron-stellar/stellar-common/pom.xml | 2 - .../stellar/common/StellarAssignment.java | 4 +- .../shell/DefaultStellarAutoCompleter.java | 220 ++++++++ .../shell/DefaultStellarShellExecutor.java | 413 ++++++++++++++ .../stellar/common/shell/PausableInput.java | 372 ------------- .../common/shell/StellarAutoCompleter.java | 45 ++ .../common/shell/StellarExecutionListeners.java | 51 ++ .../common/shell/StellarExecutionNotifier.java | 44 ++ .../stellar/common/shell/StellarExecutor.java | 327 ----------- .../stellar/common/shell/StellarResult.java | 195 +++++++ .../stellar/common/shell/StellarShell.java | 557 ------------------- .../common/shell/StellarShellExecutor.java | 79 +++ .../shell/StellarShellOptionsValidator.java | 127 ----- .../stellar/common/shell/VariableResult.java | 101 ++++ .../stellar/common/shell/cli/PausableInput.java | 372 +++++++++++++ .../stellar/common/shell/cli/StellarShell.java | 427 ++++++++++++++ .../shell/cli/StellarShellOptionsValidator.java | 127 +++++ .../shell/specials/AssignmentCommand.java | 85 +++ .../stellar/common/shell/specials/Comment.java | 53 ++ .../common/shell/specials/DocCommand.java | 105 ++++ .../shell/specials/MagicDefineGlobal.java | 86 +++ .../shell/specials/MagicListFunctions.java | 72 +++ .../common/shell/specials/MagicListGlobals.java | 54 ++ .../shell/specials/MagicListVariables.java | 88 +++ .../shell/specials/MagicUndefineGlobal.java | 70 +++ .../common/shell/specials/QuitCommand.java | 51 ++ .../common/shell/specials/SpecialCommand.java | 60 ++ .../org/apache/metron/stellar/dsl/Context.java | 12 +- .../stellar-common/src/main/scripts/stellar | 2 +- .../stellar/common/StellarCompilerTest.java | 169 ++++++ .../stellar/common/StellarInterpreterTest.java | 169 ------ .../shell/DefaultStellarAutoCompleterTest.java | 190 +++++++ .../shell/DefaultStellarShellExecutorTest.java | 298 ++++++++++ .../stellar/common/shell/StellarResultTest.java | 165 ++++++ .../shell/StellarShellOptionsValidatorTest.java | 187 ------- .../cli/StellarShellOptionsValidatorTest.java | 184 ++++++ .../common/shell/cli/StellarShellTest.java | 199 +++++++ .../shell/specials/AssignmentCommandTest.java | 212 +++++++ .../common/shell/specials/CommentTest.java | 91 +++ .../common/shell/specials/DocCommandTest.java | 86 +++ .../shell/specials/MagicDefineGlobalTest.java | 140 +++++ .../shell/specials/MagicListFunctionsTest.java | 136 +++++ .../shell/specials/MagicListGlobalsTest.java | 113 ++++ .../shell/specials/MagicListVariablesTest.java | 113 ++++ .../shell/specials/MagicUndefineGlobalTest.java | 117 ++++ metron-stellar/stellar-zeppelin/README.md | 160 ++++++ metron-stellar/stellar-zeppelin/pom.xml | 95 ++++ .../src/main/assembly/assembly.xml | 30 + .../stellar/zeppelin/StellarInterpreter.java | 200 +++++++ .../src/main/resources/interpreter-setting.json | 12 + .../zeppelin/StellarInterpreterTest.java | 192 +++++++ 62 files changed, 5835 insertions(+), 1828 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/dependencies_with_url.csv ---------------------------------------------------------------------- diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index b94a984..a1f431b 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -341,3 +341,19 @@ org.eclipse.persistence:org.eclipse.persistence.asm:jar:2.6.4:compile,EPL 1.0,ht org.eclipse.persistence:org.eclipse.persistence.core:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink org.eclipse.persistence:org.eclipse.persistence.jpa.jpql:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink org.eclipse.persistence:org.eclipse.persistence.jpa:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink + +com.google.code.gson:gson:jar:2.2:compile + org.codehaus.plexus:plexus-classworlds:jar:2.4:compile + org.codehaus.plexus:plexus-component-annotations:jar:1.5.5:compile + org.codehaus.plexus:plexus-interpolation:jar:1.14:compile + org.codehaus.plexus:plexus-utils:jar:2.0.7:compile + org.jsoup:jsoup:jar:1.6.1:compile + org.sonatype.aether:aether-api:jar:1.12:compile + org.sonatype.aether:aether-connector-file:jar:1.12:compile + org.sonatype.aether:aether-connector-wagon:jar:1.12:compile + org.sonatype.aether:aether-impl:jar:1.12:compile + org.sonatype.aether:aether-spi:jar:1.12:compile + org.sonatype.aether:aether-util:jar:1.12:compile + org.sonatype.sisu:sisu-guice:jar:no_aop:3.0.2:compile + org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile + org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-common/src/main/scripts/stellar ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar index 3f53c49..431ed7b 100644 --- a/metron-platform/metron-common/src/main/scripts/stellar +++ b/metron-platform/metron-common/src/main/scripts/stellar @@ -33,4 +33,4 @@ export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar) export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar) -java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.StellarShell "$@" +java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@" http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java index 52ff157..afac7f0 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java @@ -17,9 +17,20 @@ */ package org.apache.metron.management; -import static org.apache.metron.stellar.common.shell.StellarExecutor.CONSOLE; - import com.jakewharton.fliptables.FlipTable; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.text.WordUtils; +import org.apache.metron.stellar.common.shell.VariableResult; +import org.apache.metron.stellar.common.shell.cli.PausableInput; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.jboss.aesh.console.Console; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -31,22 +42,16 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.text.WordUtils; -import org.apache.metron.stellar.common.shell.PausableInput; -import org.apache.metron.stellar.common.shell.StellarExecutor; -import org.apache.metron.stellar.common.utils.ConversionUtils; -import org.apache.metron.stellar.dsl.BaseStellarFunction; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.ParseException; -import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.stellar.dsl.StellarFunction; -import org.jboss.aesh.console.Console; -import org.slf4j.LoggerFactory; + +import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE; public class ShellFunctions { private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static Map<String, VariableResult> getVariables(Context context) { + return (Map<String, VariableResult>) context.getCapability(Context.Capabilities.SHELL_VARIABLES).get(); + } + @Stellar( namespace = "SHELL" ,name = "MAP2TABLE" @@ -90,7 +95,7 @@ public class ShellFunctions { @Override public Object apply(List<Object> args, Context context) throws ParseException { - Map<String, StellarExecutor.VariableResult> variables = (Map<String, StellarExecutor.VariableResult>) context.getCapability(StellarExecutor.SHELL_VARIABLES).get(); + Map<String, VariableResult> variables = getVariables(context); String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"}; String[][] data = new String[variables.size()][3]; int wordWrap = -1; @@ -98,11 +103,11 @@ public class ShellFunctions { wordWrap = ConversionUtils.convert(args.get(0), Integer.class); } int i = 0; - for(Map.Entry<String, StellarExecutor.VariableResult> kv : variables.entrySet()) { - StellarExecutor.VariableResult result = kv.getValue(); + for(Map.Entry<String, VariableResult> kv : variables.entrySet()) { + VariableResult result = kv.getValue(); data[i++] = new String[] { toWrappedString(kv.getKey().toString(), wordWrap) , toWrappedString(result.getResult(), wordWrap) - , toWrappedString(result.getExpression(), wordWrap) + , toWrappedString(result.getExpression().get(), wordWrap) }; } return FlipTable.of(headers, data); @@ -139,16 +144,16 @@ public class ShellFunctions { @Override public Object apply(List<Object> args, Context context) throws ParseException { - Map<String, StellarExecutor.VariableResult> variables = (Map<String, StellarExecutor.VariableResult>) context.getCapability(StellarExecutor.SHELL_VARIABLES).get(); + Map<String, VariableResult> variables = getVariables(context); LinkedHashMap<String, String> ret = new LinkedHashMap<>(); for(Object arg : args) { if(arg == null) { continue; } String variable = (String)arg; - StellarExecutor.VariableResult result = variables.get(variable); - if(result != null && result.getExpression() != null) { - ret.put(variable, result.getExpression()); + VariableResult result = variables.get(variable); + if(result != null && result.getExpression().isPresent()) { + ret.put(variable, result.getExpression().orElseGet(() -> "")); } } return ret; @@ -177,7 +182,7 @@ public class ShellFunctions { @Override public Object apply(List<Object> args, Context context) throws ParseException { - Map<String, StellarExecutor.VariableResult> variables = (Map<String, StellarExecutor.VariableResult>) context.getCapability(StellarExecutor.SHELL_VARIABLES).get(); + Map<String, VariableResult> variables = getVariables(context); if(args.size() == 0) { return null; } @@ -185,9 +190,9 @@ public class ShellFunctions { if(variable == null) { return null; } - StellarExecutor.VariableResult result = variables.get(variable); - if(result != null && result.getExpression() != null) { - return result.getExpression(); + VariableResult result = variables.get(variable); + if(result != null && result.getExpression().isPresent()) { + return result.getExpression().get(); } return null; } http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java index 31eeafe..1920031 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java @@ -31,7 +31,6 @@ import org.apache.metron.test.utils.UnitTestHelper; import org.json.simple.parser.JSONParser; import org.json.simple.JSONObject; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java index 51a24b3..d552aa1 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java @@ -23,7 +23,7 @@ import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.stellar.common.StellarProcessor; -import org.apache.metron.stellar.common.shell.StellarExecutor; +import org.apache.metron.stellar.common.shell.VariableResult; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.DefaultVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; @@ -44,7 +44,7 @@ import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMEN public class EnrichmentConfigFunctionsTest { String configStr = emptyTransformationsConfig(); - Map<String, StellarExecutor.VariableResult> variables; + Map<String, VariableResult> variables; Context context = null; String enrichmentType = null; String group = null; @@ -73,17 +73,15 @@ public class EnrichmentConfigFunctionsTest { }); } - - @Before public void setup() { variables = ImmutableMap.of( - "upper", new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"), - "lower", new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo") + "upper", VariableResult.withExpression("FOO", "TO_UPPER('foo')"), + "lower", VariableResult.withExpression("foo", "TO_LOWER('FOO')") ); context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES, () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES, () -> variables) .build(); } @@ -151,7 +149,7 @@ public class EnrichmentConfigFunctionsTest { ); Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig)); Assert.assertEquals(1, size(stellarFunctions)); - Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper")); + Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper")); } @Test @@ -173,8 +171,8 @@ public class EnrichmentConfigFunctionsTest { ); Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig)); Assert.assertEquals(2, size(stellarFunctions)); - Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper")); - Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower")); + Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper")); + Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower")); } @Test @@ -208,7 +206,7 @@ public class EnrichmentConfigFunctionsTest { ); Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig)); Assert.assertEquals(1, size(stellarFunctions)); - Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper")); + Assert.assertEquals(variables.get("upper").getExpression().get(), get(stellarFunctions,"upper")); } @Test @@ -229,7 +227,7 @@ public class EnrichmentConfigFunctionsTest { ); Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig)); Assert.assertEquals(1, size(stellarFunctions)); - Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower")); + Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower")); } @Test @@ -270,7 +268,7 @@ public class EnrichmentConfigFunctionsTest { ); Map<String, Object> stellarFunctions = getStellarMappings(getEnrichmentConfig(newConfig)); Assert.assertEquals(1, size(stellarFunctions)); - Assert.assertEquals(variables.get("lower").getExpression(), get(stellarFunctions,"lower")); + Assert.assertEquals(variables.get("lower").getExpression().get(), get(stellarFunctions,"lower")); } /** http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java index 29c80b8..c931df5 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java @@ -19,11 +19,11 @@ package org.apache.metron.management; import com.google.common.collect.ImmutableMap; import org.apache.metron.common.configuration.IndexingConfigurations; +import org.apache.metron.stellar.common.shell.VariableResult; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.DefaultVariableResolver; import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.metron.stellar.common.StellarProcessor; -import org.apache.metron.stellar.common.shell.StellarExecutor; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -36,7 +36,7 @@ import static org.apache.metron.management.EnrichmentConfigFunctionsTest.toMap; public class IndexingConfigFunctionsTest { - Map<String, StellarExecutor.VariableResult> variables; + Map<String, VariableResult> variables; Context context = null; private Object run(String rule, Map<String, Object> variables) { @@ -47,12 +47,12 @@ public class IndexingConfigFunctionsTest { @Before public void setup() { variables = ImmutableMap.of( - "upper", new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"), - "lower", new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo") + "upper", VariableResult.withExpression("FOO", "TO_UPPER('foo')"), + "lower", VariableResult.withExpression("foo", "TO_LOWER('FOO')") ); context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES, () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES, () -> variables) .build(); } http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java index f830e62..e27a7ad 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java @@ -21,8 +21,8 @@ import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.stellar.common.shell.VariableResult; import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.common.shell.StellarExecutor; import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Before; @@ -40,17 +40,17 @@ public class ParserConfigFunctionsTest { String emptyTransformationsConfig = slurp(PARSER_CONFIGS_PATH + "/parsers/bro.json"); String existingTransformationsConfig = slurp(PARSER_CONFIGS_PATH + "/parsers/squid.json"); - Map<String, StellarExecutor.VariableResult> variables ; + Map<String, VariableResult> variables ; Context context = null; @Before public void setup() { variables = ImmutableMap.of( - "upper" , new StellarExecutor.VariableResult("TO_UPPER('foo')", "FOO"), - "lower" , new StellarExecutor.VariableResult("TO_LOWER('FOO')", "foo") + "upper" , VariableResult.withExpression("FOO", "TO_UPPER('foo')"), + "lower" , VariableResult.withExpression("foo", "TO_LOWER('FOO'") ); context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES , () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES, () -> variables) .build(); } http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java index 401454e..83c2bce 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java @@ -19,27 +19,28 @@ package org.apache.metron.management; import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.stellar.common.shell.VariableResult; import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.common.shell.StellarExecutor; import org.junit.Assert; import org.junit.Test; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; public class ShellFunctionsTest { - Map<String, StellarExecutor.VariableResult> variables = ImmutableMap.of( - "var1" , new StellarExecutor.VariableResult("TO_UPPER('casey')", "CASEY"), - "var2" , new StellarExecutor.VariableResult(null, "foo"), - "var3" , new StellarExecutor.VariableResult(null, null), - "var4" , new StellarExecutor.VariableResult("blah", null) + Map<String, VariableResult> variables = ImmutableMap.of( + "var1" , VariableResult.withExpression("CASEY", "TO_UPPER('casey')"), + "var2" , VariableResult.withValue("foo"), + "var3" , VariableResult.withValue(null), + "var4" , VariableResult.withExpression(null, "blah") ); Context context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES , () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES , () -> variables) .build(); /** ââââââââââââ¤ââââââââ¤âââââââââââââ @@ -53,10 +54,11 @@ public class ShellFunctionsTest { @Test public void testListVarsWithVars() { - Map<String, StellarExecutor.VariableResult> variables = ImmutableMap.of("foo", new StellarExecutor.VariableResult("1 + 1", 2.0)); + Map<String, VariableResult> variables = ImmutableMap.of( + "foo", VariableResult.withExpression(2.0, "1 + 1")); Context context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES , () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES , () -> variables) .build(); Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context); Assert.assertEquals(expectedListWithFoo, out); @@ -75,7 +77,7 @@ public class ShellFunctionsTest { @Test public void testListVarsWithoutVars() { Context context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES , () -> new HashMap<>()) + .with(Context.Capabilities.SHELL_VARIABLES, () -> new HashMap<>()) .build(); Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context); Assert.assertEquals(expectedEmptyList, out); http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java index 2b154d8..e281038 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java @@ -22,7 +22,7 @@ import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.RiskLevelRule; import org.apache.metron.stellar.common.StellarProcessor; -import org.apache.metron.stellar.common.shell.StellarExecutor; +import org.apache.metron.stellar.common.shell.VariableResult; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.DefaultVariableResolver; import org.apache.metron.stellar.dsl.MapVariableResolver; @@ -44,18 +44,18 @@ import static org.apache.metron.management.EnrichmentConfigFunctionsTest.toMap; public class ThreatTriageFunctionsTest { String configStr = emptyTransformationsConfig(); - Map<String, StellarExecutor.VariableResult> variables; + Map<String, VariableResult> variables; Context context = null; @Before public void setup() { variables = ImmutableMap.of( - "less", new StellarExecutor.VariableResult("1 < 2", true), - "greater", new StellarExecutor.VariableResult("1 > 2", false) + "less", VariableResult.withExpression(true, "1 < 2"), + "greater", VariableResult.withExpression(false, "1 > 2") ); context = new Context.Builder() - .with(StellarExecutor.SHELL_VARIABLES, () -> variables) + .with(Context.Capabilities.SHELL_VARIABLES, () -> variables) .build(); } @@ -127,7 +127,7 @@ public class ThreatTriageFunctionsTest { List<RiskLevelRule> triageRules = getTriageRules(newConfig); Assert.assertEquals(1, triageRules.size()); RiskLevelRule rule = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() ); Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 ); } @@ -165,11 +165,11 @@ public class ThreatTriageFunctionsTest { List<RiskLevelRule> triageRules = getTriageRules(newConfig); Assert.assertEquals(2, triageRules.size()); RiskLevelRule less = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), less.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), less.getRule() ); Assert.assertEquals(10.0, less.getScore().doubleValue(), 1e-6 ); RiskLevelRule greater = triageRules.get(1); - Assert.assertEquals(variables.get("greater").getExpression(), greater.getRule() ); + Assert.assertEquals(variables.get("greater").getExpression().get(), greater.getRule() ); Assert.assertEquals(20.0, greater.getScore().doubleValue(), 1e-6 ); } @@ -200,7 +200,7 @@ public class ThreatTriageFunctionsTest { List<RiskLevelRule> triageRules = getTriageRules(newConfig); Assert.assertEquals(1, triageRules.size()); RiskLevelRule rule = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() ); Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 ); } @@ -256,7 +256,7 @@ public class ThreatTriageFunctionsTest { List<RiskLevelRule> triageRules = getTriageRules(newConfig); Assert.assertEquals(1, triageRules.size()); RiskLevelRule rule = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() ); Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 ); } @@ -281,7 +281,7 @@ public class ThreatTriageFunctionsTest { List<RiskLevelRule> triageRules = engine.getRiskLevelRules(); Assert.assertEquals(1, triageRules.size()); RiskLevelRule rule = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), rule.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), rule.getRule() ); Assert.assertEquals(10.0, rule.getScore().doubleValue(), 1e-6 ); } @@ -320,11 +320,11 @@ public class ThreatTriageFunctionsTest { List<RiskLevelRule> triageRules = getTriageRules(newConfig); Assert.assertEquals(2, triageRules.size()); RiskLevelRule less = triageRules.get(0); - Assert.assertEquals(variables.get("less").getExpression(), less.getRule() ); + Assert.assertEquals(variables.get("less").getExpression().get(), less.getRule() ); Assert.assertEquals(10.0, less.getScore().doubleValue(), 1e-6 ); RiskLevelRule greater = triageRules.get(1); - Assert.assertEquals(variables.get("greater").getExpression(), greater.getRule() ); + Assert.assertEquals(variables.get("greater").getExpression().get(), greater.getRule() ); Assert.assertEquals(20.0, greater.getScore().doubleValue(), 1e-6 ); } http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/pom.xml ---------------------------------------------------------------------- diff --git a/metron-stellar/pom.xml b/metron-stellar/pom.xml index c2181fd..6b5cb67 100644 --- a/metron-stellar/pom.xml +++ b/metron-stellar/pom.xml @@ -43,6 +43,7 @@ <modules> <module>stellar-3rd-party-example</module> <module>stellar-common</module> + <module>stellar-zeppelin</module> </modules> <dependencies> <dependency> http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/README.md ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index 4796889..076f250 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -29,6 +29,7 @@ For a variety of components (threat intelligence triage and field transformation * [Variable Assignment](#variable-assignment) * [Magic Commands](#magic-commands) * [Advanced Usage](#advanced-usage) + * [Implementation](#implementation) * [Stellar Configuration](#stellar-configuration) @@ -1387,7 +1388,7 @@ IS_EMAIL To run the Stellar Shell directly from the Metron source code, run a command like the following. Ensure that Metron has already been built and installed with `mvn clean install -DskipTests`. ``` $ mvn exec:java \ - -Dexec.mainClass="org.apache.metron.stellar.common.shell.StellarShell" \ + -Dexec.mainClass="org.apache.metron.stellar.common.shell.cli.StellarShell" \ -pl metron-platform/metron-enrichment ... Stellar, Go! @@ -1403,7 +1404,7 @@ This can be useful for troubleshooting function resolution problems. The previo ``` $ mvn exec:java \ - -Dexec.mainClass="org.apache.metron.stellar.common.shell.StellarShell" \ + -Dexec.mainClass="org.apache.metron.stellar.common.shell.cli.StellarShell" \ -pl metron-analytics/metron-profiler ... Stellar, Go! @@ -1413,6 +1414,36 @@ Please note that functions are loading lazily in the background and will be unav ABS, APPEND_IF_MISSING, BIN, BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, CHOMP, CHOP, COUNT_MATCHES, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENDS_WITH, FILL_LEFT, FILL_RIGHT, FILTER, FORMAT, GET, GET_FIRST, GET_LAST, HLLP_ADD, HLLP_CARDINALITY, HLLP_INIT, HLLP_MERGE, IN_SUBNET, IS_DATE, IS_DOMAIN, IS_EMAIL, IS_EMPTY, IS_INTEGER, IS_IP, IS_URL, JOIN, LENGTH, LIST_ADD, MAAS_GET_ENDPOINT, MAAS_MODEL_APPLY, MAP, MAP_EXISTS, MAP_GET, MONTH, OUTLIER_MAD_ADD, OUTLIER_MAD_SCORE, OUTLIER_MAD_STATE_MERGE, PREPEND_IF_MISSING, PROFILE_FIXED, PROFILE_GET, PROFILE_WINDOW, PROTOCOL_TO_NAME, REDUCE, REGEXP_MATCH, SPLIT, STARTS_WITH, STATS_ADD, STATS_BIN, STATS_COUNT, STATS_GEOMETRIC_MEAN, STATS_INIT, STATS_KURTOSIS, STATS_MAX, STATS_MEAN, STATS_MERGE, STATS_MIN, STATS_PERCENTILE, STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS, STATS_SUM, STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, STRING_ENTROPY, SYS TEM_ENV_GET, SYSTEM_PROPERTY_GET, TO_DOUBLE, TO_EPOCH_TIMESTAMP, TO_FLOAT, TO_INTEGER, TO_LONG, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH, URL_TO_PORT, URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR ``` +### Implementation + +The Stellar Shell can be executed both from the command line and from within a Stellar Notebook. The behavior and underlying implementation of the behavior is exactly the same across these two environments. + +#### `org.apache.metron.stellar.common.shell` + +This package contains classes that are reused across both the CLI and Zeppelin shell environments. + +* `StellarShellExecutor` Executes Stellar in a shell-like environment. Provides the Stellar language extensions like variable assignment, comments, magics, and doc strings that are only accessible in the shell. + +* `StellarAutoCompleter` Handles auto-completion for Stellar. + +* `StellarExecutorListeners` An event listener that can be notified when variables, functions, and specials are defined. This is how a `StellarAutoCompleter` is notified throughout the life of a shell session. + +#### `org.apache.metron.stellar.common.shell.specials` + +All Stellar language extensions are contained within this package. + +* `SpecialCommand` The interface for all special commands. A 'special command' is anything that is not directly provided by the Stellar language itself. That includes variable assignment, comments, doc strings, magics, and quit. + +#### `org.apache.metron.stellar.common.shell.cli` + +This package contains classes that are specific to the CLI-driven REPL. + +* `StellarShell` This is the main class that drives the CLI REPL. All functionality not related to the command line interface is performed by the shared logic in `org.apache.metron.stellar.common.shell`. + +#### `org.apache.metron.stellar.zeppelin` + +This package is contained within the `stellar-zeppelin` project and performs all logic for interfacing with Zeppelin. Again, all functionality not related to Zeppelin is performed by the shared logic in `org.apache.metron.stellar.common.shell`. + ## Stellar Configuration Stellar can be configured in a variety of ways from the [Global Configuration](../../metron-platform/metron-common/README.md#global-configuration). http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/pom.xml ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml index fbb1a05..a5dd20b 100644 --- a/metron-stellar/stellar-common/pom.xml +++ b/metron-stellar/stellar-common/pom.xml @@ -29,7 +29,6 @@ <commons.config.version>1.10</commons.config.version> </properties> <dependencies> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-auth</artifactId> @@ -240,7 +239,6 @@ </execution> </executions> </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java index e77211a..587b558 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java @@ -38,7 +38,9 @@ public class StellarAssignment implements Map.Entry<String, Object>{ } public static boolean isAssignment(String statement) { - return statement != null && statement.contains(":="); + return statement != null && + statement.contains(":=") && // has the assignment operator + !statement.trim().startsWith("%"); // not a magic like %define x := 2 } public static StellarAssignment from(String statement) { http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java new file mode 100644 index 0000000..cf52955 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarAutoCompleter.java @@ -0,0 +1,220 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import org.apache.commons.collections4.IterableUtils; +import org.apache.commons.collections4.trie.PatriciaTrie; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.SortedMap; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + + +/** + * Provides auto-completion for Stellar. + */ +public class DefaultStellarAutoCompleter implements StellarAutoCompleter { + + enum OperationType { + DOC, + MAGIC, + NORMAL + } + + enum AutoCompleteType implements AutoCompleteTransformation { + FUNCTION((type, key) -> { + if(OperationType.DOC == type) { + return "?" + key; + + } else if(OperationType.NORMAL == type) { + return key + "("; + } + + return key; + }), + VARIABLE((type, key) -> key), + TOKEN((type, key) -> key); + + AutoCompleteTransformation transform; + + AutoCompleteType(AutoCompleteTransformation transform) { + this.transform = transform; + } + + @Override + public String transform(OperationType type, String key) { + return transform.transform(type, key); + } + } + + /** + * Prefix tree index of auto-completes. + */ + private PatriciaTrie<AutoCompleteType> autocompleteIndex; + + private ReadWriteLock indexLock = new ReentrantReadWriteLock(); + + public interface AutoCompleteTransformation { + String transform(OperationType type, String key); + } + + public DefaultStellarAutoCompleter() { + this.autocompleteIndex = initializeIndex(); + } + + @Override + public Iterable<String> autoComplete(String buffer) { + Iterable<String> candidates = IterableUtils.emptyIterable(); + + final String lastToken = getLastToken(buffer); + if(StringUtils.isNotEmpty(lastToken)) { + + if (isDoc(lastToken)) { + candidates = autoCompleteDoc(lastToken.substring(1)); + + } else if (isMagic(lastToken)) { + candidates = autoCompleteMagic(lastToken); + + } else { + candidates = autoCompleteNormal(lastToken); + } + } + + return candidates; + } + + /** + * Is a given expression a built-in magic? + * @param expression The expression. + */ + private boolean isMagic(String expression) { + return StringUtils.startsWith(expression, "%"); + } + + /** + * Is a given expression asking for function documentation? + * @param expression The expression. + */ + private boolean isDoc(String expression) { + return StringUtils.startsWith(expression, "?"); + } + + /** + * Auto-completes a partial Stellar expression + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + private Iterable<String> autoCompleteNormal(String buffer) { + return autoComplete(buffer, OperationType.NORMAL); + } + + /** + * Auto-completes a partial doc command. + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + private Iterable<String> autoCompleteDoc(String buffer) { + return autoComplete(buffer, OperationType.DOC); + } + + /** + * Auto-completes a partial magic commands. + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + private Iterable<String> autoCompleteMagic(String buffer) { + return autoComplete(buffer, OperationType.MAGIC); + } + + /** + * Returns a list of viable candidates for auto-completion. + * @param buffer The current buffer. + * @param opType The type of operation needing auto-completion. + * @return Viable candidates for auto-completion. + */ + private Iterable<String> autoComplete(String buffer, final OperationType opType) { + indexLock.readLock().lock(); + try { + SortedMap<String, AutoCompleteType> ret = autocompleteIndex.prefixMap(buffer); + if (ret.isEmpty()) { + return new ArrayList<>(); + } + return Iterables.transform(ret.entrySet(), kv -> kv.getValue().transform(opType, kv.getKey())); + } + finally { + indexLock.readLock().unlock(); + } + } + + /** + * Adds a candidate for auto-completing function names. + * @param name The name of the function candidate. + */ + @Override + public void addCandidateFunction(String name) { + add(name, AutoCompleteType.FUNCTION); + } + + /** + * Adds a candidate for auto-completing variable names. + * @param name The name of the function candidate. + */ + @Override + public void addCandidateVariable(String name) { + add(name, AutoCompleteType.VARIABLE); + } + + /** + * Add a candidate for auto-completion. + * @param name The name of the candidate. + * @param type The type of candidate. + */ + private void add(String name, AutoCompleteType type) { + if(StringUtils.isNotBlank(name)) { + // add the candidate to the auto-complete index + indexLock.writeLock().lock(); + try { + this.autocompleteIndex.put(name, type); + } finally { + indexLock.writeLock().unlock(); + } + } + } + + private PatriciaTrie<AutoCompleteType> initializeIndex() { + Map<String, AutoCompleteType> index = new HashMap<>(); + index.put("==", AutoCompleteType.TOKEN); + index.put(">=", AutoCompleteType.TOKEN); + index.put("<=", AutoCompleteType.TOKEN); + + return new PatriciaTrie<>(index); + } + + private static String getLastToken(String buffer) { + String lastToken = Iterables.getLast(Splitter.on(" ").split(buffer), null); + return lastToken.trim(); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java new file mode 100644 index 0000000..f83bb9e --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java @@ -0,0 +1,413 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Maps; +import org.apache.commons.collections.map.UnmodifiableMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.configuration.ConfigurationsUtils; +import org.apache.metron.stellar.common.shell.StellarExecutionListeners.FunctionDefinedListener; +import org.apache.metron.stellar.common.shell.StellarExecutionListeners.SpecialDefinedListener; +import org.apache.metron.stellar.common.shell.StellarExecutionListeners.VariableDefinedListener; +import org.apache.metron.stellar.common.shell.specials.AssignmentCommand; +import org.apache.metron.stellar.common.shell.specials.Comment; +import org.apache.metron.stellar.common.shell.specials.DocCommand; +import org.apache.metron.stellar.common.shell.specials.MagicDefineGlobal; +import org.apache.metron.stellar.common.shell.specials.MagicListFunctions; +import org.apache.metron.stellar.common.shell.specials.MagicListGlobals; +import org.apache.metron.stellar.common.shell.specials.MagicListVariables; +import org.apache.metron.stellar.common.shell.specials.MagicUndefineGlobal; +import org.apache.metron.stellar.common.shell.specials.QuitCommand; +import org.apache.metron.stellar.common.shell.specials.SpecialCommand; +import org.apache.metron.stellar.common.utils.JSONUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.MapVariableResolver; +import org.apache.metron.stellar.dsl.StellarFunctionInfo; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +import static org.apache.metron.stellar.common.configuration.ConfigurationsUtils.readGlobalConfigBytesFromZookeeper; +import static org.apache.metron.stellar.common.shell.StellarResult.noop; +import static org.apache.metron.stellar.common.shell.StellarResult.error; +import static org.apache.metron.stellar.common.shell.StellarResult.success; +import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; +import static org.apache.metron.stellar.dsl.Context.Capabilities.STELLAR_CONFIG; +import static org.apache.metron.stellar.dsl.Context.Capabilities.ZOOKEEPER_CLIENT; + +/** + * Default implementation of a StellarShellExecutor. + */ +public class DefaultStellarShellExecutor implements StellarShellExecutor { + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String SHELL_VARIABLES = "shellVariables"; + + /** + * The variables known by Stellar. + */ + private Map<String, VariableResult> variables; + + /** + * The function resolver. + */ + private FunctionResolver functionResolver; + + /** + * A Zookeeper client. Only defined if given a valid Zookeeper URL. + */ + private Optional<CuratorFramework> zkClient; + + /** + * A registry of all special commands; like %magic, ?doc, and quit. + */ + private List<SpecialCommand> specials; + + /** + * The Stellar execution context. + */ + private Context context; + + /** + * Listeners that are notified when a function is defined. + */ + private List<FunctionDefinedListener> functionListeners; + + /** + * Listeners that are notified when a variable is defined. + */ + private List<VariableDefinedListener> variableListeners; + + /** + * Listeners that are notified when a special command is defined. + */ + private List<SpecialDefinedListener> specialListeners; + + + public DefaultStellarShellExecutor( + FunctionResolver functionResolver, + Properties properties, + Optional<String> zookeeperUrl, + List<SpecialCommand> specials) throws Exception { + + this.functionListeners = new ArrayList<>(); + this.variableListeners = new ArrayList<>(); + this.specialListeners = new ArrayList<>(); + this.variables = new HashMap<>(); + this.zkClient = createZookeeperClient(zookeeperUrl); + this.context = createContext(properties, this.zkClient); + this.functionResolver = functionResolver; + this.specials = specials; + } + + public DefaultStellarShellExecutor( + FunctionResolver functionResolver, + Properties properties, + Optional<String> zookeeperUrl) throws Exception { + + this(functionResolver, properties, zookeeperUrl, defaultSpecials()); + } + + public DefaultStellarShellExecutor( + Properties properties, + Optional<String> zookeeperUrl) throws Exception { + + this(StellarFunctions.FUNCTION_RESOLVER(), properties, zookeeperUrl); + } + + /** + * The default specials that will be made available, if none are specified otherwise. + * @return The default special commands. + */ + public static List<SpecialCommand> defaultSpecials() { + return Arrays.asList( + new AssignmentCommand(), + new DocCommand(), + new QuitCommand(), + new Comment(), + new MagicListFunctions(), + new MagicListVariables(), + new MagicDefineGlobal(), + new MagicUndefineGlobal(), + new MagicListGlobals() + ); + } + + @Override + public void init() { + StellarFunctions.initialize(this.context); + + // notify listeners about the available specials + for(SpecialCommand command : specials) { + notifySpecialListeners(command); + } + + // notify listeners about the available functions + for(StellarFunctionInfo fn : functionResolver.getFunctionInfo()) { + notifyFunctionListeners(fn); + } + } + + /** + * Add a listener that will be notified when a function is defined. + * @param listener The listener to notify. + */ + @Override + public void addFunctionListener(FunctionDefinedListener listener) { + this.functionListeners.add(listener); + } + + /** + * Notify function listeners that a function has been defined. + * @param functionInfo The function that was defined. + */ + private void notifyFunctionListeners(StellarFunctionInfo functionInfo) { + for(FunctionDefinedListener listener : functionListeners) { + listener.whenFunctionDefined(functionInfo); + } + } + + /** + * Add a listener that will be notified when a variable is defined. + * @param listener The listener to notify. + */ + @Override + public void addVariableListener(VariableDefinedListener listener) { + this.variableListeners.add(listener); + } + + /** + * Notify variable listeners that a variable has been (re)defined. + * @param variableName The variable name. + * @param result The variable result. + */ + private void notifyVariableListeners(String variableName, VariableResult result) { + for(VariableDefinedListener listener : variableListeners) { + listener.whenVariableDefined(variableName, result); + } + } + + /** + * Add a listener that will be notified when a magic command is defined. + * @param listener The listener to notify. + */ + @Override + public void addSpecialListener(SpecialDefinedListener listener) { + this.specialListeners.add(listener); + } + + /** + * Notify listeners that a magic command has been defined. + * @param specialCommand The magic command. + */ + private void notifySpecialListeners(SpecialCommand specialCommand) { + for(SpecialDefinedListener listener : specialListeners) { + listener.whenSpecialDefined(specialCommand); + } + } + + @Override + public StellarResult execute(String expression) { + + // if only whitespace, there is nothing to do + expression = StringUtils.trimToEmpty(expression); + if(StringUtils.isBlank(expression)) { + return noop(); + } + + // is this a special command? + for(SpecialCommand command : specials) { + if(command.getMatcher().apply(expression)) { + return command.execute(expression, this); + } + } + + // otherwise, this must be a stellar expression + return executeStellar(expression); + } + + /** + * Retrieves the GLOBAL_CONFIG, if it exists. If it does not, it creates the GLOBAL_CONFIG + * and adds it to the Stellar execution context. + * + * @return The global configuration. + */ + @Override + public Map<String, Object> getGlobalConfig() { + Map<String, Object> globals; + Optional<Object> capability = getContext().getCapability(GLOBAL_CONFIG, false); + if (capability.isPresent()) { + globals = (Map<String, Object>) capability.get(); + + } else { + // if it does not exist, create it. this creates the global config for the current stellar executor + // session only. this does not change the global config maintained externally in zookeeper + globals = new HashMap<>(); + getContext().addCapability(GLOBAL_CONFIG, () -> globals); + } + + return globals; + } + + @Override + public void assign(String variableName, Object value, Optional<String> expression) { + + // perform the variable assignment + VariableResult varResult = VariableResult.withExpression(value, expression); + this.variables.put(variableName, varResult); + + // notify any listeners + notifyVariableListeners(variableName, varResult); + } + + @Override + public FunctionResolver getFunctionResolver() { + return functionResolver; + } + + @Override + public Map<String, VariableResult> getState() { + return UnmodifiableMap.decorate(variables); + } + + /** + * Returns all variables that have been defined. Unlike 'getState' this unwraps + * the VariableResult so that we have the actual value. + * + * @return All variables that have been defined. + */ + public Map<String, Object> getVariables() { + return Maps.transformValues(variables, (v) -> v.getResult()); + } + + @Override + public Context getContext() { + return context; + } + + /** + * Creates a Zookeeper client. + * @param zookeeperUrl The Zookeeper URL. + */ + private Optional<CuratorFramework> createZookeeperClient(Optional<String> zookeeperUrl) { + Optional<CuratorFramework> client = Optional.empty(); + + // can only create client, if have valid zookeeper URL + if(zookeeperUrl.isPresent()) { + String url = zookeeperUrl.get(); + if (StringUtils.isNotBlank(url)) { + + LOG.debug(String.format("Connecting to Zookeeper; url=%s", url)); + CuratorFramework c = ConfigurationsUtils.getClient(url); + c.start(); + client = Optional.of(c); + } + } + + return client; + } + + /** + * Creates a Context initialized with configuration stored in Zookeeper. + * @param properties Properties to configure the context. + * @param zkClient An optional Zookeeper client. + */ + private Context createContext(Properties properties, Optional<CuratorFramework> zkClient) throws Exception { + + Context.Builder contextBuilder = new Context.Builder() + .with(SHELL_VARIABLES, () -> variables) + .with(STELLAR_CONFIG, () -> properties); + + // load global configuration from zookeeper + if (zkClient.isPresent()) { + Map<String, Object> global = fetchGlobalConfig(zkClient.get()); + contextBuilder + .with(GLOBAL_CONFIG, () -> global) + .with(ZOOKEEPER_CLIENT, () -> zkClient.get()) + .with(STELLAR_CONFIG, () -> getStellarConfig(global, properties)); + } + + return contextBuilder.build(); + } + + /** + * Fetches the global configuration from Zookeeper. + * @param zkClient The Zookeeper client. + * @return The global configuration retrieved from Zookeeper. + * @throws Exception + */ + private Map<String, Object> fetchGlobalConfig(CuratorFramework zkClient) throws Exception { + byte[] raw = readGlobalConfigBytesFromZookeeper(zkClient); + return JSONUtils.INSTANCE.load( + new ByteArrayInputStream(raw), + new TypeReference<Map<String, Object>>() {}); + } + + /** + * @param globalConfig The global configuration. + * @param props Property values + * @return The Stellar configuration. + */ + private Map<String, Object> getStellarConfig(Map<String, Object> globalConfig, Properties props) { + Map<String, Object> stellarConfig = new HashMap<>(); + stellarConfig.putAll(globalConfig); + if(props != null) { + for (Map.Entry<Object, Object> kv : props.entrySet()) { + stellarConfig.put(kv.getKey().toString(), kv.getValue()); + } + } + return stellarConfig; + } + + /** + * Executes Stellar expressions. + * @param expression The expression to execute. + */ + private StellarResult executeStellar(String expression) { + StellarResult result; + + try { + // execute the stellar expression + VariableResolver variableResolver = new MapVariableResolver(getVariables()); + Object exprResult = new StellarProcessor().parse(expression, variableResolver, functionResolver, context); + result = success(exprResult); + + } catch (Throwable t) { + result = error(t); + } + + return result; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/PausableInput.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/PausableInput.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/PausableInput.java deleted file mode 100644 index 2acd058..0000000 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/PausableInput.java +++ /dev/null @@ -1,372 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.metron.stellar.common.shell; - -import java.io.IOException; -import java.io.InputStream; - -/** - * An input stream which mirrors System.in, but allows you to 'pause' and 'unpause' it. - * The Aeshell has an external thread which is constantly polling System.in. If you - * need to spawn a program externally (i.e. an editor) which shares stdin, this thread - * and the spawned program both share a buffer. This causes contention and unpredictable - * results (e.g. an input may be consumed by either the aeshell thread or the spawned program) - * - * Because you can inject an input stream into the console, we create this which can act as a - * facade to System.in under normal 'unpaused' circumstances, and when paused, turn off the - * access to System.in. This allows us to turn off access to aeshell while maintaining access - * to the external program. - * - */ -public class PausableInput extends InputStream { - InputStream in = System.in; - boolean paused = false; - private PausableInput() { - super(); - } - - /** - * Stop mirroring stdin - */ - public void pause() { - paused = true; - } - - /** - * Resume mirroring stdin. - * @throws IOException - */ - public void unpause() throws IOException { - in.read(new byte[in.available()]); - paused = false; - } - - public final static PausableInput INSTANCE = new PausableInput(); - - /** - * Reads the next byte of data from the input stream. The value byte is - * returned as an <code>int</code> in the range <code>0</code> to - * <code>255</code>. If no byte is available because the end of the stream - * has been reached, the value <code>-1</code> is returned. This method - * blocks until input data is available, the end of the stream is detected, - * or an exception is thrown. - * <p> - * <p> A subclass must provide an implementation of this method. - * - * @return the next byte of data, or <code>-1</code> if the end of the - * stream is reached. - * @throws IOException if an I/O error occurs. - */ - @Override - public int read() throws IOException { - - return in.read(); - } - - /** - * Reads some number of bytes from the input stream and stores them into - * the buffer array <code>b</code>. The number of bytes actually read is - * returned as an integer. This method blocks until input data is - * available, end of file is detected, or an exception is thrown. - * <p> - * <p> If the length of <code>b</code> is zero, then no bytes are read and - * <code>0</code> is returned; otherwise, there is an attempt to read at - * least one byte. If no byte is available because the stream is at the - * end of the file, the value <code>-1</code> is returned; otherwise, at - * least one byte is read and stored into <code>b</code>. - * <p> - * <p> The first byte read is stored into element <code>b[0]</code>, the - * next one into <code>b[1]</code>, and so on. The number of bytes read is, - * at most, equal to the length of <code>b</code>. Let <i>k</i> be the - * number of bytes actually read; these bytes will be stored in elements - * <code>b[0]</code> through <code>b[</code><i>k</i><code>-1]</code>, - * leaving elements <code>b[</code><i>k</i><code>]</code> through - * <code>b[b.length-1]</code> unaffected. - * <p> - * <p> The <code>read(b)</code> method for class <code>InputStream</code> - * has the same effect as: <pre><code> read(b, 0, b.length) </code></pre> - * - * @param b the buffer into which the data is read. - * @return the total number of bytes read into the buffer, or - * <code>-1</code> if there is no more data because the end of - * the stream has been reached. - * @throws IOException If the first byte cannot be read for any reason - * other than the end of the file, if the input stream has been closed, or - * if some other I/O error occurs. - * @throws NullPointerException if <code>b</code> is <code>null</code>. - * @see InputStream#read(byte[], int, int) - */ - @Override - public int read(byte[] b) throws IOException { - - if(paused) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return 0; - } - int ret = in.read(b); - return ret; - } - - /** - * Reads up to <code>len</code> bytes of data from the input stream into - * an array of bytes. An attempt is made to read as many as - * <code>len</code> bytes, but a smaller number may be read. - * The number of bytes actually read is returned as an integer. - * <p> - * <p> This method blocks until input data is available, end of file is - * detected, or an exception is thrown. - * <p> - * <p> If <code>len</code> is zero, then no bytes are read and - * <code>0</code> is returned; otherwise, there is an attempt to read at - * least one byte. If no byte is available because the stream is at end of - * file, the value <code>-1</code> is returned; otherwise, at least one - * byte is read and stored into <code>b</code>. - * <p> - * <p> The first byte read is stored into element <code>b[off]</code>, the - * next one into <code>b[off+1]</code>, and so on. The number of bytes read - * is, at most, equal to <code>len</code>. Let <i>k</i> be the number of - * bytes actually read; these bytes will be stored in elements - * <code>b[off]</code> through <code>b[off+</code><i>k</i><code>-1]</code>, - * leaving elements <code>b[off+</code><i>k</i><code>]</code> through - * <code>b[off+len-1]</code> unaffected. - * <p> - * <p> In every case, elements <code>b[0]</code> through - * <code>b[off]</code> and elements <code>b[off+len]</code> through - * <code>b[b.length-1]</code> are unaffected. - * <p> - * <p> The <code>read(b,</code> <code>off,</code> <code>len)</code> method - * for class <code>InputStream</code> simply calls the method - * <code>read()</code> repeatedly. If the first such call results in an - * <code>IOException</code>, that exception is returned from the call to - * the <code>read(b,</code> <code>off,</code> <code>len)</code> method. If - * any subsequent call to <code>read()</code> results in a - * <code>IOException</code>, the exception is caught and treated as if it - * were end of file; the bytes read up to that point are stored into - * <code>b</code> and the number of bytes read before the exception - * occurred is returned. The default implementation of this method blocks - * until the requested amount of input data <code>len</code> has been read, - * end of file is detected, or an exception is thrown. Subclasses are encouraged - * to provide a more efficient implementation of this method. - * - * @param b the buffer into which the data is read. - * @param off the start offset in array <code>b</code> - * at which the data is written. - * @param len the maximum number of bytes to read. - * @return the total number of bytes read into the buffer, or - * <code>-1</code> if there is no more data because the end of - * the stream has been reached. - * @throws IOException If the first byte cannot be read for any reason - * other than end of file, or if the input stream has been closed, or if - * some other I/O error occurs. - * @throws NullPointerException If <code>b</code> is <code>null</code>. - * @throws IndexOutOfBoundsException If <code>off</code> is negative, - * <code>len</code> is negative, or <code>len</code> is greater than - * <code>b.length - off</code> - * @see InputStream#read() - */ - @Override - public int read(byte[] b, int off, int len) throws IOException { - if(paused) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return 0; - } - int ret = in.read(b, off, len); - return ret; - } - - /** - * Skips over and discards <code>n</code> bytes of data from this input - * stream. The <code>skip</code> method may, for a variety of reasons, end - * up skipping over some smaller number of bytes, possibly <code>0</code>. - * This may result from any of a number of conditions; reaching end of file - * before <code>n</code> bytes have been skipped is only one possibility. - * The actual number of bytes skipped is returned. If {@code n} is - * negative, the {@code skip} method for class {@code InputStream} always - * returns 0, and no bytes are skipped. Subclasses may handle the negative - * value differently. - * <p> - * <p> The <code>skip</code> method of this class creates a - * byte array and then repeatedly reads into it until <code>n</code> bytes - * have been read or the end of the stream has been reached. Subclasses are - * encouraged to provide a more efficient implementation of this method. - * For instance, the implementation may depend on the ability to seek. - * - * @param n the number of bytes to be skipped. - * @return the actual number of bytes skipped. - * @throws IOException if the stream does not support seek, - * or if some other I/O error occurs. - */ - @Override - public long skip(long n) throws IOException { - - return in.skip(n); - } - - /** - * Returns an estimate of the number of bytes that can be read (or - * skipped over) from this input stream without blocking by the next - * invocation of a method for this input stream. The next invocation - * might be the same thread or another thread. A single read or skip of this - * many bytes will not block, but may read or skip fewer bytes. - * <p> - * <p> Note that while some implementations of {@code InputStream} will return - * the total number of bytes in the stream, many will not. It is - * never correct to use the return value of this method to allocate - * a buffer intended to hold all data in this stream. - * <p> - * <p> A subclass' implementation of this method may choose to throw an - * {@link IOException} if this input stream has been closed by - * invoking the {@link #close()} method. - * <p> - * <p> The {@code available} method for class {@code InputStream} always - * returns {@code 0}. - * <p> - * <p> This method should be overridden by subclasses. - * - * @return an estimate of the number of bytes that can be read (or skipped - * over) from this input stream without blocking or {@code 0} when - * it reaches the end of the input stream. - * @throws IOException if an I/O error occurs. - */ - @Override - public int available() throws IOException { - - return in.available(); - } - - /** - * Closes this input stream and releases any system resources associated - * with the stream. - * <p> - * <p> The <code>close</code> method of <code>InputStream</code> does - * nothing. - * - * @throws IOException if an I/O error occurs. - */ - @Override - public void close() throws IOException { - in.close(); - } - - /** - * Marks the current position in this input stream. A subsequent call to - * the <code>reset</code> method repositions this stream at the last marked - * position so that subsequent reads re-read the same bytes. - * <p> - * <p> The <code>readlimit</code> arguments tells this input stream to - * allow that many bytes to be read before the mark position gets - * invalidated. - * <p> - * <p> The general contract of <code>mark</code> is that, if the method - * <code>markSupported</code> returns <code>true</code>, the stream somehow - * remembers all the bytes read after the call to <code>mark</code> and - * stands ready to supply those same bytes again if and whenever the method - * <code>reset</code> is called. However, the stream is not required to - * remember any data at all if more than <code>readlimit</code> bytes are - * read from the stream before <code>reset</code> is called. - * <p> - * <p> Marking a closed stream should not have any effect on the stream. - * <p> - * <p> The <code>mark</code> method of <code>InputStream</code> does - * nothing. - * - * @param readlimit the maximum limit of bytes that can be read before - * the mark position becomes invalid. - * @see InputStream#reset() - */ - @Override - public synchronized void mark(int readlimit) { - in.mark(readlimit); - } - - /** - * Repositions this stream to the position at the time the - * <code>mark</code> method was last called on this input stream. - * <p> - * <p> The general contract of <code>reset</code> is: - * <p> - * <ul> - * <li> If the method <code>markSupported</code> returns - * <code>true</code>, then: - * <p> - * <ul><li> If the method <code>mark</code> has not been called since - * the stream was created, or the number of bytes read from the stream - * since <code>mark</code> was last called is larger than the argument - * to <code>mark</code> at that last call, then an - * <code>IOException</code> might be thrown. - * <p> - * <li> If such an <code>IOException</code> is not thrown, then the - * stream is reset to a state such that all the bytes read since the - * most recent call to <code>mark</code> (or since the start of the - * file, if <code>mark</code> has not been called) will be resupplied - * to subsequent callers of the <code>read</code> method, followed by - * any bytes that otherwise would have been the next input data as of - * the time of the call to <code>reset</code>. </ul> - * <p> - * <li> If the method <code>markSupported</code> returns - * <code>false</code>, then: - * <p> - * <ul><li> The call to <code>reset</code> may throw an - * <code>IOException</code>. - * <p> - * <li> If an <code>IOException</code> is not thrown, then the stream - * is reset to a fixed state that depends on the particular type of the - * input stream and how it was created. The bytes that will be supplied - * to subsequent callers of the <code>read</code> method depend on the - * particular type of the input stream. </ul></ul> - * <p> - * <p>The method <code>reset</code> for class <code>InputStream</code> - * does nothing except throw an <code>IOException</code>. - * - * @throws IOException if this stream has not been marked or if the - * mark has been invalidated. - * @see InputStream#mark(int) - * @see IOException - */ - @Override - public synchronized void reset() throws IOException { - in.reset(); - } - - /** - * Tests if this input stream supports the <code>mark</code> and - * <code>reset</code> methods. Whether or not <code>mark</code> and - * <code>reset</code> are supported is an invariant property of a - * particular input stream instance. The <code>markSupported</code> method - * of <code>InputStream</code> returns <code>false</code>. - * - * @return <code>true</code> if this stream instance supports the mark - * and reset methods; <code>false</code> otherwise. - * @see InputStream#mark(int) - * @see InputStream#reset() - */ - @Override - public boolean markSupported() { - return in.markSupported(); - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java new file mode 100644 index 0000000..35ab5ff --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarAutoCompleter.java @@ -0,0 +1,45 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +/** + * Provides auto-complete functionality for Stellar. + */ +public interface StellarAutoCompleter { + + /** + * Auto-completes based on the given buffer. + * @param buffer The partial buffer that needs auto-completed. + * @return Viable candidates for auto-completion. + */ + Iterable<String> autoComplete(String buffer); + + /** + * Adds a candidate for auto-completing function names. + * @param name The name of the function candidate. + */ + void addCandidateFunction(String name); + + /** + * Adds a candidate for auto-completing variable names. + * @param name The name of the function candidate. + */ + void addCandidateVariable(String name); +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java new file mode 100644 index 0000000..9734241 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionListeners.java @@ -0,0 +1,51 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +import org.apache.metron.stellar.common.shell.specials.SpecialCommand; +import org.apache.metron.stellar.dsl.StellarFunctionInfo; + +/** + * A listener will be notified about events that occur during the + * execution of Stellar expressions. + */ +public class StellarExecutionListeners { + + /** + * A listener that is notified when a function is defined. + */ + public interface FunctionDefinedListener { + void whenFunctionDefined(StellarFunctionInfo functionInfo); + } + + /** + * A listener that is notified when a variable is defined or redefined. + */ + public interface VariableDefinedListener { + void whenVariableDefined(String variableName, VariableResult result); + } + + /** + * A listener that is notified when a special command is defined. + */ + public interface SpecialDefinedListener { + void whenSpecialDefined(SpecialCommand magic); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java new file mode 100644 index 0000000..f4ebf2e --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutionNotifier.java @@ -0,0 +1,44 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell; + +/** + * Notifies listeners when events occur during the execution of Stellar expressions. + */ +public interface StellarExecutionNotifier { + + /** + * Add a listener that will be notified when a magic command is defined. + * @param listener The listener to notify. + */ + void addSpecialListener(StellarExecutionListeners.SpecialDefinedListener listener); + + /** + * Add a listener that will be notified when a function is defined. + * @param listener The listener to notify. + */ + void addFunctionListener(StellarExecutionListeners.FunctionDefinedListener listener); + + /** + * Add a listener that will be notified when a variable is defined. + * @param listener The listener to notify. + */ + void addVariableListener(StellarExecutionListeners.VariableDefinedListener listener); +}
