Repository: incubator-metron Updated Branches: refs/heads/master e7fed2d58 -> ef94c652c
METRON-438: Back the Stellar REPL with a readline implementation closes apache/incubator-metron#265 Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/ef94c652 Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/ef94c652 Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/ef94c652 Branch: refs/heads/master Commit: ef94c652c9b22b78bb37e6057cc42a46aad668b4 Parents: e7fed2d Author: cstella <ceste...@gmail.com> Authored: Wed Sep 21 19:22:15 2016 -0400 Committer: cstella <ceste...@gmail.com> Committed: Wed Sep 21 19:22:15 2016 -0400 ---------------------------------------------------------------------- .../docker/rpm-docker/SPECS/metron.spec | 1 + metron-platform/metron-common/README.md | 141 +++++++++++ metron-platform/metron-common/pom.xml | 11 + .../metron/common/stellar/shell/README.md | 126 ---------- .../common/stellar/shell/StellarExecutor.java | 110 ++++++++- .../common/stellar/shell/StellarShell.java | 237 ++++++++++++++----- 6 files changed, 434 insertions(+), 192 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ef94c652/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index d0720e7..7f78806 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -97,6 +97,7 @@ This package installs the Metron common files %{metron_home} %dir %{metron_home}/bin %dir %{metron_home}/lib %{metron_home}/bin/zk_load_configs.sh +%{metron_home}/bin/stellar %attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}.jar # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ef94c652/metron-platform/metron-common/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md index 367af7a..1cf9485 100644 --- a/metron-platform/metron-common/README.md +++ b/metron-platform/metron-common/README.md @@ -401,6 +401,147 @@ This will convert the timestamp field to an epoch timestamp based on the * The value in `dc2tz` associated with the value associated with field `dc`, defaulting to `UTC` +## Stellar Shell + +A REPL (Read Eval Print Loop) for the Stellar language that helps in debugging, troubleshooting and learning Stellar. The Stellar DSL (domain specific language) is used to act upon streaming data within Apache Storm. It is difficult to troubleshoot Stellar when it can only be executed within a Storm topology. This REPL is intended to help mitigate that problem by allowing a user to replicate data encountered in production, isolate initialization errors, or understand function resolution problems. + +The shell supports customization via `~/.inputrc` as it is +backed by a proper readline implementation. + +Shell-like operations are supported such as +* reverse search via ctrl-r +* autocomplete of Stellar functions and variables via tab + * NOTE: Stellar functions are read via a classpath search which + happens in the background. Until that happens, autocomplete will not include function names. +* emacs or vi keybindings for edit mode + +### Getting Started + +``` +$ $METRON_HOME/bin/stellar + +Stellar, Go! +{es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH} + +[Stellar]>>> %functions +BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, ... + +[Stellar]>>> ?PROTOCOL_TO_NAME +PROTOCOL_TO_NAME + desc: Convert the IANA protocol number to the protocol name + args: IANA Number + ret: The protocol name associated with the IANA number. + +[Stellar]>>> ip.protocol := 6 +6 +[Stellar]>>> PROTOCOL_TO_NAME(ip.protocol) +TCP +``` + +### Command Line Options + +``` +$ $METRON_HOME/bin/stellar -h +usage: stellar + -h,--help Print help + -irc,--inputrc <arg> File containing the inputrc if not the default + ~/.inputrc + -v,--variables <arg> File containing a JSON Map of variables + -z,--zookeeper <arg> Zookeeper URL + -na,--no_ansi Make the input prompt not use ANSI colors. +``` + +#### `-v, --variables` +*Optional* + +Optionally load a JSON map which contains variable assignments. This is +intended to give you the ability to save off a message from Metron and +work on it via the REPL. + +#### `-z, --zookeeper` + +*Optional* + +Attempts to connect to Zookeeper and read the Metron global configuration. Stellar functions may require the global configuration to work properly. If found, the global configuration values are printed to the console. + +``` +$ $METRON_HOME/bin/stellar -z node1:2181 +Stellar, Go! +{es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH} +[Stellar]>>> +``` + +### Variable Assignment + +Stellar has no concept of variable assignment. For testing and +debugging purposes, it is important to be able to create variables that +simulate data contained within incoming messages. The REPL has created +a means for a user to perform variable assignment outside of the core +Stellar language. This is done via the `:=` operator, such as +`foo := 1 + 1` would assign the result of the stellar expression `1 + 1` to the +variable `foo`. + +``` +[Stellar]>>> foo := 2 + 2 +4.0 +[Stellar]>>> 2 + 2 +4.0 +``` + +### Magic Commands + +The REPL has a set of magic commands that provide the REPL user with information about the Stellar execution environment. The following magic commands are supported. + +#### `%functions` + +This command lists all functions resolvable in the Stellar environment. Stellar searches the classpath for Stellar functions. This can make it difficult in some cases to understand which functions are resolvable. + +``` +[Stellar]>>> %functions +BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, +DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENDS_WITH, GET, GET_FIRST, +GET_LAST, IN_SUBNET, IS_DATE, IS_DOMAIN, IS_EMAIL, IS_EMPTY, IS_INTEGER, IS_IP, IS_URL, +JOIN, MAAS_GET_ENDPOINT, MAAS_MODEL_APPLY, MAP_EXISTS, MAP_GET, MONTH, PROTOCOL_TO_NAME, +REGEXP_MATCH, SPLIT, STARTS_WITH, STATS_ADD, STATS_COUNT, STATS_GEOMETRIC_MEAN, STATS_INIT, +STATS_KURTOSIS, STATS_MAX, STATS_MEAN, STATS_MERGE, STATS_MIN, STATS_PERCENTILE, +STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS, STATS_SUM, +STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, TO_DOUBLE, TO_EPOCH_TIMESTAMP, +TO_INTEGER, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH, URL_TO_PORT, +URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR +[Stellar]>>> +``` + +#### `%vars` + +Lists all variables in the Stellar environment. + +``` +Stellar, Go! +{es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH} +[Stellar]>>> %vars +[Stellar]>>> foo := 2 + 2 +4.0 +[Stellar]>>> %vars +foo = 4.0 +``` + +#### `?<function>` + +Returns formatted documentation of the Stellar function. Provides the description of the function along with the expected arguments. + +``` +[Stellar]>>> ?BLOOM_ADD +BLOOM_ADD + desc: Adds an element to the bloom filter passed in + args: bloom - The bloom filter, value* - The values to add + ret: Bloom Filter +[Stellar]>>> ?IS_EMAIL +IS_EMAIL + desc: Tests if a string is a valid email address + args: address - The String to test + ret: True if the string is a valid email address and false otherwise. +[Stellar]>>> +``` ##Global Configuration The format of the global enrichment is a JSON String to Object map. This is intended for http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ef94c652/metron-platform/metron-common/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml index e14cbd8..1595fdd 100644 --- a/metron-platform/metron-common/pom.xml +++ b/metron-platform/metron-common/pom.xml @@ -240,6 +240,17 @@ <artifactId>t-digest</artifactId> <version>3.1</version> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-collections4</artifactId> + <version>4.1</version> + </dependency> + <dependency> + <groupId>org.jboss.aesh</groupId> + <artifactId>aesh</artifactId> + <version>0.66.8</version> + </dependency> + </dependencies> <reporting> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ef94c652/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/README.md b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/README.md deleted file mode 100644 index 32bad31..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/README.md +++ /dev/null @@ -1,126 +0,0 @@ -# Stellar Shell - -A REPL (Read Eval Print Loop) for the Stellar language that helps in debugging, troubleshooting and learning Stellar. The Stellar DSL (domain specific language) is used to act upon streaming data within Apache Storm. It is difficult to troubleshoot Stellar when it can only be executed within a Storm topology. This REPL is intended to help mitigate that problem by allowing a user to replicate data encountered in production, isolate initialization errors, or understand function resolution problems. - -### Getting Started - -``` -$ /usr/metron/<version>/bin/stellar - -Stellar, Go! -{es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH} - ->>> %functions -BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, ... - ->>> ?PROTOCOL_TO_NAME -PROTOCOL_TO_NAME - desc: Convert the IANA protocol number to the protocol name - args: IANA Number - ret: The protocol name associated with the IANA number. - ->>> 6 -[=] 6 -[?] save as: ip.protocol - ->>> %vars -ip.protocol = 6 - ->>> PROTOCOL_TO_NAME(ip.protocol) -[=] TCP -``` - -### Command Line Options - -``` -$ /usr/metron/<version>/bin/stellar -h - -usage: stellar - -h,--help Print help - -z,--zookeeper <arg> Zookeeper URL -``` - -#### -z, --zookeeper - -*Optional* - -Attempts to connect to Zookeeper and read the Metron global configuration. Stellar functions may require the global configuration to work properly. If found, the global configuration values are printed to the console. - -``` -$ /usr/metron/<version>/bin/stellar -z node1:2181 -Stellar, Go! -{es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH} ->>> -``` - -### Variable Assignment - -Stellar has no concept of variable assignment. For testing and debugging purposes, it is important to be able to create variables that simulate data contained within incoming messages. The REPL has created a means for a user to perform variable assignment outside of the core Stellar language. - -After execution of a Stellar expression, the REPL will prompt for the name of a variable to save the expression result to. If no name is provided, the result is not saved. - -``` ->>> 2+2 -[=] 4.0 -[?] save as: foo ->>> %vars -foo = 4.0 ->>> -``` - -### Magic Commands - -The REPL has a set of magic commands that provide the REPL user with information about the Stellar execution environment. The following magic commands are supported. - -#### `%functions` - -This command lists all functions resolvable in the Stellar environment. Stellar searches the classpath for Stellar functions. This can make it difficult in some cases to understand which functions are resolvable. - -``` ->>> %functions -BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, -DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENDS_WITH, GET, GET_FIRST, -GET_LAST, IN_SUBNET, IS_DATE, IS_DOMAIN, IS_EMAIL, IS_EMPTY, IS_INTEGER, IS_IP, IS_URL, -JOIN, MAAS_GET_ENDPOINT, MAAS_MODEL_APPLY, MAP_EXISTS, MAP_GET, MONTH, PROTOCOL_TO_NAME, -REGEXP_MATCH, SPLIT, STARTS_WITH, STATS_ADD, STATS_COUNT, STATS_GEOMETRIC_MEAN, STATS_INIT, -STATS_KURTOSIS, STATS_MAX, STATS_MEAN, STATS_MERGE, STATS_MIN, STATS_PERCENTILE, -STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS, STATS_SUM, -STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, TO_DOUBLE, TO_EPOCH_TIMESTAMP, -TO_INTEGER, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH, URL_TO_PORT, -URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR ->>> -``` - -#### `%vars` - -Lists all variables in the Stellar environment. - -``` -Stellar, Go! -{es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH} ->>> %vars ->>> 2+2 -[=] 4.0 -[?] save as: foo ->>> %vars -foo = 4.0 ->>> -``` - -#### `?<function>` - -Returns formatted documentation of the Stellar function. Provides the description of the function along with the expected arguments. - -``` ->>> ?BLOOM_ADD -BLOOM_ADD - desc: Adds an element to the bloom filter passed in - args: bloom - The bloom filter, value* - The values to add - ret: Bloom Filter ->>> ?IS_EMAIL -IS_EMAIL - desc: Tests if a string is a valid email address - args: address - The String to test - ret: True if the string is a valid email address and false otherwise. ->>> -``` \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ef94c652/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/StellarExecutor.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/StellarExecutor.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/StellarExecutor.java index 8268dfc..18778b1 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/StellarExecutor.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/StellarExecutor.java @@ -21,30 +21,35 @@ package org.apache.metron.common.stellar.shell; import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Iterables; +import org.apache.commons.collections4.trie.PatriciaTrie; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.configuration.ConfigurationsUtils; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.FunctionResolver; -import org.apache.metron.common.dsl.MapVariableResolver; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.dsl.VariableResolver; +import org.apache.metron.common.dsl.*; import org.apache.metron.common.stellar.StellarProcessor; import org.apache.metron.common.utils.JSONUtils; import java.io.ByteArrayInputStream; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; +import java.util.*; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import static org.apache.metron.common.configuration.ConfigurationsUtils.readGlobalConfigBytesFromZookeeper; +import static org.apache.metron.common.stellar.shell.StellarExecutor.OperationType.DOC; +import static org.apache.metron.common.stellar.shell.StellarExecutor.OperationType.NORMAL; /** * Executes Stellar expressions and maintains state across multiple invocations. */ public class StellarExecutor { + private ReadWriteLock indexLock = new ReentrantReadWriteLock(); + + /** + * prefix tree index of autocompletes + */ + private PatriciaTrie<AutoCompleteType> autocompleteIndex; /** * The variables known by Stellar. */ @@ -65,6 +70,39 @@ public class StellarExecutor { */ private Context context; + public enum OperationType { + DOC,MAGIC,NORMAL; + } + + public interface AutoCompleteTransformation { + String transform(OperationType type, String key); + } + + public enum AutoCompleteType implements AutoCompleteTransformation{ + FUNCTION((type, key) -> { + if(type == DOC) { + return StellarShell.DOC_PREFIX + key; + } + else if(type == NORMAL) { + return key + "("; + } + return key; + }) + , VARIABLE((type, key) -> key ) + , TOKEN((type, key) -> key) + ; + AutoCompleteTransformation transform; + AutoCompleteType(AutoCompleteTransformation transform) { + this.transform = transform; + } + + @Override + public String transform(OperationType type, String key) { + return transform.transform(type, key); + } + + } + public StellarExecutor() throws Exception { this(null); } @@ -74,6 +112,49 @@ public class StellarExecutor { this.functionResolver = new StellarFunctions().FUNCTION_RESOLVER(); this.client = createClient(zookeeperUrl); this.context = createContext(); + this.autocompleteIndex = initializeIndex(); + //Asynchronously update the index with function names found from a classpath scan. + new Thread( () -> { + Iterable<StellarFunctionInfo> functions = functionResolver.getFunctionInfo(); + indexLock.writeLock().lock(); + try { + for(StellarFunctionInfo info: functions) { + String functionName = info.getName(); + autocompleteIndex.put(functionName, AutoCompleteType.FUNCTION); + } + } + finally { + System.out.println("Functions loaded, you may refer to functions now..."); + indexLock.writeLock().unlock(); + } + }).start(); + } + + private PatriciaTrie<AutoCompleteType> initializeIndex() { + Map<String, AutoCompleteType> index = new HashMap<>(); + + index.put("==", AutoCompleteType.TOKEN); + index.put(">=", AutoCompleteType.TOKEN); + index.put("<=", AutoCompleteType.TOKEN); + index.put(":=", AutoCompleteType.TOKEN); + index.put("quit", AutoCompleteType.TOKEN); + index.put(StellarShell.MAGIC_FUNCTIONS, AutoCompleteType.FUNCTION); + index.put(StellarShell.MAGIC_VARS, AutoCompleteType.FUNCTION); + return new PatriciaTrie<>(index); + } + + public Iterable<String> autoComplete(String buffer, final OperationType opType) { + indexLock.readLock().lock(); + try { + SortedMap<String, AutoCompleteType> ret = autocompleteIndex.prefixMap(buffer); + if (ret.isEmpty()) { + return new ArrayList<>(); + } + return Iterables.transform(ret.entrySet(), kv -> kv.getValue().transform(opType, kv.getKey())); + } + finally { + indexLock.readLock().unlock(); + } } /** @@ -135,6 +216,17 @@ public class StellarExecutor { */ public void assign(String variable, Object value) { this.variables.put(variable, value); + indexLock.writeLock().lock(); + try { + if (value != null) { + this.autocompleteIndex.put(variable, AutoCompleteType.VARIABLE); + } else { + this.autocompleteIndex.remove(variable); + } + } + finally { + indexLock.writeLock().unlock(); + } } public Map<String, Object> getVariables() { http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ef94c652/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/StellarShell.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/StellarShell.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/StellarShell.java index cdf7eaf..ee82b4e 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/StellarShell.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/shell/StellarShell.java @@ -20,15 +20,38 @@ package org.apache.metron.common.stellar.shell; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.PosixParser; +import org.apache.commons.collections4.trie.PatriciaTrie; import org.apache.commons.lang3.StringUtils; import org.apache.metron.common.dsl.Context; import org.apache.metron.common.dsl.StellarFunctionInfo; - +import org.apache.metron.common.utils.JSONUtils; +import org.jboss.aesh.complete.CompleteOperation; +import org.jboss.aesh.complete.Completion; +import org.jboss.aesh.console.AeshConsoleCallback; +import org.jboss.aesh.console.Console; +import org.jboss.aesh.console.ConsoleOperation; +import org.jboss.aesh.console.Prompt; +import org.jboss.aesh.console.settings.Settings; +import org.jboss.aesh.console.settings.SettingsBuilder; +import org.jboss.aesh.terminal.CharacterType; +import org.jboss.aesh.terminal.Color; +import org.jboss.aesh.terminal.TerminalCharacter; +import org.jboss.aesh.terminal.TerminalColor; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.Scanner; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -38,20 +61,37 @@ import java.util.stream.StreamSupport; * * Useful for debugging Stellar expressions. */ -public class StellarShell { +public class StellarShell extends AeshConsoleCallback implements Completion { + + private static final String WELCOME = "Stellar, Go!\n" + + "Please note that functions are loading lazily in the background and will be unavailable until loaded fully."; + private List<TerminalCharacter> EXPRESSION_PROMPT = new ArrayList<TerminalCharacter>() + {{ + add(new TerminalCharacter('[', new TerminalColor(Color.RED, Color.DEFAULT))); + add(new TerminalCharacter('S', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('t', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('e', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('l', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('l', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('a', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter('r', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.BOLD)); + add(new TerminalCharacter(']', new TerminalColor(Color.RED, Color.DEFAULT))); + add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); + add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); + add(new TerminalCharacter('>', new TerminalColor(Color.GREEN, Color.DEFAULT), CharacterType.UNDERLINE)); + add(new TerminalCharacter(' ', new TerminalColor(Color.DEFAULT, Color.DEFAULT))); + }}; - private static final String WELCOME = "Stellar, Go!"; - private static final String EXPRESSION_PROMPT = ">>> "; - private static final String RESULT_PROMPT = "[=] "; private static final String ERROR_PROMPT = "[!] "; - private static final String ASSIGN_PROMPT = "[?] save as: "; - private static final String MAGIC_PREFIX = "%"; - private static final String MAGIC_FUNCTIONS = "%functions"; - private static final String MAGIC_VARS = "%vars"; - private static final String DOC_PREFIX = "?"; + public static final String MAGIC_PREFIX = "%"; + public static final String MAGIC_FUNCTIONS = MAGIC_PREFIX + "functions"; + public static final String MAGIC_VARS = MAGIC_PREFIX + "vars"; + public static final String DOC_PREFIX = "?"; private StellarExecutor executor; + private Console console; + /** * Execute the Stellar REPL. */ @@ -69,6 +109,9 @@ public class StellarShell { // define valid command-line options Options options = new Options(); options.addOption("z", "zookeeper", true, "Zookeeper URL"); + options.addOption("v", "variables", true, "File containing a JSON Map of variables"); + options.addOption("irc", "inputrc", true, "File containing the inputrc if not the default ~/.inputrc"); + options.addOption("na", "no_ansi", false, "Make the input prompt not use ANSI colors."); options.addOption("h", "help", false, "Print help"); CommandLineParser parser = new PosixParser(); @@ -89,6 +132,31 @@ public class StellarShell { } else { executor = new StellarExecutor(); } + + if(commandLine.hasOption("v")) { + Map<String, Object> variables = JSONUtils.INSTANCE.load(new File(commandLine.getOptionValue("v")), new TypeReference<Map<String, Object>>() { + }); + for(Map.Entry<String, Object> kv : variables.entrySet()) { + executor.assign(kv.getKey(), kv.getValue()); + } + } + SettingsBuilder settings = new SettingsBuilder().enableAlias(true) + .enableMan(true) + .parseOperators(false) + ; + if(commandLine.hasOption("irc")) { + settings = settings.inputrc(new File(commandLine.getOptionValue("irc"))); + } + + console = new Console(settings.create()); + if(!commandLine.hasOption("na")) { + console.setPrompt(new Prompt(EXPRESSION_PROMPT)); + } + else { + console.setPrompt(new Prompt("[Stellar]$")); + } + console.addCompletion(this); + console.setConsoleCallback(this); } /** @@ -102,58 +170,44 @@ public class StellarShell { .getCapability(Context.Capabilities.GLOBAL_CONFIG) .ifPresent(conf -> writeLine(conf.toString())); - Scanner scanner = new Scanner(System.in); - boolean done = false; - while(!done) { - - // prompt the user for an expression - write(EXPRESSION_PROMPT); - String expression = scanner.nextLine(); - if(StringUtils.isNotBlank(expression)) { - - if(isMagic(expression)) { - handleMagic(scanner, expression); - - } else if(isDoc(expression)) { - handleDoc(scanner, expression); - - } else { - handleStellar(scanner, expression); - } - } - } + console.start(); } /** * Handles user interaction when executing a Stellar expression. - * @param scanner The scanner used to read user input. * @param expression The expression to execute. */ - private void handleStellar(Scanner scanner, String expression) { - - Object result = executeStellar(expression); + private void handleStellar(String expression) { + + Iterable<String> assignmentSplit = Splitter.on(":=").split(expression); + String stellarExpression = expression; + String variable = null; + if(Iterables.size(assignmentSplit) == 2) { + //assignment + variable = Iterables.getFirst(assignmentSplit, null); + if(variable != null) { + variable = variable.trim(); + } + stellarExpression = Iterables.getLast(assignmentSplit, null); + } + if(!stellarExpression.isEmpty()) { + stellarExpression = stellarExpression.trim(); + } + Object result = executeStellar(stellarExpression); if(result != null) { - - // echo the result - write(RESULT_PROMPT); writeLine(result.toString()); - - // assign the result to a variable? - write(ASSIGN_PROMPT); - String variable = scanner.nextLine(); - if(StringUtils.isNotBlank(variable)) { - executor.assign(variable, result); - } + } + if(variable != null) { + executor.assign(variable, result); } } /** * Handles user interaction when executing a Magic command. - * @param scanner The scanner used to read user input. - * @param expression The expression to execute. + * @param rawExpression The expression to execute. */ - private void handleMagic(Scanner scanner, String expression) { - + private void handleMagic( String rawExpression) { + String expression = rawExpression.trim(); if(MAGIC_FUNCTIONS.equals(expression)) { // list all functions @@ -177,10 +231,9 @@ public class StellarShell { /** * Handles user interaction when executing a doc command. - * @param scanner The scanner used to read user input. * @param expression The expression to execute. */ - private void handleDoc(Scanner scanner, String expression) { + private void handleDoc(String expression) { String functionName = StringUtils.substring(expression, 1); StreamSupport @@ -196,12 +249,19 @@ public class StellarShell { * @return A readable string. */ private String format(StellarFunctionInfo info) { - return String.format( - "%s\n desc: %-60s\n args: %-60s\n ret: %-60s\n", - info.getName(), - info.getDescription(), - StringUtils.join(info.getParams(), ", "), - info.getReturns()); + StringBuffer ret = new StringBuffer(); + ret.append(info.getName() + "\n"); + ret.append(String.format("Description: %-60s\n\n", info.getDescription())); + if(info.getParams().length > 0) { + ret.append("Arguments:\n"); + for(String param : info.getParams()) { + ret.append(String.format("\t%-60s\n", param)); + } + ret.append("\n"); + } + ret.append(String.format("Returns: %-60s\n", info.getReturns())); + + return ret.toString(); } /** @@ -244,6 +304,69 @@ public class StellarShell { } private void writeLine(String out) { - System.out.println(out); + console.getShell().out().println(out); + } + + @Override + public int execute(ConsoleOperation output) throws InterruptedException { + String expression = output.getBuffer().trim(); + if(StringUtils.isNotBlank(expression)) { + if(isMagic(expression)) { + handleMagic( expression); + + } else if(isDoc(expression)) { + handleDoc(expression); + + } else if (expression.equals("quit")) { + try { + console.stop(); + } catch (Throwable e) { + e.printStackTrace(); + } + } + else { + handleStellar(expression); + } + } + + return 0; + } + + @Override + public void complete(CompleteOperation completeOperation) { + if(!completeOperation.getBuffer().isEmpty()) { + String lastToken = Iterables.getLast(Splitter.on(" ").split(completeOperation.getBuffer()), null); + if(lastToken != null && !lastToken.isEmpty()) { + lastToken = lastToken.trim(); + final String lastBit = lastToken; + final boolean isDocRequest = isDoc(lastToken); + if(isDocRequest) { + lastToken = lastToken.substring(1); + } + StellarExecutor.OperationType opType = StellarExecutor.OperationType.NORMAL; + if(isDocRequest) { + opType = StellarExecutor.OperationType.DOC; + } + else if(isMagic(lastToken)) { + opType = StellarExecutor.OperationType.MAGIC; + } + Iterable<String> candidates = executor.autoComplete(lastToken, opType); + if(candidates != null && !Iterables.isEmpty(candidates)) { + completeOperation.setCompletionCandidates( Lists.newArrayList( + Iterables.transform(candidates, s -> stripOff(completeOperation.getBuffer(), lastBit) + s ) + ) + ); + } + } + } + + } + + private static String stripOff(String baseString, String lastBit) { + int index = baseString.lastIndexOf(lastBit); + if(index < 0) { + return baseString; + } + return baseString.substring(0, index); } }
