Repository: metron Updated Branches: refs/heads/master 55a737921 -> 2dd01b177
http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidatorTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidatorTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidatorTest.java new file mode 100644 index 0000000..8ca0959 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellOptionsValidatorTest.java @@ -0,0 +1,184 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.shell.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; + +public class StellarShellOptionsValidatorTest { + + @Test + public void validateOptions() throws Exception { + String[] validZHostArg = new String[]{"-z", "localhost:8888"}; + String[] validZHostArgNoPort = new String[]{"-z", "localhost"}; + String[] validZIPArgNoPort = new String[]{"-z", "10.10.10.3"}; + String[] validZHostArgList = new String[]{"-z", "localhost:8888,localhost:2181,localhost"}; + String[] validZIPArg = new String[]{"-z", "10.10.10.3:9999"}; + String[] invalidZNameArg = new String[]{"-z", "!!!@!!@!:8882"}; + String[] invalidZIPArg = new String[]{"-z", "11111.22222.10.3:3332"}; + String[] invalidZMissingNameArg = new String[]{"-z", ":8882"}; + String[] invalidZZeroPortArg = new String[]{"-z", "youtube.com:0"}; + String[] invalidZHugePortArg = new String[]{"-z", "youtube.com:75565"}; + + + String existingFileName = "./target/existsFile"; + String nonExistentFile = "./target/doesNotExist"; + + String[] validVFileArg = new String[]{"-v", existingFileName}; + String[] validIrcFileArg = new String[]{"-irc", existingFileName}; + String[] validPFileArg = new String[]{"-p", existingFileName}; + String[] invalidVFileArg = new String[]{"-v", nonExistentFile}; + String[] invalidIrcFileArg = new String[]{"-irc", nonExistentFile}; + String[] invalidPFileArg = new String[]{"-p", nonExistentFile}; + + File existingFile = new File(existingFileName); + if (!existingFile.exists()) { + existingFile.createNewFile(); + } + Options options = new Options(); + options.addOption("z", "zookeeper", true, "Zookeeper URL"); + options.addOption("v", "variables", true, "File containing a JSON Map of variables"); + options.addOption("irc", "inputrc", true, + "File containing the inputrc if not the default ~/.inputrc"); + options.addOption("na", "no_ansi", false, "Make the input prompt not use ANSI colors."); + options.addOption("h", "help", false, "Print help"); + options.addOption("p", "properties", true, "File containing Stellar properties"); + + CommandLineParser parser = new PosixParser(); + + // these should work + CommandLine commandLine = parser.parse(options, validZHostArg); + StellarShellOptionsValidator.validateOptions(commandLine); + + commandLine = parser.parse(options, validZIPArg); + StellarShellOptionsValidator.validateOptions(commandLine); + + commandLine = parser.parse(options, validVFileArg); + StellarShellOptionsValidator.validateOptions(commandLine); + + commandLine = parser.parse(options, validIrcFileArg); + StellarShellOptionsValidator.validateOptions(commandLine); + + commandLine = parser.parse(options, validPFileArg); + StellarShellOptionsValidator.validateOptions(commandLine); + + commandLine = parser.parse(options, validZHostArgNoPort); + StellarShellOptionsValidator.validateOptions(commandLine); + + commandLine = parser.parse(options, validZHostArgList); + StellarShellOptionsValidator.validateOptions(commandLine); + + commandLine = parser.parse(options, validZIPArgNoPort); + StellarShellOptionsValidator.validateOptions(commandLine); + // these should not + + boolean thrown = false; + + + try { + commandLine = parser.parse(options, invalidZNameArg); + StellarShellOptionsValidator.validateOptions(commandLine); + } catch (IllegalArgumentException e) { + thrown = true; + } + Assert.assertTrue("Did not catch failure for providing invalid host name ", thrown); + thrown = false; + + try { + commandLine = parser.parse(options, invalidZIPArg); + StellarShellOptionsValidator.validateOptions(commandLine); + } catch (IllegalArgumentException e) { + thrown = true; + } + Assert.assertTrue("Did not catch failure for providing invalid ip address ", thrown); + thrown = false; + + try { + commandLine = parser.parse(options, invalidZMissingNameArg); + StellarShellOptionsValidator.validateOptions(commandLine); + } catch (IllegalArgumentException e) { + thrown = true; + } + Assert.assertTrue("Did not catch failure for only providing port ", thrown); + thrown = false; + + try { + commandLine = parser.parse(options, invalidZZeroPortArg); + StellarShellOptionsValidator.validateOptions(commandLine); + } catch (IllegalArgumentException e) { + thrown = true; + } + Assert.assertTrue("Did not catch failure for 0 port ", thrown); + thrown = false; + + try { + commandLine = parser.parse(options, invalidZHugePortArg); + StellarShellOptionsValidator.validateOptions(commandLine); + } catch (IllegalArgumentException e) { + thrown = true; + } + Assert.assertTrue("Did not catch failure for port out of range ", thrown); + thrown = false; + + try { + commandLine = parser.parse(options, invalidVFileArg); + StellarShellOptionsValidator.validateOptions(commandLine); + } catch (IllegalArgumentException e) { + thrown = true; + } + Assert.assertTrue("Did not catch failure for passing non-existant file to -v ", thrown); + thrown = false; + + try { + commandLine = parser.parse(options, invalidVFileArg); + StellarShellOptionsValidator.validateOptions(commandLine); + } catch (IllegalArgumentException e) { + thrown = true; + } + Assert.assertTrue("Did not catch failure for passing non-existant file to -v ", thrown); + thrown = false; + + try { + commandLine = parser.parse(options, invalidIrcFileArg); + StellarShellOptionsValidator.validateOptions(commandLine); + } catch (IllegalArgumentException e) { + thrown = true; + } + Assert.assertTrue("Did not catch failure for passing non-existant file to -irc ", thrown); + thrown = false; + + try { + commandLine = parser.parse(options, invalidPFileArg); + StellarShellOptionsValidator.validateOptions(commandLine); + } catch (IllegalArgumentException e) { + thrown = true; + } + Assert.assertTrue("Did not catch failure for passing non-existant file to -p ", thrown); + thrown = false; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java new file mode 100644 index 0000000..ef2475b --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.common.shell.cli; + +import com.google.common.collect.Iterables; +import org.jboss.aesh.complete.CompleteOperation; +import org.jboss.aesh.console.AeshContext; +import org.jboss.aesh.console.ConsoleOperation; +import org.jboss.aesh.console.operator.ControlOperator; +import org.jboss.aesh.console.settings.DefaultAeshContext; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +/** + * Tests the StellarShell class. + */ +public class StellarShellTest { + + private StellarShell stellarShell; + private ByteArrayOutputStream out; + private ByteArrayOutputStream err; + + @Before + public void setup() throws Exception { + + out = new ByteArrayOutputStream(); + err = new ByteArrayOutputStream(); + + // setup streams so that we can capture stdout + System.setOut(new PrintStream(out)); + System.setErr(new PrintStream(err)); + + String[] args = new String[0]; + stellarShell = new StellarShell(args); + } + + @After + public void cleanUp() { + System.setOut(null); + System.setErr(null); + } + + /** + * @return The data written to stdout during the test (with newlines stripped out to simplify comparisons.) + */ + private String stdout() { + return out.toString().replace(System.lineSeparator(), ""); + } + + /** + * @return The data written to stdout during the test. + */ + private String stdoutWithNewlines() { + return out.toString(); + } + + + /** + * @return The data written to stderr during the test. + */ + private String stderr() { + return err.toString().replace(System.lineSeparator(), ""); + } + + /** + * @param buffer + * @return A ConsoleOperation that that StellarShell uses to drive input. + */ + private ConsoleOperation createOp(String buffer) { + return new ConsoleOperation(ControlOperator.APPEND_OUT, buffer); + } + + @Test + public void testExecuteStellar() throws Exception { + stellarShell.execute(createOp("2 + 2")); + assertEquals("4", stdout()); + } + + /** + * Ensure that Stellar lists are displayed correctly in the REPL. + */ + @Test + public void testExecuteWithStellarList() throws Exception { + stellarShell.execute(createOp("[1,2,3,4,5]")); + assertEquals("[1, 2, 3, 4, 5]", stdout()); + } + + /** + * Ensure that Stellar maps are displayed correctly in the REPL. + */ + @Test + public void testExecuteWithStellarMap() throws Exception { + stellarShell.execute(createOp("{ 'foo':2, 'key':'val' }")); + assertEquals("{foo=2, key=val}", stdout()); + } + + /** + * Ensure that 'bad' Stellar code is handled correctly by the REPL. + */ + @Test + public void testExecuteBadStellar() throws Exception { + stellarShell.execute(createOp("2 + ")); + final String expected = "[!] Unable to parse: 2 + "; + assertTrue(stdout().startsWith(expected)); + } + + /** + * The REPL should handle if no value is returned. Some Stellar expressions + * will result in no value. + */ + @Test + public void testExecuteNoop() throws Exception { + stellarShell.execute(createOp("x")); + assertEquals("", stdout()); + } + + /** + * The REPL should handle if the user chooses to quit. + */ + @Test + public void testQuit() throws Exception { + stellarShell.execute(createOp("quit")); + + // the console should not be running + assertFalse(stellarShell.getConsole().isRunning()); + } + + /** + * The REPL should handle if the user chooses to quit. + */ + @Test + public void testStart() throws Exception { + + StellarShell.main(new String[0]); + + // we should see the welcome prompt + assertTrue(stdoutWithNewlines().contains(StellarShell.WELCOME)); + } + + /** + * The REPL should support auto-completion. + */ + @Test + public void testAutoComplete() throws Exception { + + // the user's input that needs auto-completed + final String buffer = "TO_"; + + // the cursor is at the end of the buffer + int cursor = buffer.length(); + + // ask the shell to auto-complete + AeshContext context = new DefaultAeshContext(); + CompleteOperation op = new CompleteOperation(context, buffer, cursor); + stellarShell.complete(op); + + // we should have some auto-complete candidates + List<String> candidates = op.getFormattedCompletionCandidates(); + assertTrue(candidates.size() > 0); + + // validate each candidate + for(String candidate: candidates) { + String completion = buffer + candidate; + + // the auto-complete should include an open paren + assertEquals("(", completion.substring(completion.length() - 1)); + + // the candidate should be a valid, defined function + String function = completion.substring(0, completion.length() - 1); + Iterable<String> allFunctions = stellarShell.getExecutor().getFunctionResolver().getFunctions(); + String definedFunction = Iterables.find(allFunctions, (fn) -> fn.equals(function)); + assertEquals(function, definedFunction); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java new file mode 100644 index 0000000..899effb --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java @@ -0,0 +1,212 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNull; + + +/** + * Tests the AssignmentCommand class. + */ +public class AssignmentCommandTest { + + AssignmentCommand command; + StellarShellExecutor executor; + + @Before + public void setup() throws Exception { + command = new AssignmentCommand(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals(":=", command.getCommand()); + } + + @Test + public void testShouldMatch() { + List<String> inputs = Arrays.asList( + "x := 2 + 2", + " x := 2 + 2 ", + " x := 2", + " x := " + ); + for(String in : inputs) { + assertTrue("failed: " + in, command.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List<String> inputs = Arrays.asList( + "2+2", + " %define x := 2", + "x" + ); + for(String in : inputs) { + assertFalse("failed: " + in, command.getMatcher().apply(in)); + } + } + + @Test + public void testAssignment() { + StellarResult result = command.execute("x := 2 + 2", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(4, result.getValue().get()); + + // validate assignment + assertEquals(4, executor.getState().get("x").getResult()); + } + + @Test + public void testAssignments() { + + // execute a series of assignments + assertTrue(command.execute("x := 2 + 2", executor).isSuccess()); + assertTrue(command.execute("y := 2 + x", executor).isSuccess()); + assertTrue(command.execute("z := x + y", executor).isSuccess()); + + // validate assignment + assertEquals(4, executor.getState().get("x").getResult()); + assertEquals(6, executor.getState().get("y").getResult()); + assertEquals(10, executor.getState().get("z").getResult()); + } + + @Test + public void testReassignment() { + + // execute a series of assignments + assertTrue(command.execute("x := 2 + 2", executor).isSuccess()); + assertTrue(command.execute("x := 5 + 5", executor).isSuccess()); + + // validate assignment + assertEquals(10, executor.getState().get("x").getResult()); + } + + @Test + public void testAssignmentOfEmptyVar() { + + // z is not defined + StellarResult result = command.execute("x := z", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.isValueNull()); + assertFalse(result.getValue().isPresent()); + + // the value of x is null + assertNull(executor.getState().get("x").getResult()); + } + + @Test + public void testBadAssignmentExpr() { + StellarResult result = command.execute("x := 2 + ", executor); + + // validate the result + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + + // no assignment should have happened + assertFalse(executor.getState().containsKey("x")); + } + + @Test + public void testAssignNull() { + StellarResult result = command.execute("x := NULL", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.isValueNull()); + + // validate assignment + assertNull(executor.getState().get("x").getResult()); + } + + /** + * Assignment with no expression results in an empty string. Is this + * what we would expect? + */ + @Test + public void testNoAssignmentExpr() { + StellarResult result = command.execute("x := ", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // validate assignment + assertEquals("", executor.getState().get("x").getResult()); + } + + @Test + public void testAssignmentWithVar() { + + // define a variable + executor.assign("x", 10, Optional.empty()); + + // execute the assignment expression + StellarResult result = command.execute("y := x + 2", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(12, result.getValue().get()); + + // validate assignment + assertEquals(10, executor.getState().get("x").getResult()); + assertEquals(12, executor.getState().get("y").getResult()); + } + + @Test + public void testAssignmentWithOddWhitespace() { + + StellarResult result = command.execute(" x := 2 + 2", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(4, result.getValue().get()); + + // validate assignment + assertEquals(4, executor.getState().get("x").getResult()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/CommentTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/CommentTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/CommentTest.java new file mode 100644 index 0000000..0bd5baa --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/CommentTest.java @@ -0,0 +1,91 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class CommentTest { + + Comment magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new Comment(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("#", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List<String> inputs = Arrays.asList( + "#comment", + " # comment ", + " #comment" + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List<String> inputs = Arrays.asList( + "foo", + " define ", + "bar" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testComment() { + StellarResult result = magic.execute("# this is a comment ", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals("", result.getValue().get()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/DocCommandTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/DocCommandTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/DocCommandTest.java new file mode 100644 index 0000000..c3a39c4 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/DocCommandTest.java @@ -0,0 +1,86 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.dsl.functions.StringFunctions; +import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver; +import org.junit.Before; +import org.junit.Test; + +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertTrue; + +public class DocCommandTest { + + DocCommand command; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the command + command = new DocCommand(); + + // setup a function resolver - only 3 functions have been defined + SimpleFunctionResolver functionResolver = new SimpleFunctionResolver() + .withClass(StringFunctions.ToString.class) + .withClass(StringFunctions.ToLower.class) + .withClass(StringFunctions.ToUpper.class); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(functionResolver, props, Optional.empty()); + executor.init(); + } + + @Test + public void testWithFunction() { + StellarResult result = command.execute("?TO_STRING", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // validate that we have some sort of doc string + assertTrue(result.getValue().toString().length() > 0); + } + + @Test + public void testFunctionNotDefined() { + StellarResult result = command.execute("?INVALID", executor); + + // validate the result + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + } + + @Test + public void testNoFunction() { + StellarResult result = command.execute("?", executor); + + // validate the result + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobalTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobalTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobalTest.java new file mode 100644 index 0000000..4749103 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicDefineGlobalTest.java @@ -0,0 +1,140 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MagicDefineGlobalTest { + + MagicDefineGlobal magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new MagicDefineGlobal(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("%define", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List<String> inputs = Arrays.asList( + "%define", + " %define ", + "%define x := 2", + " %define x := 2 " + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List<String> inputs = Arrays.asList( + "foo", + " define ", + "bar" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testDefine() { + final int expected = 4; + + { + StellarResult result = magic.execute("%define global := 2 + 2", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + assertEquals(expected, result.getValue().get()); + + // ensure global config was updated + assertTrue(executor.getGlobalConfig().containsKey("global")); + assertEquals(expected, executor.getGlobalConfig().get("global")); + } + // + { + // get all globals + StellarResult result = executor.execute("%globals"); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + String out = ConversionUtils.convert(result.getValue().get(), String.class); + assertEquals("{global=4}", out); + } + } + + @Test + public void testNotAssignmentExpression() { + StellarResult result = magic.execute("%define 2 + 2", executor); + + // validate the result + assertTrue(result.isError()); + assertFalse(result.getValue().isPresent()); + assertTrue(result.getException().isPresent()); + + // the global config should not have changed + assertEquals(0, executor.getGlobalConfig().size()); + } + + @Test + public void testMissingExpression() { + StellarResult result = magic.execute("%define", executor); + + // validate the result + assertTrue(result.isError()); + assertFalse(result.getValue().isPresent()); + assertTrue(result.getException().isPresent()); + + // the global config should not have changed + assertEquals(0, executor.getGlobalConfig().size()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctionsTest.java new file mode 100644 index 0000000..204f9a3 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListFunctionsTest.java @@ -0,0 +1,136 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.functions.StringFunctions; +import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MagicListFunctionsTest { + + MagicListFunctions magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new MagicListFunctions(); + + // setup a function resolver - only 3 functions have been defined + SimpleFunctionResolver functionResolver = new SimpleFunctionResolver() + .withClass(StringFunctions.ToString.class) + .withClass(StringFunctions.ToLower.class) + .withClass(StringFunctions.ToUpper.class); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(functionResolver, props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("%functions", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List<String> inputs = Arrays.asList( + "%functions", + " %functions ", + "%functions FOO", + " %functions FOO " + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List<String> inputs = Arrays.asList( + "foo", + " functions ", + "bar", + "%define" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testFunctions() { + StellarResult result = magic.execute("%functions", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // there are 3 functions that should be returned + String value = ConversionUtils.convert(result.getValue().get(), String.class); + String[] functions = value.split(", "); + assertEquals(3, functions.length); + } + + @Test + public void testFunctionsWithMatch() { + StellarResult result = magic.execute("%functions UPPER", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // only 1 function; TO_UPPER should be returned + String value = ConversionUtils.convert(result.getValue().get(), String.class); + String[] functions = value.split(", "); + assertEquals(1, functions.length); + assertEquals("TO_UPPER", functions[0]); + } + + @Test + public void testFunctionsWithNoMatch() { + StellarResult result = magic.execute("%functions NOMATCH", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // no functions should be returned + String value = ConversionUtils.convert(result.getValue().get(), String.class); + String[] functions = value.trim().split(", "); + assertEquals(1, functions.length); + assertEquals("", functions[0]); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobalsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobalsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobalsTest.java new file mode 100644 index 0000000..f02bfdb --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListGlobalsTest.java @@ -0,0 +1,113 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MagicListGlobalsTest { + + MagicListGlobals magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new MagicListGlobals(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("%globals", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List<String> inputs = Arrays.asList( + "%globals", + " %globals ", + "%globals FOO", + " %globals FOO " + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List<String> inputs = Arrays.asList( + "foo", + " globals ", + "bar", + "%define" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void test() { + // define some globals + executor.getGlobalConfig().put("x", 2); + + // get all globals + StellarResult result = executor.execute("%globals"); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + String out = ConversionUtils.convert(result.getValue().get(), String.class); + assertEquals("{x=2}", out); + } + + @Test + public void testWithNoGlobals() { + // get all globals + StellarResult result = executor.execute("%globals"); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + String out = ConversionUtils.convert(result.getValue().get(), String.class); + assertEquals("{}", out); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListVariablesTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListVariablesTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListVariablesTest.java new file mode 100644 index 0000000..51ef0ac --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicListVariablesTest.java @@ -0,0 +1,113 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MagicListVariablesTest { + + MagicListVariables magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new MagicListVariables(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("%vars", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List<String> inputs = Arrays.asList( + "%vars", + " %vars ", + "%vars FOO", + " %vars FOO " + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List<String> inputs = Arrays.asList( + "foo", + " vars ", + "bar", + "%define" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void test() { + // define some vars + executor.execute("x := 2 + 2"); + StellarResult result = executor.execute("%vars"); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // validate the list of vars + String vars = ConversionUtils.convert(result.getValue().get(), String.class); + assertEquals("x = 4 via `2 + 2`", vars); + } + + @Test + public void testWithNoVars() { + // there are no vars defined + StellarResult result = executor.execute("%vars"); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // validate the list of vars + String vars = ConversionUtils.convert(result.getValue().get(), String.class); + assertEquals("", vars); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobalTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobalTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobalTest.java new file mode 100644 index 0000000..4db02f4 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/MagicUndefineGlobalTest.java @@ -0,0 +1,117 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.common.shell.specials; + +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MagicUndefineGlobalTest { + + MagicUndefineGlobal magic; + DefaultStellarShellExecutor executor; + + @Before + public void setup() throws Exception { + + // setup the %magic + magic = new MagicUndefineGlobal(); + + // setup the executor + Properties props = new Properties(); + executor = new DefaultStellarShellExecutor(props, Optional.empty()); + executor.init(); + } + + @Test + public void testGetCommand() { + assertEquals("%undefine", magic.getCommand()); + } + + @Test + public void testShouldMatch() { + List<String> inputs = Arrays.asList( + "%undefine", + " %undefine ", + "%undefine FOO", + " %undefine FOO " + ); + for(String in : inputs) { + assertTrue("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testShouldNotMatch() { + List<String> inputs = Arrays.asList( + "foo", + " undefine ", + "bar", + "%define" + ); + for(String in : inputs) { + assertFalse("failed: " + in, magic.getMatcher().apply(in)); + } + } + + @Test + public void testUndefine() { + // define a global + executor.getGlobalConfig().put("global", 22); + assertEquals(1, executor.getGlobalConfig().size()); + + // use the magic to undefine the global variable + StellarResult result = magic.execute("%undefine global", executor); + + // validate the result + assertTrue(result.isSuccess()); + assertTrue(result.getValue().isPresent()); + + // ensure that the global variable does not exist + assertEquals(0, executor.getGlobalConfig().size()); + } + + @Test + public void testWithNoVariable() { + // define a global + executor.getGlobalConfig().put("global", 22); + assertEquals(1, executor.getGlobalConfig().size()); + + // no arg specifying the var to undefine + StellarResult result = magic.execute("%undefine", executor); + + // validate the result + assertTrue(result.isError()); + assertTrue(result.getException().isPresent()); + + // ensure that the global variables were not changed + assertEquals(1, executor.getGlobalConfig().size()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-zeppelin/README.md ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-zeppelin/README.md b/metron-stellar/stellar-zeppelin/README.md new file mode 100644 index 0000000..1190658 --- /dev/null +++ b/metron-stellar/stellar-zeppelin/README.md @@ -0,0 +1,160 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +Stellar Interpreter for Apache Zeppelin +======================================= + +[Apache Zeppelin](https://zeppelin.apache.org/) is a web-based notebook that enables data-driven, interactive data analytics and collaborative documents with SQL, Scala and more. This project provides a means to run the Stellar REPL directly within a Zeppelin Notebook. + +* [Prerequisites](#prerequisites) +* [Installation](#installation) +* [Usage](#usage) + + +Prerequisites +------------- + +* [Apache Zeppelin](https://zeppelin.apache.org/) 0.7.3 + + This is tested with version 0.7.3. Other versions may work, but are not supported. + + +Installation +------------ + +Currently, you need to manually install the Stellar Interpreter in Zeppelin. In the future this step could be automated by the Metron Mpack. + +To install the Stellar Interpreter in your Apache Zeppelin installation, follow these instructions. This is paraphrased from the [Zeppelin docs](https://zeppelin.apache.org/docs/latest/development/writingzeppelininterpreter.html#install-your-interpreter-binary). + +1. Build and install Metron. Metron and its dependencies will be retrieved from your local Maven repository. + + ``` + cd $METRON_HOME + mvn clean install -DskipTests + ``` + +1. If you do not already have Zeppelin installed, [download and unpack Apache Zeppelin](https://zeppelin.apache.org/download.html). The directory in which you unpack Zeppelin will be referred to as `$ZEPPELIN_HOME`. + +1. If Zeppelin was already installed, make sure that it is not running. + +1. Create a settings directory for the Stellar interpreter. + + ``` + mkdir $ZEPPELIN_HOME/interpreter/stellar + cat <<EOF > $ZEPPELIN_HOME/interpreter/stellar/interpreter-setting.json + [ + { + "group": "stellar", + "name": "stellar", + "className": "org.apache.metron.stellar.zeppelin.StellarInterpreter", + "properties": { + } + } + ] + EOF + ``` + +1. Create a Zeppelin Site file (`$ZEPPELIN_HOME/conf/zeppelin-site.xml`). + + ``` + cp $ZEPPELIN_HOME/conf/zeppelin-site.xml.template $ZEPPELIN_HOME/conf/zeppelin-site.xml + ``` + +1. In the Zeppelin site file, add `org.apache.metron.stellar.zeppelin.StellarInterpreter` to the comma-separated list of Zeppelin interpreters under the `zeppelin.interpreters` property. + + The property will likely look-like the following. + ``` + <property> + <name>zeppelin.interpreters</name> + <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreS qlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.metron.stellar.zeppelin.StellarInterpreter</value> + <description>Comma separated interpreter configurations. First interpreter become a default</description> + </property> + ``` + +1. Start Zeppelin. + + ``` + $ZEPPELIN_HOME/bin/zeppelin-daemon.sh start + ``` + +1. Navigate to Zeppelin running at [http://localhost:8080/](http://localhost:8080/). + +1. Register the Stellar interpreter in Zeppelin. + + 1. Click on the top-right menu item labelled "Anonymous" then choose "Interpreter" in the drop-down that opens. + +1. Configure the Stellar interpreter. + + 1. Click on '**+ Create**' near the top-right. + + 1. Define the following values. + * **Interpreter Name** = `stellar` + * **Interpreter Group** = `stellar` + + 1. Under **Options**, set the following values. + * The interpreter will be instantiated **Per Note** in **isolated** process. + + 1. Under **Dependencies**, define the following fields, then click the "+" icon. Replace the Metron version as required. + * **Artifact** = `org.apache.metron:stellar-zeppelin:0.4.3` + + 1. Click "Save" + +1. Wait for the intrepreter to start. + + 1. Near the title '**stellar**', will be a status icon. This will indicate that it is downloading the dependencies. + + 1. Once the icon is shown as green, the interpreter is ready to work. + +Usage +----- + +1. Create a new notebook. + + 1. Click on "Notebook" > "Create new note". + + 1. Set the default Interpreter to `stellar`. + + When creating the notebook, if you define `stellar` as the default interpreter, then there is no need to enter `%stellar` at the top of each code block. + + If `stellar` is not the default interpreter, then you must enter `%stellar` at the top of a code block containing Stellar code. + +1. In the first block, add the following Stellar, then click Run. + + ``` + 2 in [2,3,4] + ``` + +1. In the next block, check which functions are available to you. + + ``` + %functions + ``` + + You will **only** 'see' the functions defined within `stellar-common` since that is the only library that we added to the interpreter. + +1. To see how additional functions can be added, go back to the Stellar interpreter configuration and add another dependency as follows. + + ``` + org.apache.metron:metron-statistics:0.4.3 + ``` + + Reload the Stellar interpreter and run `%functions` again. You will see the additional functions defined within the `metron-statistics` project. + +1. Auto-completion is also available for Stellar expressions. + + In another block, type 'TO_' then press the <kbd>CTRL</kbd> + <kbd>PERIOD</kbd> keys. This will trigger the auto-complete mechanism in Stellar and display a list of matching functions or variables. http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-zeppelin/pom.xml ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-zeppelin/pom.xml b/metron-stellar/stellar-zeppelin/pom.xml new file mode 100644 index 0000000..7809342 --- /dev/null +++ b/metron-stellar/stellar-zeppelin/pom.xml @@ -0,0 +1,95 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>metron-stellar</artifactId> + <groupId>org.apache.metron</groupId> + <version>0.4.3</version> + </parent> + <name>stellar-zeppelin</name> + <description>Stellar Interpreter for Apache Zeppelin</description> + <artifactId>stellar-zeppelin</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.zeppelin</groupId> + <artifactId>zeppelin-interpreter</artifactId> + <version>0.7.3</version> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <version>${global_mockito_version}</version> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>${global_jar_version}</version> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.antlr</groupId> + <artifactId>antlr4-maven-plugin</artifactId> + <version>${global_antlr_version}</version> + <configuration> + <outputDirectory>${basedir}/src/main/java</outputDirectory> + </configuration> + <executions> + <execution> + <goals> + <goal>antlr4</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptor>src/main/assembly/assembly.xml</descriptor> + </configuration> + <executions> + <execution> + <id>make-assembly</id> <!-- this is used for inheritance merges --> + <phase>package</phase> <!-- bind to the packaging phase --> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + <resources> + <resource> + <directory>src/main/resources</directory> + </resource> + </resources> + </build> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-zeppelin/src/main/assembly/assembly.xml ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-zeppelin/src/main/assembly/assembly.xml b/metron-stellar/stellar-zeppelin/src/main/assembly/assembly.xml new file mode 100644 index 0000000..090835b --- /dev/null +++ b/metron-stellar/stellar-zeppelin/src/main/assembly/assembly.xml @@ -0,0 +1,30 @@ +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<assembly> + <id>archive</id> + <formats> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${project.basedir}/target</directory> + <includes> + <include>${project.artifactId}-${project.version}.jar</include> + </includes> + <outputDirectory>lib</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + </fileSet> + </fileSets> +</assembly> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java b/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java new file mode 100644 index 0000000..58287dc --- /dev/null +++ b/metron-stellar/stellar-zeppelin/src/main/java/org/apache/metron/stellar/zeppelin/StellarInterpreter.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.zeppelin; + +import static org.apache.zeppelin.interpreter.InterpreterResult.Code.ERROR; +import static org.apache.zeppelin.interpreter.InterpreterResult.Code.SUCCESS; +import static org.apache.zeppelin.interpreter.InterpreterResult.Type.TEXT; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.metron.stellar.common.shell.DefaultStellarAutoCompleter; +import org.apache.metron.stellar.common.shell.DefaultStellarShellExecutor; +import org.apache.metron.stellar.common.shell.StellarAutoCompleter; +import org.apache.metron.stellar.common.shell.StellarResult; +import org.apache.metron.stellar.common.shell.StellarShellExecutor; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A Zeppelin Interpreter for Stellar. + */ +public class StellarInterpreter extends Interpreter { + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * Executes the Stellar expressions. + * + * <p>Zeppelin will handle isolation and how the same executor is or is not used across + * multiple notebooks. This is configurable by the user. + * + * <p>See https://zeppelin.apache.org/docs/latest/manual/interpreters.html#interpreter-binding-mode. + */ + private StellarShellExecutor executor; + + /** + * Handles auto-completion for Stellar expressions. + */ + private StellarAutoCompleter autoCompleter; + + public StellarInterpreter(Properties properties) { + super(properties); + this.autoCompleter = new DefaultStellarAutoCompleter(); + } + + @Override + public void open() { + try { + executor = createExecutor(); + + } catch (Exception e) { + LOG.error("Unable to create a StellarShellExecutor", e); + } + } + + @Override + public void close() { + // nothing to do + } + + @Override + public InterpreterResult interpret(final String input, InterpreterContext context) { + InterpreterResult result; + try { + + // execute the input + StellarResult stellarResult = executor.execute(input); + if(stellarResult.isSuccess()) { + + // on success - if no result, use a blank value + Object value = stellarResult.getValue().orElse(""); + String text = value.toString(); + result = new InterpreterResult(SUCCESS, TEXT, text); + + } else if(stellarResult.isError()) { + + // an error occurred + Optional<Throwable> e = stellarResult.getException(); + String message = getErrorMessage(e, input); + result = new InterpreterResult(ERROR, TEXT, message); + + } else { + + // should never happen + throw new IllegalStateException("Unexpected error. result=" + stellarResult); + } + + } catch(Throwable t) { + + // unexpected exception + String message = getErrorMessage(Optional.of(t), input); + result = new InterpreterResult(ERROR, TEXT, message); + } + + return result; + } + + @Override + public void cancel(InterpreterContext context) { + // there is no way to cancel the execution of a Stellar expression + } + + @Override + public FormType getFormType() { + return FormType.SIMPLE; + } + + @Override + public int getProgress(InterpreterContext context) { + // unable to provide progress + return 0; + } + + @Override + public List<InterpreterCompletion> completion(String buf, int cursor) { + + // use the autoCompleter to return a list of completes to Zeppelin + List<InterpreterCompletion> completes = new ArrayList<>(); + for(String candidate : autoCompleter.autoComplete(buf)) { + completes.add(new InterpreterCompletion(candidate, candidate)); + } + + return completes; + } + + /** + * Generates an error message that is shown to the user. + * + * @param e An optional exception that occurred. + * @param input The user input that led to the error condition. + * @return An error message for the user. + */ + private String getErrorMessage(Optional<Throwable> e, String input) { + String message; + if(e.isPresent()) { + + // base the error message on the exception + String error = ExceptionUtils.getRootCauseMessage(e.get()); + String trace = ExceptionUtils.getStackTrace(e.get()); + message = error + System.lineSeparator() + trace; + + } else { + // no exception provided; create generic error message + message = "Invalid expression: " + input; + } + + return message; + } + + /** + * Create an executor that will run the Stellar code for the Zeppelin Notebook. + * @return The stellar executor. + */ + private StellarShellExecutor createExecutor() throws Exception { + + Properties props = getProperty(); + StellarShellExecutor executor = new DefaultStellarShellExecutor(props, Optional.empty()); + + // register the auto-completer to be notified + executor.addSpecialListener((magic) -> autoCompleter.addCandidateFunction(magic.getCommand())); + executor.addFunctionListener((fn) -> autoCompleter.addCandidateFunction(fn.getName())); + executor.addVariableListener((name, val) -> autoCompleter.addCandidateVariable(name)); + + executor.init(); + return executor; + } + + /** + * Returns the executor used to execute Stellar expressions. + * @return The executor of Stellar expressions. + */ + public StellarShellExecutor getExecutor() { + return executor; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-zeppelin/src/main/resources/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-zeppelin/src/main/resources/interpreter-setting.json b/metron-stellar/stellar-zeppelin/src/main/resources/interpreter-setting.json new file mode 100644 index 0000000..cbd974a --- /dev/null +++ b/metron-stellar/stellar-zeppelin/src/main/resources/interpreter-setting.json @@ -0,0 +1,12 @@ +[ + { + "group": "stellar", + "name": "stellar", + "className": "org.apache.metron.stellar.zeppelin.StellarInterpreter", + "properties": { + }, + "editor": { + "completionKey": "TAB" + } + } +] http://git-wip-us.apache.org/repos/asf/metron/blob/2dd01b17/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java new file mode 100644 index 0000000..363938e --- /dev/null +++ b/metron-stellar/stellar-zeppelin/src/test/java/org/apache/metron/stellar/zeppelin/StellarInterpreterTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.zeppelin; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import com.google.common.collect.Iterables; +import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.Properties; + +/** + * Tests the StellarInterpreter. + */ +public class StellarInterpreterTest { + + private StellarInterpreter interpreter; + private InterpreterContext context; + + @Before + public void setup() { + Properties props = new Properties(); + interpreter = new StellarInterpreter(props); + interpreter.open(); + + context = mock(InterpreterContext.class); + } + + /** + * Ensure that we can run Stellar code in the interpreter. + */ + @Test + public void testExecuteStellar() { + InterpreterResult result = interpreter.interpret("2 + 2", context); + + // validate the result + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + + // validate the message + InterpreterResultMessage message = result.message().get(0); + assertEquals("4", message.getData()); + assertEquals(InterpreterResult.Type.TEXT, message.getType()); + } + + /** + * Ensure that Stellar lists are displayed correctly in Zeppelin. + */ + @Test + public void testExecuteWithStellarList() { + final String expected = "[1, 2, 3, 4, 5]"; + InterpreterResult result = interpreter.interpret("[1,2,3,4,5]", context); + + // validate the result + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + + // validate the message + InterpreterResultMessage message = result.message().get(0); + assertEquals(expected, message.getData()); + assertEquals(InterpreterResult.Type.TEXT, message.getType()); + } + + /** + * Ensure that Stellar maps are displayed correctly in Zeppelin. + */ + @Test + public void testExecuteWithStellarMap() { + final String expected = "{foo=2, key=val}"; + InterpreterResult result = interpreter.interpret("{ 'foo':2, 'key':'val' }", context); + + // validate the result + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + + // validate the message + InterpreterResultMessage message = result.message().get(0); + assertEquals(expected, message.getData()); + assertEquals(InterpreterResult.Type.TEXT, message.getType()); + } + + /** + * Ensure that 'bad' Stellar code is handled correctly by the interpreter. + */ + @Test + public void testExecuteBadStellar() { + InterpreterResult result = interpreter.interpret("2 + ", context); + + // validate the result + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertEquals(1, result.message().size()); + + // validate the message + InterpreterResultMessage message = result.message().get(0); + assertTrue(message.getData().length() > 0); + assertEquals(InterpreterResult.Type.TEXT, message.getType()); + } + + /** + * The interpreter should handle if no value is returned. Some Stellar expressions + * will result in no value. + */ + @Test + public void testExecuteNoop() { + + // x is undefined and will have no result + InterpreterResult result = interpreter.interpret("x", context); + + // validate the result + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + + // validate the message + InterpreterResultMessage message = result.message().get(0); + assertEquals(0, message.getData().length()); + assertEquals(InterpreterResult.Type.TEXT, message.getType()); + } + + /** + * The interpreter should support auto-completion. + */ + @Test + public void testAutoCompletion() { + + // the user's input that needs auto-completed + final String buffer = "TO_"; + + // the cursor is at the end of the buffer + int cursor = buffer.length(); + + List<InterpreterCompletion> completions = interpreter.completion(buffer, cursor); + + // expect some completions to be offered + assertTrue(completions.size() > 0); + + for(InterpreterCompletion iCompletion: completions) { + String completion = iCompletion.getValue(); + + // the auto-complete should include an open paren + assertEquals("(", completion.substring(completion.length() - 1)); + + // the candidate should be a valid, defined function + String function = completion.substring(0, completion.length() - 1); + Iterable<String> allFunctions = interpreter.getExecutor().getFunctionResolver().getFunctions(); + String definedFunction = Iterables.find(allFunctions, (fn) -> StringUtils.equals(fn, function)); + assertEquals(function, definedFunction); + } + } + + /** + * What happens when we have nothing useful to auto-complete? + */ + @Test + public void testAutoCompletionWithNoCompletions() { + + // the user's input that needs auto-completed + final String buffer = "NOTHING_AUTOCOMPLETES_THIS_"; + + // the cursor is at the end of the buffer + int cursor = buffer.length(); + + // perform auto-completion + List<InterpreterCompletion> completions = interpreter.completion(buffer, cursor); + + // expect no completions + assertEquals(0, completions.size()); + } +}
