http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java new file mode 100644 index 0000000..871055a --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java @@ -0,0 +1,739 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.dsl.functions; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.*; + +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate; + +@SuppressWarnings("ALL") +public class BasicStellarTest { + + @Stellar( + description="throw exception", + name="THROW", + params = { + "message - exception message" + }, + returns="nothing" + ) + public static class Throw implements StellarFunction { + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + throw new IllegalStateException(Joiner.on(" ").join(args)); + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } + } + + @Stellar( + description="Always returns true", + name="RET_TRUE", + params = { + "arg* - Any set of args you wish to give it (including the empty set), they're ignored." + }, + returns="true" + ) + public static class TrueFunc implements StellarFunction { + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + return true; + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } + } + + @Test + public void ensureDocumentation() { + ClassLoader classLoader = getClass().getClassLoader(); + int numFound = 0; + for (Class<?> clazz : new ClasspathFunctionResolver().resolvables()) { + if (clazz.isAnnotationPresent(Stellar.class)) { + numFound++; + Stellar annotation = clazz.getAnnotation(Stellar.class); + Assert.assertFalse("Must specify a name for " + clazz.getName(),StringUtils.isEmpty(annotation.name())); + Assert.assertFalse("Must specify a description annotation for " + clazz.getName(),StringUtils.isEmpty(annotation.description())); + Assert.assertTrue("Must specify a non-empty params for " + clazz.getName(), annotation.params().length > 0); + Assert.assertTrue("Must specify a non-empty params for " + clazz.getName(), StringUtils.isNoneEmpty(annotation.params())); + Assert.assertFalse("Must specify a returns annotation for " + clazz.getName(), StringUtils.isEmpty(annotation.returns())); + } + } + Assert.assertTrue(numFound > 0); + } + + @Test + public void testEscapedLiterals() { + Assert.assertEquals("'bar'", run("\"'bar'\"", new HashMap<>())); + Assert.assertEquals("'BAR'", run("TO_UPPER('\\'bar\\'')", new HashMap<>())); + Assert.assertEquals("\"bar\"", run("\"\\\"bar\\\"\"", new HashMap<>())); + Assert.assertEquals("\"bar\"", run("'\"bar\"'", new HashMap<>())); + Assert.assertEquals("\"BAR\"", run("TO_UPPER(\"\\\"bar\\\"\")", new HashMap<>())); + Assert.assertEquals("bar \\ foo", run("'bar \\\\ foo'", new HashMap<>())); + Assert.assertEquals("bar \\\\ foo", run("'bar \\\\\\\\ foo'", new HashMap<>())); + Assert.assertEquals("bar\nfoo", run("'bar\\nfoo'", new HashMap<>())); + Assert.assertEquals("bar\n\nfoo", run("'bar\\n\\nfoo'", new HashMap<>())); + Assert.assertEquals("bar\tfoo", run("'bar\\tfoo'", new HashMap<>())); + Assert.assertEquals("bar\t\tfoo", run("'bar\\t\\tfoo'", new HashMap<>())); + Assert.assertEquals("bar\rfoo", run("'bar\\rfoo'", new HashMap<>())); + Assert.assertEquals("'bar'", run("'\\'bar\\''", new HashMap<>())); + } + + @Test + public void testVariableResolution() { + { + String query = "bar:variable"; + Assert.assertEquals("bar", run(query, ImmutableMap.of("bar:variable", "bar"))); + Assert.assertEquals("grok", run(query, ImmutableMap.of("bar:variable", "grok"))); + } + { + String query = "JOIN(['foo', bar:variable], '')"; + Assert.assertEquals("foobar", run(query, ImmutableMap.of("bar:variable", "bar"))); + Assert.assertEquals("foogrok", run(query, ImmutableMap.of("bar:variable", "grok"))); + } + { + String query = "MAP_GET('bar', { 'foo' : 1, 'bar' : bar:variable})"; + Assert.assertEquals("bar", run(query, ImmutableMap.of("bar:variable", "bar"))); + Assert.assertEquals("grok", run(query, ImmutableMap.of("bar:variable", "grok"))); + } + } + + @Test + public void testIfThenElseBug1() { + String query = "50 + (true == true ? 10 : 20)"; + Assert.assertEquals(60, run(query, new HashMap<>())); + } + + @Test + public void testIfThenElseBug2() { + String query = "50 + (true == false ? 10 : 20)"; + Assert.assertEquals(70, run(query, new HashMap<>())); + } + + @Test + public void testIfThenElseBug3() { + String query = "50 * (true == false ? 2 : 10) + 20"; + Assert.assertEquals(520, run(query, new HashMap<>())); + } + + @Test + public void testIfThenElseBug4() { + String query = "TO_INTEGER(true == true ? 10.0 : 20.0 )"; + Assert.assertEquals(10, run(query, new HashMap<>())); + } + + @Test + public void testVariablesUsed() { + StellarProcessor processor = new StellarProcessor(); + { + Assert.assertEquals(new HashSet<>(), processor.variablesUsed("if 1 < 2 then 'one' else 'two'")); + } + { + Assert.assertEquals(ImmutableSet.of("one") + , processor.variablesUsed("if 1 < 2 then one else 'two'")); + } + { + Assert.assertEquals(ImmutableSet.of("one", "two") + , processor.variablesUsed("if 1 < 2 then one else two")); + } + { + Assert.assertEquals(ImmutableSet.of("bar") + , processor.variablesUsed("MAP_GET('foo', { 'foo' : bar})")); + } + } + + + @Test + public void testFunctionEmptyArgs() { + { + String query = "STARTS_WITH(casey, 'case') or MAP_EXISTS()"; + Assert.assertTrue((Boolean)run(query, ImmutableMap.of("casey", "casey"))); + } + { + String query = "true or MAP_EXISTS()"; + Assert.assertTrue((Boolean)run(query, new HashMap<>())); + } + { + String query = "MAP_EXISTS() or true"; + Assert.assertTrue((Boolean)run(query, new HashMap<>())); + } + } + @Test + public void testNull() { + { + String query = "if 1 < 2 then NULL else true"; + Assert.assertNull(run(query, new HashMap<>())); + } + { + String query = "1 < 2 ? NULL : true"; + Assert.assertNull(run(query, new HashMap<>())); + } + { + String query = "null == null ? true : false"; + Assert.assertTrue((Boolean)run(query, new HashMap<>())); + } + } + + @Test + public void testMapConstant() { + { + String query = "MAP_GET('bar', { 'foo' : 1, 'bar' : 'bar'})"; + Assert.assertEquals("bar", run(query, new HashMap<>())); + } + { + String query = "MAP_GET('blah', { 'blah' : 1 < 2 })"; + Assert.assertEquals(true, run(query, new HashMap<>())); + } + { + String query = "MAP_GET('blah', { 'blah' : not(STARTS_WITH(casey, 'case')) })"; + Assert.assertEquals(false, run(query, ImmutableMap.of("casey", "casey"))); + } + { + String query = "MAP_GET('blah', { 'blah' : one })"; + Assert.assertEquals(1, run(query, ImmutableMap.of("one", 1))); + } + { + String query = "MAP_GET('blah', { 'blah' : null })"; + Assert.assertNull(run(query, new HashMap<>())); + } + { + String query = "MAP_GET('BLAH', { TO_UPPER('blah') : null })"; + Assert.assertNull(run(query, new HashMap<>())); + } + { + String query = "MAP_GET('BLAH', { TO_UPPER('blah') : 1 < 2 })"; + Assert.assertEquals(true, run(query, new HashMap<>())); + } + } + + @Test + public void testIfThenElse() { + { + String query = "if STARTS_WITH(casey, 'case') then 'one' else 'two'"; + Assert.assertEquals("one", run(query, ImmutableMap.of("casey", "casey"))); + } + { + String query = "if 1 < 2 then 'one' else 'two'"; + Assert.assertEquals("one", run(query, new HashMap<>())); + } + { + String query = "if 1 + 1 < 2 then 'one' else 'two'"; + Assert.assertEquals("two", run(query, new HashMap<>())); + } + { + String query = "if 1 + 1 <= 2 AND 1 + 2 in [3] then 'one' else 'two'"; + Assert.assertEquals("one", run(query, new HashMap<>())); + } + { + String query = "if 1 + 1 <= 2 AND (1 + 2 in [3]) then 'one' else 'two'"; + Assert.assertEquals("one", run(query, new HashMap<>())); + } + { + String query = "if not(1 < 2) then 'one' else 'two'"; + Assert.assertEquals("two", run(query, new HashMap<>())); + } + { + String query = "if 1 == 1.0000001 then 'one' else 'two'"; + Assert.assertEquals("two", run(query, new HashMap<>())); + } + { + String query = "if one < two then 'one' else 'two'"; + Assert.assertEquals("one", run(query, ImmutableMap.of("one", 1, "two", 2))); + } + { + String query = "if one == very_nearly_one then 'one' else 'two'"; + Assert.assertEquals("two", run(query, ImmutableMap.of("one", 1, "very_nearly_one", 1.0000001))); + } + { + String query = "if one == very_nearly_one OR one == very_nearly_one then 'one' else 'two'"; + Assert.assertEquals("two", run(query, ImmutableMap.of("one", 1, "very_nearly_one", 1.0000001))); + } + { + String query = "if one == very_nearly_one OR one != very_nearly_one then 'one' else 'two'"; + Assert.assertEquals("one", run(query, ImmutableMap.of("one", 1, "very_nearly_one", 1.0000001))); + } + { + String query = "if one != very_nearly_one OR one == very_nearly_one then 'one' else 'two'"; + Assert.assertEquals("one", run(query, ImmutableMap.of("one", 1, "very_nearly_one", 1.0000001))); + } + { + String query = "if 'foo' in ['foo'] OR one == very_nearly_one then 'one' else 'two'"; + Assert.assertEquals("one", run(query, ImmutableMap.of("one", 1, "very_nearly_one", 1.0000001))); + } + { + String query = "if ('foo' in ['foo']) OR one == very_nearly_one then 'one' else 'two'"; + Assert.assertEquals("one", run(query, ImmutableMap.of("one", 1, "very_nearly_one", 1.0000001))); + } + { + String query = "if not('foo' in ['foo']) OR one == very_nearly_one then 'one' else 'two'"; + Assert.assertEquals("two", run(query, ImmutableMap.of("one", 1, "very_nearly_one", 1.0000001))); + } + { + String query = "if not('foo' in ['foo'] OR one == very_nearly_one) then 'one' else 'two'"; + Assert.assertEquals("two", run(query, ImmutableMap.of("one", 1, "very_nearly_one", 1.0000001))); + } + { + String query = "1 < 2 ? 'one' : 'two'"; + Assert.assertEquals("one", run(query, new HashMap<>())); + } + { + String query = "1 < 2 ? TO_UPPER('one') : 'two'"; + Assert.assertEquals("ONE", run(query, new HashMap<>())); + } + { + String query = "1 < 2 ? one : 'two'"; + Assert.assertEquals("one", run(query, ImmutableMap.of("one", "one"))); + } + { + String query = "1 < 2 ? one*3 : 'two'"; + Assert.assertTrue(Math.abs(3 - (int) run(query, ImmutableMap.of("one", 1))) < 1e-6); + } + { + String query = "1 < 2 AND 1 < 2 ? one*3 : 'two'"; + Assert.assertTrue(Math.abs(3 - (int) run(query, ImmutableMap.of("one", 1))) < 1e-6); + } + { + String query = "1 < 2 AND 1 > 2 ? one*3 : 'two'"; + Assert.assertEquals("two", run(query, ImmutableMap.of("one", 1))); + } + { + String query = "1 > 2 AND 1 < 2 ? one*3 : 'two'"; + Assert.assertEquals("two", run(query, ImmutableMap.of("one", 1))); + } + { + String query = "1 < 2 AND 'foo' in ['', 'foo'] ? one*3 : 'two'"; + Assert.assertEquals(3, run(query, ImmutableMap.of("one", 1))); + } + { + String query = "1 < 2 AND ('foo' in ['', 'foo']) ? one*3 : 'two'"; + Assert.assertEquals(3, run(query, ImmutableMap.of("one", 1))); + } + { + String query = "'foo' in ['', 'foo'] ? one*3 : 'two'"; + Assert.assertEquals(3, run(query, ImmutableMap.of("one", 1))); + } + } + + @Test + public void testInNotIN(){ + HashMap variables = new HashMap<>(); + boolean thrown = false; + try{ + Object o = run("in in ['','in']" ,variables ); + }catch(ParseException pe) { + thrown = true; + } + Assert.assertTrue(thrown); + thrown = false; + + try{ + Assert.assertEquals(true,run("'in' in ['','in']" ,variables )); + }catch(ParseException pe) { + thrown = true; + } + Assert.assertFalse(thrown); + } + + @Test + public void testHappyPath() { + String query = "TO_UPPER(TRIM(foo))"; + Assert.assertEquals("CASEY", run(query, ImmutableMap.of("foo", "casey "))); + } + + @Test + public void testLengthString(){ + String query = "LENGTH(foo)"; + Assert.assertEquals(5, run(query,ImmutableMap.of("foo","abcde"))); + } + @Test + public void testLengthCollection(){ + String query = "LENGTH(foo)"; + Collection c = Arrays.asList(1,2,3,4,5); + Assert.assertEquals(5, run(query,ImmutableMap.of("foo",c))); + } + + @Test + public void testEmptyLengthString(){ + String query = "LENGTH(foo)"; + Assert.assertEquals(0,run(query,ImmutableMap.of("foo",""))); + } + @Test + public void testEmptyLengthCollection(){ + String query = "LENGTH(foo)"; + Collection c = new ArrayList(); + Assert.assertEquals(0,run(query,ImmutableMap.of("foo",c))); + } + @Test + public void testNoVarLength(){ + String query = "LENGTH(foo)"; + Assert.assertEquals(0,run(query,ImmutableMap.of())); + } + + @Test + public void testJoin() { + String query = "JOIN( [ TO_UPPER(TRIM(foo)), 'bar' ], ',')"; + Assert.assertEquals("CASEY,bar", run(query, ImmutableMap.of("foo", "casey "))); + } + + @Test + public void testSplit() { + String query = "JOIN( SPLIT(foo, ':'), ',')"; + Assert.assertEquals("casey,bar", run(query, ImmutableMap.of("foo", "casey:bar"))); + } + + @Test + public void testMapGet() { + String query = "MAP_GET(dc, dc2tz, 'UTC')"; + Assert.assertEquals("UTC" + , run(query, ImmutableMap.of("dc", "nyc" + ,"dc2tz", ImmutableMap.of("la", "PST") + ) + ) + ); + Assert.assertEquals("EST" + , run(query, ImmutableMap.of("dc", "nyc" + ,"dc2tz", ImmutableMap.of("nyc", "EST") + ) + ) + ); + } + + @Test + public void testTLDExtraction() { + String query = "DOMAIN_TO_TLD(foo)"; + Assert.assertEquals("co.uk", run(query, ImmutableMap.of("foo", "www.google.co.uk"))); + } + + @Test + public void testTLDRemoval() { + String query = "DOMAIN_REMOVE_TLD(foo)"; + Assert.assertEquals("www.google", run(query, ImmutableMap.of("foo", "www.google.co.uk"))); + } + + @Test + public void testSubdomainRemoval() { + String query = "DOMAIN_REMOVE_SUBDOMAINS(foo)"; + Assert.assertEquals("google.co.uk", run(query, ImmutableMap.of("foo", "www.google.co.uk"))); + Assert.assertEquals("google.com", run(query, ImmutableMap.of("foo", "www.google.com"))); + } + + @Test + public void testURLToHost() { + String query = "URL_TO_HOST(foo)"; + Assert.assertEquals("www.google.co.uk", run(query, ImmutableMap.of("foo", "http://www.google.co.uk/my/path";))); + } + + @Test + public void testURLToPort() { + String query = "URL_TO_PORT(foo)"; + Assert.assertEquals(80, run(query, ImmutableMap.of("foo", "http://www.google.co.uk/my/path";))); + } + + @Test + public void testURLToProtocol() { + String query = "URL_TO_PROTOCOL(foo)"; + Assert.assertEquals("http", run(query, ImmutableMap.of("foo", "http://www.google.co.uk/my/path";))); + } + + @Test + public void testURLToPath() { + String query = "URL_TO_PATH(foo)"; + Assert.assertEquals("/my/path", run(query, ImmutableMap.of("foo", "http://www.google.co.uk/my/path";))); + } + + @Ignore //until field transformations avail to Stellar + @Test + public void testProtocolToName() { + String query = "PROTOCOL_TO_NAME(protocol)"; + Assert.assertEquals("TCP", run(query, ImmutableMap.of("protocol", "6"))); + Assert.assertEquals("TCP", run(query, ImmutableMap.of("protocol", 6))); + Assert.assertEquals(null, run(query, ImmutableMap.of("foo", 6))); + Assert.assertEquals("chicken", run(query, ImmutableMap.of("protocol", "chicken"))); + } + + @Test + public void testDateConversion() { + long expected =1452013350000L; + { + String query = "TO_EPOCH_TIMESTAMP(foo, 'yyyy-MM-dd HH:mm:ss', 'UTC')"; + Assert.assertEquals(expected, run(query, ImmutableMap.of("foo", "2016-01-05 17:02:30"))); + } + { + String query = "TO_EPOCH_TIMESTAMP(foo, 'yyyy-MM-dd HH:mm:ss')"; + Long ts = (Long) run(query, ImmutableMap.of("foo", "2016-01-05 17:02:30")); + //is it within 24 hours of the UTC? + Assert.assertTrue(Math.abs(ts - expected) < 8.64e+7); + } + } + + @Test + public void testToString() { + Assert.assertEquals("5", run("TO_STRING(foo)", ImmutableMap.of("foo", 5))); + } + + @Test + public void testToInteger() { + Assert.assertEquals(5, run("TO_INTEGER(foo)", ImmutableMap.of("foo", "5"))); + Assert.assertEquals(5, run("TO_INTEGER(foo)", ImmutableMap.of("foo", 5))); + } + + @Test + public void testToDouble() { + Assert.assertEquals(5.1d, run("TO_DOUBLE(foo)", ImmutableMap.of("foo", 5.1d))); + Assert.assertEquals(5.1d, run("TO_DOUBLE(foo)", ImmutableMap.of("foo", "5.1"))); + } + + @Test + public void testGet() { + Map<String, Object> variables = ImmutableMap.of("foo", "www.google.co.uk"); + Assert.assertEquals("www", run("GET_FIRST(SPLIT(DOMAIN_REMOVE_TLD(foo), '.'))", variables)); + Assert.assertEquals("www", run("GET(SPLIT(DOMAIN_REMOVE_TLD(foo), '.'), 0)", variables)); + Assert.assertEquals("google", run("GET_LAST(SPLIT(DOMAIN_REMOVE_TLD(foo), '.'))", variables)); + Assert.assertEquals("google", run("GET(SPLIT(DOMAIN_REMOVE_TLD(foo), '.'), 1)", variables)); + } + + @Test + public void testBooleanOps() throws Exception { + final Map<String, String> variableMap = new HashMap<String, String>() {{ + put("foo", "casey"); + put("empty", ""); + put("spaced", "metron is great"); + }}; + Assert.assertFalse(runPredicate("not('casey' == foo and true)", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("not(not('casey' == foo and true))", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("('casey' == foo) && ( false != true )", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("('casey' == foo) and (FALSE == TRUE)", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("'casey' == foo and FALSE", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("'casey' == foo and true", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("true", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("TRUE", v -> variableMap.get(v))); + } + + @Test + public void testInCollection() throws Exception { + final Map<String, String> variableMap = new HashMap<String, String>() {{ + put("foo", "casey"); + put("empty", ""); + }}; + Assert.assertTrue(runPredicate("foo in [ 'casey', 'david' ]", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("foo in [ ]", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("foo in [ foo, 'david' ]", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("foo in [ 'casey', 'david' ] and 'casey' == foo", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("foo in [ 'casey', 'david' ] and foo == 'casey'", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("foo in [ 'casey' ]", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("foo not in [ 'casey', 'david' ]", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("foo not in [ 'casey', 'david' ] and 'casey' == foo", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("null in [ null, 'something' ]", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("null not in [ null, 'something' ]", v -> variableMap.get(v))); + } + + @Test + public void testInMap() throws Exception { + final Map<String, String> variableMap = new HashMap<String, String>() {{ + put("foo", "casey"); + put("empty", ""); + }}; + Assert.assertTrue(runPredicate("'casey' in { foo : 5 }", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("'casey' not in { foo : 5 }", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("foo in { foo : 5 }", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("foo not in { foo : 5 }", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("'foo' in { 'foo' : 5 }", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("'foo' not in { 'foo' : 5 }", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("foo in { 'casey' : 5 }", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("foo not in { 'casey' : 5 }", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("empty in { foo : 5 }", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("empty not in { foo : 5 }", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("'foo' in { }", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("null in { 'foo' : 5 }", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("null not in { 'foo' : 5 }", v -> variableMap.get(v))); + } + + @Test + public void testInString() throws Exception { + final Map<String, String> variableMap = new HashMap<String, String>() {{ + put("foo", "casey"); + put("empty", ""); + }}; + Assert.assertTrue(runPredicate("'case' in foo", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("'case' not in foo", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("'case' in empty", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("'case' not in empty", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("'case' in [ foo ]", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("'case' not in [ foo ]", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("null in foo", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("null not in foo", v -> variableMap.get(v))); + } + + @Test + public void inNestedInStatement() throws Exception { + final Map<String, String> variableMap = new HashMap<>(); + + Assert.assertTrue(runPredicate("('grok' not in 'foobar') == true", variableMap::get)); + Assert.assertTrue(runPredicate("'grok' not in ('foobar' == true)", variableMap::get)); + Assert.assertFalse(runPredicate("'grok' in 'grokbar' == true", variableMap::get)); + Assert.assertTrue(runPredicate("false in 'grokbar' == true", variableMap::get)); + + Assert.assertTrue(runPredicate("('foo' in 'foobar') == true", variableMap::get)); + Assert.assertFalse(runPredicate("'foo' in ('foobar' == true)", variableMap::get)); + Assert.assertTrue(runPredicate("'grok' not in 'grokbar' == true", variableMap::get)); + Assert.assertTrue(runPredicate("false in 'grokbar' == true", variableMap::get)); + Assert.assertTrue(runPredicate("'foo' in ['foo'] AND 'bar' in ['bar']", variableMap::get)); + Assert.assertTrue(runPredicate("('foo' in ['foo']) AND 'bar' in ['bar']", variableMap::get)); + Assert.assertTrue(runPredicate("'foo' in ['foo'] AND ('bar' in ['bar'])", variableMap::get)); + Assert.assertTrue(runPredicate("('foo' in ['foo']) AND ('bar' in ['bar'])", variableMap::get)); + Assert.assertTrue(runPredicate("('foo' in ['foo'] AND 'bar' in ['bar'])", variableMap::get)); + } + + @Test + public void testExists() throws Exception { + final Map<String, String> variableMap = new HashMap<String, String>() {{ + put("foo", "casey"); + put("empty", ""); + put("spaced", "metron is great"); + }}; + Assert.assertTrue(runPredicate("exists(foo)", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("exists(bar)", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("exists(bar) or true", v -> variableMap.get(v))); + } + + @Test + public void testMapFunctions_advanced() throws Exception { + final Map<String, Object> variableMap = new HashMap<String, Object>() {{ + put("foo", "casey"); + put("bar", "bar.casey.grok"); + put("ip", "192.168.0.1"); + put("empty", ""); + put("spaced", "metron is great"); + put("myMap", ImmutableMap.of("casey", "apple")); + }}; + Assert.assertTrue(runPredicate("MAP_EXISTS(foo, myMap)", v -> variableMap.get(v))); + } + + @Test + public void testLogicalFunctions() throws Exception { + final Map<String, String> variableMap = new HashMap<String, String>() {{ + put("foo", "casey"); + put("ip", "192.168.0.1"); + put("ip_src_addr", "192.168.0.1"); + put("ip_dst_addr", "10.0.0.1"); + put("other_ip", "10.168.0.1"); + put("empty", ""); + put("spaced", "metron is great"); + }}; + Assert.assertTrue(runPredicate("IN_SUBNET(ip, '192.168.0.0/24')", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("IN_SUBNET(ip, '192.168.0.0/24', '11.0.0.0/24')", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("IN_SUBNET(ip, '192.168.0.0/24', '11.0.0.0/24') in [true]", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("true in IN_SUBNET(ip, '192.168.0.0/24', '11.0.0.0/24')", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("IN_SUBNET(ip_dst_addr, '192.168.0.0/24', '11.0.0.0/24')", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("IN_SUBNET(other_ip, '192.168.0.0/24')", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("IN_SUBNET(blah, '192.168.0.0/24')", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("true and STARTS_WITH(foo, 'ca')", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("true and STARTS_WITH(TO_UPPER(foo), 'CA')", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("(true and STARTS_WITH(TO_UPPER(foo), 'CA')) || true", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("true and ENDS_WITH(foo, 'sey')", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("not(IN_SUBNET(ip_src_addr, '192.168.0.0/24') and IN_SUBNET(ip_dst_addr, '192.168.0.0/24'))", v-> variableMap.get(v))); + Assert.assertTrue(runPredicate("IN_SUBNET(ip_src_addr, '192.168.0.0/24')", v-> variableMap.get(v))); + Assert.assertFalse(runPredicate("not(IN_SUBNET(ip_src_addr, '192.168.0.0/24'))", v-> variableMap.get(v))); + Assert.assertFalse(runPredicate("IN_SUBNET(ip_dst_addr, '192.168.0.0/24')", v-> variableMap.get(v))); + Assert.assertTrue(runPredicate("not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24'))", v-> variableMap.get(v))); + } + + @Test + public void testShortCircuit_conditional() throws Exception { + Assert.assertEquals("foo", run("if true then 'foo' else (if false then 'bar' else 'grok')", new HashMap<>())); + Assert.assertEquals("foo", run("if true_var != null && true_var then 'foo' else (if false then 'bar' else 'grok')", ImmutableMap.of("true_var", true))); + Assert.assertEquals("foo", run("if true then 'foo' else THROW('expression')", new HashMap<>())); + Assert.assertEquals("foo", run("true ? 'foo' : THROW('expression')", new HashMap<>())); + Assert.assertEquals("foo", run("if false then THROW('exception') else 'foo'", new HashMap<>())); + Assert.assertEquals("foo", run("false ? THROW('exception') : 'foo'", new HashMap<>())); + Assert.assertEquals(true, run("RET_TRUE(if true then 'foo' else THROW('expression'))", new HashMap<>())); + Assert.assertEquals("foo", run("if true or (true or THROW('if exception')) then 'foo' else THROW('expression')", new HashMap<>())); + Assert.assertEquals("foo", run("if true or (false or THROW('if exception')) then 'foo' else THROW('expression')", new HashMap<>())); + Assert.assertEquals("foo", run("if NOT(true or (false or THROW('if exception'))) then THROW('expression') else 'foo'", new HashMap<>())); + Assert.assertEquals("foo", run("if NOT('metron' in [ 'metron', 'metronicus'] ) then THROW('expression') else 'foo'", new HashMap<>())); + } + + @Test + public void testShortCircuit_boolean() throws Exception { + Assert.assertTrue(runPredicate("'metron' in ['metron', 'metronicus', 'mortron'] or (true or THROW('exception'))", x -> null)); + Assert.assertTrue(runPredicate("true or (true or THROW('exception'))", x -> null)); + Assert.assertTrue(runPredicate("true or (false or THROW('exception'))", x -> null)); + Assert.assertTrue(runPredicate("TO_UPPER('foo') == 'FOO' or (true or THROW('exception'))", x -> null)); + Assert.assertFalse(runPredicate("false and (true or THROW('exception'))", x -> null)); + Assert.assertTrue(runPredicate("true or false or false or true", x -> null)); + Assert.assertFalse(runPredicate("false or (false and THROW('exception'))", x -> null)); + Assert.assertTrue(runPredicate("'casey' == 'casey' or THROW('exception')", x -> null)); + Assert.assertTrue(runPredicate("TO_UPPER('casey') == 'CASEY' or THROW('exception')", x -> null)); + Assert.assertTrue(runPredicate("NOT(TO_UPPER('casey') != 'CASEY') or THROW('exception')", x -> null)); + Assert.assertTrue(runPredicate("(TO_UPPER('casey') == 'CASEY') or THROW('exception')", x -> null)); + Assert.assertFalse(runPredicate("NOT(NOT(TO_UPPER('casey') != 'CASEY') or THROW('exception'))", x -> null)); + Assert.assertFalse(runPredicate("NOT(NOT(TO_UPPER('casey') != 'CASEY')) and THROW('exception')", x -> null)); + Assert.assertTrue(runPredicate("RET_TRUE('foo') or THROW('exception')", x -> null)); + Assert.assertFalse(runPredicate("NOT(foo == null or THROW('exception')) and THROW('and exception')", x -> null)); + Assert.assertTrue(runPredicate("(foo == null or THROW('exception') ) or THROW('and exception')", x -> null)); + Assert.assertTrue(runPredicate("( RET_TRUE('foo', true, false) or ( foo == null or THROW('exception') ) or THROW('and exception')) or THROW('or exception')", x -> null)); + } + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void non_boolean_predicate_throws_exception() { + final Map<String, String> variableMap = new HashMap<String, String>() {{ + put("protocol", "http"); + }}; + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("The rule 'TO_UPPER(protocol)' does not return a boolean value."); + runPredicate("TO_UPPER(protocol)", v -> variableMap.get(v)); + } +}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ConversionFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ConversionFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ConversionFunctionsTest.java new file mode 100644 index 0000000..099d540 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ConversionFunctionsTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.dsl.functions; + +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +public class ConversionFunctionsTest { + + @Test + public void conversionFunctionsShouldProperlyConvertToSpecificType() throws Exception { + assertEquals(1D, new ConversionFunctions.TO_DOUBLE().apply(Collections.singletonList(1))); + assertEquals(1F, new ConversionFunctions.TO_FLOAT().apply(Collections.singletonList(1.0D))); + assertEquals(1, new ConversionFunctions.TO_INTEGER().apply(Collections.singletonList(1.0D))); + assertEquals(1L, new ConversionFunctions.TO_LONG().apply(Collections.singletonList(1F))); + } + + @Test + public void conversionFunctionsShouldProperlyHandleNull() throws Exception { + assertEquals(null, new ConversionFunctions.TO_DOUBLE().apply(Collections.singletonList(null))); + assertEquals(null, new ConversionFunctions.TO_FLOAT().apply(Collections.singletonList(null))); + assertEquals(null, new ConversionFunctions.TO_INTEGER().apply(Collections.singletonList(null))); + assertEquals(null, new ConversionFunctions.TO_LONG().apply(Collections.singletonList(null))); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctionsTest.java new file mode 100644 index 0000000..735b803 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctionsTest.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.dsl.functions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.hamcrest.CoreMatchers; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; + +public class DataStructureFunctionsTest { + + @Test + public void is_empty_handles_happy_path() { + DataStructureFunctions.IsEmpty isEmpty = new DataStructureFunctions.IsEmpty(); + { + boolean empty = (boolean) isEmpty.apply(ImmutableList.of("hello")); + Assert.assertThat("should be false", empty, CoreMatchers.equalTo(false)); + } + { + boolean empty = (boolean) isEmpty.apply(ImmutableList.of(ImmutableList.of("hello", "world"))); + Assert.assertThat("should be false", empty, CoreMatchers.equalTo(false)); + } + { + boolean empty = (boolean) isEmpty.apply(ImmutableList.of(1)); + Assert.assertThat("should be false", empty, CoreMatchers.equalTo(false)); + } + } + + @Test + public void is_empty_handles_empty_values() { + DataStructureFunctions.IsEmpty isEmpty = new DataStructureFunctions.IsEmpty(); + { + boolean empty = (boolean) isEmpty.apply(ImmutableList.of()); + Assert.assertThat("should be true", empty, CoreMatchers.equalTo(true)); + } + { + boolean empty = (boolean) isEmpty.apply(null); + Assert.assertThat("should be true", empty, CoreMatchers.equalTo(true)); + } + { + boolean empty = (boolean) isEmpty.apply(ImmutableList.of("")); + Assert.assertThat("should be true", empty, CoreMatchers.equalTo(true)); + } + } + + @Test + public void listAdd_number() { + for(String expr : ImmutableList.of("LIST_ADD(my_list, 1)" + ,"LIST_ADD([], 1)" + ,"LIST_ADD([], val)" + ) + ) + { + Object o = run(expr, ImmutableMap.of("my_list", new ArrayList<>(), "val", 1)); + Assert.assertTrue(o instanceof List); + List<Number> result = (List<Number>) o; + Assert.assertEquals(1, result.size()); + Assert.assertEquals(1, result.get(0)); + } + } + + @Test + public void listAdd_mixed() { + for(String expr : ImmutableList.of("LIST_ADD(my_list, 1)" + ,"LIST_ADD(['foo'], 1)" + ,"LIST_ADD(['foo'], val)" + ) + ) + { + ArrayList<Object> list = new ArrayList<>(); + list.add("foo"); + Object o = run(expr, ImmutableMap.of("my_list", list, "val", 1)); + Assert.assertTrue(o instanceof List); + List<Object> result = (List<Object>) o; + Assert.assertEquals(2, result.size()); + Assert.assertEquals("foo", result.get(0)); + Assert.assertEquals(1, result.get(1)); + } + } + + @Test + public void listAdd_number_nonempty() { + for(String expr : ImmutableList.of("LIST_ADD(my_list, 2)" + ,"LIST_ADD([1], 2)" + ,"LIST_ADD([1], val)" + ) + ) + { + ArrayList<Integer> list = new ArrayList<>(); + list.add(1); + Object o = run(expr, ImmutableMap.of("my_list", list, "val", 2)); + Assert.assertTrue(o instanceof List); + List<Number> result = (List<Number>) o; + Assert.assertEquals(2, result.size()); + Assert.assertEquals(1, result.get(0)); + Assert.assertEquals(2, result.get(1)); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java new file mode 100644 index 0000000..a36f8ff --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java @@ -0,0 +1,232 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.dsl.functions; + +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.junit.Before; +import org.junit.Test; + +import java.util.Calendar; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests the DateFunctions class. + */ +public class DateFunctionsTest { + + private Map<String, Object> variables = new HashMap<>(); + private Calendar calendar; + + /** + * Runs a Stellar expression. + * @param expr The expression to run. + */ + private Object run(String expr) { + StellarProcessor processor = new StellarProcessor(); + assertTrue(processor.validate(expr)); + return processor.parse(expr, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT()); + } + + /** + * Thu Aug 25 2016 09:27:10 EST + */ + private long AUG2016 = 1472131630748L; + + @Before + public void setup() { + variables.put("epoch", AUG2016); + calendar = Calendar.getInstance(); + } + + @Test + public void testDayOfWeek() { + Object result = run("DAY_OF_WEEK(epoch)"); + assertEquals(Calendar.THURSDAY, result); + } + + /** + * If no argument, then return the current day of week. + */ + @Test + public void testDayOfWeekNow() { + Object result = run("DAY_OF_WEEK()"); + assertEquals(calendar.get(Calendar.DAY_OF_WEEK), result); + } + + /** + * If refer to variable that does not exist, expect null returned. + */ + @Test + public void testDayOfWeekNull() { + Object result = run("DAY_OF_WEEK(nada)"); + assertEquals(null, result); + } + + @Test + public void testWeekOfMonth() { + Object result = run("WEEK_OF_MONTH(epoch)"); + assertEquals(4, result); + } + + /** + * If no argument, then return the current week of month. + */ + @Test + public void testWeekOfMonthNow() { + Object result = run("WEEK_OF_MONTH()"); + assertEquals(calendar.get(Calendar.WEEK_OF_MONTH), result); + } + + /** + * If refer to variable that does not exist, expect null returned. + */ + @Test + public void testWeekOfMonthNull() { + Object result = run("WEEK_OF_MONTH(nada)"); + assertEquals(null, result); + } + + @Test + public void testMonth() { + Object result = run("MONTH(epoch)"); + assertEquals(Calendar.AUGUST, result); + } + + /** + * If no argument, then return the current month. + */ + @Test + public void testMonthNow() { + Object result = run("MONTH()"); + assertEquals(calendar.get(Calendar.MONTH), result); + } + + /** + * If refer to variable that does not exist, expect null returned. + */ + @Test + public void testMonthNull() { + Object result = run("MONTH(nada)"); + assertEquals(null, result); + } + + @Test + public void testYear() { + Object result = run("YEAR(epoch)"); + assertEquals(2016, result); + } + + /** + * If no argument, then return the current year. + */ + @Test + public void testYearNow() { + Object result = run("YEAR()"); + assertEquals(calendar.get(Calendar.YEAR), result); + } + + /** + * If refer to variable that does not exist, expect null returned. + */ + @Test + public void testYearNull() { + Object result = run("YEAR(nada)"); + assertEquals(null, result); + } + + @Test + public void testDayOfMonth() { + Object result = run("DAY_OF_MONTH(epoch)"); + assertEquals(25, result); + } + + /** + * If no argument, then return the current day of month. + */ + @Test + public void testDayOfMonthNow() { + Object result = run("DAY_OF_MONTH()"); + assertEquals(calendar.get(Calendar.DAY_OF_MONTH), result); + } + + /** + * If refer to variable that does not exist, expect null returned. + */ + @Test + public void testDayOfMonthNull() { + Object result = run("DAY_OF_MONTH(nada)"); + assertEquals(null, result); + } + + @Test + public void testWeekOfYear() { + Object result = run("WEEK_OF_YEAR(epoch)"); + assertEquals(35, result); + } + + /** + * If no argument, then return the current week of year. + */ + @Test + public void testWeekOfYearNow() { + Object result = run("WEEK_OF_YEAR()"); + assertEquals(calendar.get(Calendar.WEEK_OF_YEAR), result); + } + + /** + * If refer to variable that does not exist, expect null returned. + */ + @Test + public void testWeekOfYearNull() { + Object result = run("WEEK_OF_YEAR(nada)"); + assertEquals(null, result); + } + + @Test + public void testDayOfYear() { + Object result = run("DAY_OF_YEAR(epoch)"); + assertEquals(238, result); + } + + /** + * If no argument, then return the current day of year. + */ + @Test + public void testDayOfYearNow() { + Object result = run("DAY_OF_YEAR()"); + assertEquals(calendar.get(Calendar.DAY_OF_YEAR), result); + } + + /** + * If refer to variable that does not exist, expect null returned. + */ + @Test + public void testDayOfYearNull() { + Object result = run("DAY_OF_YEAR(nada)"); + assertEquals(null, result); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctionsTest.java new file mode 100644 index 0000000..3eec2b0 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctionsTest.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.dsl.functions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.hamcrest.CoreMatchers; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; + +public class FunctionalFunctionsTest { + + @Test + public void testRecursive() { + for (String expr : ImmutableList.of( "MAP(list, inner_list -> REDUCE(inner_list, (x, y) -> x + y, 0) )" + , "MAP(list, (inner_list) -> REDUCE(inner_list, (x, y) -> x + y, 0) )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("list", ImmutableList.of(ImmutableList.of(1, 2, 3), ImmutableList.of(4, 5, 6)))); + Assert.assertTrue(o instanceof List); + List<Number> result = (List<Number>) o; + Assert.assertEquals(2, result.size()); + Assert.assertEquals(6, result.get(0)); + Assert.assertEquals(15, result.get(1)); + } + } + + @Test + public void testMap_null() { + for (String expr : ImmutableList.of( "MAP([ 1, 2, null], x -> if x == null then 0 else 2*x )" + , "MAP([ 1, 2, null], x -> x == null ? 0 : 2*x )" + , "MAP([ 1, foo, baz], x -> x == null ? 0 : 2*x )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", 2, "bar", 3)); + Assert.assertTrue(o instanceof List); + List<String> result = (List<String>) o; + Assert.assertEquals(3, result.size()); + Assert.assertEquals(2, result.get(0)); + Assert.assertEquals(4, result.get(1)); + Assert.assertEquals(0, result.get(2)); + } + } + + + @Test + public void testMap() { + for (String expr : ImmutableList.of( "MAP([ 'foo', 'bar'], (x) -> TO_UPPER(x) )" + , "MAP([ foo, 'bar'], (x) -> TO_UPPER(x) )" + , "MAP([ foo, bar], (x) -> TO_UPPER(x) )" + , "MAP([ foo, bar], x -> TO_UPPER(x) )" + , "MAP([ foo, bar], x -> true?TO_UPPER(x):THROW('error') )" + , "MAP([ foo, bar], x -> false?THROW('error'):TO_UPPER(x) )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", "foo", "bar", "bar")); + Assert.assertTrue(o instanceof List); + List<String> result = (List<String>) o; + Assert.assertEquals(2, result.size()); + Assert.assertEquals("FOO", result.get(0)); + Assert.assertEquals("BAR", result.get(1)); + } + } + + + @Test + public void testMap_conditional() { + for (String expr : ImmutableList.of("MAP([ 'foo', 'bar'], (item) -> item == 'foo' )" + ,"MAP([ foo, bar], (item) -> item == 'foo' )" + ,"MAP([ foo, bar], (item) -> item == foo )" + ,"MAP([ foo, bar], item -> item == foo )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", "foo", "bar", "bar")); + Assert.assertTrue(o instanceof List); + List<Boolean> result = (List<Boolean>) o; + Assert.assertEquals(2, result.size()); + Assert.assertEquals(true, result.get(0)); + Assert.assertEquals(false, result.get(1)); + } + } + + @Test + public void testFilter() { + for (String expr : ImmutableList.of("FILTER([ 'foo', 'bar'], (item) -> item == 'foo' )" + ,"FILTER([ 'foo', bar], (item) -> item == 'foo' )" + ,"FILTER([ foo, bar], (item) -> item == 'foo' )" + ,"FILTER([ foo, bar], (item) -> (item == 'foo' && true) )" + ,"FILTER([ foo, bar], (item) -> if item == 'foo' then true else false )" + ,"FILTER([ foo, bar], item -> if item == 'foo' then true else false )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", "foo", "bar", "bar")); + Assert.assertTrue(o instanceof List); + List<String> result = (List<String>) o; + Assert.assertEquals(1, result.size()); + Assert.assertEquals("foo", result.get(0)); + } + } + + + @Test + public void testFilter_shortcircuit() { + for (String expr : ImmutableList.of("FILTER([ 'foo'], item -> item == 'foo' or THROW('exception') )" + ,"FILTER([ 'foo'], (item) -> item == 'foo' or THROW('exception') )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", "foo", "bar", "bar")); + Assert.assertTrue(o instanceof List); + List<String> result = (List<String>) o; + Assert.assertEquals(1, result.size()); + Assert.assertEquals("foo", result.get(0)); + } + } + + @Test + public void testFilter_null() { + for (String expr : ImmutableList.of("FILTER([ 'foo', null], item -> item == null )" + ,"FILTER([ 'foo', baz], (item) -> item == null )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", "foo", "bar", "bar")); + Assert.assertTrue(o instanceof List); + List<String> result = (List<String>) o; + Assert.assertEquals(1, result.size()); + Assert.assertEquals(null, result.get(0)); + } + } + + @Test + public void testFilter_notnull() { + for (String expr : ImmutableList.of("FILTER([ 'foo', null], item -> item != null )" + ,"FILTER([ 'foo', baz], (item) -> item != null )" + ,"FILTER([ foo, baz], (item) -> item != null )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", "foo", "bar", "bar")); + Assert.assertTrue(o instanceof List); + List<String> result = (List<String>) o; + Assert.assertEquals(1, result.size()); + Assert.assertEquals("foo", result.get(0)); + } + } + + @Test + public void testFilter_none() { + for (String expr : ImmutableList.of( "FILTER([ foo, bar], () -> false )" + , "FILTER([ 'foo', 'bar'], (item)-> false )" + ,"FILTER([ 'foo', bar], (item ) -> false )" + ,"FILTER([ foo, bar], (item) -> false )" + ,"FILTER([ foo, bar], item -> false )" + + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", "foo", "bar", "bar")); + Assert.assertTrue(o instanceof List); + List<String> result = (List<String>) o; + Assert.assertEquals(0, result.size()); + } + } + + @Test + public void testFilter_all() { + for (String expr : ImmutableList.of("FILTER([ 'foo', 'bar'], (item) -> true )" + ,"FILTER([ 'foo', bar], (item) -> true )" + ,"FILTER([ foo, bar], (item) -> true )" + ,"FILTER([ foo, bar], item -> true )" + ,"FILTER([ foo, bar], ()-> true )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", "foo", "bar", "bar")); + Assert.assertTrue(o instanceof List); + List<String> result = (List<String>) o; + Assert.assertEquals(2, result.size()); + Assert.assertEquals("foo", result.get(0)); + Assert.assertEquals("bar", result.get(1)); + } + } + + @Test + public void testReduce_null() { + for (String expr : ImmutableList.of("REDUCE([ 1, 2, 3, null], (x, y) -> if y != null then x + y else x , 0 )" + ,"REDUCE([ foo, bar, 3, baz], (sum, y) -> if y != null then sum + y else sum, 0 )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", 1, "bar", 2)); + Assert.assertTrue(o instanceof Number); + Number result = (Number) o; + Assert.assertEquals(6, result.intValue()); + } + } + + @Test + public void testReduce() { + for (String expr : ImmutableList.of("REDUCE([ 1, 2, 3 ], (x, y) -> x + y , 0 )" + ,"REDUCE([ foo, bar, 3 ], (x, y) -> x + y , 0 )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", 1, "bar", 2)); + Assert.assertTrue(o instanceof Number); + Number result = (Number) o; + Assert.assertEquals(6, result.intValue()); + } + } + + @Test + public void testReduce_on_various_list_sizes() { + { + String expr = "REDUCE([ 1, 2, 3, 4 ], (x, y) -> x + y , 0 )"; + Object o = run(expr, ImmutableMap.of()); + Assert.assertTrue(o instanceof Number); + Number result = (Number) o; + Assert.assertEquals(10, result.intValue()); + } + { + String expr = "REDUCE([ 1, 2 ], (x, y) -> x + y , 0 )"; + Object o = run(expr, ImmutableMap.of()); + Assert.assertTrue(o instanceof Number); + Number result = (Number) o; + Assert.assertEquals(3, result.intValue()); + } + { + String expr = "REDUCE([ 1 ], (x, y) -> x + y , 0 )"; + Object o = run(expr, ImmutableMap.of()); + Assert.assertTrue(o instanceof Number); + Number result = (Number) o; + Assert.assertEquals(1, result.intValue()); + } + } + + @Test + public void testReduce_NonNumeric() { + for (String expr : ImmutableList.of("REDUCE([ 'foo', 'bar', 'grok'], (x, y) -> LIST_ADD(x, y), [] )" + ) + ) + { + Object o = run(expr, ImmutableMap.of("foo", 1, "bar", 2)); + Assert.assertTrue(o instanceof List); + List<String> result = (List<String>) o; + Assert.assertEquals(3, result.size()); + Assert.assertEquals("foo", result.get(0)); + Assert.assertEquals("bar", result.get(1)); + Assert.assertEquals("grok", result.get(2)); + } + } + + @Test + public void testReduce_returns_null_when_less_than_3_args() { + { + String expr = "REDUCE([ 1, 2, 3 ], (x, y) -> LIST_ADD(x, y))"; + Assert.assertThat(run(expr, ImmutableMap.of()), CoreMatchers.equalTo(null)); + } + { + String expr = "REDUCE([ 1, 2, 3 ])"; + Assert.assertThat(run(expr, ImmutableMap.of()), CoreMatchers.equalTo(null)); + } + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/MathFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/MathFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/MathFunctionsTest.java new file mode 100644 index 0000000..d427fb4 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/MathFunctionsTest.java @@ -0,0 +1,47 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.stellar.dsl.functions; + +import com.google.common.collect.ImmutableMap; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; + +public class MathFunctionsTest { + public static Object run(String rule, Map<String, Object> variables) { + Context context = Context.EMPTY_CONTEXT(); + StellarProcessor processor = new StellarProcessor(); + Assert.assertTrue(rule + " not valid.", processor.validate(rule, context)); + return processor.parse(rule, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), context); + } + + @Test + public void testAbs() { + Assert.assertEquals((Double)run("ABS(value)", ImmutableMap.of("value", 0)), 0, 1e-7); + Assert.assertTrue(Double.isNaN((Double)run("ABS(value)", ImmutableMap.of("value", Double.NaN)))); + Assert.assertEquals((Double)run("ABS(value)", ImmutableMap.of("value", 10.5)), 10.5, 1e-7); + Assert.assertEquals((Double)run("ABS(value)", ImmutableMap.of("value", -10.5)), 10.5, 1e-7); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java new file mode 100644 index 0000000..ab6ca82 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java @@ -0,0 +1,430 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.dsl.functions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.commons.collections4.map.HashedMap; +import org.apache.metron.stellar.dsl.ParseException; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Calendar; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate; + +public class StringFunctionsTest { + + @Test + public void testStringFunctions() throws Exception { + final Map<String, String> variableMap = new HashMap<String, String>() {{ + put("foo", "casey"); + put("ip", "192.168.0.1"); + put("empty", ""); + put("spaced", "metron is great"); + }}; + Assert.assertTrue(runPredicate("true and TO_UPPER(foo) == 'CASEY'", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("foo in [ TO_LOWER('CASEY'), 'david' ]", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("TO_UPPER(foo) in [ TO_UPPER('casey'), 'david' ] and IN_SUBNET(ip, '192.168.0.0/24')", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("TO_LOWER(foo) in [ TO_UPPER('casey'), 'david' ]", v -> variableMap.get(v))); + } + + @Test + public void testStringFunctions_advanced() throws Exception { + final Map<String, Object> variableMap = new HashMap<String, Object>() {{ + put("foo", "casey"); + put("bar", "bar.casey.grok"); + put("ip", "192.168.0.1"); + put("empty", ""); + put("spaced", "metron is great"); + put("myList", ImmutableList.of("casey", "apple", "orange")); + }}; + Assert.assertTrue(runPredicate("foo in SPLIT(bar, '.')", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("foo in SPLIT(ip, '.')", v -> variableMap.get(v))); + Assert.assertTrue(runPredicate("foo in myList", v -> variableMap.get(v))); + Assert.assertFalse(runPredicate("foo not in myList", v -> variableMap.get(v))); + } + + @Test + public void testLeftRightFills() throws Exception { + final Map<String, Object> variableMap = new HashMap<String, Object>() {{ + put("foo", null); + put("bar", null); + put("notInt", "oh my"); + }}; + + //LEFT + Object left = run("FILL_LEFT('123','X', 10)", new HashedMap()); + Assert.assertNotNull(left); + Assert.assertEquals(10, ((String) left).length()); + Assert.assertEquals("XXXXXXX123", (String) left); + + //RIGHT + Object right = run("FILL_RIGHT('123','X', 10)", new HashedMap()); + Assert.assertNotNull(right); + Assert.assertEquals(10, ((String) right).length()); + Assert.assertEquals("123XXXXXXX", (String) right); + + //INPUT ALREADY LENGTH + Object same = run("FILL_RIGHT('123','X', 3)", new HashedMap()); + Assert.assertEquals(3, ((String) same).length()); + Assert.assertEquals("123", (String) same); + + //INPUT BIGGER THAN LENGTH + Object tooBig = run("FILL_RIGHT('1234567890','X', 3)", new HashedMap()); + Assert.assertEquals(10, ((String) tooBig).length()); + Assert.assertEquals("1234567890", (String) tooBig); + + //NULL VARIABLES + boolean thrown = false; + try { + run("FILL_RIGHT('123',foo,bar)", variableMap); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("are both required")); + } + Assert.assertTrue(thrown); + thrown = false; + + // NULL LENGTH + try { + run("FILL_RIGHT('123','X',bar)", variableMap); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("are both required")); + } + Assert.assertTrue(thrown); + thrown = false; + + // NULL FILL + try { + run("FILL_RIGHT('123',foo, 7)", variableMap); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("are both required")); + } + Assert.assertTrue(thrown); + thrown = false; + + // NON INTEGER LENGTH + try { + run("FILL_RIGHT('123','X', 'z' )", new HashedMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("not a valid Integer")); + } + Assert.assertTrue(thrown); + thrown = false; + + // EMPTY STRING PAD + try { + Object returnValue = run("FILL_RIGHT('123','', 10 )", new HashedMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("cannot be an empty")); + } + Assert.assertTrue(thrown); + thrown = false; + + //MISSING LENGTH PARAMETER + try { + run("FILL_RIGHT('123',foo)", variableMap); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("expects three")); + } + Assert.assertTrue(thrown); + } + + @Test + public void shannonEntropyTest() throws Exception { + //test empty string + Assert.assertEquals(0.0, (Double) run("STRING_ENTROPY('')", new HashMap<>()), 0.0); + Assert.assertEquals(0.0, (Double) run("STRING_ENTROPY(foo)", ImmutableMap.of("foo", "")), 0.0); + + /* + Now consider the string aaaaaaaaaabbbbbccccc or 10 a's followed by 5 b's and 5 c's. + The probabilities of each character is as follows: + p(a) = 1/2 + p(b) = 1/4 + p(c) = 1/4 + so the shannon entropy should be + -p(a)*log_2(p(a)) - p(b)*log_2(p(b)) - p(c)*log_2(p(c)) = + -0.5*-1 - 0.25*-2 - 0.25*-2 = 1.5 + */ + Assert.assertEquals(1.5, (Double) run("STRING_ENTROPY(foo)", ImmutableMap.of("foo", "aaaaaaaaaabbbbbccccc")), 0.0); + } + + @Test + public void testFormat() throws Exception { + + Map<String, Object> vars = ImmutableMap.of( + "cal", new Calendar.Builder().setDate(2017, 02, 02).build(), + "x", 234, + "y", 3); + + Assert.assertEquals("no args", run("FORMAT('no args')", vars)); + Assert.assertEquals("234.0", run("FORMAT('%.1f', TO_DOUBLE(234))", vars)); + Assert.assertEquals("000234", run("FORMAT('%06d', 234)", vars)); + Assert.assertEquals("03 2,2017", run("FORMAT('%1$tm %1$te,%1$tY', cal)", vars)); + Assert.assertEquals("234 > 3", run("FORMAT('%d > %d', x, y)", vars)); + Assert.assertEquals("missing: null", run("FORMAT('missing: %d', missing)", vars)); + } + + /** + * FORMAT - Not passing a format string will throw an exception + */ + @Test(expected = ParseException.class) + public void testFormatWithNoArguments() throws Exception { + run("FORMAT()", Collections.emptyMap()); + } + + /** + * FORMAT - Forgetting to pass an argument required by the format string will throw an exception. + */ + @Test(expected = ParseException.class) + public void testFormatWithMissingArguments() throws Exception { + run("FORMAT('missing arg: %d')", Collections.emptyMap()); + } + + + /** + * CHOMP StringFunction + * + * @throws Exception + */ + @Test + public void testChomp() throws Exception { + Assert.assertEquals("abc", run("CHOMP('abc')", new HashedMap())); + Assert.assertEquals("abc", run("CHOMP(msg)", ImmutableMap.of("msg", "abc\r\n"))); + Assert.assertEquals("", run("CHOMP(msg)", ImmutableMap.of("msg", "\n"))); + Assert.assertEquals("", run("CHOMP('')", new HashedMap())); + Assert.assertEquals(null, run("CHOMP(msg)", new HashedMap())); + Assert.assertEquals(null, run("CHOMP(null)", new HashedMap())); + + // No input + boolean thrown = false; + try { + run("CHOMP()", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("missing argument")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Integer input + try { + run("CHOMP(123)", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("cannot be cast")); + } + Assert.assertTrue(thrown); + + } + + /** + * CHOP StringFunction + * + * @throws Exception + */ + @Test + public void testChop() throws Exception { + Assert.assertEquals("ab", run("CHOP('abc')", new HashedMap())); + Assert.assertEquals(null, run("CHOP(null)", new HashedMap())); + Assert.assertEquals(null, run("CHOP(msg)", new HashedMap())); + Assert.assertEquals("abc", run("CHOP(msg)", ImmutableMap.of("msg", "abc\r\n"))); + Assert.assertEquals("", run("CHOP(msg)", ImmutableMap.of("msg", ""))); + Assert.assertEquals("", run("CHOP(msg)", ImmutableMap.of("msg", "\n"))); + Assert.assertEquals("", run("CHOP('')", new HashedMap())); + + // No input + boolean thrown = false; + try { + run("CHOP()", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("missing argument")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Integer input + try { + run("CHOP(123)", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("cannot be cast")); + } + Assert.assertTrue(thrown); + + } + + /** + * PREPEND_IF_MISSING StringFunction + */ + @Test + public void testPrependIfMissing() throws Exception { + Assert.assertEquals("xyzabc", run("PREPEND_IF_MISSING('abc', 'xyz')", new HashedMap())); + Assert.assertEquals("xyzXYZabc", run("PREPEND_IF_MISSING('XYZabc', 'xyz', 'mno')", new HashedMap())); + Assert.assertEquals("mnoXYZabc", run("PREPEND_IF_MISSING('mnoXYZabc', 'xyz', 'mno')", new HashedMap())); + Assert.assertEquals(null, run("PREPEND_IF_MISSING(null, null, null)", new HashedMap())); + Assert.assertEquals("xyz", run("PREPEND_IF_MISSING('', 'xyz', null)", new HashedMap())); + + // No input + boolean thrown = false; + try { + run("PREPEND_IF_MISSING()", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("incorrect arguments")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Incorrect number of arguments - 1 + try { + run("PREPEND_IF_MISSING('abc')", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("incorrect arguments")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Incorrect number of arguments - 2 + try { + run("PREPEND_IF_MISSING('abc', 'def', 'ghi', 'jkl')", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("incorrect arguments")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Integer input + try { + run("PREPEND_IF_MISSING(123, 'abc')", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("cannot be cast")); + } + Assert.assertTrue(thrown); + + } + + /** + * APPEND_IF_MISSING StringFunction + */ + @Test + public void testAppendIfMissing() throws Exception { + Assert.assertEquals("apachemetron", run("APPEND_IF_MISSING('apache', 'metron')", new HashedMap())); + Assert.assertEquals("abcXYZxyz", run("APPEND_IF_MISSING('abcXYZ', 'xyz', 'mno')", new HashedMap())); + Assert.assertEquals(null, run("APPEND_IF_MISSING(null, null, null)", new HashedMap())); + Assert.assertEquals("xyz", run("APPEND_IF_MISSING('', 'xyz', null)", new HashedMap())); + + // No input + boolean thrown = false; + try { + run("APPEND_IF_MISSING()", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("incorrect arguments")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Incorrect number of arguments - 1 + try { + run("APPEND_IF_MISSING('abc')", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("incorrect arguments")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Incorrect number of arguments - 2 + try { + run("APPEND_IF_MISSING('abc', 'def', 'ghi', 'jkl')", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("incorrect arguments")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Integer input + try { + run("APPEND_IF_MISSING(123, 'abc')", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("cannot be cast")); + } + Assert.assertTrue(thrown); + + } + + /** + * COUNT_MATCHES StringFunction + */ + @Test + public void testCountMatches() throws Exception { + Assert.assertEquals(0, (int) run("COUNT_MATCHES(null, '*')", new HashedMap())); + Assert.assertEquals(2, (int) run("COUNT_MATCHES('apachemetron', 'e')", new HashedMap())); + Assert.assertEquals(2, (int) run("COUNT_MATCHES('anand', 'an')", new HashedMap())); + Assert.assertEquals(0, (int) run("COUNT_MATCHES('abcd', null)", new HashedMap())); + + // No input + boolean thrown = false; + try { + run("COUNT_MATCHES()", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("incorrect arguments")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Incorrect number of arguments - 1 + try { + run("COUNT_MATCHES('abc')", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("incorrect arguments")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Integer input + try { + run("COUNT_MATCHES(123, 456)", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("cannot be cast")); + } + Assert.assertTrue(thrown); + + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SystemFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SystemFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SystemFunctionsTest.java new file mode 100644 index 0000000..9dff323 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SystemFunctionsTest.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.dsl.functions; + +import com.google.common.collect.ImmutableList; +import org.apache.metron.stellar.common.system.Environment; +import org.junit.Test; + +import java.util.ArrayList; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SystemFunctionsTest { + + @Test + public void smoke_test_non_mocked_env() { + SystemFunctions.EnvGet envGet = new SystemFunctions.EnvGet(); + String envVal = (String) envGet.apply(ImmutableList.of("ENV_GET_VAR")); + assertThat("Value should not exist", envVal, equalTo(null)); + } + + @Test + public void env_get_returns_value() { + Environment env = mock(Environment.class); + when(env.get("ENV_GET_VAR")).thenReturn("ENV_GET_VALUE"); + SystemFunctions.EnvGet envGet = new SystemFunctions.EnvGet(env); + String envVal = (String) envGet.apply(ImmutableList.of("ENV_GET_VAR")); + assertThat("Value should match", envVal, equalTo("ENV_GET_VALUE")); + } + + @Test + public void env_get_returns_null_if_key_is_not_string() { + SystemFunctions.EnvGet envGet = new SystemFunctions.EnvGet(); + String envVal = (String) envGet.apply(ImmutableList.of(new ArrayList())); + assertThat("Value should be null", envVal, equalTo(null)); + } + + @Test + public void property_get_returns_value() { + System.getProperties().put("ENV_GET_VAR", "ENV_GET_VALUE"); + SystemFunctions.PropertyGet propertyGet = new SystemFunctions.PropertyGet(); + String propertyVal = (String) propertyGet.apply(ImmutableList.of("ENV_GET_VAR")); + assertThat("Value should match", propertyVal, equalTo("ENV_GET_VALUE")); + } + + @Test + public void property_get_nonexistent_returns_null() { + SystemFunctions.PropertyGet propertyGet = new SystemFunctions.PropertyGet(); + String propertyVal = (String) propertyGet.apply(ImmutableList.of("PROPERTY_MISSING")); + assertThat("Value should not exist", propertyVal, equalTo(null)); + } + + @Test + public void property_get_returns_null_if_key_is_not_string() { + SystemFunctions.PropertyGet propertyGet = new SystemFunctions.PropertyGet(); + String propertyVal = (String) propertyGet.apply(ImmutableList.of(new ArrayList())); + assertThat("Value should be null", propertyVal, equalTo(null)); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java new file mode 100644 index 0000000..72cd53a --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.dsl.functions.resolver; + +import com.google.common.collect.Lists; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.metron.stellar.dsl.Context; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.File; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; + +import static org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver.Config.*; + +public class ClasspathFunctionResolverTest { + + private static List<String> expectedFunctions; + + @BeforeClass + public static void setup() { + + // search the entire classpath for functions - provides a baseline to test against + Properties config = new Properties(); + // use a permissive regex that should not filter anything + ClasspathFunctionResolver resolver = create(config); + + expectedFunctions = Lists.newArrayList(resolver.getFunctions()); + } + + /** + * Create a function resolver to test. + * @param config The configuration for Stellar. + */ + public static ClasspathFunctionResolver create(Properties config) { + ClasspathFunctionResolver resolver = new ClasspathFunctionResolver(); + + Context context = new Context.Builder() + .with(Context.Capabilities.STELLAR_CONFIG, () -> config) + .build(); + resolver.initialize(context); + + return resolver; + } + + @Test + public void testInclude() { + + // setup - include all `org.apache.metron.*` functions + Properties config = new Properties(); + config.put(STELLAR_SEARCH_INCLUDES_KEY.param(), "org.apache.metron.*"); + + // execute + ClasspathFunctionResolver resolver = create(config); + List<String> actual = Lists.newArrayList(resolver.getFunctions()); + + // validate - should have found all of the functions + Assert.assertEquals(expectedFunctions, actual); + } + + @Test + public void testWithMultipleIncludes() { + + // setup - include all of the common and management functions, which is most of them + Properties config = new Properties(); + config.put(STELLAR_SEARCH_INCLUDES_KEY.param(), "org.apache.metron.common.*, org.apache.metron.management.*"); + + // execute + ClasspathFunctionResolver resolver = create(config); + List<String> actual = Lists.newArrayList(resolver.getFunctions()); + + // validate - should have found all of the functions + Assert.assertTrue(actual.size() > 0); + Assert.assertTrue(actual.size() <= expectedFunctions.size()); + } + + @Test + public void testExclude() { + + // setup - exclude all `org.apache.metron.*` functions + Properties config = new Properties(); + config.put(STELLAR_SEARCH_EXCLUDES_KEY.param(), "org.apache.metron.*"); + + // use a permissive regex that should not filter anything + ClasspathFunctionResolver resolver = create(config); + List<String> actual = Lists.newArrayList(resolver.getFunctions()); + + // both should have resolved the same functions + Assert.assertEquals(0, actual.size()); + } + + @Ignore //until relocate dependencies for src/test/classpath-resources/custom-1.0-SNAPSHOT.jar + @Test + public void testExternalLocal() throws FileSystemException, ClassNotFoundException { + File jar = new File("src/test/classpath-resources"); + Assert.assertTrue(jar.exists()); + Properties config = new Properties(); + config.put(STELLAR_VFS_PATHS.param(), jar.toURI() + "/.*.jar"); + + ClasspathFunctionResolver resolver = create(config); + HashSet<String> functions = new HashSet<>(Lists.newArrayList(resolver.getFunctions())); + Assert.assertTrue(functions.contains("NOW")); + } + + + @Ignore //until relocate dependencies for src/test/classpath-resources/custom-1.0-SNAPSHOT.jar + @Test + public void testExternalHDFS() throws FileSystemException, ClassNotFoundException { + /* TODO: this needs to test HDFS, not local filesystem */ + File jar = new File("src/test/classpath-resources"); + Assert.assertTrue(jar.exists()); + Properties config = new Properties(); + config.put(STELLAR_VFS_PATHS.param(), jar.toURI() + "/.*.jar"); + + ClasspathFunctionResolver resolver = create(config); + HashSet<String> functions = new HashSet<>(Lists.newArrayList(resolver.getFunctions())); + Assert.assertTrue(functions.contains("NOW")); + } +}
