http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml index cecf464..9e8b300 100644 --- a/metron-analytics/metron-profiler/pom.xml +++ b/metron-analytics/metron-profiler/pom.xml @@ -132,7 +132,7 @@ <dependency> <groupId>com.esotericsoftware</groupId> <artifactId>kryo-shaded</artifactId> - <version>3.0.3</version> + <version>${global_kryo_version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java index 695f7b7..da33080 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java @@ -24,7 +24,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import org.apache.metron.common.bolt.ConfiguredProfilerBolt; import org.apache.metron.common.configuration.profiler.ProfileConfig; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.profiler.ProfileBuilder; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.profiler.clock.WallClock; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java index 9ad145a..c0716bf 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java @@ -20,20 +20,19 @@ package org.apache.metron.profiler.bolt; -import org.apache.metron.profiler.stellar.DefaultStellarExecutor; +import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor; +import org.apache.metron.stellar.common.StellarStatefulExecutor; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; -import org.apache.commons.lang.StringUtils; import org.apache.metron.common.bolt.ConfiguredProfilerBolt; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.common.configuration.profiler.ProfilerConfig; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.profiler.stellar.StellarExecutor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -64,7 +63,7 @@ public class ProfileSplitterBolt extends ConfiguredProfilerBolt { /** * Executes Stellar code. */ - private StellarExecutor executor; + private StellarStatefulExecutor executor; /** * @param zookeeperUrl The Zookeeper URL that contains the configuration for this bolt. @@ -78,7 +77,7 @@ public class ProfileSplitterBolt extends ConfiguredProfilerBolt { super.prepare(stormConf, context, collector); this.collector = collector; this.parser = new JSONParser(); - this.executor = new DefaultStellarExecutor(); + this.executor = new DefaultStellarStatefulExecutor(); initializeStellar(); } @@ -160,11 +159,11 @@ public class ProfileSplitterBolt extends ConfiguredProfilerBolt { declarer.declare(new Fields("entity", "profile", "message")); } - public StellarExecutor getExecutor() { + public StellarStatefulExecutor getExecutor() { return executor; } - public void setExecutor(StellarExecutor executor) { + public void setExecutor(StellarStatefulExecutor executor) { this.executor = executor; } } http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java index 935fe57..1d66905 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java @@ -28,7 +28,6 @@ import org.apache.metron.profiler.ProfileBuilder; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.test.bolt.BaseBoltTest; import org.apache.storm.Constants; -import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; import org.json.simple.JSONObject; @@ -41,7 +40,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.concurrent.TimeUnit; -import static org.apache.metron.common.utils.ConversionUtils.convert; +import static org.apache.metron.stellar.common.utils.ConversionUtils.convert; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java index bdeef0b..17d6827 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileHBaseMapperTest.java @@ -25,14 +25,10 @@ import org.apache.storm.tuple.Tuple; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.profiler.hbase.RowKeyBuilder; -import org.apache.metron.profiler.stellar.DefaultStellarExecutor; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.ArgumentCaptor; -import java.util.Arrays; -import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java index 0879835..91cbf32 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java @@ -20,11 +20,11 @@ package org.apache.metron.profiler.bolt; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.profiler.stellar.DefaultStellarExecutor; +import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor; import org.apache.metron.test.bolt.BaseBoltTest; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -35,7 +35,6 @@ import java.io.IOException; import java.util.HashMap; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; import static org.mockito.Matchers.refEq; import static org.mockito.Mockito.*; @@ -143,7 +142,7 @@ public class ProfileSplitterBoltTest extends BaseBoltTest { bolt.setCuratorFramework(client); bolt.setTreeCache(cache); bolt.getConfigurations().updateProfilerConfig(profilerConfig.getBytes("UTF-8")); - bolt.setExecutor(new DefaultStellarExecutor()); + bolt.setExecutor(new DefaultStellarStatefulExecutor()); bolt.prepare(new HashMap<>(), topologyContext, outputCollector); return bolt; @@ -231,7 +230,7 @@ public class ProfileSplitterBoltTest extends BaseBoltTest { /** * What happens when invalid Stella code is used for 'onlyif'? */ - @Test(expected = org.apache.metron.common.dsl.ParseException.class) + @Test(expected = org.apache.metron.stellar.dsl.ParseException.class) public void testOnlyIfInvalid() throws Exception { // setup http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-statistics/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/pom.xml b/metron-analytics/metron-statistics/pom.xml index 6489ae4..2aba75c 100644 --- a/metron-analytics/metron-statistics/pom.xml +++ b/metron-analytics/metron-statistics/pom.xml @@ -30,7 +30,7 @@ <dependency> <groupId>com.esotericsoftware</groupId> <artifactId>kryo</artifactId> - <version>3.0.3</version> + <version>${global_kryo_version}</version> </dependency> <dependency> <groupId>org.apache.metron</groupId> @@ -39,10 +39,8 @@ </dependency> <dependency> <groupId>org.apache.metron</groupId> - <artifactId>metron-common</artifactId> + <artifactId>stellar-common</artifactId> <version>${project.parent.version}</version> - <type>test-jar</type> - <scope>test</scope> </dependency> <dependency> <groupId>org.apache.commons</groupId> http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/BinFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/BinFunctions.java b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/BinFunctions.java new file mode 100644 index 0000000..103d421 --- /dev/null +++ b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/BinFunctions.java @@ -0,0 +1,71 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.statistics; + +import org.apache.metron.stellar.dsl.*; + +import java.util.List; +import java.util.function.Function; + +import static org.apache.metron.stellar.common.utils.ConversionUtils.convert; + +public class BinFunctions { + + /** + * Calculates the statistical bin that a value falls in. + */ + @Stellar(name = "BIN" + , description = "Computes the bin that the value is in given a set of bounds." + , params = { + "value - The value to bin" + , "bounds - A list of value bounds (excluding min and max) in sorted order." + } + ,returns = "Which bin N the value falls in such that bound(N-1) < value <= bound(N). " + + "No min and max bounds are provided, so values smaller than the 0'th bound go in the 0'th bin, " + + "and values greater than the last bound go in the M'th bin." + ) + public static class Bin extends BaseStellarFunction { + + public static int getBin(double value, int numBins, Function<Integer, Double> boundFunc) { + double lastBound = Double.NEGATIVE_INFINITY; + for(int bin = 0; bin < numBins;++bin) { + double bound = boundFunc.apply(bin); + if(lastBound > bound ) { + throw new IllegalStateException("Your bins must be non-decreasing"); + } + if(value <= bound) { + return bin; + } + lastBound = bound; + } + return numBins; + } + + @Override + public Object apply(List<Object> args) { + Double value = convert(args.get(0), Double.class); + final List<Number> bins = args.size() > 1?convert(args.get(1), List.class):null; + if ( value == null || bins == null || bins.size() == 0) { + return -1; + } + return getBin(value, bins.size(), bin -> bins.get(bin).doubleValue()); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/MathFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/MathFunctions.java b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/MathFunctions.java deleted file mode 100644 index 9364095..0000000 --- a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/MathFunctions.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.metron.statistics; - -import org.apache.metron.common.dsl.*; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; - -import static org.apache.metron.common.utils.ConversionUtils.convert; - -public class MathFunctions { - - @Stellar(name="ABS" - ,description="Returns the absolute value of a number." - ,params = { - "number - The number to take the absolute value of" - } - , returns="The absolute value of the number passed in." - ) - public static class Abs implements StellarFunction{ - - @Override - public Object apply(List<Object> args, Context context) throws ParseException { - if(args.size() < 1) { - return Double.NaN; - } - Number n = (Number)args.get(0); - if(n == null) { - return Double.NaN; - } - return Math.abs(n.doubleValue()); - } - - @Override - public void initialize(Context context) { - - } - - @Override - public boolean isInitialized() { - return true; - } - } - - /** - * Calculates the statistical bin that a value falls in. - */ - @Stellar(name = "BIN" - , description = "Computes the bin that the value is in given a set of bounds." - , params = { - "value - The value to bin" - , "bounds - A list of value bounds (excluding min and max) in sorted order." - } - ,returns = "Which bin N the value falls in such that bound(N-1) < value <= bound(N). " + - "No min and max bounds are provided, so values smaller than the 0'th bound go in the 0'th bin, " + - "and values greater than the last bound go in the M'th bin." - ) - public static class Bin extends BaseStellarFunction { - - public static int getBin(double value, int numBins, Function<Integer, Double> boundFunc) { - double lastBound = Double.NEGATIVE_INFINITY; - for(int bin = 0; bin < numBins;++bin) { - double bound = boundFunc.apply(bin); - if(lastBound > bound ) { - throw new IllegalStateException("Your bins must be non-decreasing"); - } - if(value <= bound) { - return bin; - } - lastBound = bound; - } - return numBins; - } - - @Override - public Object apply(List<Object> args) { - Double value = convert(args.get(0), Double.class); - final List<Number> bins = args.size() > 1?convert(args.get(1), List.class):null; - if ( value == null || bins == null || bins.size() == 0) { - return -1; - } - return getBin(value, bins.size(), bin -> bins.get(bin).doubleValue()); - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StellarStatisticsFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StellarStatisticsFunctions.java b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StellarStatisticsFunctions.java index 2223b47..d02734e 100644 --- a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StellarStatisticsFunctions.java +++ b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StellarStatisticsFunctions.java @@ -21,15 +21,14 @@ package org.apache.metron.statistics; import com.google.common.collect.ImmutableList; -import org.apache.metron.common.dsl.BaseStellarFunction; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.common.utils.ConversionUtils; -import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.apache.metron.common.utils.ConversionUtils.convert; +import static org.apache.metron.stellar.common.utils.ConversionUtils.convert; /** * Provides Stellar functions that can calculate summary statistics on @@ -477,7 +476,7 @@ public class StellarStatisticsFunctions { if (stats == null || value == null || bins.size() == 0) { return -1; } - return MathFunctions.Bin.getBin(value, bins.size(), bin -> stats.getPercentile(bins.get(bin).doubleValue())); + return BinFunctions.Bin.getBin(value, bins.size(), bin -> stats.getPercentile(bins.get(bin).doubleValue())); } } http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctions.java b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctions.java index f123494..8bee982 100644 --- a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctions.java +++ b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctions.java @@ -17,9 +17,9 @@ */ package org.apache.metron.statistics.approximation; -import org.apache.metron.common.dsl.BaseStellarFunction; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.common.utils.ConversionUtils; import java.util.ArrayList; import java.util.List; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationFunctions.java b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationFunctions.java index fdb4b33..7f088a9 100644 --- a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationFunctions.java +++ b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationFunctions.java @@ -19,10 +19,10 @@ */ package org.apache.metron.statistics.outlier; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; import org.apache.metron.statistics.OnlineStatisticsProvider; import java.util.ArrayList; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/BinFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/BinFunctionsTest.java b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/BinFunctionsTest.java new file mode 100644 index 0000000..97c3959 --- /dev/null +++ b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/BinFunctionsTest.java @@ -0,0 +1,53 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.statistics; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.common.StellarProcessor; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; + +public class BinFunctionsTest { + public static Object run(String rule, Map<String, Object> variables) { + Context context = Context.EMPTY_CONTEXT(); + StellarProcessor processor = new StellarProcessor(); + Assert.assertTrue(rule + " not valid.", processor.validate(rule, context)); + return processor.parse(rule, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), context); + } + + @Test + public void testBin() { + Assert.assertEquals(run("BIN(value, bounds)", ImmutableMap.of("value", 0, "bounds", ImmutableList.of(10, 20, 30))), 0); + Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 0)), 0); + Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 9)), 0); + Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 10)), 0); + Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 11)), 1); + Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 19)), 1); + Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 21)), 2); + Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 29)), 2); + Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 31)), 3); + Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 1000)), 3); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/MathFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/MathFunctionsTest.java b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/MathFunctionsTest.java deleted file mode 100644 index c615966..0000000 --- a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/MathFunctionsTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.metron.statistics; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.stellar.StellarProcessor; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Map; - -public class MathFunctionsTest { - public static Object run(String rule, Map<String, Object> variables) { - Context context = Context.EMPTY_CONTEXT(); - StellarProcessor processor = new StellarProcessor(); - Assert.assertTrue(rule + " not valid.", processor.validate(rule, context)); - return processor.parse(rule, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), context); - } - - @Test - public void testAbs() { - Assert.assertEquals((Double)run("ABS(value)", ImmutableMap.of("value", 0)), 0, 1e-7); - Assert.assertTrue(Double.isNaN((Double)run("ABS(value)", ImmutableMap.of("value", Double.NaN)))); - Assert.assertEquals((Double)run("ABS(value)", ImmutableMap.of("value", 10.5)), 10.5, 1e-7); - Assert.assertEquals((Double)run("ABS(value)", ImmutableMap.of("value", -10.5)), 10.5, 1e-7); - } - - @Test - public void testBin() { - Assert.assertEquals(run("BIN(value, bounds)", ImmutableMap.of("value", 0, "bounds", ImmutableList.of(10, 20, 30))), 0); - Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 0)), 0); - Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 9)), 0); - Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 10)), 0); - Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 11)), 1); - Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 19)), 1); - Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 21)), 2); - Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 29)), 2); - Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 31)), 3); - Assert.assertEquals(run("BIN(value, [ 10, 20, 30 ])", ImmutableMap.of("value", 1000)), 3); - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java index 029edb9..bbedeca 100644 --- a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java +++ b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/StellarStatisticsFunctionsTest.java @@ -26,10 +26,9 @@ import org.apache.commons.math3.random.GaussianRandomGenerator; import org.apache.commons.math3.random.MersenneTwister; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.stellar.StellarProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.common.utils.SerDeUtils; import org.junit.Assert; import org.junit.Before; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsIntegrationTest.java b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsIntegrationTest.java index cd9cae4..7e67e1c 100644 --- a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsIntegrationTest.java +++ b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsIntegrationTest.java @@ -18,7 +18,7 @@ package org.apache.metron.statistics.approximation; import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.common.utils.StellarProcessorUtils; +import org.apache.metron.stellar.common.utils.StellarProcessorUtils; import org.junit.Assert; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationTest.java b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationTest.java index 952da8d..2676d55 100644 --- a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationTest.java +++ b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/outlier/MedianAbsoluteDeviationTest.java @@ -24,9 +24,9 @@ import org.apache.commons.math3.distribution.TDistribution; import org.apache.commons.math3.random.GaussianRandomGenerator; import org.apache.commons.math3.random.MersenneTwister; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.stellar.StellarProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.common.utils.SerDeUtils; import org.junit.Assert; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-deployment/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 75af150..245f87e 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -115,6 +115,7 @@ Components in the RPMs: - metron-parsers - metron-pcap - metron-solr +- stellar-common ### Prerequisites - Docker. The image detailed in: `metron-deployment/packaging/docker/rpm-docker/README.md` will automatically be built (or rebuilt if necessary). http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java index 321dc48..1f9af3f 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java @@ -19,13 +19,12 @@ package org.apache.metron.rest.service.impl; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.configuration.ConfigurationsUtils; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.StellarFunctionInfo; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.StellarFunctionInfo; +import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.metron.common.field.transformation.FieldTransformations; -import org.apache.metron.common.stellar.StellarProcessor; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.rest.model.StellarFunctionDescription; import org.apache.metron.rest.model.SensorParserContext; import org.apache.metron.rest.service.StellarService; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/3rdPartyStellar.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/3rdPartyStellar.md b/metron-platform/metron-common/3rdPartyStellar.md deleted file mode 100644 index a4d3d2b..0000000 --- a/metron-platform/metron-common/3rdPartyStellar.md +++ /dev/null @@ -1,160 +0,0 @@ -# Custom Stellar Functions - -Metron is fundamentally a programmable, extensible system -and Stellar is the extension language. We have some great Stellar functions -available out of the box and we'll be adding more over time, but they may -not quite scratch quite your particular itch. - -Of course, we'd love to have your contribution inside of Metron if you think it -general purpose enough, but not every function is general-purpose or it may rely -on libraries those licenses aren't acceptable for an Apache project. In that case, then you will -be wondering how to add your custom function to a running instance of Metron. - -## Building Your Own Function - -Let's say that I need a function that returns the current time in milliseconds -since the epoch. I notice that there's nothing like that currently in Metron, -so I embark on the adventure of adding it for my cluster. - -I will presume that you have an installed Metron into your local maven repo via `mvn install` . In the future, when we publish to a maven repo, -you will not need this. I will depend on 0.4.0 for the -purpose of this demonstration - -### Hack, Hack, Hack - -I like to use Maven, so we'll use that for this demonstration, but you can use whatever -build system that you like. Here's my favorite way to build a project with groupId `com.mycompany.stellar` -and artifactId of `tempus` -`mvn archetype:create -DgroupId=com.mycompany.stellar -DartifactId=tempus -DarchetypeArtifactId=maven-archetype-quickstart` - -First, we should depend on `metron-common` and we can do that by adjusting the `pom.xml` just created: -``` -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <groupId>com.mycompany.stellar</groupId> - <artifactId>tempus</artifactId> - <version>1.0-SNAPSHOT</version> - <packaging>jar</packaging> - - <name>Stellar Time Functions</name> - <url>http://mycompany.com</url> - - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - </properties> - - <dependencies> - <dependency> - <groupId>org.apache.metron</groupId> - <artifactId>metron-common</artifactId> - <version>0.4.0</version> - <!-- NOTE: We will want to depend on the deployed common on the classpath. --> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>3.8.1</version> - <scope>test</scope> - </dependency> - </dependencies> -</project> -``` - -Let's add our implementation in `src/main/java/com/mycompany/stellar/TimeFunctions.java` with the following content: -``` -package com.notmetron.stellar; - -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; - -import java.util.List; - -public class TimeFunction { - @Stellar( name="NOW", - description = "Right now!", - params = {}, - returns="Timestamp" - ) - public static class Now implements StellarFunction { - - public Object apply(List<Object> list, Context context) throws ParseException { - return System.currentTimeMillis(); - } - - public void initialize(Context context) { } - - public boolean isInitialized() { - return true; - } - } -} -``` - -Now we can build the project via `mvn package` which will create a `target/tempus-1.0-SNAPSHOT.jar` file. - -## Install the Function - -Now that we have a jar with our custom function, we must make Metron aware of it. - -### Deploy the Jar - -First you need to place the jar in HDFS, if we have it on an access node, one way to do that is: -* `hadoop fs -put tempus-1.0-SNAPSHOT.jar /apps/metron/stellar` -This presumes that: -* you've standardized on `/apps/metron/stellar` as the location for custom jars -* you are running the command from an access node with the `hadoop` command installed -* you are running from a user that has write access to `/apps/metron/stellar` - -### Set Global Config - -You may not need this if your Metron administrator already has this setup. - -With that dispensed with, we need to ensure that Metron knows to look at that location. -We need to ensure that the `stellar.function.paths` property in the `global.json` is in place that makes Metron aware -to look for Stellar functions in `/apps/metron/stellar` on HDFS. - -This property looks like, the following for a vagrant install -``` -{ - "es.clustername": "metron", - "es.ip": "node1", - "es.port": "9300", - "es.date.format": "yyyy.MM.dd.HH", - "stellar.function.paths" : "hdfs://node1:8020/apps/metron/stellar/.*.jar", -} -``` - -The `stellar.function.paths` property takes a comma separated list of URIs or URIs with regex expressions at the end. -Also, note path is prefaced by the HDFS default name, which, if you do not know, can be found by executing, -`hdfs getconf -confKey fs.default.name`, such as -``` -[root@node1 ~]# hdfs getconf -confKey fs.default.name -hdfs://node1:8020 -``` -### Use the Function - -Now that we have deployed the function, if we want to use it, -any running topologies that use Stellar will need to be restarted. - -Beyond that, let's take a look at it in the REPL: -``` -Stellar, Go! -Please note that functions are loading lazily in the background and will be unavailable until loaded fully. -{es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH, stellar.function.paths=hdfs://node1:8020/apps/metron/stellar/.*.jar, profiler.client.period.duration=1, profiler.client.period.duration.units=MINUTES} -[Stellar]>>> # Get the help for NOW -[Stellar]>>> ?NOW -Functions loaded, you may refer to functions now... -NOW -Description: Right now! - -Returns: Timestamp -[Stellar]>>> # Try to run the NOW function, which we added: -[Stellar]>>> NOW() -1488400515655 -[Stellar]>>> # Looks like I got a timestamp, success! -```
