Repository: metron Updated Branches: refs/heads/feature/METRON-1699-create-batch-profiler f5eaef3c2 -> 9455c4ee3
http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java b/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java new file mode 100644 index 0000000..94c8fc8 --- /dev/null +++ b/metron-analytics/metron-profiler-repl/src/test/java/org/apache/metron/profiler/repl/StandAloneProfilerTest.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.profiler.repl; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.configuration.profiler.ProfilerConfig; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.profiler.ProfileMeasurement; +import org.apache.metron.stellar.dsl.Context; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests the StandAloneProfiler class. + */ +public class StandAloneProfilerTest { + + /** + * { + * "profiles": [ + * ] + * } + */ + @Multiline + private String noProfiles; + + /** + * { + * "profiles": [ + * { + * "profile": "profile1", + * "foreach": "'global'", + * "init": { "count": 0 }, + * "update": { "count": "count + 1" }, + * "result": "count" + * } + * ] + * } + */ + @Multiline + private String oneProfile; + + /** + * { + * "profiles": [ + * { + * "profile": "profile1", + * "foreach": "'global1'", + * "result": "'result'" + * }, + * { + * "profile": "profile2", + * "foreach": "'global2'", + * "result": "'result'" + * } + * ] + * } + */ + @Multiline + private String twoProfiles; + + /** + * { + * "ip_src_addr": "10.0.0.1", + * "ip_dst_addr": "10.0.0.20", + * "protocol": "HTTP", + * "timestamp": 2222222222222, + * } + */ + @Multiline + private String messageJson; + + private JSONObject message; + + private long periodDurationMillis = TimeUnit.MINUTES.toMillis(15); + + private Context context = Context.EMPTY_CONTEXT(); + + @Before + public void setup() throws Exception { + + // parse the input message + JSONParser parser = new JSONParser(); + message = (JSONObject) parser.parse(messageJson); + } + + @Test + public void testWithOneProfile() throws Exception { + + StandAloneProfiler profiler = createProfiler(oneProfile); + profiler.apply(message); + profiler.apply(message); + profiler.apply(message); + + List<ProfileMeasurement> measurements = profiler.flush(); + assertEquals(1, measurements.size()); + + // expect 1 measurement for the 1 profile that has been defined + ProfileMeasurement m = measurements.get(0); + assertEquals("profile1", m.getProfileName()); + assertEquals(3, m.getProfileValue()); + } + + + @Test + public void testWithTwoProfiles() throws Exception { + + StandAloneProfiler profiler = createProfiler(twoProfiles); + profiler.apply(message); + profiler.apply(message); + profiler.apply(message); + + List<ProfileMeasurement> measurements = profiler.flush(); + assertEquals(2, measurements.size()); + + // expect 2 measurements, 1 for each profile + List<String> expected = Arrays.asList(new String[] { "profile1", "profile2" }); + { + ProfileMeasurement m = measurements.get(0); + assertTrue(expected.contains(m.getProfileName())); + assertEquals("result", m.getProfileValue()); + } + { + ProfileMeasurement m = measurements.get(1); + assertTrue(expected.contains(m.getProfileName())); + assertEquals("result", m.getProfileValue()); + } + } + + /** + * The message count and route count will always be equal, if there is only one + * profile defined. The message count and route count can be different when there + * are multiple profiles defined that each use the same message. + */ + @Test + public void testRouteAndMessageCounters() throws Exception { + { + StandAloneProfiler profiler = createProfiler(noProfiles); + + profiler.apply(message); + assertEquals(1, profiler.getMessageCount()); + assertEquals(0, profiler.getRouteCount()); + + profiler.apply(message); + assertEquals(2, profiler.getMessageCount()); + assertEquals(0, profiler.getRouteCount()); + + profiler.apply(message); + assertEquals(3, profiler.getMessageCount()); + assertEquals(0, profiler.getRouteCount()); + } + { + StandAloneProfiler profiler = createProfiler(oneProfile); + + profiler.apply(message); + assertEquals(1, profiler.getMessageCount()); + assertEquals(1, profiler.getRouteCount()); + + profiler.apply(message); + assertEquals(2, profiler.getMessageCount()); + assertEquals(2, profiler.getRouteCount()); + + profiler.apply(message); + assertEquals(3, profiler.getMessageCount()); + assertEquals(3, profiler.getRouteCount()); + } + { + StandAloneProfiler profiler = createProfiler(twoProfiles); + + profiler.apply(message); + assertEquals(1, profiler.getMessageCount()); + assertEquals(2, profiler.getRouteCount()); + + profiler.apply(message); + assertEquals(2, profiler.getMessageCount()); + assertEquals(4, profiler.getRouteCount()); + + profiler.apply(message); + assertEquals(3, profiler.getMessageCount()); + assertEquals(6, profiler.getRouteCount()); + } + } + + @Test + public void testProfileCount() throws Exception { + { + StandAloneProfiler profiler = createProfiler(noProfiles); + assertEquals(0, profiler.getProfileCount()); + } + { + StandAloneProfiler profiler = createProfiler(oneProfile); + assertEquals(1, profiler.getProfileCount()); + } + { + StandAloneProfiler profiler = createProfiler(twoProfiles); + assertEquals(2, profiler.getProfileCount()); + } + } + + /** + * Creates a ProfilerConfig based on a string containing JSON. + * + * @param configAsJSON The config as JSON. + * @return The ProfilerConfig. + * @throws Exception + */ + private ProfilerConfig toProfilerConfig(String configAsJSON) throws Exception { + + InputStream in = new ByteArrayInputStream(configAsJSON.getBytes("UTF-8")); + return JSONUtils.INSTANCE.load(in, ProfilerConfig.class); + } + + /** + * Creates the StandAloneProfiler + * + * @param profileJson The Profiler configuration to use as a String containing JSON. + * @throws Exception + */ + private StandAloneProfiler createProfiler(String profileJson) throws Exception { + + // the TTL and max routes need not be bounded + long profileTimeToLiveMillis = Long.MAX_VALUE; + long maxNumberOfRoutes = Long.MAX_VALUE; + + ProfilerConfig config = toProfilerConfig(profileJson); + return new StandAloneProfiler(config, periodDurationMillis, profileTimeToLiveMillis, maxNumberOfRoutes, context); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-analytics/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/pom.xml b/metron-analytics/pom.xml index dc72f3b..37ee2b0 100644 --- a/metron-analytics/pom.xml +++ b/metron-analytics/pom.xml @@ -47,6 +47,7 @@ <module>metron-profiler-client</module> <module>metron-profiler-common</module> <module>metron-profiler-spark</module> + <module>metron-profiler-repl</module> </modules> <dependencies> <dependency> http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml index eae756a..a5a3a26 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml @@ -410,6 +410,9 @@ <name>metron-profiler-spark</name> </package> <package> + <name>metron-profiler-repl</name> + </package> + <package> <name>metron-indexing</name> </package> <package> http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-deployment/packaging/docker/deb-docker/pom.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/deb-docker/pom.xml b/metron-deployment/packaging/docker/deb-docker/pom.xml index 90f7810..a0df09a 100644 --- a/metron-deployment/packaging/docker/deb-docker/pom.xml +++ b/metron-deployment/packaging/docker/deb-docker/pom.xml @@ -138,6 +138,12 @@ </includes> </resource> <resource> + <directory>${metron_dir}/metron-analytics/metron-profiler-repl/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> + <resource> <directory>${metron_dir}/metron-interface/metron-rest/target/</directory> <includes> <include>*.tar.gz</include> http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 47bcad3..cee9cd9 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -59,6 +59,7 @@ Source12: metron-maas-service-%{full_version}-archive.tar.gz Source13: metron-alerts-%{full_version}-archive.tar.gz Source14: metron-performance-%{full_version}-archive.tar.gz Source15: metron-profiler-spark-%{full_version}-archive.tar.gz +Source16: metron-profiler-repl-%{full_version}-archive.tar.gz %description Apache Metron provides a scalable advanced security analytics framework @@ -97,6 +98,7 @@ tar -xzf %{SOURCE12} -C %{buildroot}%{metron_home} tar -xzf %{SOURCE13} -C %{buildroot}%{metron_home} tar -xzf %{SOURCE14} -C %{buildroot}%{metron_home} tar -xzf %{SOURCE15} -C %{buildroot}%{metron_home} +tar -xzf %{SOURCE16} -C %{buildroot}%{metron_home} install %{buildroot}%{metron_home}/bin/metron-management-ui %{buildroot}/etc/init.d/ install %{buildroot}%{metron_home}/bin/metron-alerts-ui %{buildroot}/etc/init.d/ @@ -560,6 +562,23 @@ This package installs the Metron Profiler for Spark %{metron_home} # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +%package profiler-repl +Summary: Metron Profiler for the Stellar REPL +Group: Applications/Internet +Provides: profiler-repl = %{version} + +%description profiler-repl +This package installs the Metron Profiler for the Stellar REPL %{metron_home} + +%files profiler-repl +%defattr(-,root,root,755) +%dir %{metron_root} +%dir %{metron_home} +%dir %{metron_home}/lib +%attr(0644,root,root) %{metron_home}/lib/metron-profiler-repl-%{full_version}.jar + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + %post config chkconfig --add metron-management-ui chkconfig --add metron-alerts-ui @@ -569,6 +588,8 @@ chkconfig --del metron-management-ui chkconfig --del metron-alerts-ui %changelog +* Tue Aug 21 2018 Apache Metron <d...@metron.apache.org> - 0.5.1 +- Add Profiler for REPL * Tue Aug 14 2018 Apache Metron <d...@metron.apache.org> - 0.5.1 - Add Profiler for Spark * Thu Feb 1 2018 Apache Metron <d...@metron.apache.org> - 0.4.3 http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-deployment/packaging/docker/rpm-docker/pom.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml index 1ea8d46..0d9c4d1 100644 --- a/metron-deployment/packaging/docker/rpm-docker/pom.xml +++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml @@ -174,6 +174,12 @@ </includes> </resource> <resource> + <directory>${metron_dir}/metron-analytics/metron-profiler-repl/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> + <resource> <directory>${metron_dir}/metron-interface/metron-rest/target/</directory> <includes> <include>*.tar.gz</include> http://git-wip-us.apache.org/repos/asf/metron/blob/9455c4ee/metron-platform/metron-common/src/main/scripts/stellar ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar index 431ed7b..c831c62 100644 --- a/metron-platform/metron-common/src/main/scripts/stellar +++ b/metron-platform/metron-common/src/main/scripts/stellar @@ -33,4 +33,5 @@ export METRON_VERSION=${project.version} export METRON_HOME=/usr/metron/$METRON_VERSION export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar) export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar) -java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@" +export PROFILER_LIB=$(find $METRON_HOME/lib/ -name metron-profiler-repl*.jar) +java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
