Repository: metron Updated Branches: refs/heads/feature/METRON-1136-extensions-parsers 5f7454e4a -> f6458a380
METRON-1136 Track Master (ottobackwards) closes apache/metron#725 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f6458a38 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f6458a38 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f6458a38 Branch: refs/heads/feature/METRON-1136-extensions-parsers Commit: f6458a380ef77632897949687607b8aefa21b997 Parents: 5f7454e Author: ottobackwards <[email protected]> Authored: Thu Aug 31 12:42:47 2017 -0400 Committer: otto <[email protected]> Committed: Thu Aug 31 12:42:47 2017 -0400 ---------------------------------------------------------------------- .../metron-profiler-client/README.md | 21 +++-- .../client/stellar/ProfilerFunctions.java | 35 ++++++-- .../client/stellar/ProfilerFunctionsTest.java | 93 ++++++++++++++++++-- .../metron/profiler/DefaultProfileBuilder.java | 53 +++++++---- .../metron/profiler/StandAloneProfiler.java | 40 +++++++++ .../profiler/DefaultProfileBuilderTest.java | 2 +- metron-analytics/metron-profiler/README.md | 3 + .../METRON/CURRENT/service_advisor.py | 2 +- .../docker/rpm-docker/SPECS/metron.spec | 23 +++++ .../packaging/docker/rpm-docker/pom.xml | 6 ++ metron-stellar/stellar-common/README.md | 10 ++- .../stellar/common/shell/StellarShell.java | 55 +++++++----- 12 files changed, 285 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-analytics/metron-profiler-client/README.md ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/README.md b/metron-analytics/metron-profiler-client/README.md index a8b5a55..27aa3f4 100644 --- a/metron-analytics/metron-profiler-client/README.md +++ b/metron-analytics/metron-profiler-client/README.md @@ -418,8 +418,11 @@ Follow these steps in the Stellar REPL to see how it can be used to help create ``` [Stellar]>>> profiler := PROFILER_INIT(conf) [Stellar]>>> profiler - org.apache.metron.profiler.StandAloneProfiler@4f8ef473 + Profiler{1 profile(s), 0 messages(s), 0 route(s)} ``` + The profiler itself will show the number of profiles defined, the number of messages applied, and the number of routes taken. + + A route is defined when a message is applied to a specific profile. If a message is applied and not needed by any profile, then there are no routes. If a message is needed by one profile, then one route has been defined. If a message is needed by two profiles, then two routes have been defined. 1. Create a message to simulate the type of telemetry that you expect to be profiled. As an example, in the editor copy/paste the JSON below. ``` @@ -436,14 +439,18 @@ Follow these steps in the Stellar REPL to see how it can be used to help create 1. Apply some telemetry messages to your profiles. The following applies the same message 3 times. ``` [Stellar]>>> PROFILER_APPLY(message, profiler) - org.apache.metron.profiler.StandAloneProfiler@4f8ef473 - + Profiler{1 profile(s), 1 messages(s), 1 route(s)} + ``` + ``` [Stellar]>>> PROFILER_APPLY(message, profiler) - org.apache.metron.profiler.StandAloneProfiler@4f8ef473 - + Profiler{1 profile(s), 2 messages(s), 2 route(s)} + ``` + ``` [Stellar]>>> PROFILER_APPLY(message, profiler) - org.apache.metron.profiler.StandAloneProfiler@4f8ef473 + Profiler{1 profile(s), 3 messages(s), 3 route(s)} ``` + + It is also possible to apply multiple messages at once. This is useful when testing against a larger set of data. To do this, create a string that contains a JSON array of messages and pass that to the `PROFILER_APPLY` function. 1. Flush the Profiler to see what has been calculated. A flush is what occurs at the end of each 15 minute period in the Profiler. The result is a list of profile measurements. Each measurement is a map containing detailed information about the profile data that has been generated. ``` @@ -455,4 +462,4 @@ Follow these steps in the Stellar REPL to see how it can be used to help create This profile simply counts the number of messages by IP source address. Notice that the value is '3' for the entity '10.0.0.1' as we applied 3 messages with an 'ip_src_addr' of '10.0.0.1'. There will always be one measurement for each [profile, entity] pair. -1. If you are unhappy with the data that has been generated, then 'wash, rinse and repeat' this process. After you are satisfied with the data being generated by the profile, then follow the [Getting Started](../metron-profiler#getting-started) guide to use the profile against your live, streaming data in a Metron cluster. \ No newline at end of file +1. If you are unhappy with the data that has been generated, then 'wash, rinse and repeat' this process. Once you are happy with the profile that was created, follow the [Getting Started](../metron-profiler#getting-started) guide to use the profile against your live, streaming data in a Metron cluster. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java index 827e1c4..8df5ca8 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java @@ -28,6 +28,8 @@ import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.storm.shade.org.apache.commons.lang.ClassUtils; +import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.slf4j.LoggerFactory; @@ -107,7 +109,7 @@ public class ProfilerFunctions { name="APPLY", description="Apply a message to a local profile runner.", params={ - "message", "The message to apply.", + "message(s)", "The message to apply. A JSON list can be used to apply multiple messages.", "profiler", "A local profile runner returned by PROFILER_INIT." }, returns="The local profile runner." @@ -129,16 +131,33 @@ public class ProfilerFunctions { @Override public Object apply(List<Object> args, Context context) throws ParseException { - // user must provide the json telemetry message + // user must provide the message as a string String arg0 = Util.getArg(0, String.class, args); if(arg0 == null) { throw new IllegalArgumentException(format("expected string, found null")); } - // parse the message - JSONObject message; + // there could be one or more messages + List<JSONObject> messages = new ArrayList<>(); try { - message = (JSONObject) parser.parse(arg0); + Object parsedArg0 = parser.parse(arg0); + if(parsedArg0 instanceof JSONObject) { + // if there is only one message + messages.add((JSONObject) parsedArg0); + + } else if(parsedArg0 instanceof JSONArray) { + // there are multiple messages + JSONArray jsonArray = (JSONArray) parsedArg0; + for(Object json: jsonArray) { + if(json instanceof JSONObject) { + messages.add((JSONObject) json); + + } else { + throw new IllegalArgumentException(format("invalid message: found '%s', expected JSONObject", + ClassUtils.getShortClassName(json, "null"))); + } + } + } } catch(org.json.simple.parser.ParseException e) { throw new IllegalArgumentException("invalid message", e); @@ -147,10 +166,12 @@ public class ProfilerFunctions { // user must provide the stand alone profiler StandAloneProfiler profiler = Util.getArg(1, StandAloneProfiler.class, args); try { - profiler.apply(message); + for(JSONObject message : messages) { + profiler.apply(message); + } } catch(ExecutionException e) { - throw new IllegalArgumentException(e); + throw new IllegalArgumentException(format("Failed to apply message; error=%s", e.getMessage()), e); } return profiler; http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java index 9cc8046..bad3efe 100644 --- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java +++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java @@ -58,6 +58,28 @@ public class ProfilerFunctionsTest { private String message; /** + * [ + * { + * "ip_src_addr": "10.0.0.1", + * "ip_dst_addr": "10.0.0.2", + * "source.type": "test", + * }, + * { + * "ip_src_addr": "10.0.0.1", + * "ip_dst_addr": "10.0.0.2", + * "source.type": "test", + * }, + * { + * "ip_src_addr": "10.0.0.1", + * "ip_dst_addr": "10.0.0.2", + * "source.type": "test", + * } + * ] + */ + @Multiline + private String messages; + + /** * { * "profiles": [ * { @@ -108,7 +130,9 @@ public class ProfilerFunctionsTest { state.put("config", "{ \"profiles\" : [] }"); StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); assertNotNull(profiler); - assertEquals(0, profiler.getConfig().getProfiles().size()); + assertEquals(0, profiler.getProfileCount()); + assertEquals(0, profiler.getMessageCount()); + assertEquals(0, profiler.getRouteCount()); } @Test @@ -116,7 +140,9 @@ public class ProfilerFunctionsTest { state.put("config", helloWorldProfilerDef); StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); assertNotNull(profiler); - assertEquals(1, profiler.getConfig().getProfiles().size()); + assertEquals(1, profiler.getProfileCount()); + assertEquals(0, profiler.getMessageCount()); + assertEquals(0, profiler.getRouteCount()); } @Test(expected = IllegalArgumentException.class) @@ -144,7 +170,9 @@ public class ProfilerFunctionsTest { StandAloneProfiler profiler = executor.execute(expression, state, StandAloneProfiler.class); assertNotNull(profiler); - assertEquals(1, profiler.getConfig().getProfiles().size()); + assertEquals(1, profiler.getProfileCount()); + assertEquals(0, profiler.getMessageCount()); + assertEquals(0, profiler.getRouteCount()); } @Test @@ -158,19 +186,74 @@ public class ProfilerFunctionsTest { // apply a message to the profiler state.put("message", message); StandAloneProfiler result = run("PROFILER_APPLY(message, profiler)", StandAloneProfiler.class); + + // validate + assertSame(profiler, result); + assertEquals(1, profiler.getProfileCount()); + assertEquals(1, profiler.getMessageCount()); + assertEquals(1, profiler.getRouteCount()); + } + + @Test + public void testProfilerApplyWithMultipleMessages() { + + // initialize the profiler + state.put("config", helloWorldProfilerDef); + StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); + state.put("profiler", profiler); + + // apply a message to the profiler + state.put("messages", messages); + StandAloneProfiler result = run("PROFILER_APPLY(messages, profiler)", StandAloneProfiler.class); + + // validate assertSame(profiler, result); + assertEquals(1, profiler.getProfileCount()); + assertEquals(3, profiler.getMessageCount()); + assertEquals(3, profiler.getRouteCount()); + } + + @Test + public void testProfilerApplyWithEmptyList() { + + // initialize the profiler + state.put("config", helloWorldProfilerDef); + StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); + state.put("profiler", profiler); + + // apply a message to the profiler + state.put("messages", "[ ]"); + StandAloneProfiler result = run("PROFILER_APPLY(messages, profiler)", StandAloneProfiler.class); + + // validate + assertSame(profiler, result); + assertEquals(1, profiler.getProfileCount()); + assertEquals(0, profiler.getMessageCount()); + assertEquals(0, profiler.getRouteCount()); } @Test(expected = IllegalArgumentException.class) - public void testProfilerApplyNoArgs() { + public void testProfilerApplyWithNoArgs() { run("PROFILER_APPLY()", StandAloneProfiler.class); } @Test(expected = IllegalArgumentException.class) - public void testProfilerApplyInvalidArg() { + public void testProfilerApplyWithInvalidArg() { run("PROFILER_APPLY(undefined)", StandAloneProfiler.class); } + @Test(expected = IllegalArgumentException.class) + public void testProfilerApplyWithNullMessage() { + + // initialize the profiler + state.put("config", helloWorldProfilerDef); + StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class); + state.put("profiler", profiler); + + // there is no 'messages' variable + StandAloneProfiler result = run("PROFILER_APPLY(messages, profiler)", StandAloneProfiler.class); + } + @Test public void testProfilerFlush() { http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java index 1f352d0..2e34160 100644 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java +++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java @@ -27,9 +27,11 @@ import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.collections4.ListUtils; @@ -245,17 +247,28 @@ public class DefaultProfileBuilder implements ProfileBuilder, Serializable { * @param expressionType The type of expression; init, update, result. Provides additional context if expression execution fails. */ private void assign(Map<String, String> expressions, Map<String, Object> transientState, String expressionType) { - try { - // execute each of the 'update' expressions - MapUtils.emptyIfNull(expressions) - .forEach((var, expr) -> executor.assign(var, expr, transientState)); + // for each expression... + for(Map.Entry<String, String> entry : MapUtils.emptyIfNull(expressions).entrySet()) { + String var = entry.getKey(); + String expr = entry.getValue(); + + try { + // assign the result of the expression to the variable + executor.assign(var, expr, transientState); - } catch(ParseException e) { + } catch (Throwable e) { - // make it brilliantly clear that one of the 'update' expressions is bad - String msg = format("Bad '%s' expression: error=%s, profile=%s, entity=%s", expressionType, e.getMessage(), profileName, entity); - throw new ParseException(msg, e); + // in-scope variables = persistent state maintained by the profiler + the transient state + Set<String> variablesInScope = new HashSet<>(); + variablesInScope.addAll(transientState.keySet()); + variablesInScope.addAll(executor.getState().keySet()); + + String msg = format("Bad '%s' expression: error='%s', expr='%s', profile='%s', entity='%s', variables-available='%s'", + expressionType, e.getMessage(), expr, profileName, entity, variablesInScope); + LOG.error(msg, e); + throw new ParseException(msg, e); + } } } @@ -269,14 +282,24 @@ public class DefaultProfileBuilder implements ProfileBuilder, Serializable { private List<Object> execute(List<String> expressions, Map<String, Object> transientState, String expressionType) { List<Object> results = new ArrayList<>(); - try { - ListUtils.emptyIfNull(expressions) - .forEach((expr) -> results.add(executor.execute(expr, transientState, Object.class))); + for(String expr: ListUtils.emptyIfNull(expressions)) { + try { + // execute an expression + Object result = executor.execute(expr, transientState, Object.class); + results.add(result); + + } catch (Throwable e) { - } catch (Throwable e) { - String msg = format("Bad '%s' expression: error=%s, profile=%s, entity=%s", expressionType, e.getMessage(), profileName, entity); - LOG.error(msg, e); - throw new ParseException(msg, e); + // in-scope variables = persistent state maintained by the profiler + the transient state + Set<String> variablesInScope = new HashSet<>(); + variablesInScope.addAll(transientState.keySet()); + variablesInScope.addAll(executor.getState().keySet()); + + String msg = format("Bad '%s' expression: error='%s', expr='%s', profile='%s', entity='%s', variables-available='%s'", + expressionType, e.getMessage(), expr, profileName, entity, variablesInScope); + LOG.error(msg, e); + throw new ParseException(msg, e); + } } return results; http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/StandAloneProfiler.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/StandAloneProfiler.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/StandAloneProfiler.java index cf034c8..6db7079 100644 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/StandAloneProfiler.java +++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/StandAloneProfiler.java @@ -53,12 +53,28 @@ public class StandAloneProfiler { */ private MessageDistributor distributor; + /** + * Counts the number of messages that have been applied. + */ + private int messageCount; + + /** + * Counts the number of routes. + * + * If a message is not needed by any profiles, then there are 0 routes. + * If a message is needed by 1 profile then there is 1 route. + * If a message is needed by 2 profiles then there are 2 routes. + */ + private int routeCount; + public StandAloneProfiler(ProfilerConfig config, long periodDurationMillis, Context context) { this.context = context; this.config = config; this.router = new DefaultMessageRouter(context); // the period TTL does not matter in this context this.distributor = new DefaultMessageDistributor(periodDurationMillis, Long.MAX_VALUE); + this.messageCount = 0; + this.routeCount = 0; } /** @@ -72,6 +88,18 @@ public class StandAloneProfiler { for(MessageRoute route : routes) { distributor.distribute(message, route, context); } + + routeCount += routes.size(); + messageCount += 1; + } + + @Override + public String toString() { + return "Profiler{" + + getProfileCount() + " profile(s), " + + getMessageCount() + " messages(s), " + + getRouteCount() + " route(s)" + + '}'; } /** @@ -85,4 +113,16 @@ public class StandAloneProfiler { public ProfilerConfig getConfig() { return config; } + + public int getProfileCount() { + return (config == null) ? 0: config.getProfiles().size(); + } + + public int getMessageCount() { + return messageCount; + } + + public int getRouteCount() { + return routeCount; + } } http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java index 71ef982..d25b7ff 100644 --- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java +++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultProfileBuilderTest.java @@ -600,7 +600,7 @@ public class DefaultProfileBuilderTest { * "init": { "x": "0" }, * "update": { "x": "x + 1" }, * "result": "x", - * "groupBy": ["2 / 0"] + * "groupBy": ["nonexistant"] * } */ @Multiline http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-analytics/metron-profiler/README.md ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/README.md b/metron-analytics/metron-profiler/README.md index f27af59..9b908f6 100644 --- a/metron-analytics/metron-profiler/README.md +++ b/metron-analytics/metron-profiler/README.md @@ -200,6 +200,7 @@ A common use case would be grouping by day of week. This allows a contiguous sc ``` The expression can reference any of these variables. +* Any variable defined by the profile in its `init` or `update` expressions. * `profile` The name of the profile. * `entity` The name of the entity being profiled. * `start` The start time of the profile period in epoch milliseconds. @@ -281,6 +282,8 @@ In the following example, three values, the minimum, the maximum and the mean ar A numeric value that defines how many days the profile data is retained. After this time, the data expires and is no longer accessible. If no value is defined, the data does not expire. +The REPL can be a powerful for developing profiles. Read all about [Developing Profiles](../metron-profiler-client/#developing_profiles). + ## Configuring the Profiler The Profiler runs as an independent Storm topology. The configuration for the Profiler topology is stored in local filesystem at `$METRON_HOME/config/profiler.properties`. http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py index 2fb1ab0..1a9d195 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py @@ -43,7 +43,7 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0] metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0] - metronProfilerMaster = self.getHosts(componentsList, "METRON_PROFILER")[0] + metronProfilerHost = self.getHosts(componentsList, "METRON_PROFILER")[0] metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0] metronRESTHost = self.getHosts(componentsList, "METRON_REST")[0] http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index b95e6b0..c617842 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -73,6 +73,7 @@ Source21: metron-parser-squid-assembly-%{full_version}-archive.tar.gz Source22: metron-parser-websphere-assembly-%{full_version}-archive.tar.gz Source23: metron-parser-yaf-assembly-%{full_version}-archive.tar.gz Source24: metron-management-%{full_version}-archive.tar.gz +Source25: metron-maas-service-%{full_version}-archive.tar.gz %description Apache Metron provides a scalable advanced security analytics framework @@ -139,6 +140,7 @@ tar -xzf %{SOURCE21} -C %{buildroot}%{metron_extensions_etc_parsers}/squid tar -xzf %{SOURCE22} -C %{buildroot}%{metron_extensions_etc_parsers}/websphere tar -xzf %{SOURCE23} -C %{buildroot}%{metron_extensions_etc_parsers}/yaf tar -xzf %{SOURCE24} -C %{buildroot}%{metron_home} +tar -xzf %{SOURCE25} -C %{buildroot}%{metron_home} # move the bundles from config to extensions lib mv %{buildroot}%{metron_extensions_etc_parsers}/asa/lib/*.bundle %{buildroot}%{metron_extensions_lib}/ @@ -868,7 +870,28 @@ chkconfig --del metron-management-ui # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +%package maas-service +Summary: Metron MaaS service +Group: Application/Internet +Provides: maas-service = %{version} + +%description maas-service +This package install the Metron MaaS Service files %{metron_home} + +%files maas-service +%defattr(-,root,root,755) +%dir %{metron_root} +%dir %{metron_home} +%dir %{metron_home}/bin +%{metron_home}/bin/maas_service.sh +%{metron_home}/bin/maas_deploy.sh +%attr(0644,root,root) %{metron_home}/lib/metron-maas-service-%{full_version}-uber.jar + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + %changelog +* Tue Aug 29 2017 Apache Metron <[email protected]> - 0.4.1 +- Add Metron MaaS service * Thu Jun 29 2017 Apache Metron <[email protected]> - 0.4.1 - Add Metron Management jar * Thu May 15 2017 Apache Metron <[email protected]> - 0.4.0 http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-deployment/packaging/docker/rpm-docker/pom.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml index 6991e7e..db5f288 100644 --- a/metron-deployment/packaging/docker/rpm-docker/pom.xml +++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml @@ -173,6 +173,12 @@ <include>*.tar.gz</include> </includes> </resource> + <resource> + <directory>${metron_dir}/metron-analytics/metron-maas-service/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> <!-- extensions --> <resource> <directory>${metron_dir}/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-assembly/target/</directory> http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-stellar/stellar-common/README.md ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index 8746e60..926132e 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -1052,7 +1052,7 @@ The REPL has a set of magic commands that provide the REPL user with information #### `%functions` -This command lists all functions resolvable in the Stellar environment. Stellar searches the classpath for Stellar functions. This can make it difficult in some cases to understand which functions are resolvable. +This command lists all functions resolvable in the Stellar environment. ``` [Stellar]>>> %functions @@ -1066,7 +1066,13 @@ STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS, STATS STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, TO_DOUBLE, TO_EPOCH_TIMESTAMP, TO_FLOAT, TO_INTEGER, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH, URL_TO_PORT, URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR -[Stellar]>>> +``` + +The list of functions returned can also be filtered by passing an argument. Only the functions containing the argument as a substring will be returned. + +``` +[Stellar]>>> %functions NET +IN_SUBNET ``` #### `%vars` http://git-wip-us.apache.org/repos/asf/metron/blob/f6458a38/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java index 0d2f0c3..a860778 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java @@ -54,6 +54,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -283,36 +284,44 @@ public class StellarShell extends AeshConsoleCallback implements Completion { } /** - * Handles user interaction when executing a Magic command. + * Executes a magic expression. * @param rawExpression The expression to execute. */ private void handleMagic( String rawExpression) { - String expression = rawExpression.trim(); - if(MAGIC_FUNCTIONS.equals(expression)) { + String[] expression = rawExpression.trim().split(" "); - // list all functions + String command = expression[0]; + if(MAGIC_FUNCTIONS.equals(command)) { + + // if '%functions FOO' then show only functions that contain 'FOO' + Predicate<String> nameFilter = (name -> true); + if(expression.length > 1) { + nameFilter = (name -> name.contains(expression[1])); + } + + // list available functions String functions = StreamSupport .stream(executor.getFunctionResolver().getFunctionInfo().spliterator(), false) .map(info -> String.format("%s", info.getName())) + .filter(nameFilter) .sorted() .collect(Collectors.joining(", ")); writeLine(functions); - } else if(MAGIC_VARS.equals(expression)) { + } else if(MAGIC_VARS.equals(command)) { // list all variables - executor.getVariables() .forEach((k,v) -> writeLine(String.format("%s = %s", k, v))); } else { - writeLine(ERROR_PROMPT + "undefined magic command: " + expression); + writeLine(ERROR_PROMPT + "undefined magic command: " + rawExpression); } } /** - * Handles user interaction when executing a doc command. - * @param expression The expression to execute. + * Executes a doc expression. + * @param expression The doc expression to execute. */ private void handleDoc(String expression) { @@ -325,6 +334,17 @@ public class StellarShell extends AeshConsoleCallback implements Completion { } /** + * Executes a quit. + */ + private void handleQuit() { + try { + console.stop(); + } catch (Throwable e) { + e.printStackTrace(); + } + } + + /** * Formats the Stellar function info object into a readable string. * @param info The stellar function info object. * @return A readable string. @@ -380,16 +400,12 @@ public class StellarShell extends AeshConsoleCallback implements Completion { handleDoc(expression); } else if (expression.equals("quit")) { - try { - console.stop(); - } catch (Throwable e) { - e.printStackTrace(); - } - } - else if(expression.charAt(0) == '#') { - return 0; - } - else { + handleQuit(); + + } else if(expression.charAt(0) == '#') { + return 0; // comment, do nothing + + } else { handleStellar(expression); } } @@ -424,7 +440,6 @@ public class StellarShell extends AeshConsoleCallback implements Completion { } } } - } private static String stripOff(String baseString, String lastBit) {
