Repository: metron Updated Branches: refs/heads/master d64afbc3f -> 1d89c3183
METRON-1118 Ignore Profile with Bad 'onlyif' or 'foreach' Expressions (nickwallen) closes apache/metron#705 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1d89c318 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1d89c318 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1d89c318 Branch: refs/heads/master Commit: 1d89c3183720ec1ad295b56a7d5ef52aca19dc17 Parents: d64afbc Author: nickwallen <[email protected]> Authored: Thu Aug 24 10:05:15 2017 -0400 Committer: nickallen <[email protected]> Committed: Thu Aug 24 10:05:15 2017 -0400 ---------------------------------------------------------------------- .../metron/profiler/DefaultMessageRouter.java | 39 ++++++++- .../profiler/DefaultMessageRouterTest.java | 88 ++++++++++++++++++++ .../profiler/bolt/ProfileSplitterBoltTest.java | 9 +- 3 files changed, 128 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/1d89c318/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageRouter.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageRouter.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageRouter.java index 5a32e69..d1a1a3b 100644 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageRouter.java +++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageRouter.java @@ -27,10 +27,16 @@ import org.apache.metron.stellar.common.StellarStatefulExecutor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.StellarFunctions; import org.json.simple.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; + +import static java.lang.String.format; /** * Routes incoming telemetry messages. @@ -40,6 +46,8 @@ import java.util.Map; */ public class DefaultMessageRouter implements MessageRouter { + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + /** * Executes Stellar code. */ @@ -62,21 +70,44 @@ public class DefaultMessageRouter implements MessageRouter { @Override public List<MessageRoute> route(JSONObject message, ProfilerConfig config, Context context) { List<MessageRoute> routes = new ArrayList<>(); - @SuppressWarnings("unchecked") - final Map<String, Object> state = (Map<String, Object>) message; // attempt to route the message to each of the profiles for (ProfileConfig profile: config.getProfiles()) { + Optional<MessageRoute> route = routeToProfile(message, profile); + route.ifPresent(routes::add); + } + + return routes; + } + /** + * Creates a route if a message is needed by a profile. + * @param message The message that needs routed. + * @param profile The profile that may need the message. + * @return A MessageRoute if the message is needed by the profile. + */ + private Optional<MessageRoute> routeToProfile(JSONObject message, ProfileConfig profile) { + Optional<MessageRoute> route = Optional.empty(); + + // allow the profile to access the fields defined within the message + @SuppressWarnings("unchecked") + final Map<String, Object> state = (Map<String, Object>) message; + + try { // is this message needed by this profile? if (executor.execute(profile.getOnlyif(), state, Boolean.class)) { // what is the name of the entity in this message? String entity = executor.execute(profile.getForeach(), state, String.class); - routes.add(new MessageRoute(profile, entity)); + route = Optional.of(new MessageRoute(profile, entity)); } + + } catch(Throwable e) { + // log an error and move on. ignore bad profiles. + String msg = format("error while executing profile; profile='%s', error='%s'", profile.getProfile(), e.getMessage()); + LOG.error(msg, e); } - return routes; + return route; } } http://git-wip-us.apache.org/repos/asf/metron/blob/1d89c318/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultMessageRouterTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultMessageRouterTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultMessageRouterTest.java index d0c0fbc..534f155 100644 --- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultMessageRouterTest.java +++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/DefaultMessageRouterTest.java @@ -119,6 +119,62 @@ public class DefaultMessageRouterTest { @Multiline private String exclusiveProfile; + /** + * { + * "profiles": [ + * { + * "profile": "bad-profile", + * "foreach": "2 / 0", + * "init": { "x": "0" }, + * "update": { "x": "x + 1" }, + * "result": "x" + * } + * ] + * } + */ + @Multiline + private String badForeach; + + /** + * { + * "profiles": [ + * { + * "profile": "bad-profile", + * "onlyif": "2 / 0", + * "foreach": "ip_src_addr", + * "init": { "x": "0" }, + * "update": { "x": "x + 1" }, + * "result": "x" + * } + * ] + * } + */ + @Multiline + private String badOnlyif; + + /** + * { + * "profiles": [ + * { + * "profile": "bad-profile", + * "foreach": "2 / 0", + * "init": { "x": "0" }, + * "update": { "x": "x + 1" }, + * "result": "x" + * }, + * { + * "profile": "good-profile", + * "foreach": "ip_src_addr", + * "init": { "x": "0" }, + * "update": { "x": "x + 1" }, + * "result": "x" + * } + * ] + * } + */ + @Multiline + private String goodAndBad; + private DefaultMessageRouter router; private Context context; @@ -180,4 +236,36 @@ public class DefaultMessageRouterTest { List<MessageRoute> routes = router.route(messageOne, createConfig(exclusiveProfile), context); assertEquals(0, routes.size()); } + + /** + * If a profile has a bad foreach expression, any exceptions need caught and the profile needs to be ignored. + */ + @Test + public void testWithBadForeachExpression() throws Exception { + List<MessageRoute> routes = router.route(messageOne, createConfig(badForeach), context); + assertEquals(0, routes.size()); + } + + /** + * If a profile has a bad foreach expression, any exceptions need caught and the profile needs to be ignored. + */ + @Test + public void testWithBadOnlyifExpression() throws Exception { + List<MessageRoute> routes = router.route(messageOne, createConfig(badForeach), context); + assertEquals(0, routes.size()); + } + + /** + * What happens if there are good and bad profiles? The good profiles need routes, the bad profiles need + * to be ignored. + */ + @Test + public void testWithGoodAndBad() throws Exception { + List<MessageRoute> routes = router.route(messageOne, createConfig(goodAndBad), context); + + assertEquals(1, routes.size()); + MessageRoute route1 = routes.get(0); + assertEquals("good-profile", route1.getProfileDefinition().getProfile()); + assertEquals(messageOne.get("ip_src_addr"), route1.getEntity()); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/1d89c318/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java index fbdc73a..d51401f 100644 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java @@ -227,15 +227,16 @@ public class ProfileSplitterBoltTest extends BaseBoltTest { } /** - * What happens when invalid Stella code is used for 'onlyif'? + * What happens when invalid Stella code is used for 'onlyif'? The invalid profile should be ignored. */ - @Test(expected = org.apache.metron.stellar.dsl.ParseException.class) + @Test public void testOnlyIfInvalid() throws Exception { // setup ProfileSplitterBolt bolt = createBolt(onlyIfInvalid); - - // execute bolt.execute(tuple); + + // a tuple should NOT be emitted for the downstream profile builder + verify(outputCollector, times(0)).emit(any(Values.class)); } }
