http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java new file mode 100644 index 0000000..2802cdd --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.util.concurrent.UncheckedExecutionException; +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.TokenStream; + +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ErrorListener; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +import org.apache.metron.stellar.common.evaluators.ArithmeticEvaluator; +import org.apache.metron.stellar.common.evaluators.ComparisonExpressionWithOperatorEvaluator; +import org.apache.metron.stellar.common.evaluators.NumberLiteralEvaluator; +import org.apache.metron.stellar.common.generated.StellarLexer; +import org.apache.metron.stellar.common.generated.StellarParser; + +import static org.apache.commons.lang3.StringUtils.isEmpty; + +/** + * The base implementation of a Stellar processor. This is used to evaluate Stellar expressions. + * + * @param <T> The type that the processor expects to return after processing a Stellar expression. + * @see StellarProcessor + * @see StellarPredicateProcessor + */ +public class BaseStellarProcessor<T> { + public static final int DEFAULT_CACHE_SIZE = 500; + public static final int DEFAULT_EXPIRY_TIME = 10; + public static final TimeUnit DEFAULT_EXPIRY_TIME_UNITS = TimeUnit.MINUTES; + + /** + * The default expression cache. This is used when the expression cache is not otherwise specified. + */ + private static Cache<String, StellarCompiler.Expression> defaultExpressionCache; + static { + defaultExpressionCache = createCache(DEFAULT_CACHE_SIZE, DEFAULT_EXPIRY_TIME, DEFAULT_EXPIRY_TIME_UNITS); + } + /** + * The class containing the type that the Stellar expression being processed will evaluate to. + */ + private Class<T> clazz; + + /** + * @param clazz The class containing the type that the Stellar expression being processed will evaluate to. + */ + Cache<String, StellarCompiler.Expression> expressionCache; + + /** + * Create a default stellar processor. This processor uses the static expression cache. + */ + BaseStellarProcessor(final Class<T> clazz) { + this(clazz, defaultExpressionCache); + } + + BaseStellarProcessor(final Class<T> clazz, int cacheSize, int expiryTime, TimeUnit expiryUnit) { + this(clazz, createCache(cacheSize, expiryTime, expiryUnit)); + } + + BaseStellarProcessor(final Class<T> clazz, Cache<String, StellarCompiler.Expression> expressionCache) { + this.clazz = clazz; + this.expressionCache = expressionCache; + } + + static Cache<String, StellarCompiler.Expression> createCache( int cacheSize + , int expiryTime + , TimeUnit expiryUnit + ) { + CacheLoader<String, StellarCompiler.Expression> loader = new CacheLoader<String, StellarCompiler.Expression>() { + @Override + public StellarCompiler.Expression load(String key) throws Exception { + return compile(key); + } + }; + return CacheBuilder.newBuilder() + .maximumSize(cacheSize) + .expireAfterAccess(expiryTime, expiryUnit) + .build(loader); + } + + /** + * Parses the given rule and returns a set of variables that are used in the given Stellar expression, {@code rule}. + * + * @param rule The Stellar expression to find out what variables are used. + * @return A set of variables used in the given Stellar expression. + */ + public Set<String> variablesUsed(final String rule) { + if (rule == null || isEmpty(rule.trim())) { + return null; + } + StellarCompiler.Expression expression = null; + try { + expression = expressionCache.get(rule, () -> compile(rule)); + } catch (ExecutionException e) { + throw new ParseException("Unable to parse: " + rule + " due to: " + e.getMessage(), e); + } + return expression.variablesUsed; + } + + /** + * Parses and evaluates the given Stellar expression, {@code rule}. + * @param rule The Stellar expression to parse and evaluate. + * @param variableResolver The {@link VariableResolver} to determine values of variables used in the Stellar expression, {@code rule}. + * @param functionResolver The {@link FunctionResolver} to determine values of functions used in the Stellar expression, {@code rule}. + * @param context The context used during validation. + * @return The value of the evaluated Stellar expression, {@code rule}. + */ + public T parse(final String rule, final VariableResolver variableResolver, final FunctionResolver functionResolver, final Context context) { + StellarCompiler.Expression expression = null; + if (rule == null || isEmpty(rule.trim())) { + return null; + } + try { + expression = expressionCache.get(rule, () -> compile(rule)); + } catch (ExecutionException|UncheckedExecutionException e) { + throw new ParseException("Unable to parse: " + rule + " due to: " + e.getMessage(), e); + } + return clazz.cast(expression.apply(new StellarCompiler.ExpressionState(context, functionResolver, variableResolver))); + } + + /** + * Parses and evaluates the given Stellar expression, {@code rule}. + * @param rule The Stellar expression to parse and evaluate. + * @return The Expression, which can be reevaluated without reparsing in different Contexts and Resolvers. + */ + public static StellarCompiler.Expression compile(final String rule) { + if (rule == null || isEmpty(rule.trim())) { + return null; + } + + ANTLRInputStream input = new ANTLRInputStream(rule); + StellarLexer lexer = new StellarLexer(input); + lexer.removeErrorListeners(); + lexer.addErrorListener(new ErrorListener()); + TokenStream tokens = new CommonTokenStream(lexer); + StellarParser parser = new StellarParser(tokens); + + StellarCompiler treeBuilder = new StellarCompiler( + ArithmeticEvaluator.INSTANCE, + NumberLiteralEvaluator.INSTANCE, + ComparisonExpressionWithOperatorEvaluator.INSTANCE + ); + parser.addParseListener(treeBuilder); + parser.removeErrorListeners(); + parser.addErrorListener(new ErrorListener()); + parser.transformation(); + return treeBuilder.getExpression(); + } + + /** + * This method determines if a given rule is valid or not. If the given rule is valid then true + * will be returned otherwise a {@link ParseException} is thrown. If it is desired to return a boolean + * whether the rule is valid or not, use the {@link #validate(String, boolean, Context) validate} method. It is important + * to note that all variables will resolve to 'null.' + * + * @param rule The rule to validate. + * @return If the given rule is valid then true otherwise an exception is thrown. + * @throws ParseException If the rule is invalid an exception of this type is thrown. + */ + public boolean validate(final String rule) throws ParseException { + return validate(rule, true, Context.EMPTY_CONTEXT()); + } + + /** + * Validates a given Stellar expression based on given context. + * @param rule The Stellar expression to validate. + * @param context The context used to validate the Stellar expression. + * @return If valid Stellar expression true, otherwise an exception will be thrown. + * @throws ParseException The exception containing the information as to why the expression is not valid. + */ + public boolean validate(final String rule, final Context context) throws ParseException { + return validate(rule, true, context); + } + + /** + * Here it is not desirable to add our custom listener. It is not the intent to evaluate the rule. + * The rule is only meant to be validated. Validate in this instance means check whether or not the + * rule is syntactically valid and whether the functions used exist. For example, it will not check + * for variables that are not defined. Currently all variables resolve to 'null.' This is mainly to + * make sure things function as expected when values are null. + * + * @param rule The Stellar transformation to validate. + * @param throwException If true an invalid Stellar transformation will throw a {@link ParseException} otherwise a boolean will be returned. + * @param context The Stellar context to be used when validating the Stellar transformation. + * @return If {@code throwException} is true and {@code rule} is invalid a {@link ParseException} is thrown. If + * {@code throwException} is false and {@code rule} is invalid then false is returned. Otherwise true if {@code rule} is valid, + * false if {@code rule} is invalid. + * @throws ParseException Thrown if {@code rule} is invalid and {@code throwException} is true. + */ + public boolean validate(final String rule, final boolean throwException, final Context context) throws ParseException { + if (rule == null || isEmpty(rule.trim())) { + return true; + } + + try { + parse(rule, x -> null, StellarFunctions.FUNCTION_RESOLVER(), context); + } catch (Throwable t) { + if (throwException) { + throw new ParseException("Unable to parse " + rule + ": " + t.getMessage(), t); + } else { + return false; + } + } + + return true; + } + +}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BooleanOp.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BooleanOp.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BooleanOp.java new file mode 100644 index 0000000..925131e --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BooleanOp.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common; + +public interface BooleanOp { + boolean op(boolean left, boolean right); +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/Constants.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/Constants.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/Constants.java new file mode 100644 index 0000000..50ad830 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/Constants.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.common; + +import java.util.HashMap; +import java.util.Map; + +public class Constants { + + public static final String ZOOKEEPER_ROOT = "/metron"; + public static final String ZOOKEEPER_TOPOLOGY_ROOT = ZOOKEEPER_ROOT + "/topology"; + public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000; +/* public static final String SENSOR_TYPE = "source.type"; + public static final String ENRICHMENT_TOPIC = "enrichments"; + public static final String INDEXING_TOPIC = "indexing"; + public static final String SIMPLE_HBASE_ENRICHMENT = "hbaseEnrichment"; + public static final String SIMPLE_HBASE_THREAT_INTEL = "hbaseThreatIntel"; */ + public static final String ERROR_STREAM = "error"; + public static final String GUID = "guid"; + + public interface Field { + String getName(); + } + public enum Fields implements Field { + SRC_ADDR("ip_src_addr") + ,SRC_PORT("ip_src_port") + ,DST_ADDR("ip_dst_addr") + ,DST_PORT("ip_dst_port") + ,PROTOCOL("protocol") + ,TIMESTAMP("timestamp") + ,ORIGINAL("original_string") + ,INCLUDES_REVERSE_TRAFFIC("includes_reverse_traffic") + ; + private static Map<String, Fields> nameToField; + + static { + nameToField = new HashMap<>(); + for (Fields f : Fields.values()) { + nameToField.put(f.getName(), f); + } + } + + private String name; + + Fields(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public static Fields fromString(String fieldName) { + return nameToField.get(fieldName); + } + } + + public enum ErrorFields { + MESSAGE("message") + ,FAILED_SENSOR_TYPE("failed_sensor_type") + ,ERROR_TYPE("error_type") + ,EXCEPTION("exception") + ,STACK("stack") + ,TIMESTAMP("timestamp") + ,HOSTNAME("hostname") + ,RAW_MESSAGE("raw_message") + ,RAW_MESSAGE_BYTES("raw_message_bytes") + ,ERROR_FIELDS("error_fields") + ,ERROR_HASH("error_hash") + ; + + private String name; + + ErrorFields(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + public enum ErrorType { + + PARSER_ERROR("parser_error") + ,DEFAULT_ERROR("error") + ; + + private String type; + + ErrorType(String type) { + this.type = type; + } + + public String getType() { + return type; + } + } + +} + http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/DefaultStellarStatefulExecutor.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/DefaultStellarStatefulExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/DefaultStellarStatefulExecutor.java new file mode 100644 index 0000000..86222c6 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/DefaultStellarStatefulExecutor.java @@ -0,0 +1,162 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common; + +import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang.ClassUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +import org.apache.metron.stellar.dsl.MapVariableResolver; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.dsl.VariableResolver; +import org.apache.metron.stellar.common.utils.ConversionUtils; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +/** + * The default implementation of a StellarStatefulExecutor. + */ +public class DefaultStellarStatefulExecutor implements StellarStatefulExecutor, Serializable { + + /** + * The current state of the Stellar execution environment. + */ + private Map<String, Object> state; + + /** + * Provides additional context for initializing certain Stellar functions. For + * example, references to Zookeeper or HBase. + */ + private Context context; + + /** + * Responsible for function resolution. + */ + private FunctionResolver functionResolver; + + public DefaultStellarStatefulExecutor() { + this(StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT()); + } + + public DefaultStellarStatefulExecutor(FunctionResolver functionResolver, Context context) { + clearState(); + this.context = context; + this.functionResolver = functionResolver; + } + + /** + * @param initialState Initial state loaded into the execution environment. + */ + public DefaultStellarStatefulExecutor(Map<String, Object> initialState) { + this(); + this.state = new HashMap<>(initialState); + } + + /** + * The current state of the Stellar execution environment. + */ + @Override + public Map<String, Object> getState() { + return ImmutableMap.copyOf(state); + } + + /** + * Execute an expression and assign the result to a variable. The variable is maintained + * in the context of this executor and is available to all subsequent expressions. + * + * @param variable The name of the variable to assign to. + * @param expression The expression to execute. + * @param transientState Additional state available to the expression. This most often represents + * the values available to the expression from an individual message. The state + * maps a variable name to a variable's value. + */ + @Override + public void assign(String variable, String expression, Map<String, Object> transientState) { + Object result = execute(expression, transientState); + if(result == null || variable == null) { + return; + } + state.put(variable, result); + } + + @Override + public void assign(String variable, Object value) { + if(value == null || variable == null) { + return; + } + state.put(variable, value); + } + + /** + * Execute a Stellar expression and return the result. The internal state of the executor + * is not modified. + * + * @param expression The expression to execute. + * @param state Additional state available to the expression. This most often represents + * the values available to the expression from an individual message. The state + * maps a variable name to a variable's value. + * @param clazz The expected type of the expression's result. + * @param <T> The expected type of the expression's result. + */ + @Override + public <T> T execute(String expression, Map<String, Object> state, Class<T> clazz) { + Object resultObject = execute(expression, state); + + // perform type conversion, if necessary + T result = ConversionUtils.convert(resultObject, clazz); + if (result == null) { + throw new IllegalArgumentException(String.format("Unexpected type: expected=%s, actual=%s, expression=%s", + clazz.getSimpleName(), ClassUtils.getShortClassName(resultObject,"null"), expression)); + } + + return result; + } + + @Override + public void clearState() { + this.state = new HashMap<>(); + } + + @Override + public void setContext(Context context) { + this.context = context; + } + + public void setFunctionResolver(FunctionResolver functionResolver) { + this.functionResolver = functionResolver; + } + + /** + * Execute a Stellar expression. + * + * @param expression The expression to execute. + * @param transientState Additional state available to the expression. This most often represents + * the values available to the expression from an individual message. The state + * maps a variable name to a variable's value. + */ + private Object execute(String expression, Map<String, Object> transientState) { + VariableResolver variableResolver = new MapVariableResolver(state, transientState); + StellarProcessor processor = new StellarProcessor(); + return processor.parse(expression, variableResolver, functionResolver, context); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/FrameContext.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/FrameContext.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/FrameContext.java new file mode 100644 index 0000000..9e06837 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/FrameContext.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.common; + +public enum FrameContext { + BOOLEAN_AND, + BOOLEAN_OR; + + public static class Context { + private FrameContext variety; + public Context(FrameContext variety) { + this.variety = variety; + } + + public FrameContext getVariety() { + return variety; + } + + @Override + public String toString() { + return "Context{" + + "variety=" + variety + + '}'; + } + } + + public Context create() { + return new Context(this); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/LambdaExpression.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/LambdaExpression.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/LambdaExpression.java new file mode 100644 index 0000000..06171fa --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/LambdaExpression.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common; + +import org.apache.metron.stellar.dsl.Token; +import org.apache.metron.stellar.dsl.VariableResolver; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; +import java.util.Map; +import java.util.HashMap; + + +public class LambdaExpression extends StellarCompiler.Expression { + StellarCompiler.ExpressionState state; + List<String> variables; + public LambdaExpression(List<String> variables, Deque<Token<?>> tokenDeque, StellarCompiler.ExpressionState state) { + super(tokenDeque); + this.state = state; + this.variables = variables; + } + + @Override + public Deque<Token<?>> getTokenDeque() { + Deque<Token<?>> ret = new ArrayDeque<>(super.getTokenDeque().size()); + for(Token<?> token : super.getTokenDeque()) { + ret.add(token); + } + return ret; + } + + public Object apply(List<Object> variableArgs) { + Map<String, Object> lambdaVariables = new HashMap<>(); + int i = 0; + for(;i < Math.min(variables.size(),variableArgs.size()) ;++i) { + lambdaVariables.put(variables.get(i), variableArgs.get(i)); + } + for(;i < variables.size();++i) { + lambdaVariables.put(variables.get(i), null); + } + + VariableResolver variableResolver = variable -> lambdaVariables.getOrDefault(variable + , state.variableResolver.resolve(variable) + ); + StellarCompiler.ExpressionState localState = new StellarCompiler.ExpressionState( + state.context + , state.functionResolver + , variableResolver); + return apply(localState); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java new file mode 100644 index 0000000..e77211a --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarAssignment.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.common; + + +import java.util.Map; + +public class StellarAssignment implements Map.Entry<String, Object>{ + private String variable; + private String statement; + + public StellarAssignment(String variable, String statement) { + this.variable = variable; + this.statement = statement; + } + + public String getVariable() { + return variable; + } + + public String getStatement() { + return statement; + } + + public static boolean isAssignment(String statement) { + return statement != null && statement.contains(":="); + } + + public static StellarAssignment from(String statement) { + if(statement == null || statement.length() == 0) { + return new StellarAssignment(null, null); + } + char prev = statement.charAt(0); + char curr; + String variable = "" + prev; + String s = null; + boolean isAssignment = false; + for(int i = 1;i < statement.length();++i,prev=curr) { + curr = statement.charAt(i); + if(prev == ':' && curr == '=') { + isAssignment = true; + variable = variable.substring(0, variable.length() - 1); + s = ""; + continue; + } + if(!isAssignment) { + variable += curr; + } + else { + s += curr; + } + } + + if(!isAssignment) { + s = variable; + variable = null; + } + + if(s != null) { + s = s.trim(); + } + if(variable != null) { + variable = variable.trim(); + } + return new StellarAssignment(variable, s); + } + + /** + * Returns the key corresponding to this entry. + * + * @return the key corresponding to this entry + * @throws IllegalStateException implementations may, but are not + * required to, throw this exception if the entry has been + * removed from the backing map. + */ + @Override + public String getKey() { + return variable; + } + + /** + * Returns the value corresponding to this entry. If the mapping + * has been removed from the backing map (by the iterator's + * <tt>remove</tt> operation), the results of this call are undefined. + * + * @return the value corresponding to this entry + * @throws IllegalStateException implementations may, but are not + * required to, throw this exception if the entry has been + * removed from the backing map. + */ + @Override + public Object getValue() { + return statement; + } + + /** + * Replaces the value corresponding to this entry with the specified + * value (optional operation). (Writes through to the map.) The + * behavior of this call is undefined if the mapping has already been + * removed from the map (by the iterator's <tt>remove</tt> operation). + * + * @param value new value to be stored in this entry + * @return old value corresponding to the entry + * @throws UnsupportedOperationException if the <tt>put</tt> operation + * is not supported by the backing map + * @throws ClassCastException if the class of the specified value + * prevents it from being stored in the backing map + * @throws NullPointerException if the backing map does not permit + * null values, and the specified value is null + * @throws IllegalArgumentException if some property of this value + * prevents it from being stored in the backing map + * @throws IllegalStateException implementations may, but are not + * required to, throw this exception if the entry has been + * removed from the backing map. + */ + @Override + public String setValue(Object value) { + throw new UnsupportedOperationException("Assignments are immutable."); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java new file mode 100644 index 0000000..8f2b9c0 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java @@ -0,0 +1,719 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.common; + +import org.antlr.v4.runtime.ParserRuleContext; +import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.Token; +import org.apache.metron.stellar.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +import org.apache.metron.stellar.common.evaluators.ArithmeticEvaluator; +import org.apache.metron.stellar.common.evaluators.ComparisonExpressionWithOperatorEvaluator; +import org.apache.metron.stellar.common.evaluators.NumberLiteralEvaluator; +import org.apache.metron.stellar.common.generated.StellarBaseListener; +import org.apache.metron.stellar.common.generated.StellarParser; +import com.google.common.base.Joiner; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.metron.stellar.dsl.FunctionMarker; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.stellar.common.utils.ConversionUtils; + +import java.io.Serializable; +import java.util.*; + +import static java.lang.String.format; + +public class StellarCompiler extends StellarBaseListener { + private static Token<?> EXPRESSION_REFERENCE = new Token<>(null, Object.class); + private static Token<?> LAMBDA_VARIABLES = new Token<>(null, Object.class); + + private Expression expression; + private final ArithmeticEvaluator arithmeticEvaluator; + private final NumberLiteralEvaluator numberLiteralEvaluator; + private final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator; + + public interface ShortCircuitOp {} + + public static class ShortCircuitFrame {} + public static class BooleanArg implements ShortCircuitOp {} + public static class IfExpr implements ShortCircuitOp {} + public static class ThenExpr implements ShortCircuitOp {} + public static class ElseExpr implements ShortCircuitOp {} + public static class EndConditional implements ShortCircuitOp {} + + public static class ExpressionState { + Context context; + FunctionResolver functionResolver; + VariableResolver variableResolver; + public ExpressionState(Context context + , FunctionResolver functionResolver + , VariableResolver variableResolver + ) { + this.context = context; + this.variableResolver = variableResolver; + this.functionResolver = functionResolver; + } + } + + public static class Expression implements Serializable { + final Deque<Token<?>> tokenDeque; + final Deque<FrameContext.Context> multiArgumentState; + final Set<String> variablesUsed; + public Expression(Deque<Token<?>> tokenDeque) { + this.tokenDeque = tokenDeque; + this.variablesUsed = new HashSet<>(); + this.multiArgumentState = new ArrayDeque<>(); + } + + public void clear() { + tokenDeque.clear(); + variablesUsed.clear(); + multiArgumentState.clear(); + } + + public Deque<Token<?>> getTokenDeque() { + return tokenDeque; + } + + public Object apply(ExpressionState state) { + Deque<Token<?>> instanceDeque = new ArrayDeque<>(); + { + boolean skipElse = false; + Token<?> token = null; + for (Iterator<Token<?>> it = getTokenDeque().descendingIterator(); it.hasNext(); ) { + token = it.next(); + //if we've skipped an else previously, then we need to skip the deferred tokens associated with the else. + if(skipElse && token.getUnderlyingType() == ElseExpr.class) { + while(it.hasNext()) { + token = it.next(); + if(token.getUnderlyingType() == EndConditional.class) { + break; + } + } + skipElse = false; + } + /* + curr is the current value on the stack. This is the non-deferred actual evaluation for this expression + and with the current context. + */ + Token<?> curr = instanceDeque.peek(); + if( curr != null + && curr.getValue() != null && curr.getValue() instanceof Boolean + && ShortCircuitOp.class.isAssignableFrom(token.getUnderlyingType()) + ) { + //if we have a boolean as the current value and the next non-contextual token is a short circuit op + //then we need to short circuit possibly + if(token.getUnderlyingType() == BooleanArg.class) { + if (curr.getMultiArgContext() != null + && curr.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_OR + && (Boolean) (curr.getValue()) + ) { + //short circuit the or + FrameContext.Context context = curr.getMultiArgContext(); + shortCircuit(it, context); + } else if (curr.getMultiArgContext() != null + && curr.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_AND + && !(Boolean) (curr.getValue()) + ) { + //short circuit the and + FrameContext.Context context = curr.getMultiArgContext(); + shortCircuit(it, context); + } + } + else if(token.getUnderlyingType() == IfExpr.class) { + //short circuit the if/then/else + instanceDeque.pop(); + if((Boolean)curr.getValue()) { + //choose then + skipElse = true; + } + else { + //choose else + while(it.hasNext()) { + Token<?> t = it.next(); + if(t.getUnderlyingType() == ElseExpr.class) { + break; + } + } + } + } + } + if (token.getUnderlyingType() == DeferredFunction.class) { + DeferredFunction func = (DeferredFunction) token.getValue(); + func.apply(instanceDeque, state); + } + else if(token.getUnderlyingType() != ShortCircuitFrame.class + && !ShortCircuitOp.class.isAssignableFrom(token.getUnderlyingType()) + ) { + instanceDeque.push(token); + } + + } + } + + if (instanceDeque.isEmpty()) { + throw new ParseException("Invalid predicate: Empty stack."); + } + Token<?> token = instanceDeque.pop(); + if (instanceDeque.isEmpty()) { + return token.getValue(); + } + if (instanceDeque.isEmpty()) { + throw new ParseException("Invalid parse, stack not empty: " + Joiner.on(',').join(instanceDeque)); + } else { + throw new ParseException("Invalid parse, found " + token); + } + } + + public void shortCircuit(Iterator<Token<?>> it, FrameContext.Context context) { + while (it.hasNext()) { + Token<?> token = it.next(); + if (token.getUnderlyingType() == ShortCircuitFrame.class && token.getMultiArgContext() == context) { + break; + } + } + } + } + + interface DeferredFunction { + void apply( Deque<Token<?>> tokenDeque + , ExpressionState state + ); + } + + public StellarCompiler( + final ArithmeticEvaluator arithmeticEvaluator, + final NumberLiteralEvaluator numberLiteralEvaluator, + final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator + ){ + this(new Expression(new ArrayDeque<>()), arithmeticEvaluator, numberLiteralEvaluator, comparisonExpressionWithOperatorEvaluator); + } + + public StellarCompiler( + final Expression expression, + final ArithmeticEvaluator arithmeticEvaluator, + final NumberLiteralEvaluator numberLiteralEvaluator, + final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator + ){ + this.expression = expression; + this.arithmeticEvaluator = arithmeticEvaluator; + this.numberLiteralEvaluator = numberLiteralEvaluator; + this.comparisonExpressionWithOperatorEvaluator = comparisonExpressionWithOperatorEvaluator; + } + + @Override + public void enterTransformation(StellarParser.TransformationContext ctx) { + expression.clear(); + } + + private boolean handleIn(final Token<?> left, final Token<?> right) { + Object key = right.getValue(); + + + if (left.getValue() != null) { + if (left.getValue() instanceof String && key instanceof String) { + return ((String) left.getValue()).contains(key.toString()); + } + else if (left.getValue() instanceof Collection) { + return ((Collection) left.getValue()).contains(key); + } + else if (left.getValue() instanceof Map) { + return ((Map) left.getValue()).containsKey(key); + } + else { + if (key == null) { + return key == left.getValue(); + } + else { + return key.equals(left.getValue()); + } + } + } else { + return false; + } + } + + @Override + public void exitNullConst(StellarParser.NullConstContext ctx) { + expression.tokenDeque.push(new Token<>(null, Object.class, getArgContext())); + } + + @Override + public void exitArithExpr_plus(StellarParser.ArithExpr_plusContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair(tokenDeque); + tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.addition(context), p)); + }, DeferredFunction.class, context)); + } + + @Override + public void exitArithExpr_minus(StellarParser.ArithExpr_minusContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair(tokenDeque); + tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.subtraction(context), p)); + }, DeferredFunction.class, context)); + } + + @Override + public void exitArithExpr_div(StellarParser.ArithExpr_divContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair(tokenDeque); + tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.division(context), p)); + }, DeferredFunction.class, context)); + } + + @Override + public void exitArithExpr_mul(StellarParser.ArithExpr_mulContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + Pair<Token<? extends Number>, Token<? extends Number>> p = getArithExpressionPair(tokenDeque); + tokenDeque.push(arithmeticEvaluator.evaluate(ArithmeticEvaluator.ArithmeticEvaluatorFunctions.multiplication(context), p)); + }, DeferredFunction.class, context)); + } + + @SuppressWarnings("unchecked") + private Pair<Token<? extends Number>, Token<? extends Number>> getArithExpressionPair(Deque<Token<?>> tokenDeque) { + Token<? extends Number> right = (Token<? extends Number>) popDeque(tokenDeque); + Token<? extends Number> left = (Token<? extends Number>) popDeque(tokenDeque); + return Pair.of(left, right); + } + + @Override + public void exitIf_expr(StellarParser.If_exprContext ctx) { + expression.tokenDeque.push(new Token<>(new IfExpr(), IfExpr.class, getArgContext())); + } + + @Override + public void enterThen_expr(StellarParser.Then_exprContext ctx) { + expression.tokenDeque.push(new Token<>(new ThenExpr(), ThenExpr.class, getArgContext())); + } + + @Override + public void enterElse_expr(StellarParser.Else_exprContext ctx) { + expression.tokenDeque.push(new Token<>(new ElseExpr(), ElseExpr.class, getArgContext())); + } + + @Override + public void exitElse_expr(StellarParser.Else_exprContext ctx) { + expression.tokenDeque.push(new Token<>(new EndConditional(), EndConditional.class, getArgContext())); + } + + @Override + public void exitInExpressionStatement(StellarParser.InExpressionStatementContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + Token<?> left = popDeque(tokenDeque); + Token<?> right = popDeque(tokenDeque); + tokenDeque.push(new Token<>(handleIn(left, right), Boolean.class, context)); + }, DeferredFunction.class, context)); + } + + + @Override + public void exitNInExpressionStatement(StellarParser.NInExpressionStatementContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + Token<?> left = popDeque(tokenDeque); + Token<?> right = popDeque(tokenDeque); + tokenDeque.push(new Token<>(!handleIn(left, right), Boolean.class, context)); + }, DeferredFunction.class, context)); + } + + @Override + public void exitNotFunc(StellarParser.NotFuncContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + Token<Boolean> arg = (Token<Boolean>) popDeque(tokenDeque); + tokenDeque.push(new Token<>(!arg.getValue(), Boolean.class, context)); + }, DeferredFunction.class, context)); + } + + + @Override + public void exitVariable(StellarParser.VariableContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + tokenDeque.push(new Token<>(state.variableResolver.resolve(ctx.getText()), Object.class, context)); + }, DeferredFunction.class, context)); + expression.variablesUsed.add(ctx.getText()); + } + + @Override + public void exitStringLiteral(StellarParser.StringLiteralContext ctx) { + String rawToken = ctx.getText(); + String literal = StringEscapeUtils.UNESCAPE_JSON.translate(rawToken); + expression.tokenDeque.push(new Token<>(literal.substring(1, literal.length()-1), String.class, getArgContext())); + } + + @Override + public void exitIntLiteral(StellarParser.IntLiteralContext ctx) { + expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); + } + + @Override + public void exitDoubleLiteral(StellarParser.DoubleLiteralContext ctx) { + expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); + } + + @Override + public void exitFloatLiteral(StellarParser.FloatLiteralContext ctx) { + expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); + } + + @Override + public void exitLongLiteral(StellarParser.LongLiteralContext ctx) { + expression.tokenDeque.push(numberLiteralEvaluator.evaluate(ctx, getArgContext())); + } + + @Override + public void enterB_expr(StellarParser.B_exprContext ctx) { + //Enter is not guaranteed to be called by Antlr for logical labels, so we need to + //emulate it like this. See https://github.com/antlr/antlr4/issues/802 + if(ctx.getParent() instanceof StellarParser.LogicalExpressionOrContext) { + expression.multiArgumentState.push(FrameContext.BOOLEAN_OR.create()); + } + else if(ctx.getParent() instanceof StellarParser.LogicalExpressionAndContext) { + expression.multiArgumentState.push(FrameContext.BOOLEAN_AND.create()); + } + } + + @Override + public void exitB_expr(StellarParser.B_exprContext ctx) { + if(ctx.getParent() instanceof StellarParser.LogicalExpressionOrContext + || ctx.getParent() instanceof StellarParser.LogicalExpressionAndContext + ) + { + //we want to know when the argument to the boolean expression is complete + expression.tokenDeque.push(new Token<>(new BooleanArg(), BooleanArg.class, getArgContext())); + } + } + + @Override + public void exitLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx) { + final FrameContext.Context context = getArgContext(); + popArgContext(); + final FrameContext.Context parentContext = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + Token<?> left = popDeque(tokenDeque); + Token<?> right = popDeque(tokenDeque); + tokenDeque.push(new Token<>(booleanOp(left, right, (l, r) -> l && r, "&&"), Boolean.class, parentContext)); + }, DeferredFunction.class, context)); + expression.tokenDeque.push(new Token<>(new ShortCircuitFrame(), ShortCircuitFrame.class, context)); + } + + @Override + public void exitLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx) { + final FrameContext.Context context = getArgContext(); + popArgContext(); + final FrameContext.Context parentContext = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + Token<?> left = popDeque(tokenDeque); + Token<?> right = popDeque(tokenDeque); + + tokenDeque.push(new Token<>(booleanOp(left, right, (l, r) -> l || r, "||"), Boolean.class, parentContext)); + }, DeferredFunction.class, context)); + expression.tokenDeque.push(new Token<>(new ShortCircuitFrame(), ShortCircuitFrame.class, context)); + } + + @Override + public void exitLogicalConst(StellarParser.LogicalConstContext ctx) { + Boolean b; + switch (ctx.getText().toUpperCase()) { + case "TRUE": + b = true; + break; + case "FALSE": + b = false; + break; + default: + throw new ParseException("Unable to process " + ctx.getText() + " as a boolean constant"); + } + expression.tokenDeque.push(new Token<>(b, Boolean.class, getArgContext())); + } + + private boolean booleanOp(final Token<?> left, final Token<?> right, final BooleanOp op, final String opName) { + Boolean l = ConversionUtils.convert(left.getValue(), Boolean.class); + Boolean r = ConversionUtils.convert(right.getValue(), Boolean.class); + if (l == null || r == null) { + throw new ParseException("Unable to operate on " + left.getValue() + " " + opName + " " + right.getValue() + ", null value"); + } + return op.op(l, r); + } + + + @Override + public void enterSingle_lambda_variable(StellarParser.Single_lambda_variableContext ctx) { + enterLambdaVariables(); + } + + @Override + public void exitSingle_lambda_variable(StellarParser.Single_lambda_variableContext ctx) { + exitLambdaVariables(); + } + + @Override + public void enterLambda_variables(StellarParser.Lambda_variablesContext ctx) { + enterLambdaVariables(); + } + + @Override + public void exitLambda_variables(StellarParser.Lambda_variablesContext ctx) { + exitLambdaVariables(); + } + + @Override + public void exitLambda_variable(StellarParser.Lambda_variableContext ctx) { + expression.tokenDeque.push(new Token<>(ctx.getText(), String.class, getArgContext())); + } + + private void enterLambdaVariables() { + expression.tokenDeque.push(LAMBDA_VARIABLES); + } + + private void exitLambdaVariables() { + Token<?> t = expression.tokenDeque.pop(); + LinkedList<String> variables = new LinkedList<>(); + for(; !expression.tokenDeque.isEmpty() && t != LAMBDA_VARIABLES; t = expression.tokenDeque.pop()) { + variables.addFirst(t.getValue().toString()); + } + expression.tokenDeque.push(new Token<>(variables, List.class, getArgContext())); + } + + private void enterLambda() { + expression.tokenDeque.push(EXPRESSION_REFERENCE); + } + + private void exitLambda(boolean hasArgs) { + final FrameContext.Context context = getArgContext(); + Token<?> t = expression.tokenDeque.pop(); + final Deque<Token<?>> instanceDeque = new ArrayDeque<>(); + for(; !expression.tokenDeque.isEmpty() && t != EXPRESSION_REFERENCE; t = expression.tokenDeque.pop()) { + instanceDeque.addLast(t); + } + final List<String> variables = hasArgs? (List<String>) instanceDeque.removeLast().getValue() :new ArrayList<>(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + LambdaExpression expr = new LambdaExpression(variables, instanceDeque, state); + tokenDeque.push(new Token<>(expr, Object.class, context)); + }, DeferredFunction.class, context) ); + } + + @Override + public void enterLambda_with_args(StellarParser.Lambda_with_argsContext ctx) { + enterLambda(); + } + + @Override + public void exitLambda_with_args(StellarParser.Lambda_with_argsContext ctx) { + exitLambda(true); + } + + @Override + public void enterLambda_without_args(StellarParser.Lambda_without_argsContext ctx) { + enterLambda(); + } + + @Override + public void exitLambda_without_args(StellarParser.Lambda_without_argsContext ctx) { + exitLambda(false); + } + + @Override + public void exitTransformationFunc(StellarParser.TransformationFuncContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + // resolve and initialize the function + String functionName = ctx.getChild(0).getText(); + StellarFunction function = resolveFunction(state.functionResolver, functionName); + initializeFunction(state.context, function, functionName); + + // fetch the args, execute, and push result onto the stack + List<Object> args = getFunctionArguments(popDeque(tokenDeque)); + Object result = function.apply(args, state.context); + tokenDeque.push(new Token<>(result, Object.class, context)); + }, DeferredFunction.class, context)); + } + + /** + * Get function arguments. + * @param token The token containing the function arguments. + * @return + */ + @SuppressWarnings("unchecked") + private List<Object> getFunctionArguments(final Token<?> token) { + if (token.getUnderlyingType().equals(List.class)) { + return (List<Object>) token.getValue(); + + } else { + throw new ParseException("Unable to process in clause because " + token.getValue() + " is not a set"); + } + } + + /** + * Resolves a function by name. + * @param funcName + * @return + */ + private StellarFunction resolveFunction(FunctionResolver functionResolver, String funcName) { + try { + return functionResolver.apply(funcName); + + } catch (Exception e) { + String valid = Joiner.on(',').join(functionResolver.getFunctions()); + String error = format("Unable to resolve function named '%s'. Valid functions are %s", funcName, valid); + throw new ParseException(error, e); + } + } + + /** + * Initialize a Stellar function. + * @param function The function to initialize. + * @param functionName The name of the functions. + */ + private void initializeFunction(Context context, StellarFunction function, String functionName) { + try { + if (!function.isInitialized()) { + function.initialize(context); + } + } catch (Throwable t) { + String error = format("Unable to initialize function '%s'", functionName); + throw new ParseException(error, t); + } + } + + @Override + public void exitExistsFunc(StellarParser.ExistsFuncContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + String variable = ctx.getChild(2).getText(); + boolean exists = state.variableResolver.resolve(variable) != null; + tokenDeque.push(new Token<>(exists, Boolean.class, context)); + }, DeferredFunction.class, context)); + String variable = ctx.getChild(2).getText(); + expression.variablesUsed.add(variable); + } + + @Override + public void enterFunc_args(StellarParser.Func_argsContext ctx) { + expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class, getArgContext())); + } + + @Override + public void exitFunc_args(StellarParser.Func_argsContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>((tokenDeque, state) -> { + LinkedList<Object> args = new LinkedList<>(); + while (true) { + Token<?> token = popDeque(tokenDeque); + if (token.getUnderlyingType().equals(FunctionMarker.class)) { + break; + } else { + args.addFirst(token.getValue()); + } + } + tokenDeque.push(new Token<>(args, List.class, context)); + }, DeferredFunction.class, context)); + } + + @Override + public void enterMap_entity(StellarParser.Map_entityContext ctx) { + expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class, getArgContext())); + } + + @Override + public void exitMap_entity(StellarParser.Map_entityContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + HashMap<String, Object> args = new HashMap<>(); + Object value = null; + for (int i = 0; true; i++) { + Token<?> token = popDeque(tokenDeque); + if (token.getUnderlyingType().equals(FunctionMarker.class)) { + break; + } else { + if (i % 2 == 0) { + value = token.getValue(); + } else { + args.put(token.getValue() + "", value); + } + } + } + tokenDeque.push(new Token<>(args, Map.class, context)); + }, DeferredFunction.class, context)); + } + + @Override + public void exitList_entity(StellarParser.List_entityContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + LinkedList<Object> args = new LinkedList<>(); + while (true) { + Token<?> token = popDeque(tokenDeque); + if (token.getUnderlyingType().equals(FunctionMarker.class)) { + break; + } else { + args.addFirst(token.getValue()); + } + } + tokenDeque.push(new Token<>(args, List.class, context)); + }, DeferredFunction.class, context)); + } + + + + @Override + public void exitComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx) { + final FrameContext.Context context = getArgContext(); + expression.tokenDeque.push(new Token<>( (tokenDeque, state) -> { + StellarParser.Comp_operatorContext op = ctx.comp_operator(); + Token<?> right = popDeque(tokenDeque); + Token<?> left = popDeque(tokenDeque); + + tokenDeque.push(comparisonExpressionWithOperatorEvaluator.evaluate(left, right, (StellarParser.ComparisonOpContext) op, context)); + }, DeferredFunction.class, context)); + } + + @Override + public void enterList_entity(StellarParser.List_entityContext ctx) { + expression.tokenDeque.push(new Token<>(new FunctionMarker(), FunctionMarker.class, getArgContext())); + } + + private void popArgContext() { + if(!expression.multiArgumentState.isEmpty()) { + expression.multiArgumentState.pop(); + } + } + + private FrameContext.Context getArgContext() { + return expression.multiArgumentState.isEmpty()?null:expression.multiArgumentState.peek(); + } + + private Token<?> popDeque(Deque<Token<?>> tokenDeque) { + if (tokenDeque.isEmpty()) { + throw new ParseException("Unable to pop an empty stack"); + } + return tokenDeque.pop(); + } + + public Expression getExpression() {return expression;} + +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarPredicateProcessor.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarPredicateProcessor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarPredicateProcessor.java new file mode 100644 index 0000000..c27d0cd --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarPredicateProcessor.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common; + + +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +import org.apache.metron.stellar.dsl.VariableResolver; + +import java.util.concurrent.TimeUnit; + +import static org.apache.commons.lang3.StringUtils.isEmpty; + +/** + * The Stellar Predicate Processor is intended to allow for specific predicate transformations using the Stellar + * domain specific language. In contrast to the StellarProcessor, which is a general purpose transformation + * tool, the output of the stellar statement is always a boolean. In java parlance, this is like a + * java.util.function.Predicate + */ + +public class StellarPredicateProcessor extends BaseStellarProcessor<Boolean> { + + /** + * Create a default stellar processor. This processor uses the static expression cache. + */ + public StellarPredicateProcessor() { + super(Boolean.class); + } + + public StellarPredicateProcessor(int cacheSize, int expiryTime, TimeUnit expiryUnit) { + super(Boolean.class, cacheSize, expiryTime, expiryUnit); + } + @Override + public Boolean parse( String rule + , VariableResolver variableResolver + , FunctionResolver functionResolver + , Context context + ) + { + if(rule == null || isEmpty(rule.trim())) { + return true; + } + try { + return super.parse(rule, variableResolver, functionResolver, context); + } catch (ClassCastException e) { + // predicate must return boolean + throw new IllegalArgumentException(String.format("The rule '%s' does not return a boolean value.", rule), e); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarProcessor.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarProcessor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarProcessor.java new file mode 100644 index 0000000..946a260 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarProcessor.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common; + +import java.util.concurrent.TimeUnit; + +/** + * The Stellar Processor is intended to allow for general transformations using the Stellar + * domain specific language. In contrast to the StellarPredicateProcessor where + * the output of the stellar statement is always a boolean, this is intended for use when you + * need non-predicate transformation. In java parlance, this is similar to a java.util.function.Function + * rather than a java.util.function.Predicate + */ +public class StellarProcessor extends BaseStellarProcessor<Object> { + + /** + * Create a default stellar processor. This processor uses the static expression cache. + */ + public StellarProcessor() { + super(Object.class); + } + + /** + * Create a stellar processor with a new expression cache. NOTE: This object should be reused to prevent + * performance regressions. + * @param cacheSize + * @param expiryTime + * @param expiryUnit + */ + public StellarProcessor(int cacheSize, int expiryTime, TimeUnit expiryUnit) { + super(Object.class, cacheSize, expiryTime, expiryUnit); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarStatefulExecutor.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarStatefulExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarStatefulExecutor.java new file mode 100644 index 0000000..9595d6e --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarStatefulExecutor.java @@ -0,0 +1,81 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.stellar.common; + +import org.apache.metron.stellar.dsl.Context; + +import java.util.Map; + +/** + * Executes Stellar expressions and maintains state across multiple invocations. + */ +public interface StellarStatefulExecutor { + + /** + * Assign a variable a specific value. + * @param variable The variable name. + * @param value The value to assign to the variable. + */ + void assign(String variable, Object value); + + /** + * Execute an expression and assign the result to a variable. The variable is maintained + * in the context of this executor and is available to all subsequent expressions. + * + * @param variable The name of the variable to assign to. + * @param expression The expression to execute. + * @param state Additional state available to the expression. This most often represents + * the values available to the expression from an individual message. The state + * maps a variable name to a variable's value. + */ + void assign(String variable, String expression, Map<String, Object> state); + + /** + * Execute a Stellar expression and return the result. The internal state of the executor + * is not modified. + * + * @param expression The expression to execute. + * @param state Additional state available to the expression. This most often represents + * the values available to the expression from an individual message. The state + * maps a variable name to a variable's value. + * @param clazz The expected type of the expression's result. + * @param <T> The expected type of the expression's result. + */ + <T> T execute(String expression, Map<String, Object> state, Class<T> clazz); + + /** + * The current state of the Stellar execution environment. + */ + Map<String, Object> getState(); + + /** + * Removes all state from the execution environment. + */ + void clearState(); + + /** + * Sets the Context for the Stellar execution environment. This provides global data used + * to initialize Stellar functions. + * + * @param context The Stellar context. + */ + void setContext(Context context); +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/Microbenchmark.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/Microbenchmark.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/Microbenchmark.java new file mode 100644 index 0000000..3a1b4b9 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/Microbenchmark.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.common.benchmark; + +import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +import org.apache.metron.stellar.common.StellarProcessor; + +import java.util.function.Consumer; + +public class Microbenchmark { + + public static class StellarStatement { + String expression; + VariableResolver variableResolver; + FunctionResolver functionResolver; + Context context; + } + + public static DescriptiveStatistics run(StellarStatement statement, int warmupRounds, int benchmarkRounds ) + { + run(warmupRounds, statement, ts -> {}); + final DescriptiveStatistics stats = new DescriptiveStatistics(); + run(benchmarkRounds, statement, ts -> { stats.addValue(ts);}); + return stats; + } + + private static void run(int numTimes, StellarStatement statement, Consumer<Long> func) { + StellarProcessor processor = new StellarProcessor(); + for(int i = 0;i < numTimes;++i) { + long start = System.nanoTime(); + processor.parse(statement.expression, statement.variableResolver, statement.functionResolver, statement.context); + func.accept((System.nanoTime() - start)/1000); + } + } + + public static String describe(DescriptiveStatistics stats, Double[] percentiles){ + StringBuilder sb = new StringBuilder(); + sb.append(String.format("round: mean of %dms [+-%d], measured %d rounds;\n", + (long)stats.getMean(), + (long)stats.getStandardDeviation(), stats.getN() )); + sb.append("\tMin - " + (long)stats.getMin() + "\n"); + for(double pctile : percentiles) { + sb.append("\t" + pctile + " - " + stats.getPercentile(pctile) + "\n"); + } + sb.append("\tMax - " + (long)stats.getMax()); + return sb.toString(); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java new file mode 100644 index 0000000..69e5297 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.common.benchmark; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.io.Files; +import org.apache.commons.cli.*; +import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.MapVariableResolver; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.common.utils.JSONUtils; +import org.apache.metron.stellar.common.utils.cli.OptionHandler; + +import javax.annotation.Nullable; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.Charset; +import java.util.*; + +public class StellarMicrobenchmark { + + public static int DEFAULT_WARMUP = 100; + public static int DEFAULT_NUM_TIMES = 1000; + public static Double[] DEFAULT_PERCENTILES = new Double[] { + 50d, 75d, 95d, 99d + }; + + enum BenchmarkOptions { + HELP("h", new OptionHandler<BenchmarkOptions>() { + + @Nullable + @Override + public Option apply(@Nullable String s) { + return new Option(s, "help", false, "Generate Help screen"); + } + }), + WARMUP("w", new OptionHandler<BenchmarkOptions>() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "warmup", true, "Number of times for warmup per expression. Default: " + DEFAULT_WARMUP); + o.setArgName("NUM"); + o.setRequired(false); + return o; + } + + @Override + public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }), + PERCENTILES("p", new OptionHandler<BenchmarkOptions>() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "percentiles", true + , "Percentiles to calculate per run. Default: " + Joiner.on(",").join(Arrays.asList(DEFAULT_PERCENTILES)) + ); + o.setArgName("NUM"); + o.setRequired(false); + return o; + } + + @Override + public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }), + NUM_TIMES("n", new OptionHandler<BenchmarkOptions>() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "num_times", true, "Number of times to run per expression (after warmup). Default: " + DEFAULT_NUM_TIMES); + o.setArgName("NUM"); + o.setRequired(false); + return o; + } + + @Override + public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }), + EXPRESSIONS("e", new OptionHandler<BenchmarkOptions>() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "expressions", true, "Stellar expressions"); + o.setArgName("FILE"); + o.setRequired(false); + return o; + } + + @Override + public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }), + VARIABLES("v", new OptionHandler<BenchmarkOptions>() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "variables", true, "File containing a JSON Map of variables to use"); + o.setArgName("FILE"); + o.setRequired(false); + return o; + } + + @Override + public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }), + OUTPUT("o", new OptionHandler<BenchmarkOptions>() { + @Nullable + @Override + public Option apply(@Nullable String s) { + Option o = new Option(s, "output", true, "File to write output."); + o.setArgName("FILE"); + o.setRequired(false); + return o; + } + + @Override + public Optional<Object> getValue(BenchmarkOptions option, CommandLine cli) { + return Optional.ofNullable(option.get(cli).trim()); + } + }) + ; + ; + Option option; + String shortCode; + OptionHandler<BenchmarkOptions> handler; + BenchmarkOptions(String shortCode, OptionHandler<BenchmarkOptions> optionHandler) { + this.shortCode = shortCode; + this.handler = optionHandler; + this.option = optionHandler.apply(shortCode); + } + + public boolean has(CommandLine cli) { + return cli.hasOption(shortCode); + } + + public String get(CommandLine cli) { + return cli.getOptionValue(shortCode); + } + + public static CommandLine parse(CommandLineParser parser, String[] args) { + try { + CommandLine cli = parser.parse(getOptions(), args); + if(HELP.has(cli)) { + printHelp(); + System.exit(0); + } + return cli; + } catch (org.apache.commons.cli.ParseException e) { + System.err.println("Unable to parse args: " + Joiner.on(' ').join(args)); + e.printStackTrace(System.err); + printHelp(); + System.exit(-1); + return null; + } + } + + public static EnumMap<BenchmarkOptions, Optional<Object>> createConfig(CommandLine cli) { + EnumMap<BenchmarkOptions, Optional<Object> > ret = new EnumMap<>(BenchmarkOptions.class); + for(BenchmarkOptions option : values()) { + ret.put(option, option.handler.getValue(option, cli)); + } + return ret; + } + + public static void printHelp() { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( "StellarBenchmark", getOptions()); + } + + public static Options getOptions() { + Options ret = new Options(); + for(BenchmarkOptions o : BenchmarkOptions.values()) { + ret.addOption(o.option); + } + return ret; + } + } + + public static void main(String... argv) throws IOException { + CommandLine cli = BenchmarkOptions.parse(new PosixParser(), argv); + if(!BenchmarkOptions.EXPRESSIONS.has(cli)) { + throw new IllegalStateException("You must at least specify an expressions file."); + } + File expressionsFile = new File(BenchmarkOptions.EXPRESSIONS.get(cli)); + Optional<File> variablesFile = Optional.ofNullable(!BenchmarkOptions.VARIABLES.has(cli) + ?null + :new File(BenchmarkOptions.VARIABLES.get(cli)) + ); + Optional<File> output = Optional.ofNullable(!BenchmarkOptions.OUTPUT.has(cli) + ?null + :new File(BenchmarkOptions.OUTPUT.get(cli)) + ); + List<String> lines = Files.readLines(expressionsFile, Charset.defaultCharset()); + Map<String, Object> variables = new HashMap<>(); + if(variablesFile.isPresent()) { + variables = JSONUtils.INSTANCE.load(new FileInputStream(variablesFile.get()), new TypeReference<Map<String, Object>>() { + }); + } + int numTimes = DEFAULT_NUM_TIMES; + if(BenchmarkOptions.NUM_TIMES.has(cli)) { + numTimes = Integer.parseInt(BenchmarkOptions.NUM_TIMES.get(cli)); + } + int warmup = DEFAULT_WARMUP; + if(BenchmarkOptions.WARMUP.has(cli)) { + warmup = Integer.parseInt(BenchmarkOptions.WARMUP.get(cli)); + } + Double[] percentiles = DEFAULT_PERCENTILES; + if(BenchmarkOptions.PERCENTILES.has(cli)) { + List<Double> percentileList = new ArrayList<>(); + for(String token : Splitter.on(",").split(BenchmarkOptions.PERCENTILES.get(cli))) { + if(token.trim().isEmpty()) { + continue; + } + Double d = Double.parseDouble(token.trim()); + percentileList.add(d); + } + percentiles = (Double[])percentileList.toArray(); + } + PrintWriter out = new PrintWriter(System.out); + if(output.isPresent()) { + out = new PrintWriter(output.get()); + } + for(String statement : lines) { + if(statement.trim().startsWith("#") || statement.trim().isEmpty()) { + continue; + } + Microbenchmark.StellarStatement s = new Microbenchmark.StellarStatement(); + s.context = Context.EMPTY_CONTEXT(); + s.expression = statement; + s.functionResolver = StellarFunctions.FUNCTION_RESOLVER(); + s.variableResolver = new MapVariableResolver(variables); + DescriptiveStatistics stats = Microbenchmark.run(s, warmup, numTimes); + out.println("Expression: " + statement); + out.println(Microbenchmark.describe(stats, percentiles)); + } + if(argv.length > 2) { + out.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java new file mode 100644 index 0000000..16cde83 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.configuration; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Function; +import org.apache.metron.stellar.common.Constants; +import org.apache.metron.stellar.common.utils.JSONUtils; + +import java.io.IOException; +import java.util.Map; + +public enum ConfigurationType implements Function<String, Object> { + + GLOBAL("global",".", s -> { + try { + return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() { + }); + } catch (IOException e) { + throw new RuntimeException("Unable to load " + s, e); + } + }); + + String name; + String directory; + String zookeeperRoot; + Function<String,?> deserializer; + + ConfigurationType(String name, String directory, Function<String, ?> deserializer) { + this.name = name; + this.directory = directory; + this.zookeeperRoot = Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name; + this.deserializer = deserializer; + } + + public String getName() { + return name; + } + + public String getDirectory() { + return directory; + } + + public Object deserialize(String s) { + return deserializer.apply(s); + } + + @Override + public Object apply(String s) { + return deserialize(s); + } + + public String getZookeeperRoot() { + return zookeeperRoot; + } +}
