http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/GrammarUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/GrammarUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/GrammarUtils.java deleted file mode 100644 index e65af94..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/GrammarUtils.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.metron.common.dsl; - -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.Token; - -import java.util.ArrayList; -import java.util.List; - -public class GrammarUtils { - - public static String toSyntaxTree(ParseTree tree) { - return new AST(tree).toString(); - } - - /** - * This is a utility class to walk the parse tree for an antlr ParseTree - */ - private static class AST { - - private final Object payload; - private final List<AST> children; - - public AST(ParseTree tree) { - this(null, tree); - } - - private AST(AST ast, ParseTree tree) { - this(ast, tree, new ArrayList<>()); - } - - private AST(AST parent, ParseTree tree, List<AST> children) { - this.payload = getPayload(tree); - this.children = children; - if (parent == null) { - walk(tree, this); - } - else { - parent.children.add(this); - } - } - - private Object getPayload(ParseTree tree) { - if (tree.getChildCount() == 0) { - return tree.getPayload(); - } - else { - String ruleName = tree.getClass().getSimpleName().replace("Context", ""); - return Character.toLowerCase(ruleName.charAt(0)) + ruleName.substring(1); - } - } - - private static void walk(ParseTree tree, AST ast) { - if (tree.getChildCount() == 0) { - new AST(ast, tree); - } - else if (tree.getChildCount() == 1) { - walk(tree.getChild(0), ast); - } - else if (tree.getChildCount() > 1) { - for (int i = 0; i < tree.getChildCount(); i++) { - AST temp = new AST(ast, tree.getChild(i)); - if (!(temp.payload instanceof Token)) { - walk(tree.getChild(i), temp); - } - } - } - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - AST ast = this; - List<AST> firstStack = new ArrayList<>(); - firstStack.add(ast); - List<List<AST>> childListStack = new ArrayList<>(); - childListStack.add(firstStack); - while (!childListStack.isEmpty()) { - List<AST> childStack = childListStack.get(childListStack.size() - 1); - if (childStack.isEmpty()) { - childListStack.remove(childListStack.size() - 1); - } - else { - ast = childStack.remove(0); - String caption; - if (ast.payload instanceof Token) { - Token token = (Token) ast.payload; - caption = String.format("TOKEN[type: %s, text: %s]", - token.getType(), token.getText().replace("\n", "\\n")); - } - else { - caption = String.valueOf(ast.payload); - } - String indent = ""; - for (int i = 0; i < childListStack.size() - 1; i++) { - indent += (childListStack.get(i).size() > 0) ? "| " : " "; - } - builder.append(indent) - .append(childStack.isEmpty() ? "'- " : "|- ") - .append(caption) - .append("\n"); - - if (ast.children.size() > 0) { - List<AST> children = new ArrayList<>(); - for (int i = 0; i < ast.children.size(); i++) { - children.add(ast.children.get(i)); - } - childListStack.add(children); - } - } - } - return builder.toString(); - } - } -}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java deleted file mode 100644 index f180e80..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.common.dsl; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class MapVariableResolver implements VariableResolver { - - List<Map> variableMappings = new ArrayList<>(); - - public MapVariableResolver(Map variableMappingOne, Map... variableMapping) { - if(variableMappingOne != null) { - variableMappings.add(variableMappingOne); - } - for(Map m : variableMapping) { - if(m != null) { - this.variableMappings.add(m); - } - } - } - @Override - public Object resolve(String variable) { - for(Map variableMapping : variableMappings) { - Object o = variableMapping.get(variable); - if(o != null) { - return o; - } - } - return null; - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/ParseException.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/ParseException.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/ParseException.java deleted file mode 100644 index 20c65c6..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/ParseException.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.common.dsl; - -public class ParseException extends RuntimeException { - public ParseException(String reason) { - super(reason); - } - public ParseException(String reason, Throwable t) { - super(reason, t); - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Predicate2StellarFunction.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Predicate2StellarFunction.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Predicate2StellarFunction.java deleted file mode 100644 index 2f08285..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Predicate2StellarFunction.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.common.dsl; - -import java.util.List; -import java.util.function.Predicate; - -public class Predicate2StellarFunction extends BaseStellarFunction { - Predicate<List<Object>> pred; - public Predicate2StellarFunction(Predicate<List<Object>> pred) { - this.pred = pred; - } - - @Override - public Object apply(List<Object> objects) { - return pred.test(objects); - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Stellar.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Stellar.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Stellar.java deleted file mode 100644 index d55d36b..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Stellar.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.common.dsl; - -import org.atteo.classindex.IndexAnnotated; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Retention(RetentionPolicy.RUNTIME) -@IndexAnnotated -public @interface Stellar { - String namespace() default ""; - String name(); - String description() default ""; - String returns() default ""; - String[] params() default {}; -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java deleted file mode 100644 index 2f52f47..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.common.dsl; - -import java.util.List; - -public interface StellarFunction { - Object apply(List<Object> args, Context context) throws ParseException; - void initialize(Context context); - boolean isInitialized(); -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctionInfo.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctionInfo.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctionInfo.java deleted file mode 100644 index 6e36f76..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctionInfo.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.common.dsl; - -import java.util.Arrays; -import java.util.List; - -/** - * Describes a Stellar function. - */ -public class StellarFunctionInfo { - - /** - * The name of the function. - */ - String name; - - /** - * A description of the function. Used for documentation purposes. - */ - String description; - - /** - * A description of what the function returns. Used for documentation purposes. - */ - String returns; - - /** - * The function parameters. Used for documentation purposes. - */ - String[] params; - - /** - * The actual function that can be executed. - */ - StellarFunction function; - - public StellarFunctionInfo(String description, String name, String[] params, String returns, StellarFunction function) { - this.description = description; - this.name = name; - this.params = params; - this.function = function; - this.returns = returns; - } - - public String getReturns() { - return returns; - } - - public String getDescription() { - return description; - } - - public String getName() { - return name; - } - - public String[] getParams() { - return params; - } - - public StellarFunction getFunction() { - return function; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - StellarFunctionInfo that = (StellarFunctionInfo) o; - - if (name != null ? !name.equals(that.name) : that.name != null) return false; - if (description != null ? !description.equals(that.description) : that.description != null) return false; - if (returns != null ? !returns.equals(that.returns) : that.returns != null) return false; - // Probably incorrect - comparing Object[] arrays with Arrays.equals - if (!Arrays.equals(params, that.params)) return false; - return function != null ? function.equals(that.function) : that.function == null; - - } - - @Override - public int hashCode() { - int result = name != null ? name.hashCode() : 0; - result = 31 * result + (description != null ? description.hashCode() : 0); - result = 31 * result + (returns != null ? returns.hashCode() : 0); - result = 31 * result + Arrays.hashCode(params); - result = 31 * result + (function != null ? function.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "StellarFunctionInfo{" + - "name='" + name + '\'' + - ", description='" + description + '\'' + - ", returns='" + returns + '\'' + - ", params=" + Arrays.toString(params) + - ", function=" + function + - '}'; - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java deleted file mode 100644 index 745ab86..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.common.dsl; - -import org.apache.metron.common.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver; - -public class StellarFunctions { - - public static FunctionResolver FUNCTION_RESOLVER() { - return SingletonFunctionResolver.getInstance(); - } - - public static void initialize(Context context) { - SingletonFunctionResolver.getInstance().initialize(context); - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Token.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Token.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Token.java deleted file mode 100644 index cf168a4..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Token.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.common.dsl; - -import org.apache.metron.common.stellar.FrameContext; - -public class Token<T> { - T value; - Class<T> underlyingType; - FrameContext.Context multiArgContext; - - public Token(T value, Class<T> clazz) { - this(value, clazz, null); - } - - public Token(T value, Class<T> clazz, FrameContext.Context multiArgContext) { - this.value = value; - this.underlyingType = clazz; - this.multiArgContext = multiArgContext; - } - - public FrameContext.Context getMultiArgContext() { - return multiArgContext; - } - - public T getValue() { - return value; - } - public Class<T> getUnderlyingType() { - return underlyingType; - } - - @Override - public String toString() { - return "" + value; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Token<?> token = (Token<?>) o; - - if (getValue() != null ? !getValue().equals(token.getValue()) : token.getValue() != null) return false; - return getUnderlyingType() != null ? getUnderlyingType().equals(token.getUnderlyingType()) : token.getUnderlyingType() == null; - - } - - @Override - public int hashCode() { - int result = getValue() != null ? getValue().hashCode() : 0; - result = 31 * result + (getUnderlyingType() != null ? getUnderlyingType().hashCode() : 0); - return result; - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/VariableResolver.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/VariableResolver.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/VariableResolver.java deleted file mode 100644 index 5c5b23f..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/VariableResolver.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.common.dsl; - -public interface VariableResolver { - Object resolve(String variable); -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/ConversionFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/ConversionFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/ConversionFunctions.java deleted file mode 100644 index 4ddff99..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/ConversionFunctions.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.common.dsl.functions; - -import org.apache.metron.common.dsl.BaseStellarFunction; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.utils.ConversionUtils; - -import java.util.List; - -public class ConversionFunctions { - - public static class Cast<T> extends BaseStellarFunction { - Class<T> clazz; - public Cast(Class<T> clazz) { - this.clazz = clazz; - } - - @Override - public Object apply(List<Object> strings ) { - return strings.get(0) == null?null: ConversionUtils.convert(strings.get(0), clazz); - } - } - - @Stellar(name="TO_INTEGER" - , description="Transforms the first argument to an integer" - , params = { "input - Object of string or numeric type"} - , returns = "Integer version of the first argument" - ) - public static class TO_INTEGER extends Cast<Integer> { - - public TO_INTEGER() { - super(Integer.class); - } - } - - @Stellar(name="TO_DOUBLE" - , description="Transforms the first argument to a double precision number" - , params = { "input - Object of string or numeric type"} - , returns = "Double version of the first argument" - ) - public static class TO_DOUBLE extends Cast<Double> { - - public TO_DOUBLE() { - super(Double.class); - } - } - - @Stellar(name="TO_LONG" - , description="Transforms the first argument to a long integer" - , params = { "input - Object of string or numeric type"} - , returns = "Long version of the first argument" - ) - public static class TO_LONG extends Cast<Long> { - - public TO_LONG() { super(Long.class); } - } - - @Stellar(name="TO_FLOAT" - , description="Transforms the first argument to a float" - , params = { "input - Object of string or numeric type"} - , returns = "Float version of the first argument" - ) - public static class TO_FLOAT extends Cast<Float> { - - public TO_FLOAT() { super(Float.class); } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DataStructureFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DataStructureFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DataStructureFunctions.java deleted file mode 100644 index c0f2e8c..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DataStructureFunctions.java +++ /dev/null @@ -1,220 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.common.dsl.functions; - -import org.apache.metron.common.dsl.BaseStellarFunction; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.utils.BloomFilter; -import org.apache.metron.common.utils.ConversionUtils; -import org.apache.metron.common.utils.SerDeUtils; - -import java.util.Collection; -import java.util.List; - -public class DataStructureFunctions { - - @Stellar(name="ADD" - , namespace="BLOOM" - , description="Adds an element to the bloom filter passed in" - , params = { "bloom - The bloom filter" - , "value* - The values to add" - } - , returns = "Bloom Filter" - ) - public static class BloomAdd extends BaseStellarFunction { - - @Override - public Object apply(List<Object> args) { - BloomFilter<Object> filter = (BloomFilter)args.get(0); - for(int i = 1;i < args.size();++i) { - Object arg = args.get(i); - if(arg != null) { - filter.add(args.get(i)); - } - } - return filter; - } - } - - @Stellar(name="EXISTS" - , namespace="BLOOM" - , description="If the bloom filter contains the value" - , params = { "bloom - The bloom filter" - , "value - The value to check" - } - , returns = "True if the filter might contain the value and false otherwise" - ) - public static class BloomExists extends BaseStellarFunction { - - @Override - public Object apply(List<Object> args) { - if(args.size() == 0) { - return false; - } - BloomFilter<Object> filter = (BloomFilter)args.get(0); - if(args.size() > 1) { - Object arg = args.get(1); - if(arg == null) { - return false; - } - return filter.mightContain(arg); - } - return false; - } - } - - @Stellar(name="INIT" - , namespace="BLOOM" - , description="Returns an empty bloom filter" - , params = { "expectedInsertions - The expected insertions" - , "falsePositiveRate - The false positive rate you are willing to tolerate" - } - , returns = "Bloom Filter" - ) - public static class BloomInit extends BaseStellarFunction { - - @Override - public Object apply(List<Object> args) { - int expectedInsertions = 100000; - float falsePositiveRate = 0.01f; - if(args.size() > 1) { - expectedInsertions = ConversionUtils.convert(args.get(0), Integer.class); - } - if(args.size() > 2) { - falsePositiveRate= ConversionUtils.convert(args.get(1), Float.class); - } - return new BloomFilter<>(SerDeUtils.SERIALIZER, expectedInsertions, falsePositiveRate); - } - } - - @Stellar( name="MERGE" - , namespace="BLOOM" - , description="Returns a merged bloom filter" - , params = { "bloomfilters - A list of bloom filters to merge" - } - , returns = "Bloom Filter or null if the list is empty" - ) - public static class BloomMerge extends BaseStellarFunction { - - @Override - public Object apply(List<Object> args) { - if(args.size() > 0) { - Object firstArg = args.get(0); - if(firstArg instanceof List) { - BloomFilter ret = null; - for(Object bf : (List)firstArg) { - if(bf instanceof BloomFilter) { - if(ret == null) { - ret = (BloomFilter)bf; - } - else { - ret.merge((BloomFilter)bf); - } - } - } - return ret; - } - else { - return null; - } - } - return null; - } - } - - @Stellar(name="IS_EMPTY" - , description="Returns true if string or collection is empty or null and false if otherwise." - , params = { "input - Object of string or collection type (for example, list)"} - , returns = "True if the string or collection is empty or null and false if otherwise." - ) - public static class IsEmpty extends BaseStellarFunction { - - @Override - public Object apply(List<Object> list) { - if(null == list || list.size() == 0) { - return true; - } - Object o = list.get(0); - if(o instanceof Collection) { - return ((Collection)o).isEmpty(); - } - else if(o instanceof String) { - String val = (String) list.get(0); - return val == null || val.isEmpty() ? true : false; - } - else { - return o == null; - } - } - } - @Stellar(name="ADD" - ,namespace="LIST" - , description="Adds an element to a list." - , params = { "list - List to add element to." - , "element - Element to add to list" - } - , returns = "Resulting list with the item added at the end." - ) - - public static class ListAdd extends BaseStellarFunction { - @Override - public Object apply(List<Object> list) { - if (list.size() == 0) { - return null; - } - Object o = list.get(0); - if(list.size() == 1) { - return o; - } - if(o instanceof List) { - List l = (List)o; - Object arg = list.get(1); - l.add(arg); - return l; - } - else { - return o; - } - } - } - - @Stellar(name="LENGTH" - , description="Returns the length of a string or size of a collection. Returns 0 for empty or null Strings" - , params = { "input - Object of string or collection type (e.g. list)"} - , returns = "Integer" - ) - public static class Length extends BaseStellarFunction { - @Override - public Object apply(List<Object> list) { - if(list.size() == 0) { - return 0; - } - Object o = list.get(0); - if(o instanceof Collection) { - return ((Collection)o).size(); - } - else if(o instanceof String) { - String val = (String) list.get(0); - return val == null || val.isEmpty() ? 0 : val.length(); - } - else { - return 0; - } - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DateFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DateFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DateFunctions.java deleted file mode 100644 index 071a8c5..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/DateFunctions.java +++ /dev/null @@ -1,368 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.common.dsl.functions; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import org.apache.metron.common.dsl.BaseStellarFunction; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.utils.ConversionUtils; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.List; -import java.util.Optional; -import java.util.TimeZone; -import java.util.concurrent.ExecutionException; - -/** - * Stellar data functions. - */ -public class DateFunctions { - - private static class TimezonedFormat { - - private String format; - private Optional<String> timezone; - - public TimezonedFormat(String format, String timezone) { - this.format = format; - this.timezone = Optional.of(timezone); - } - - public TimezonedFormat(String format) { - this.format = format; - this.timezone = Optional.empty(); - } - - public SimpleDateFormat toDateFormat() { - return createFormat(format, timezone); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TimezonedFormat that = (TimezonedFormat) o; - - if (format != null ? !format.equals(that.format) : that.format != null) return false; - return timezone != null ? timezone.equals(that.timezone) : that.timezone == null; - } - - @Override - public int hashCode() { - int result = format != null ? format.hashCode() : 0; - result = 31 * result + (timezone != null ? timezone.hashCode() : 0); - return result; - } - } - - private static LoadingCache<TimezonedFormat, ThreadLocal<SimpleDateFormat>> formatCache = - CacheBuilder.newBuilder().build( - new CacheLoader<TimezonedFormat, ThreadLocal<SimpleDateFormat>>() { - @Override - public ThreadLocal<SimpleDateFormat> load(final TimezonedFormat format) throws Exception { - return new ThreadLocal<SimpleDateFormat>() { - @Override - public SimpleDateFormat initialValue() { - return format.toDateFormat(); - } - }; - } - }); - - public static SimpleDateFormat createFormat(String format, Optional<String> timezone) { - SimpleDateFormat sdf = new SimpleDateFormat(format); - if(timezone.isPresent()) { - sdf.setTimeZone(TimeZone.getTimeZone(timezone.get())); - } - return sdf; - } - - public static long getEpochTime(String date, String format, Optional<String> timezone) throws ExecutionException, ParseException { - TimezonedFormat fmt; - if(timezone.isPresent()) { - fmt = new TimezonedFormat(format, timezone.get()); - } else { - fmt = new TimezonedFormat(format); - } - SimpleDateFormat sdf = formatCache.get(fmt).get(); - return sdf.parse(date).getTime(); - } - - - /** - * Stellar Function: TO_EPOCH_TIMESTAMP - */ - @Stellar( name="TO_EPOCH_TIMESTAMP" - , description="Returns the epoch timestamp of the dateTime in the specified format. " + - "If the format does not have a timestamp and you wish to assume a " + - "given timestamp, you may specify the timezone optionally." - , params = { "dateTime - DateTime in String format" - , "format - DateTime format as a String" - , "timezone - Optional timezone in String format" - } - , returns = "Epoch timestamp") - public static class ToTimestamp extends BaseStellarFunction { - @Override - public Object apply(List<Object> objects) { - Object dateObj = objects.get(0); - Object formatObj = objects.get(1); - Object tzObj = null; - if(objects.size() >= 3) { - tzObj = objects.get(2); - } - if(dateObj != null && formatObj != null) { - try { - Optional<String> tz = (tzObj == null) ? Optional.empty() : Optional.of(tzObj.toString()); - return getEpochTime(dateObj.toString(), formatObj.toString(), tz); - - } catch (ExecutionException | ParseException e) { - return null; - } - } - return null; - } - } - - /** - * Gets the value from a list of arguments. - * - * If the argument at the specified position does not exist, a default value will be returned. - * If the argument at the specified position exists, but cannot be coerced to the right type, null is returned. - * Otherwise, the argument value is returned. - * - * @param args A list of arguments. - * @param position The position of the argument to get. - * @param clazz The type of class expected. - * @param defaultValue The default value. - * @param <T> The expected type of the argument. - */ - private static <T> T getOrDefault(List<Object> args, int position, Class<T> clazz, T defaultValue) { - T result = defaultValue; - if(args.size() > position) { - result = ConversionUtils.convert(args.get(position), clazz); - } - return result; - } - - /** - * Stellar Function: DAY_OF_WEEK - * - * The numbered day within the week. The first day of the week, Sunday, has a value of 1. - * - * If no argument is supplied, returns the current day of week. - */ - @Stellar( name="DAY_OF_WEEK" - , description="The numbered day within the week. The first day of the week, Sunday, has a value of 1." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The numbered day within the week.") - public static class DayOfWeek extends BaseStellarFunction { - @Override - public Object apply(List<Object> args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.DAY_OF_WEEK); - } - } - - /** - * Stellar Function: DAY_OF_MONTH - * - * The day within the month. The first day within the month has a value of 1. - */ - @Stellar( name="DAY_OF_MONTH" - , description="The numbered day within the month. The first day within the month has a value of 1." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The numbered day within the month.") - public static class DayOfMonth extends BaseStellarFunction { - @Override - public Object apply(List<Object> args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.DAY_OF_MONTH); - } - } - - /** - * Stellar Function: WEEK_OF_MONTH - * - * The numbered week within the month. The first week has a value of 1. - */ - @Stellar( name="WEEK_OF_MONTH" - , description="The numbered week within the month. The first week within the month has a value of 1." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The numbered week within the month.") - public static class WeekOfMonth extends BaseStellarFunction { - @Override - public Object apply(List<Object> args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.WEEK_OF_MONTH); - } - } - - /** - * Stellar Function: WEEK_OF_YEAR - * - * The numbered week within the year. The first week in the year has a value of 1. - */ - @Stellar( name="WEEK_OF_YEAR" - , description="The numbered week within the year. The first week in the year has a value of 1." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The numbered week within the year.") - public static class WeekOfYear extends BaseStellarFunction { - @Override - public Object apply(List<Object> args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.WEEK_OF_YEAR); - } - } - - /** - * Stellar Function: MONTH - * - * A number representing the month. The first month, January, has a value of 0. - */ - @Stellar( name="MONTH" - , description="The number representing the month. The first month, January, has a value of 0." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The current month (0-based).") - public static class MonthOfYear extends BaseStellarFunction { - @Override - public Object apply(List<Object> args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.MONTH); - } - } - - /** - * Stellar Function: YEAR - * - * The calendar year. - */ - @Stellar( name="YEAR" - , description="The number representing the year. " - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The current year" - ) - public static class Year extends BaseStellarFunction { - @Override - public Object apply(List<Object> args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.YEAR); - } - } - - /** - * Stellar Function: DAY_OF_YEAR - * - * The day number within the year. The first day of the year has value of 1. - */ - @Stellar( name="DAY_OF_YEAR" - , description="The day number within the year. The first day of the year has value of 1." - , params = { "dateTime - The datetime as a long representing the milliseconds since unix epoch" - } - , returns = "The day number within the year." - ) - public static class DayOfYear extends BaseStellarFunction { - @Override - public Object apply(List<Object> args) { - - // expects epoch millis, otherwise defaults to current time - Long epochMillis = getOrDefault(args, 0, Long.class, System.currentTimeMillis()); - if(epochMillis == null) { - return null; // invalid argument - } - - // create a calendar - Calendar calendar = Calendar.getInstance(); - calendar.setTimeInMillis(epochMillis); - - return calendar.get(Calendar.DAY_OF_YEAR); - } - } -} - http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/FunctionalFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/FunctionalFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/FunctionalFunctions.java deleted file mode 100644 index 40e8863..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/FunctionalFunctions.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.common.dsl.functions; - -import org.apache.metron.common.dsl.BaseStellarFunction; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.stellar.LambdaExpression; - -import java.util.ArrayList; -import java.util.List; - -public class FunctionalFunctions { - - @Stellar(name="MAP" - , description="Applies lambda expression to a list of arguments. e.g. MAP( [ 'foo', 'bar' ] , ( x ) -> TO_UPPER(x) ) would yield [ 'FOO', 'BAR' ]" - , params = { - "list - List of arguments." - ,"transform_expression - The lambda expression to apply. This expression is assumed to take one argument." - } - , returns = "The input list transformed item-wise by the lambda expression." - ) - public static class Map extends BaseStellarFunction { - - @Override - public Object apply(List<Object> args) { - List<Object> input = (List<Object>) args.get(0); - LambdaExpression expression = (LambdaExpression)args.get(1); - if(input == null || expression == null) { - return input; - } - List<Object> ret = new ArrayList<>(); - for(Object o : input) { - ret.add(expression.apply(listOf(o))); - } - return ret; - } - } - - @Stellar(name="FILTER" - , description="Applies a filter in the form of a lambda expression to a list. e.g. FILTER( [ 'foo', 'bar' ] , (x) -> x == 'foo') would yield [ 'foo']" - , params = { - "list - List of arguments." - ,"predicate - The lambda expression to apply. This expression is assumed to take one argument and return a boolean." - } - , returns = "The input list filtered by the predicate." - ) - public static class Filter extends BaseStellarFunction { - - @Override - public Object apply(List<Object> args) { - List<Object> input = (List<Object>) args.get(0); - LambdaExpression expression = (LambdaExpression) args.get(1); - if(input == null || expression == null) { - return input; - } - List<Object> ret = new ArrayList<>(); - for(Object o : input) { - Object result = expression.apply(listOf(o)); - if(result != null && result instanceof Boolean && (Boolean)result) { - ret.add(o); - } - } - return ret; - } - } - - @Stellar(name="REDUCE" - , description="Reduces a list by a binary lambda expression. That is, the expression takes two arguments. Usage example: REDUCE( [ 1, 2, 3 ] , (x, y) -> x + y) would sum the input list, yielding 6." - , params = { - "list - List of arguments." - ,"binary_operation - The lambda expression function to apply to reduce the list. It is assumed that this takes two arguments, the first being the running total and the second being an item from the list." - ,"initial_value - The initial value to use." - } - , returns = "The reduction of the list." - ) - public static class Reduce extends BaseStellarFunction { - - @Override - public Object apply(List<Object> args) { - List<Object> input = (List<Object>) args.get(0); - if(input == null || args.size() < 3) { - return null; - } - LambdaExpression expression = (LambdaExpression) args.get(1); - - Object runningResult = args.get(2); - if(expression == null || runningResult == null) { - return null; - } - for(int i = 0;i < input.size();++i) { - Object rhs = input.get(i); - runningResult = expression.apply(listOf(runningResult, rhs)); - } - return runningResult; - } - } - - private static List<Object> listOf(Object... vals) { - List<Object> ret = new ArrayList<>(vals.length); - for(int i = 0;i < vals.length;++i) { - ret.add(vals[i]); - } - return ret; - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java deleted file mode 100644 index b862a57..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java +++ /dev/null @@ -1,325 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.common.dsl.functions; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import org.apache.curator.framework.CuratorFramework; -import org.apache.hadoop.security.authorize.Service; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; -import org.apache.metron.common.utils.JSONUtils; -import org.apache.metron.maas.config.Endpoint; -import org.apache.metron.maas.config.MaaSConfig; -import org.apache.metron.maas.config.ModelEndpoint; -import org.apache.metron.maas.discovery.ServiceDiscoverer; -import org.apache.metron.maas.util.ConfigUtil; -import org.apache.metron.maas.util.RESTUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -public class MaaSFunctions { - protected static final Logger LOG = LoggerFactory.getLogger(MaaSFunctions.class); - private static class ModelCacheKey { - String name; - String version; - String method; - Map<String, String> args; - public ModelCacheKey(String name, String version, String method, Map<String, String> args) { - this.name = name; - this.version = version; - this.method = method; - this.args = args; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ModelCacheKey that = (ModelCacheKey) o; - - if (name != null ? !name.equals(that.name) : that.name != null) return false; - if (version != null ? !version.equals(that.version) : that.version != null) return false; - if (method != null ? !method.equals(that.method) : that.method != null) return false; - return args != null ? args.equals(that.args) : that.args == null; - - } - - @Override - public int hashCode() { - int result = name != null ? name.hashCode() : 0; - result = 31 * result + (version != null ? version.hashCode() : 0); - result = 31 * result + (method != null ? method.hashCode() : 0); - result = 31 * result + (args != null ? args.hashCode() : 0); - return result; - } - } - - @Stellar(name="MODEL_APPLY" - , namespace="MAAS" - , description = "Returns the output of a model deployed via Model as a Service. NOTE: Results are cached locally for 10 minutes." - , params = { "endpoint - A map containing the name, version, and url for the REST endpoint" - , "function - The optional endpoint path; default is 'apply'" - , "model_args - A Dictionary of arguments for the model (these become request params)" - } - , returns = "The output of the model deployed as a REST endpoint in Map form. Assumes REST endpoint returns a JSON Map." - ) - public static class ModelApply implements StellarFunction { - private boolean isInitialized = false; - private ServiceDiscoverer discoverer; - private Cache<ModelCacheKey, Map<String, Object> > resultCache; - public ModelApply() { - resultCache = CacheBuilder.newBuilder() - .concurrencyLevel(4) - .weakKeys() - .maximumSize(100000) - .expireAfterWrite(10, TimeUnit.MINUTES) - .build(); - } - - @Override - public Object apply(List<Object> args, Context context) throws ParseException { - if(args.size() < 2) { - throw new ParseException("Unable to execute model_apply. " + - "Expected arguments: endpoint_map:map, " + - " [endpoint method:string], model_args:map" - ); - } - if(!isInitialized) { - return null; - } - int i = 0; - if(args.size() == 0) { - return null; - } - Object endpointObj = args.get(i++); - Map endpoint = null; - String modelName; - String modelVersion; - String modelUrl; - if(endpointObj instanceof Map) { - endpoint = (Map)endpointObj; - modelName = endpoint.get("name") + ""; - modelVersion = endpoint.get("version") + ""; - modelUrl = endpoint.get("url") + ""; - } - else { - return null; - } - String modelFunction = "apply"; - Map<String, String> modelArgs = new HashMap<>(); - if(args.get(i) instanceof String) { - String func = (String)args.get(i); - if(endpoint.containsKey("endpoint:" + func)) { - modelFunction = "" + endpoint.get("endpoint:" + func); - } - else { - modelFunction = func; - } - i++; - } - - if(args.get(i) instanceof Map) { - if(endpoint.containsKey("endpoint:apply")) { - modelFunction = "" + endpoint.get("endpoint:apply"); - } - modelArgs = (Map)args.get(i); - } - if( modelName == null - || modelVersion == null - || modelFunction == null - ) { - return null; - } - ModelCacheKey cacheKey = new ModelCacheKey(modelName, modelVersion, modelFunction, modelArgs); - Map<String, Object> ret = resultCache.getIfPresent(cacheKey); - if(ret != null) { - return ret; - } - else { - String url = modelUrl; - if (url.endsWith("/")) { - url = url.substring(0, url.length() - 1); - } - if (modelFunction.startsWith("/")) { - modelFunction = modelFunction.substring(1); - } - try { - URL u = new URL(url + "/" + modelFunction); - - String results = RESTUtil.INSTANCE.getRESTJSONResults(u, modelArgs); - ret = JSONUtils.INSTANCE.load(results, new TypeReference<Map<String, Object>>() { - }); - resultCache.put(cacheKey, ret); - return ret; - } catch (Exception e) { - LOG.error(e.getMessage(), e); - if (discoverer != null) { - try { - URL u = new URL(modelUrl); - discoverer.blacklist(u); - } catch (MalformedURLException e1) { - } - } - } - } - return null; - } - - @Override - public synchronized void initialize(Context context) { - - try { - Optional<ServiceDiscoverer> discovererOpt = (Optional) (context.getCapability(Context.Capabilities.SERVICE_DISCOVERER)); - if (discovererOpt.isPresent()) { - discoverer = discovererOpt.get(); - } - else { - Optional<Object> clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); - CuratorFramework client = null; - if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { - client = (CuratorFramework) clientOptional.get(); - } else { - throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client."); - } - discoverer = createDiscoverer(client); - } - } - catch(Exception ex) { - LOG.error(ex.getMessage(), ex); - } - finally { - //We always want to set initialize to true because we don't want to keep trying to initialize over and over - isInitialized = true; - } - } - - @Override - public boolean isInitialized() { - return isInitialized; - } - } - - private static ServiceDiscoverer createDiscoverer(CuratorFramework client) throws Exception { - MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class); - ServiceDiscoverer discoverer = new ServiceDiscoverer(client, config.getServiceRoot()); - discoverer.start(); - return discoverer; - } - - @Stellar(name="GET_ENDPOINT" - , namespace="MAAS" - , description="Inspects ZooKeeper and returns a map containing the name, version and url for the model referred to by the input parameters." - , params = { - "model_name - The name of the model" - ,"model_version - The optional version of the model. If the model version is not specified, the most current version is used." - } - , returns = "A map containing the name, version, and url for the REST endpoint (fields named name, version and url). " + - "Note that the output of this function is suitable for input into the first argument of MAAS_MODEL_APPLY." - ) - public static class GetEndpoint implements StellarFunction { - ServiceDiscoverer discoverer; - private boolean isInitialized = false; - private boolean isValidState = false; - - @Override - public Object apply(List<Object> args, Context context) throws ParseException { - if(!isValidState) { - LOG.error("Invalid state: Unable to find ServiceDiscoverer service."); - return null; - } - String modelName = null; - String modelVersion = null; - if(args.size() >= 1) { - modelName = args.get(0).toString(); - } - if(args.size() >= 2) - { - modelVersion = args.get(1).toString(); - } - if(modelName == null) { - return null; - } - try { - ModelEndpoint ep = null; - if (modelVersion == null) { - ep = discoverer.getEndpoint(modelName); - } else { - ep = discoverer.getEndpoint(modelName, modelVersion); - } - return ep == null ? null : endpointToMap(ep.getName(), ep.getVersion(), ep.getEndpoint()); - } - catch(Exception ex) { - LOG.error("Unable to discover endpoint: " + ex.getMessage(), ex); - return null; - } - } - - public static Map<String, String> endpointToMap(String name, String version, Endpoint ep) { - Map<String, String> ret = new HashMap<>(); - ret.put("url", ep.getUrl()); - ret.put("name", name); - ret.put("version", version); - for(Map.Entry<String, String> kv : ep.getFunctions().entrySet()) { - ret.put("endpoint:" + kv.getKey(), kv.getValue()); - } - return ret; - } - - @Override - public synchronized void initialize(Context context) { - try { - Optional<Object> clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); - CuratorFramework client = null; - if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { - client = (CuratorFramework) clientOptional.get(); - } else { - throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client."); - } - try { - discoverer = createDiscoverer(client); - context.addCapability(Context.Capabilities.SERVICE_DISCOVERER, () -> discoverer); - isValidState = true; - } catch (Exception e) { - LOG.error(e.getMessage(), e); - throw new IllegalStateException("Unable to initialize MAAS_GET_ENDPOINT", e); - } - } - finally { - isInitialized = true; - } - } - - @Override - public boolean isInitialized() { - return isInitialized; - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MapFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MapFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MapFunctions.java deleted file mode 100644 index 3329516..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MapFunctions.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.common.dsl.functions; - -import org.apache.metron.common.dsl.BaseStellarFunction; -import org.apache.metron.common.dsl.Stellar; - -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -public class MapFunctions { - - @Stellar(name="EXISTS" - ,namespace="MAP" - , description="Checks for existence of a key in a map." - , params = { - "key - The key to check for existence" - ,"map - The map to check for existence of the key" - } - , returns = "True if the key is found in the map and false if otherwise." - ) - public static class MapExists extends BaseStellarFunction { - - @Override - public Object apply(List<Object> list) { - if(list.size() < 2) { - return false; - } - Object key = list.get(0); - Object mapObj = list.get(1); - if(key != null && mapObj != null && mapObj instanceof Map) { - return ((Map)mapObj).containsKey(key); - } - return false; - } - } - - @Stellar(name="GET" - ,namespace="MAP" - , description="Gets the value associated with a key from a map" - , params = { - "key - The key" - ,"map - The map" - ,"default - Optionally the default value to return if the key is not in the map." - } - , returns = "The object associated with the key in the map. If no value is associated with the key and default is specified, then default is returned. If no value is associated with the key or default, then null is returned." - ) - public static class MapGet extends BaseStellarFunction { - @Override - public Object apply(List<Object> objects) { - Object keyObj = objects.get(0); - Object mapObj = objects.get(1); - Object defaultObj = null; - if(objects.size() >= 3) { - defaultObj = objects.get(2); - } - if(keyObj == null || mapObj == null) { - return defaultObj; - } - Map<Object, Object> map = (Map)mapObj; - Object ret = map.get(keyObj); - if(ret == null && defaultObj != null) { - return defaultObj; - } - return ret; - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/NetworkFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/NetworkFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/NetworkFunctions.java deleted file mode 100644 index 2dc92c9..0000000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/NetworkFunctions.java +++ /dev/null @@ -1,293 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.common.dsl.functions; - -import com.google.common.base.Splitter; -import com.google.common.collect.Iterables; -import com.google.common.net.InternetDomainName; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.net.util.SubnetUtils; -import org.apache.metron.common.dsl.BaseStellarFunction; -import org.apache.metron.common.dsl.Stellar; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.List; - -public class NetworkFunctions { - @Stellar(name="IN_SUBNET" - ,description = "Returns true if an IP is within a subnet range." - ,params = { - "ip - The IP address in string form" - ,"cidr+ - One or more IP ranges specified in CIDR notation (for example 192.168.0.0/24)" - } - ,returns = "True if the IP address is within at least one of the network ranges and false if otherwise" - ) - public static class InSubnet extends BaseStellarFunction { - - @Override - public Object apply(List<Object> list) { - if(list.size() < 2) { - throw new IllegalStateException("IN_SUBNET expects at least two args: [ip, cidr1, cidr2, ...]" - + " where cidr is the subnet mask in cidr form" - ); - } - String ip = (String) list.get(0); - if(ip == null) { - return false; - } - boolean inSubnet = false; - for(int i = 1;i < list.size() && !inSubnet;++i) { - String cidr = (String) list.get(i); - if(cidr == null) { - continue; - } - inSubnet |= new SubnetUtils(cidr).getInfo().isInRange(ip); - } - - return inSubnet; - } - } - - @Stellar(name="REMOVE_SUBDOMAINS" - ,namespace = "DOMAIN" - ,description = "Removes the subdomains from a domain." - , params = { - "domain - Fully qualified domain name" - } - , returns = "The domain without the subdomains. " + - "(for example, DOMAIN_REMOVE_SUBDOMAINS('mail.yahoo.com') yields 'yahoo.com')" - ) - public static class RemoveSubdomains extends BaseStellarFunction { - - @Override - public Object apply(List<Object> objects) { - if(objects.isEmpty()) { - return null; - } - Object dnObj = objects.get(0); - InternetDomainName idn = toDomainName(dnObj); - if(idn != null) { - String dn = dnObj.toString(); - String tld = extractTld(idn, dn); - if(!StringUtils.isEmpty(dn)) { - String suffix = safeSubstring(dn, 0, dn.length() - tld.length()); - String hostnameWithoutTLD = safeSubstring(suffix, 0, suffix.length() - 1); - if(hostnameWithoutTLD == null) { - return dn; - } - String hostnameWithoutSubsAndTLD = Iterables.getLast(Splitter.on(".").split(hostnameWithoutTLD), null); - if(hostnameWithoutSubsAndTLD == null) { - return null; - } - return hostnameWithoutSubsAndTLD + "." + tld; - } - } - return null; - } - } - - @Stellar(name="REMOVE_TLD" - ,namespace = "DOMAIN" - ,description = "Removes the top level domain (TLD) suffix from a domain." - , params = { - "domain - Fully qualified domain name" - } - , returns = "The domain without the TLD. " + - "(for example, DOMAIN_REMOVE_TLD('mail.yahoo.co.uk') yields 'mail.yahoo')" - ) - public static class RemoveTLD extends BaseStellarFunction { - @Override - public Object apply(List<Object> objects) { - Object dnObj = objects.get(0); - InternetDomainName idn = toDomainName(dnObj); - if(idn != null) { - String dn = dnObj.toString(); - String tld = extractTld(idn, dn); - String suffix = safeSubstring(dn, 0, dn.length() - tld.length()); - if(StringUtils.isEmpty(suffix)) { - return suffix; - } - else { - return suffix.substring(0, suffix.length() - 1); - } - } - return null; - } - } - - @Stellar(name="TO_TLD" - ,namespace = "DOMAIN" - ,description = "Extracts the top level domain from a domain" - , params = { - "domain - Fully qualified domain name" - } - , returns = "The TLD of the domain. " + - "(for example, DOMAIN_TO_TLD('mail.yahoo.co.uk') yields 'co.uk')" - ) - public static class ExtractTLD extends BaseStellarFunction { - @Override - public Object apply(List<Object> objects) { - Object dnObj = objects.get(0); - InternetDomainName idn = toDomainName(dnObj); - return extractTld(idn, dnObj + ""); - } - } - - @Stellar(name="TO_PORT" - ,namespace="URL" - ,description = "Extract the port from a URL. " + - "If the port is not explicitly stated in the URL, then an implicit port is inferred based on the protocol." - , params = { - "url - URL in string form" - } - , returns = "The port used in the URL as an integer (for example, URL_TO_PORT('http://www.yahoo.com/foo') would yield 80)" - ) - public static class URLToPort extends BaseStellarFunction { - @Override - public Object apply(List<Object> objects) { - URL url = toUrl(objects.get(0)); - if(url == null) { - return null; - } - int port = url.getPort(); - return port >= 0?port:url.getDefaultPort(); - } - } - - @Stellar(name="TO_PATH" - ,namespace="URL" - ,description = "Extract the path from a URL." - , params = { - "url - URL in String form" - } - , returns = "The path from the URL as a String. e.g. URL_TO_PATH('http://www.yahoo.com/foo') would yield 'foo'") - public static class URLToPath extends BaseStellarFunction { - @Override - public Object apply(List<Object> objects) { - URL url = toUrl(objects.get(0)); - return url == null?null:url.getPath(); - } - } - - @Stellar(name="TO_HOST" - ,namespace="URL" - ,description = "Extract the hostname from a URL." - , params = { - "url - URL in String form" - } - , returns = "The hostname from the URL as a String. e.g. URL_TO_HOST('http://www.yahoo.com/foo') would yield 'www.yahoo.com'" - ) - public static class URLToHost extends BaseStellarFunction { - - @Override - public Object apply(List<Object> objects) { - URL url = toUrl(objects.get(0)); - return url == null?null:url.getHost(); - } - } - - @Stellar(name="TO_PROTOCOL" - ,namespace="URL" - ,description = "Extract the protocol from a URL." - , params = { - "url - URL in String form" - } - , returns = "The protocol from the URL as a String. e.g. URL_TO_PROTOCOL('http://www.yahoo.com/foo') would yield 'http'") - public static class URLToProtocol extends BaseStellarFunction { - - @Override - public Object apply(List<Object> objects) { - URL url = toUrl(objects.get(0)); - return url == null?null:url.getProtocol(); - } - } - - /** - * Extract the TLD. If the domain is a normal domain, then we can handle the TLD via the InternetDomainName object. - * If it is not, then we default to returning the last segment after the final '.' - * @param idn - * @param dn - * @return The TLD of the domain - */ - private static String extractTld(InternetDomainName idn, String dn) { - - if(idn != null && idn.hasPublicSuffix()) { - return idn.publicSuffix().toString(); - } - else if(dn != null) { - StringBuffer tld = new StringBuffer(""); - for(int idx = dn.length() -1;idx >= 0;idx--) { - char c = dn.charAt(idx); - if(c == '.') { - break; - } - else { - tld.append(dn.charAt(idx)); - } - } - return tld.reverse().toString(); - } - else { - return null; - } - } - - private static String safeSubstring(String val, int start, int end) { - if(!StringUtils.isEmpty(val)) { - return val.substring(start, end); - } - return null; - } - - private static InternetDomainName toDomainName(Object dnObj) { - if(dnObj != null) { - if(dnObj instanceof String) { - String dn = dnObj.toString(); - try { - return InternetDomainName.from(dn); - } - catch(IllegalArgumentException iae) { - return null; - } - } - else { - throw new IllegalArgumentException(dnObj + " is not a string and therefore also not a domain."); - } - } - return null; - } - - private static URL toUrl(Object urlObj) { - if(urlObj == null) { - return null; - } - if(urlObj instanceof String) { - String url = urlObj.toString(); - try { - return new URL(url); - } catch (MalformedURLException e) { - return null; - } - } - else { - throw new IllegalArgumentException(urlObj + " is not a string and therefore also not a URL."); - } - } -}
