Repository: metron Updated Branches: refs/heads/master 0cc0e2ee0 -> 6ffff0299
METRON-1209: Make stellar repl take logging properties, like other CLI apps in metron closes apache/incubator-metron#772 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6ffff029 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6ffff029 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6ffff029 Branch: refs/heads/master Commit: 6ffff0299e75664dc36bbd014668a019cd1d00f2 Parents: 0cc0e2e Author: cstella <ceste...@gmail.com> Authored: Tue Oct 3 10:06:12 2017 -0400 Committer: cstella <ceste...@gmail.com> Committed: Tue Oct 3 10:06:12 2017 -0400 ---------------------------------------------------------------------- metron-stellar/pom.xml | 1 + .../stellar-3rd-party-example/README.md | 6 +++ .../stellar-3rd-party-example/pom.xml | 39 ++++++++++++++++ .../metron/stellar/external/TimeFunction.java | 44 +++++++++++++++++++ .../stellar/common/shell/StellarShell.java | 19 +++++--- .../resolver/ClasspathFunctionResolver.java | 10 ++++- .../classpath-resources/custom-1.0-SNAPSHOT.jar | Bin 3570 -> 3968 bytes .../resolver/ClasspathFunctionResolverTest.java | 15 ------- 8 files changed, 110 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/pom.xml ---------------------------------------------------------------------- diff --git a/metron-stellar/pom.xml b/metron-stellar/pom.xml index fd62786..075ceb5 100644 --- a/metron-stellar/pom.xml +++ b/metron-stellar/pom.xml @@ -41,6 +41,7 @@ </license> </licenses> <modules> + <module>stellar-3rd-party-example</module> <module>stellar-common</module> </modules> <dependencies> http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-3rd-party-example/README.md ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-3rd-party-example/README.md b/metron-stellar/stellar-3rd-party-example/README.md new file mode 100644 index 0000000..14c489e --- /dev/null +++ b/metron-stellar/stellar-3rd-party-example/README.md @@ -0,0 +1,6 @@ +# Introduction + +This is just a sample 3rd party stellar function (which is used in our +unit testing). If Stellar changes its API (e.g. the annotation changes +classes), then this will need to be built and the output jar of this (in +`target`) will need to be placed in `../stellar-common/src/test/classpath-resources/`. http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-3rd-party-example/pom.xml ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-3rd-party-example/pom.xml b/metron-stellar/stellar-3rd-party-example/pom.xml new file mode 100644 index 0000000..119892c --- /dev/null +++ b/metron-stellar/stellar-3rd-party-example/pom.xml @@ -0,0 +1,39 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>metron-stellar</artifactId> + <groupId>org.apache.metron</groupId> + <version>0.4.1</version> + </parent> + <artifactId>stellar-3rd-party-example</artifactId> + <packaging>jar</packaging> + + <name>3rd party Functions (just for tests)</name> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-3rd-party-example/src/main/java/org/apache/metron/stellar/external/TimeFunction.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-3rd-party-example/src/main/java/org/apache/metron/stellar/external/TimeFunction.java b/metron-stellar/stellar-3rd-party-example/src/main/java/org/apache/metron/stellar/external/TimeFunction.java new file mode 100644 index 0000000..887531e --- /dev/null +++ b/metron-stellar/stellar-3rd-party-example/src/main/java/org/apache/metron/stellar/external/TimeFunction.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.external; + +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import java.util.List; + +public class TimeFunction { + @Stellar( name="NOW", + description = "Right now!", + params = {}, + returns="Timestamp" + ) + public static class Now implements StellarFunction { + + public Object apply(List<Object> list, Context context) throws ParseException { + return System.currentTimeMillis(); + } + + public void initialize(Context context) { } + + public boolean isInitialized() { + return true; + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java index 0002c5a..6f23a5e 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java @@ -24,12 +24,9 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.PosixParser; +import org.apache.commons.cli.*; import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.PropertyConfigurator; import org.apache.metron.stellar.common.StellarAssignment; import org.apache.metron.stellar.common.utils.JSONUtils; import org.apache.metron.stellar.dsl.StellarFunctionInfo; @@ -124,7 +121,12 @@ public class StellarShell extends AeshConsoleCallback implements Completion { options.addOption("na", "no_ansi", false, "Make the input prompt not use ANSI colors."); options.addOption("h", "help", false, "Print help"); options.addOption("p", "properties", true, "File containing Stellar properties"); - + { + Option o = new Option("l", "log4j", true, "The log4j properties file to load"); + o.setArgName("FILE"); + o.setRequired(false); + options.addOption(o); + } CommandLineParser parser = new PosixParser(); CommandLine commandLine = parser.parse(options, args); @@ -141,7 +143,10 @@ public class StellarShell extends AeshConsoleCallback implements Completion { System.out.println(e.getMessage()); System.exit(1); } - + //setting up logging if specified + if(commandLine.hasOption("l")) { + PropertyConfigurator.configure(commandLine.getOptionValue("l")); + } console = createConsole(commandLine); executor = createExecutor(commandLine, console, getStellarProperties(commandLine)); loadVariables(commandLine, executor); http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java index 41de982..85aa015 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java @@ -234,7 +234,9 @@ public class ClasspathFunctionResolver extends BaseFunctionResolver { else { cls = new ClassLoader[this.classLoaders.size()]; for (int i = 0; i < this.classLoaders.size(); ++i) { - cls[i] = this.classLoaders.get(i); + ClassLoader cl = this.classLoaders.get(i); + LOG.debug("Using classloader: "+ cl.getClass().getCanonicalName()); + cls[i] = cl; } } @@ -253,9 +255,13 @@ public class ClasspathFunctionResolver extends BaseFunctionResolver { Set<Class<? extends StellarFunction>> ret = new HashSet<>(); for(ClassLoader cl : cls) { for(Class<?> c : ClassIndex.getAnnotated(Stellar.class, cl)) { - if(StellarFunction.class.isAssignableFrom(c) && filterBuilder.apply(c.getCanonicalName())) { + LOG.debug("{}: Found class: {}", cl.getClass().getCanonicalName(), c.getCanonicalName()); + boolean isAssignable = StellarFunction.class.isAssignableFrom(c); + boolean isFiltered = filterBuilder.apply(c.getCanonicalName()); + if( isAssignable && isFiltered ) { String className = c.getName(); if(!classes.contains(className)) { + LOG.debug("{}: Added class: {}", cl.getClass().getCanonicalName(), className); ret.add((Class<? extends StellarFunction>) c); classes.add(className); } http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-common/src/test/classpath-resources/custom-1.0-SNAPSHOT.jar ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/classpath-resources/custom-1.0-SNAPSHOT.jar b/metron-stellar/stellar-common/src/test/classpath-resources/custom-1.0-SNAPSHOT.jar index 7007eda..9ce87e7 100644 Binary files a/metron-stellar/stellar-common/src/test/classpath-resources/custom-1.0-SNAPSHOT.jar and b/metron-stellar/stellar-common/src/test/classpath-resources/custom-1.0-SNAPSHOT.jar differ http://git-wip-us.apache.org/repos/asf/metron/blob/6ffff029/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java index 72cd53a..1d37f99 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolverTest.java @@ -109,7 +109,6 @@ public class ClasspathFunctionResolverTest { Assert.assertEquals(0, actual.size()); } - @Ignore //until relocate dependencies for src/test/classpath-resources/custom-1.0-SNAPSHOT.jar @Test public void testExternalLocal() throws FileSystemException, ClassNotFoundException { File jar = new File("src/test/classpath-resources"); @@ -122,18 +121,4 @@ public class ClasspathFunctionResolverTest { Assert.assertTrue(functions.contains("NOW")); } - - @Ignore //until relocate dependencies for src/test/classpath-resources/custom-1.0-SNAPSHOT.jar - @Test - public void testExternalHDFS() throws FileSystemException, ClassNotFoundException { - /* TODO: this needs to test HDFS, not local filesystem */ - File jar = new File("src/test/classpath-resources"); - Assert.assertTrue(jar.exists()); - Properties config = new Properties(); - config.put(STELLAR_VFS_PATHS.param(), jar.toURI() + "/.*.jar"); - - ClasspathFunctionResolver resolver = create(config); - HashSet<String> functions = new HashSet<>(Lists.newArrayList(resolver.getFunctions())); - Assert.assertTrue(functions.contains("NOW")); - } }
