METRON-1708 Run the Batch Profiler in Spark (nickwallen) closes apache/metron#1161
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c6d0721b Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c6d0721b Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c6d0721b Branch: refs/heads/master Commit: c6d0721b8f02752cea9f91acb29042ca9ddf0bc9 Parents: 3bfbf01 Author: nickwallen <n...@nickallen.org> Authored: Mon Aug 27 13:47:28 2018 -0400 Committer: nickallen <nickal...@apache.org> Committed: Mon Aug 27 13:47:28 2018 -0400 ---------------------------------------------------------------------- metron-analytics/metron-profiler-spark/pom.xml | 15 ++ .../src/main/assembly/assembly.xml | 60 +++++++ .../src/main/config/batch-profiler.properties | 20 +++ .../profiler/spark/cli/BatchProfilerCLI.java | 174 +++++++++++++++++++ .../spark/cli/BatchProfilerCLIOptions.java | 144 +++++++++++++++ .../src/main/scripts/start_batch_profiler.sh | 32 ++++ .../spark/cli/BatchProfilerCLITest.java | 61 +++++++ .../src/test/resources/globals.json | 3 + .../src/test/resources/profiles-empty.json | 5 + .../resources/profiles-no-timestamp-field.json | 12 ++ .../src/test/resources/profiles.json | 13 ++ 11 files changed, 539 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/pom.xml b/metron-analytics/metron-profiler-spark/pom.xml index 93ce08a..2d5ec98 100644 --- a/metron-analytics/metron-profiler-spark/pom.xml +++ b/metron-analytics/metron-profiler-spark/pom.xml @@ -190,6 +190,21 @@ </execution> </executions> </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptor>src/main/assembly/assembly.xml</descriptor> + </configuration> + <executions> + <execution> + <id>make-assembly</id> <!-- this is used for inheritance merges --> + <phase>package</phase> <!-- bind to the packaging phase --> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml new file mode 100644 index 0000000..02f97eb --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml @@ -0,0 +1,60 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + ~ + --> + +<assembly> + <id>archive</id> + <formats> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${project.basedir}/src/main/config</directory> + <outputDirectory>config</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + <excludes> + <exclude>**/*.formatted</exclude> + <exclude>**/*.filtered</exclude> + </excludes> + <fileMode>0644</fileMode> + <lineEnding>unix</lineEnding> + <filtered>true</filtered> + </fileSet> + <fileSet> + <directory>${project.basedir}/src/main/scripts</directory> + <outputDirectory>bin</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + <excludes> + <exclude>**/*.formatted</exclude> + <exclude>**/*.filtered</exclude> + </excludes> + <fileMode>0755</fileMode> + <lineEnding>unix</lineEnding> + <filtered>true</filtered> + </fileSet> + <fileSet> + <directory>${project.basedir}/target</directory> + <includes> + <include>${project.artifactId}-${project.version}.jar</include> + </includes> + <outputDirectory>lib</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + </fileSet> + </fileSets> +</assembly> http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties b/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties new file mode 100644 index 0000000..c651791 --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +spark.master=local +spark.app.name=Batch Profiler http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java new file mode 100644 index 0000000..bdcf231 --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java @@ -0,0 +1,174 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.profiler.spark.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.io.IOUtils; +import org.apache.metron.common.configuration.profiler.ProfilerConfig; +import org.apache.metron.profiler.spark.BatchProfiler; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.Serializable; +import java.lang.invoke.MethodHandles; +import java.util.Properties; + +import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.CONFIGURATION_FILE; +import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.GLOBALS_FILE; +import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.PROFILE_DEFN_FILE; +import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.parse; + +/** + * The main entry point which launches the Batch Profiler in Spark. + * + * With this class the Batch Profiler can be submitted using the following command. + * + * <pre>{@code + * $SPARK_HOME/bin/spark-submit \ + * --class org.apache.metron.profiler.spark.cli.BatchProfilerCLI \ + * --properties-file spark.properties \ + * metron-profiler-spark-<version>.jar \ + * --config profiler.properties \ + * --globals global.properties \ + * --profiles profiles.json + * }</pre> + */ +public class BatchProfilerCLI implements Serializable { + + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static Properties globals; + public static Properties profilerProps; + public static ProfilerConfig profiles; + + public static void main(String[] args) throws IOException, org.apache.commons.cli.ParseException { + // parse the command line + CommandLine commandLine = parseCommandLine(args); + profilerProps = handleProfilerProperties(commandLine); + globals = handleGlobals(commandLine); + profiles = handleProfileDefinitions(commandLine); + + // the batch profiler must use 'event time' + if(!profiles.getTimestampField().isPresent()) { + throw new IllegalArgumentException("The Batch Profiler must use event time. The 'timestampField' must be defined."); + } + + // one or more profiles must be defined + if(profiles.getProfiles().size() == 0) { + throw new IllegalArgumentException("No profile definitions found."); + } + + SparkSession spark = SparkSession + .builder() + .config(new SparkConf()) + .getOrCreate(); + + BatchProfiler profiler = new BatchProfiler(); + long count = profiler.run(spark, profilerProps, globals, profiles); + LOG.info("Profiler produced {} profile measurement(s)", count); + } + + /** + * Load the Stellar globals from a file. + * + * @param commandLine The command line. + */ + private static Properties handleGlobals(CommandLine commandLine) throws IOException { + Properties globals = new Properties(); + if(GLOBALS_FILE.has(commandLine)) { + String globalsPath = GLOBALS_FILE.get(commandLine); + + LOG.info("Loading global properties from '{}'", globalsPath); + globals.load(new FileInputStream(globalsPath)); + + LOG.info("Globals = {}", globals); + } + return globals; + } + + /** + * Load the Profiler configuration from a file. + * + * @param commandLine The command line. + */ + private static Properties handleProfilerProperties(CommandLine commandLine) throws IOException { + Properties config = new Properties(); + if(CONFIGURATION_FILE.has(commandLine)) { + String propertiesPath = CONFIGURATION_FILE.get(commandLine); + + LOG.info("Loading profiler properties from '{}'", propertiesPath); + config.load(new FileInputStream(propertiesPath)); + + LOG.info("Properties = {}", config.toString()); + } + return config; + } + + /** + * Load the profile definitions from a file. + * + * @param commandLine The command line. + */ + private static ProfilerConfig handleProfileDefinitions(CommandLine commandLine) throws IOException { + ProfilerConfig profiles; + if(PROFILE_DEFN_FILE.has(commandLine)) { + String profilePath = PROFILE_DEFN_FILE.get(commandLine); + + LOG.info("Loading profiles from '{}'", profilePath); + String contents = IOUtils.toString(new FileInputStream(profilePath)); + + profiles = ProfilerConfig.fromJSON(contents); + LOG.info("Loaded {} profile(s)", profiles.getProfiles().size()); + + } else { + throw new IllegalArgumentException("No profile(s) defined"); + } + return profiles; + } + + /** + * Parse the command line arguments submitted by the user. + * @param args The command line arguments to parse. + * @throws org.apache.commons.cli.ParseException + */ + private static CommandLine parseCommandLine(String[] args) throws ParseException { + CommandLineParser parser = new PosixParser(); + return parse(parser, args); + } + + public static Properties getGlobals() { + return globals; + } + + public static Properties getProfilerProps() { + return profilerProps; + } + + public static ProfilerConfig getProfiles() { + return profiles; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java new file mode 100644 index 0000000..f5dfe12 --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java @@ -0,0 +1,144 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.profiler.spark.cli; + +import com.google.common.base.Joiner; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import java.util.function.Supplier; + +/** + * Defines the command line interface (CLI) options accepted by the Batch + * Profiler. + */ +public enum BatchProfilerCLIOptions { + + PROFILE_DEFN_FILE(() -> { + Option o = new Option("p", "profiles", true, "Path to a file containing profile definitions."); + o.setRequired(true); + return o; + }), + + CONFIGURATION_FILE(() -> { + Option o = new Option("c", "config", true, "Path to the profiler properties file."); + o.setRequired(false); + return o; + }), + + GLOBALS_FILE(() -> { + Option o = new Option("g", "globals", true, "Path to the Stellar global config file."); + o.setRequired(false); + return o; + }), + + HELP(() -> { + Option o = new Option("h", "help", false, "Usage instructions."); + o.setRequired(false); + return o; + }); + + private Option option; + + BatchProfilerCLIOptions(Supplier<Option> optionSupplier) { + this.option = optionSupplier.get(); + } + + /** + * Returns true, if the command line contains this option. + * + * @param cli The command line to parse. + */ + public boolean has(CommandLine cli) { + return cli.hasOption(option.getOpt()); + } + + /** + * Returns the option value from the command line. + * + * @param cli The command line to parse. + */ + public String get(CommandLine cli) { + return cli.getOptionValue(option.getOpt()); + } + + /** + * Returns the option value from the command line. The default value is + * returned if the value is not provided on the command line. + * + * @param cli The command line to parse. + * @param defaultValue The default value. + */ + public String get(CommandLine cli, String defaultValue) { + return has(cli) ? cli.getOptionValue(option.getOpt()) : defaultValue; + } + + /** + * Extracts the valid command line options from a given command line. + * + * @param parser The command line parser. + * @param args The command line to parse. + * @return The command line options. + * @throws ParseException If the command line cannot be parsed successfully. + */ + public static CommandLine parse(CommandLineParser parser, String[] args) throws ParseException { + try { + CommandLine cli = parser.parse(getOptions(), args); + if(HELP.has(cli)) { + printHelp(); + System.exit(0); + } + return cli; + + } catch (ParseException e) { + System.err.println("invalid arguments: " + Joiner.on(' ').join(args)); + e.printStackTrace(System.err); + printHelp(); + throw e; + } + } + + /** + * Print a help screen. + */ + public static void printHelp() { + HelpFormatter formatter = new HelpFormatter(); + String header = "options:"; + String footer = ""; + String cmd = String.format("spark-submit --class %s --properties-file [spark.properties] [jar] [options]", + BatchProfilerCLI.class.getCanonicalName()); + formatter.printHelp(cmd, header, getOptions(), footer); + } + + /** + * Returns all valid CLI options. + */ + public static Options getOptions() { + Options allOptions = new Options(); + for(BatchProfilerCLIOptions o : BatchProfilerCLIOptions.values()) { + allOptions.addOption(o.option); + } + return allOptions; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh new file mode 100644 index 0000000..c489af7 --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +METRON_VERSION=${project.version} +METRON_HOME=/usr/metron/${METRON_VERSION} +PROFILER_JAR=${METRON_HOME}/lib/${project.artifactId}-${METRON_VERSION}.jar +MAIN_CLASS=org.apache.metron.profiler.spark.cli.BatchProfilerCLI +PROFILER_PROPS=${PROFILER_PROPS:-"${METRON_HOME}/config/batch-profiler.properties"} +PROFILES_FILE=${PROFILES:-"${METRON_HOME}/config/zookeeper/profiler.json"} +SPARK_HOME=${SPARK_HOME:-"/usr/hdp/current/spark2-client"} + +${SPARK_HOME}/bin/spark-submit \ + --class ${MAIN_CLASS} \ + --properties-file ${PROFILER_PROPS} \ + ${PROFILER_JAR} \ + --config ${PROFILER_PROPS} \ + --profiles ${PROFILES_FILE} http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java new file mode 100644 index 0000000..c27495e --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.profiler.spark.cli; + +import org.apache.commons.cli.MissingOptionException; +import org.junit.Test; + +/** + * Tests the {@link BatchProfilerCLI} class. + */ +public class BatchProfilerCLITest { + + /** + * The user must provide a Profiler configuration that defines the 'timestampField'. The + * Batch Profiler only operates using event time, not processing time. + */ + @Test(expected = IllegalArgumentException.class) + public void mustDefineTimestampField() throws Exception { + String[] args = new String[] { + "--profiles", "src/test/resources/profiles-no-timestamp-field.json" + }; + BatchProfilerCLI.main(args); + } + + /** + * The user must define the -p, --profiles option. The Profiler cannot work without profiles. + */ + @Test(expected = MissingOptionException.class) + public void mustDefineProfilesOption() throws Exception { + String[] args = new String[] {}; + BatchProfilerCLI.main(args); + } + + /** + * If the profile definition contains no valid profiles, we have a problem. + */ + @Test(expected = IllegalArgumentException.class) + public void mustDefineProfiles() throws Exception { + String[] args = new String[] { + "--profiles", "src/test/resources/profiles-empty.json" + }; + BatchProfilerCLI.main(args); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/globals.json ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/globals.json b/metron-analytics/metron-profiler-spark/src/test/resources/globals.json new file mode 100644 index 0000000..2e295f1 --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/test/resources/globals.json @@ -0,0 +1,3 @@ +{ + "global-key": "global-value" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json new file mode 100644 index 0000000..ce57978 --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json @@ -0,0 +1,5 @@ +{ + "profiles": [ + ], + "timestampField":"timestamp" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json new file mode 100644 index 0000000..e47a6b5 --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json @@ -0,0 +1,12 @@ +{ + "profiles": [ + { + "profile": "hello-world", + "onlyif": "exists(ip_src_addr)", + "foreach": "ip_src_addr", + "init": { "count": "0" }, + "update": { "count": "count + 1" }, + "result": "count" + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json b/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json new file mode 100644 index 0000000..ac77f4b --- /dev/null +++ b/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json @@ -0,0 +1,13 @@ +{ + "profiles": [ + { + "profile": "hello-world", + "onlyif": "exists(ip_src_addr)", + "foreach": "ip_src_addr", + "init": { "count": "0" }, + "update": { "count": "count + 1" }, + "result": "count" + } + ], + "timestampField":"timestamp" +} \ No newline at end of file