This is an automated email from the ASF dual-hosted git repository.
nickallen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 2283487 METRON-1986 Batch Profiler Fails to Resolve Stats Stellar
Functions (nickwallen) closes apache/metron#1328
2283487 is described below
commit 22834879017355fa456f42a6e71a3ed44d0b43fe
Author: nickwallen <n...@nickallen.org>
AuthorDate: Thu Feb 14 17:33:22 2019 -0500
METRON-1986 Batch Profiler Fails to Resolve Stats Stellar Functions
(nickwallen) closes apache/metron#1328
---
metron-analytics/metron-profiler-spark/pom.xml | 37 ++++++++--
.../spark/BatchProfilerIntegrationTest.java | 83 ++++++++++++++++------
2 files changed, 90 insertions(+), 30 deletions(-)
diff --git a/metron-analytics/metron-profiler-spark/pom.xml
b/metron-analytics/metron-profiler-spark/pom.xml
index 000d884..40bd551 100644
--- a/metron-analytics/metron-profiler-spark/pom.xml
+++ b/metron-analytics/metron-profiler-spark/pom.xml
@@ -52,7 +52,6 @@
<groupId>org.apache.metron</groupId>
<artifactId>metron-profiler-client</artifactId>
<version>${project.parent.version}</version>
- <scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.antlr</groupId>
@@ -62,6 +61,11 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
<artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
<exclusions>
@@ -182,8 +186,7 @@
</excludes>
</artifactSet>
<transformers>
- <transformer
-
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources>
<resource>.yaml</resource>
<resource>LICENSE.txt</resource>
@@ -191,16 +194,36 @@
<resource>NOTICE.txt</resource>
</resources>
</transformer>
- <transformer
-
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer
-
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass></mainClass>
</transformer>
+ <!--
+ The ClassIndex library is used to create
an index of all Stellar functions at
+ compile-time. This index is packaged in
the JAR as
+
`META-INF/annotations/org.apache.metron.stellar.dsl.Stellar`.
+
+ When creating a shaded jar using the Maven
Shade Plugin, the index files created by
+ ClassIndex are overwritten by default.
ClassIndex provides a special transformer for
+ Maven which merges the index files instead
of overwriting them.
+
+ Without using this transformer, the index
files are overwritten and Stellar function
+ resolution will not work.
+
+
https://github.com/atteo/classindex#making-shaded-jar
+ -->
+ <transformer
implementation="org.atteo.classindex.ClassIndexTransformer"/>
</transformers>
</configuration>
</execution>
</executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.atteo.classindex</groupId>
+ <artifactId>classindex-transformer</artifactId>
+ <version>${global_classindex_version}</version>
+ </dependency>
+ </dependencies>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
diff --git
a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
index 9ea151a..b36cf8c 100644
---
a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
+++
b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
@@ -70,29 +70,6 @@ import static org.junit.Assert.assertTrue;
public class BatchProfilerIntegrationTest {
private static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- /**
- * {
- * "timestampField": "timestamp",
- * "profiles": [
- * {
- * "profile": "count-by-ip",
- * "foreach": "ip_src_addr",
- * "init": { "count": 0 },
- * "update": { "count" : "count + 1" },
- * "result": "count"
- * },
- * {
- * "profile": "total-count",
- * "foreach": "'total'",
- * "init": { "count": 0 },
- * "update": { "count": "count + 1" },
- * "result": "count"
- * }
- * ]
- * }
- */
- @Multiline
- private static String profileJson;
private static SparkSession spark;
private Properties profilerProperties;
private Properties readerProperties;
@@ -152,6 +129,30 @@ public class BatchProfilerIntegrationTest {
}
/**
+ * {
+ * "timestampField": "timestamp",
+ * "profiles": [
+ * {
+ * "profile": "count-by-ip",
+ * "foreach": "ip_src_addr",
+ * "init": { "count": 0 },
+ * "update": { "count" : "count + 1" },
+ * "result": "count"
+ * },
+ * {
+ * "profile": "total-count",
+ * "foreach": "'total'",
+ * "init": { "count": 0 },
+ * "update": { "count": "count + 1" },
+ * "result": "count"
+ * }
+ * ]
+ * }
+ */
+ @Multiline
+ private static String profileJson;
+
+ /**
* This test uses the Batch Profiler to seed two profiles using archived
telemetry.
*
* The first profile counts the number of messages by 'ip_src_addr'. The
second profile counts the total number
@@ -328,6 +329,42 @@ public class BatchProfilerIntegrationTest {
}
/**
+ * {
+ * "timestampField": "timestamp",
+ * "profiles": [
+ * {
+ * "profile": "count-by-ip",
+ * "foreach": "ip_src_addr",
+ * "init": { "count": "STATS_INIT()" },
+ * "update": { "count" : "STATS_ADD(count, 1)" },
+ * "result": "TO_INTEGER(STATS_COUNT(count))"
+ * },
+ * {
+ * "profile": "total-count",
+ * "foreach": "'total'",
+ * "init": { "count": "STATS_INIT()" },
+ * "update": { "count": "STATS_ADD(count, 1)" },
+ * "result": "TO_INTEGER(STATS_COUNT(count))"
+ * }
+ * ]
+ * }
+ */
+ @Multiline
+ private static String statsProfileJson;
+
+ @Test
+ public void testBatchProfilerWithStatsFunctions() throws Exception {
+ profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), JSON.toString());
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(),
"src/test/resources/telemetry.json");
+
+ BatchProfiler profiler = new BatchProfiler();
+ profiler.run(spark, profilerProperties, getGlobals(), readerProperties,
fromJSON(statsProfileJson));
+
+ // the profiles do the exact same counting, but using the STATS functions
+ validateProfiles();
+ }
+
+ /**
* Validates the profiles that were built.
*
* These tests use the Batch Profiler to seed two profiles with archived
telemetry. The first profile
