metron git commit: METRON-1552: Add gzip file validation check to the geo loader (mmiklavc via mmiklavc) closes apache/metron#1011

[mmiklavcic]

Repository: metron
Updated Branches:
  refs/heads/master 9ce4ba5a9 -> 20eaed239


METRON-1552: Add gzip file validation check to the geo loader (mmiklavc via 
mmiklavc) closes apache/metron#1011


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/20eaed23
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/20eaed23
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/20eaed23

Branch: refs/heads/master
Commit: 20eaed239b2552d0823d34f571b63d941c352bc9
Parents: 9ce4ba5
Author: mmiklavc 
Authored: Tue May 15 11:12:07 2018 -0600
Committer: Michael Miklavcic 
Committed: Tue May 15 11:12:07 2018 -0600

--
 .../common/utils/CompressionStrategies.java | 100 +++
 .../common/utils/CompressionStrategy.java   |  52 ++
 .../common/utils/CompressionUtilsTest.java  |  62 
 .../nonbulk/geo/GeoEnrichmentLoader.java|  70 +
 .../nonbulk/geo/GeoEnrichmentLoaderTest.java|  40 ++--
 5 files changed, 298 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/20eaed23/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
new file mode 100644
index 000..f9c53c8
--- /dev/null
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.utils;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipException;
+
+/*
+ * Factory to provide various compression strategies.
+ */
+public enum CompressionStrategies implements CompressionStrategy {
+
+  GZIP(new CompressionStrategy() {
+@Override
+public void compress(File inFile, File outFile) throws IOException {
+  try (FileInputStream fis = new FileInputStream(inFile);
+  FileOutputStream fos = new FileOutputStream(outFile);
+  GZIPOutputStream gzipOS = new GZIPOutputStream(fos)) {
+byte[] buffer = new byte[1024];
+int len;
+while ((len = fis.read(buffer)) != -1) {
+  gzipOS.write(buffer, 0, len);
+}
+  }
+}
+
+@Override
+public void decompress(File inFile, File outFile) throws IOException {
+  try (FileInputStream fis = new FileInputStream(inFile);
+  GZIPInputStream gis = new GZIPInputStream(fis);
+  FileOutputStream fos = new FileOutputStream(outFile)) {
+byte[] buffer = new byte[1024];
+int len;
+while ((len = gis.read(buffer)) != -1) {
+  fos.write(buffer, 0, len);
+}
+  }
+
+}
+
+@Override
+public boolean test(File gzipFile) {
+  try (FileInputStream fis = new FileInputStream(gzipFile);
+  GZIPInputStream gis = new GZIPInputStream(fis)) {
+byte[] buffer = new byte[1024];
+// this will throw an exception on malformed file
+gis.read(buffer);
+  } catch (ZipException | EOFException e) {
+return false;
+  } catch (IOException e) {
+throw new IllegalStateException("Error occurred while attempting to 
validate gzip file", e);
+  }
+  return true;
+}
+  });
+
+  private CompressionStrategy strategy;
+
+  CompressionStrategies(CompressionStrategy strategy) {
+this.strategy = strategy;
+  }
+
+  @Override
+  public void compress(File inFile, File outFile) throws IOException {
+strategy.compress(inFile, outFile);
+  }
+
+  @Override
+  

[38/50] [abbrv] metron git commit: METRON-1528: Fix missing file in metron.spec (mmiklavc via mmiklavc) closes apache/metron#996

[otto]

METRON-1528: Fix missing file in metron.spec (mmiklavc via mmiklavc) closes 
apache/metron#996


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a41611b1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a41611b1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a41611b1

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: a41611b1ae2bb17fa9333ef6f965749652e95538
Parents: e094914
Author: mmiklavc 
Authored: Wed Apr 18 07:35:01 2018 -0600
Committer: Michael Miklavcic 
Committed: Wed Apr 18 07:35:01 2018 -0600

--
 metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/a41611b1/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec 
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 1f40105..15469d9 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -123,6 +123,7 @@ This package installs the Metron common files %{metron_home}
 %dir %{metron_home}/lib
 %{metron_home}/bin/zk_load_configs.sh
 %{metron_home}/bin/stellar
+%{metron_home}/bin/cluster_info.py
 %{metron_home}/config/zookeeper/global.json
 %attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}.jar
 

[36/50] [abbrv] metron git commit: METRON-1502 Upgrade Doxia plugin to 1.8 (justinleet) closes apache/metron#974

[otto]

METRON-1502 Upgrade Doxia plugin to 1.8 (justinleet) closes apache/metron#974


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/daf543b1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/daf543b1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/daf543b1

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: daf543b10423c2cfc2b740e39647ef4eb3863872
Parents: 08252f5
Author: justinleet 
Authored: Tue Apr 17 13:32:23 2018 -0400
Committer: leet 
Committed: Tue Apr 17 13:32:23 2018 -0400

--
 metron-deployment/amazon-ec2/README.md  |  88 +-
 metron-deployment/packaging/ambari/README.md| 168 +--
 .../packaging/packer-build/README.md|   2 +-
 metron-interface/metron-rest/README.md  |  56 +++
 metron-platform/metron-enrichment/README.md |   2 +-
 metron-sensors/pycapa/README.md |  84 +-
 site-book/pom.xml   |   4 +-
 .../src-resources/templates/site.xml.template   |   6 +-
 8 files changed, 206 insertions(+), 204 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/daf543b1/metron-deployment/amazon-ec2/README.md
--
diff --git a/metron-deployment/amazon-ec2/README.md 
b/metron-deployment/amazon-ec2/README.md
index 73a3d70..b2efc9e 100644
--- a/metron-deployment/amazon-ec2/README.md
+++ b/metron-deployment/amazon-ec2/README.md
@@ -46,39 +46,39 @@ Any platform that supports these tools is suitable, but the 
following instructio
 
 1. Install Homebrew by running the following command in a terminal.  Refer to 
the  [Homebrew](http://brew.sh/) home page for the latest installation 
instructions.
 
-  ```
-  /usr/bin/ruby -e "$(curl -fsSL 
https://raw.githubusercontent.com/Homebrew/install/master/install)"
-  ```
+```
+/usr/bin/ruby -e "$(curl -fsSL 
https://raw.githubusercontent.com/Homebrew/install/master/install)"
+```
 
 2. With Homebrew installed, run the following command in a terminal to install 
all of the required tools and dependencies.
 
-  ```
-  brew update
-  brew tap caskroom/versions
-  brew cask install java8 vagrant virtualbox
-  brew install maven git node
-  ```
+```
+brew update
+brew tap caskroom/versions
+brew cask install java8 vagrant virtualbox
+brew install maven git node
+```
 
 3. Install Ansible by following the instructions 
[here](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip).
 
 4. Ensure that a public SSH key is located at `~/.ssh/id_rsa.pub`.
 
-  ```
-  $ cat ~/.ssh/id_rsa.pub
-  ssh-rsa 
B3NzaC1yc2EDAQABAAABAQChv5GJxPjR39UJV7VY17ivbLVlxFrH7UHwh1Jsjem4d1eYiAtde5N2y65/HRNxWbhYli9ED8k0/MRP92ejewucEbrPNq5mytPqdC4IvZ98Ln2GbqTDwvlP3T7xa/wYFOpFsOmXXql8216wSrnrS4f3XK7ze34S6/VmY+lsBYnr3dzyj8sG/mexpJgFS/w83mWJV0e/ryf4Hd7P6DZ5fO+nmTXfKNK22ga4ctcnbZ+toYcPL+ODCh8598XCKVo97XjwF5OxN3vl1p1HHguo3cHB4H1OIaqX5mUt59gFIZcAXUME89PO6NUiZDd3RTstpf125nQVkQAHu2fvW96/f037
 nick@localhost
-  ```
+```
+$ cat ~/.ssh/id_rsa.pub
+ssh-rsa 
B3NzaC1yc2EDAQABAAABAQChv5GJxPjR39UJV7VY17ivbLVlxFrH7UHwh1Jsjem4d1eYiAtde5N2y65/HRNxWbhYli9ED8k0/MRP92ejewucEbrPNq5mytPqdC4IvZ98Ln2GbqTDwvlP3T7xa/wYFOpFsOmXXql8216wSrnrS4f3XK7ze34S6/VmY+lsBYnr3dzyj8sG/mexpJgFS/w83mWJV0e/ryf4Hd7P6DZ5fO+nmTXfKNK22ga4ctcnbZ+toYcPL+ODCh8598XCKVo97XjwF5OxN3vl1p1HHguo3cHB4H1OIaqX5mUt59gFIZcAXUME89PO6NUiZDd3RTstpf125nQVkQAHu2fvW96/f037
 nick@localhost
+```
 
-  If this file does not exist, run the following command at a terminal and 
accept all defaults.  Only the public key, not the private key, will be 
uploaded to Amazon and configured on each host to enable SSH connectivity.  
While it is possible to create and use an alternative key those details will 
not be covered.  
+If this file does not exist, run the following command at a terminal and 
accept all defaults.  Only the public key, not the private key, will be 
uploaded to Amazon and configured on each host to enable SSH connectivity.  
While it is possible to create and use an alternative key those details will 
not be covered.  
 
-  ```
-  ssh-keygen -t rsa
-  ```
+```
+ssh-keygen -t rsa
+```
 
 5. Ensure the JAVA_HOME environment variable is set
 
-   ```
-   export 
JAVA_HOME="/Library/Java/JavaVirtualMachines/jdk1.8.0_162.jdk/Contents/Home"
-   ```
+```
+export 
JAVA_HOME="/Library/Java/JavaVirtualMachines/jdk1.8.0_162.jdk/Contents/Home"
+```
 
Notice: You must replace the path with the installed JDK version path
 
@@ -103,20 +103,20 @@ Having successfully created your Amazon Web Services 
account, hopefully you will
 
 1. Use the Amazon access key by exporting its values via the shell's 

[43/50] [abbrv] metron git commit: METRON-1543 Unable to Set Parser Output Topic in Sensor Config (nickwallen) closes apache/metron#1007

[otto]

METRON-1543 Unable to Set Parser Output Topic in Sensor Config (nickwallen) 
closes apache/metron#1007


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3bb926df
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3bb926df
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3bb926df

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3bb926df5d253a907bbf8dab4b76b78dd32993ea
Parents: 2b4f0b8
Author: nickwallen 
Authored: Wed May 2 15:06:03 2018 -0400
Committer: nickallen 
Committed: Wed May 2 15:06:03 2018 -0400

--
 .../org/apache/metron/common/Constants.java |  10 +-
 .../configuration/SensorParserConfig.java   | 410 +++
 .../parsers/topology/ParserTopologyBuilder.java | 139 ---
 .../parsers/topology/ParserTopologyCLI.java | 147 +--
 .../components/ParserTopologyComponent.java |  80 ++--
 .../parsers/topology/ParserTopologyCLITest.java | 122 --
 ...pleHbaseEnrichmentWriterIntegrationTest.java |  69 ++--
 .../integration/WriterBoltIntegrationTest.java  | 109 ++---
 .../apache/metron/writer/kafka/KafkaWriter.java |   5 +
 9 files changed, 676 insertions(+), 415 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/3bb926df/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index b939a92..12b541c 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -17,9 +17,7 @@
  */
 package org.apache.metron.common;
 
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 public class Constants {
@@ -37,9 +35,17 @@ public class Constants {
   public static final String SIMPLE_HBASE_THREAT_INTEL = "hbaseThreatIntel";
   public static final String GUID = "guid";
 
+  /**
+   * The key in the global configuration that defines the global parser error 
topic.
+   *
+   * This value is used only if the error topic is left undefined in a 
sensor's parser configuration.
+   */
+  public static final String PARSER_ERROR_TOPIC_GLOBALS_KEY = 
"parser.error.topic";
+
   public interface Field {
 String getName();
   }
+
   public enum Fields implements Field {
  SRC_ADDR("ip_src_addr")
 ,SRC_PORT("ip_src_port")

http://git-wip-us.apache.org/repos/asf/metron/blob/3bb926df/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
index d347481..1dfb045 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
@@ -18,6 +18,9 @@
 package org.apache.metron.common.configuration;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.metron.common.utils.JSONUtils;
 
 import java.io.IOException;
@@ -27,35 +30,171 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+/**
+ * The configuration object that defines a parser for a given sensor.  Each
+ * sensor has its own parser configuration.
+ */
 public class SensorParserConfig implements Serializable {
 
+  /**
+   * The class name of the parser.
+   */
   private String parserClassName;
+
+  /**
+   * Allows logic to be defined to filter or ignore messages.  Messages that 
have been
+   * filtered will not be parsed.
+   *
+   * This should be a fully qualified name of a class that implements the
+   * org.apache.metron.parsers.interfaces.MessageFilter interface.
+   */
   private String filterClassName;
+
+  /**
+   * The input topic containing the sensor telemetry to parse.
+   */
   private String sensorTopic;
+
+  /**
+   * The output topic where the parsed telemetry will be written.
+   */
+  private String outputTopic;
+
+  /**
+   * The error topic where errors are written to.
+   */
+  private String errorTopic;
+
+  /**
+   * The 

[46/50] [abbrv] metron git commit: METRON-1184 EC2 Deployment - Updating control_path to accommodate for Linux (Ahmed Shah via ottobackwards) closes apache/metron#754

[otto]

METRON-1184 EC2 Deployment - Updating control_path to accommodate for Linux 
(Ahmed Shah via ottobackwards) closes apache/metron#754


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ca4644ba
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ca4644ba
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ca4644ba

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ca4644ba5e94c579fed6a7c7d1e8bea05fe9ba9c
Parents: 1b1a45b
Author: Ahmed Shah 
Authored: Wed May 9 14:22:50 2018 -0400
Committer: otto 
Committed: Wed May 9 14:22:50 2018 -0400

--
 metron-deployment/amazon-ec2/ansible.cfg | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/ca4644ba/metron-deployment/amazon-ec2/ansible.cfg
--
diff --git a/metron-deployment/amazon-ec2/ansible.cfg 
b/metron-deployment/amazon-ec2/ansible.cfg
index 1f4f0ea..0fe3343 100644
--- a/metron-deployment/amazon-ec2/ansible.cfg
+++ b/metron-deployment/amazon-ec2/ansible.cfg
@@ -24,5 +24,8 @@ forks = 20
 log_path = ./ansible.log
 
 # fix for "ssh throws 'unix domain socket too long' " problem
+#[ssh_connection]
+#control_path = ~/.ssh/ansible-ssh-%%C
+
 [ssh_connection]
-control_path = ~/.ssh/ansible-ssh-%%C
+control_path = ~/.ssh/ansbile-ssh-%%h-%%r

[31/50] [abbrv] metron git commit: METRON-1522 Fix the typo errors at profile debugger readme (MohanDV via nickwallen) closes apache/metron#992

[otto]

METRON-1522 Fix the typo errors at profile debugger readme  (MohanDV via 
nickwallen) closes apache/metron#992


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1d3e7fcd
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1d3e7fcd
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1d3e7fcd

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 1d3e7fcd7db64e7cb6d986802240fcbae174ae91
Parents: f8b7c58
Author: MohanDV 
Authored: Fri Apr 13 16:50:14 2018 -0400
Committer: nickallen 
Committed: Fri Apr 13 16:50:14 2018 -0400

--
 metron-analytics/metron-profiler/README.md | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/1d3e7fcd/metron-analytics/metron-profiler/README.md
--
diff --git a/metron-analytics/metron-profiler/README.md 
b/metron-analytics/metron-profiler/README.md
index 218ec66..01918aa 100644
--- a/metron-analytics/metron-profiler/README.md
+++ b/metron-analytics/metron-profiler/README.md
@@ -152,8 +152,8 @@ Creating and refining profiles is an iterative process.  
Iterating against a liv
* If a message is needed by two profiles, then two routes have been 
followed.
 
```
-   [Stellar]>>> p := PROFILER_INIT(conf)
-   [Stellar]>>> p
+   [Stellar]>>> profiler := PROFILER_INIT(conf)
+   [Stellar]>>> profiler
Profiler{1 profile(s), 0 messages(s), 0 route(s)}
```
 
@@ -172,11 +172,11 @@ Creating and refining profiles is an iterative process.  
Iterating against a liv
 1. Apply the message to your Profiler, as many times as you like.
 
```
-   [Stellar]>>> PROFILER_APPLY(msg, p)
+   [Stellar]>>> PROFILER_APPLY(msg, profiler)
Profiler{1 profile(s), 1 messages(s), 1 route(s)}
```
```
-   [Stellar]>>> PROFILER_APPLY(msg, p)
+   [Stellar]>>> PROFILER_APPLY(msg, profiler)
Profiler{1 profile(s), 2 messages(s), 2 route(s)}
```
 
@@ -205,7 +205,7 @@ Creating and refining profiles is an iterative process.  
Iterating against a liv
```
Apply those 10 messages to your profile(s).
```
-   [Stellar]>>> PROFILER_APPLY(msgs, p)
+   [Stellar]>>> PROFILER_APPLY(msgs, profiler)
  Profiler{1 profile(s), 10 messages(s), 10 route(s)}
```
 

[37/50] [abbrv] metron git commit: METRON-1445: Update performance tuning guide with more explicit parameter instructions (mmiklavc via mmiklavc) closes apache/metron#988

[otto]

METRON-1445: Update performance tuning guide with more explicit parameter 
instructions (mmiklavc via mmiklavc) closes apache/metron#988


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e0949142
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e0949142
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e0949142

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: e0949142dd682a84e59fea09066de8024f106f13
Parents: daf543b
Author: mmiklavc 
Authored: Tue Apr 17 12:31:37 2018 -0600
Committer: Michael Miklavcic 
Committed: Tue Apr 17 12:31:37 2018 -0600

--
 metron-platform/Performance-tuning-guide.md | 244 +++-
 metron-platform/metron-common/README.md |  32 ++
 .../src/main/scripts/cluster_info.py| 389 +++
 3 files changed, 659 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/e0949142/metron-platform/Performance-tuning-guide.md
--
diff --git a/metron-platform/Performance-tuning-guide.md 
b/metron-platform/Performance-tuning-guide.md
index 7d79ace..e2d1ae2 100644
--- a/metron-platform/Performance-tuning-guide.md
+++ b/metron-platform/Performance-tuning-guide.md
@@ -17,6 +17,14 @@ limitations under the License.
 -->
 # Metron Performance Tuning Guide
 
+- [Overview](#overview)
+- [General Tuning Suggestions](#general-tuning-suggestions)
+- [Component Tuning Levers](#component-tuning-levers)
+- [Use Case Specific Tuning Suggestions](#use-case-specific-tuning-suggestions)
+- [Debugging](#debugging)
+- [Issues](#issues)
+- [Reference](#reference)
+
 ## Overview
 
 This document provides guidance from our experiences tuning the Apache Metron 
Storm topologies for maximum performance. You'll find
@@ -31,20 +39,33 @@ pipe, and the majority of these options assist in tweaking 
the various pipe widt
 
 ## General Tuning Suggestions
 
+### Storm Executors vs. Tasks
+
 Note that there is currently no method for specifying the number of tasks from 
the number of executors in Flux topologies (enrichment,
  indexing). By default, the number of tasks will equal the number of 
executors. Logically, setting the number of tasks equal to the number
 of executors is sensible. Storm enforces num executors <= num tasks. The 
reason you might set the number of tasks higher than the number of
 executors is for future performance tuning and rebalancing without the need to 
bring down your topologies. The number of tasks is fixed
 at topology startup time whereas the number of executors can be increased up 
to a maximum value equal to the number of tasks.
 
-When configuring Storm Kafka spouts, we found that the default values for 
poll.timeout.ms, offset.commit.period.ms, and max.uncommitted.offsets worked 
well in nearly all cases.
-As a general rule, it was optimal to set spout parallelism equal to the number 
of partitions used in your Kafka topic. Any greater
+### Kafka Spout Configuration
+
+When configuring Storm Kafka spouts, we found that the default values for
+
+- `poll.timeout.ms`
+- `offset.commit.period.ms`
+- `max.uncommitted.offsets `
+
+worked well in nearly all cases. As a general rule, it was optimal to set 
spout parallelism equal to the number of partitions used in your Kafka topic. 
Any greater
 parallelism will leave you with idle consumers since Kafka limits the max 
number of consumers to the number of partitions. This is
 important because Kafka has certain ordering guarantees for message delivery 
per partition that would not be possible if more than
 one consumer in a given consumer group were able to read from that partition.
 
 ## Component Tuning Levers
 
+### High Level Overview
+
+There are a number of levers that can be set while tuning a Metron cluster. 
The main services to interact with for performance tuning are: Kafka, Storm, 
HDFS, and indexing (Elasticsearch or Solr). For each service, here is a high 
level breakdown of the major knobs and levers that can be modified while tuning 
your cluster.
+
 - Kafka
 - Number partitions
 - Storm
@@ -70,12 +91,15 @@ for more details.
 
 ### Storm Tuning
 
+ Overview
+
 There are quite a few options you will be confronted with when tuning your 
Storm topologies and this is largely trial and error. As a general rule of 
thumb,
 we recommend starting with the defaults and smaller numbers in terms of 
parallelism while iteratively working up until the desired performance is 
achieved.
 You will find the offset lag tool indispensable while verifying your settings.
 
 We won't go into a full discussion about Storm's architecture - see references 
section for more info - but there are some general rules of thumb 

[40/50] [abbrv] metron git commit: METRON-1529 CONFIG_GET Fails to Retrieve Latest Config When Run in Zeppelin REPL (nickwallen) closes apache/metron#997

[otto]

METRON-1529 CONFIG_GET Fails to Retrieve Latest Config When Run in Zeppelin 
REPL (nickwallen) closes apache/metron#997


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/37e3fd32
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/37e3fd32
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/37e3fd32

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 37e3fd32c256ddc129eb7c1363d78e9095a39748
Parents: b5bf9a9
Author: nickwallen 
Authored: Wed Apr 25 09:27:18 2018 -0400
Committer: nickallen 
Committed: Wed Apr 25 09:27:18 2018 -0400

--
 .../configuration/ConfigurationsUtils.java  | 123 +++-
 .../management/ConfigurationFunctions.java  | 564 ++-
 .../management/ConfigurationFunctionsTest.java  | 424 ++
 .../shell/DefaultStellarShellExecutor.java  |   4 +-
 .../common/utils/StellarProcessorUtils.java | 135 +++--
 5 files changed, 825 insertions(+), 425 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/37e3fd32/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
index a89db63..c7b39f0 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
@@ -27,6 +27,7 @@ import com.fasterxml.jackson.databind.JsonNode;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.PrintStream;
 import java.lang.invoke.MethodHandles;
 import java.nio.file.Files;
@@ -45,6 +46,7 @@ import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.metron.common.Constants;
 import 
org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.StellarFunctions;
@@ -235,12 +237,99 @@ public class ConfigurationsUtils {
   );
   }
 
+  /**
+   * Reads the global configuration stored in Zookeeper.
+   *
+   * @param client The Zookeeper client.
+   * @return The global configuration, if one exists.  Otherwise, null.
+   * @throws Exception
+   */
+  public static Map 
readGlobalConfigFromZookeeper(CuratorFramework client) throws Exception {
+Map config = null;
+
+Optional bytes = 
readFromZookeeperSafely(GLOBAL.getZookeeperRoot(), client);
+if(bytes.isPresent()) {
+  InputStream in = new ByteArrayInputStream(bytes.get());
+  config = JSONUtils.INSTANCE.load(in, JSONUtils.MAP_SUPPLIER);
+}
+
+return config;
+  }
+
+  /**
+   * Reads the Indexing configuration from Zookeeper.
+   *
+   * @param sensorType The type of sensor.
+   * @param client The Zookeeper client.
+   * @return The indexing configuration for the given sensor type, if one 
exists.  Otherwise, null.
+   * @throws Exception
+   */
+  public static Map 
readSensorIndexingConfigFromZookeeper(String sensorType, CuratorFramework 
client) throws Exception {
+Map config = null;
+
+Optional bytes = 
readFromZookeeperSafely(INDEXING.getZookeeperRoot() + "/" + sensorType, client);
+if(bytes.isPresent()) {
+  InputStream in = new ByteArrayInputStream(bytes.get());
+  config = JSONUtils.INSTANCE.load(in, JSONUtils.MAP_SUPPLIER);
+}
+
+return config;
+  }
+
+  /**
+   * Reads the Enrichment configuration from Zookeeper.
+   *
+   * @param sensorType The type of sensor.
+   * @param client The Zookeeper client.
+   * @return The Enrichment configuration for the given sensor type, if one 
exists. Otherwise, null.
+   * @throws Exception
+   */
   public static SensorEnrichmentConfig 
readSensorEnrichmentConfigFromZookeeper(String sensorType, CuratorFramework 
client) throws Exception {
-return JSONUtils.INSTANCE.load(new 
ByteArrayInputStream(readFromZookeeper(ENRICHMENT.getZookeeperRoot() + "/" + 
sensorType, client)), SensorEnrichmentConfig.class);
+SensorEnrichmentConfig config = null;
+
+Optional bytes = 
readFromZookeeperSafely(ENRICHMENT.getZookeeperRoot() + "/" + sensorType, 

[45/50] [abbrv] metron git commit: METRON-1530 Default proxy config settings in metron-contrib need to be updated (sardell via merrimanr) closes apache/metron#998

[otto]

METRON-1530 Default proxy config settings in metron-contrib need to be updated 
(sardell via merrimanr) closes apache/metron#998


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1b1a45b7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1b1a45b7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1b1a45b7

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 1b1a45b7e9c2b8c451f970028b4fe310ef9b7894
Parents: ac05638
Author: sardell 
Authored: Wed May 9 09:43:58 2018 -0500
Committer: merrimanr 
Committed: Wed May 9 09:43:58 2018 -0500

--
 metron-interface/metron-alerts/proxy.conf.json | 9 -
 metron-interface/metron-config/proxy.conf.json | 4 ++--
 2 files changed, 2 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/1b1a45b7/metron-interface/metron-alerts/proxy.conf.json
--
diff --git a/metron-interface/metron-alerts/proxy.conf.json 
b/metron-interface/metron-alerts/proxy.conf.json
index 27d1ee9..612bd67 100644
--- a/metron-interface/metron-alerts/proxy.conf.json
+++ b/metron-interface/metron-alerts/proxy.conf.json
@@ -6,14 +6,5 @@
   "/logout": {
 "target": "http://node1:8082;,
 "secure": false
-  },
-  "/search": {
-"target": "http://node1:9200;,
-"pathRewrite": {"^/search" : ""},
-"secure": false
-  },
-  "/_cluster": {
-"target": "http://node1:9200;,
-"secure": false
   }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/1b1a45b7/metron-interface/metron-config/proxy.conf.json
--
diff --git a/metron-interface/metron-config/proxy.conf.json 
b/metron-interface/metron-config/proxy.conf.json
index 29466cc..612bd67 100644
--- a/metron-interface/metron-config/proxy.conf.json
+++ b/metron-interface/metron-config/proxy.conf.json
@@ -1,10 +1,10 @@
 {
   "/api/v1": {
-"target": "http://localhost:8080;,
+"target": "http://node1:8082;,
 "secure": false
   },
   "/logout": {
-"target": "http://localhost:8080;,
+"target": "http://node1:8082;,
 "secure": false
   }
 }

[32/50] [abbrv] metron git commit: METRON-1515: Errors loading stellar functions currently bomb the entire topology, they should be recoverable closes apache/incubator-metron#985

[otto]

METRON-1515: Errors loading stellar functions currently bomb the entire 
topology, they should be recoverable closes apache/incubator-metron#985


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3fcbf8b4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3fcbf8b4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3fcbf8b4

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3fcbf8b4e4e38f9c50842b8af857092b091c7c40
Parents: 1d3e7fc
Author: cstella 
Authored: Mon Apr 16 15:12:11 2018 -0400
Committer: cstella 
Committed: Mon Apr 16 15:12:11 2018 -0400

--
 .../resolver/ClasspathFunctionResolver.java | 45 +++-
 .../resolver/ClasspathFunctionResolverTest.java | 30 +
 2 files changed, 65 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/3fcbf8b4/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
index 85aa015..b17233a 100644
--- 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
@@ -34,6 +34,7 @@ import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.Stellar;
 import org.apache.metron.stellar.dsl.StellarFunction;
 
+import org.atteo.classindex.ClassFilter;
 import org.atteo.classindex.ClassIndex;
 import org.reflections.util.FilterBuilder;
 
@@ -219,6 +220,17 @@ public class ClasspathFunctionResolver extends 
BaseFunctionResolver {
 }
   }
 
+  protected Iterable getStellarClasses(ClassLoader cl) {
+return ClassIndex.getAnnotated(Stellar.class, cl);
+  }
+
+  protected boolean includeClass(Class c, FilterBuilder filterBuilder)
+  {
+boolean isAssignable = StellarFunction.class.isAssignableFrom(c);
+boolean isFiltered = filterBuilder.apply(c.getCanonicalName());
+return isAssignable && isFiltered;
+  }
+
   /**
* Returns a set of classes that should undergo further interrogation for 
resolution
* (aka discovery) of Stellar functions.
@@ -254,16 +266,29 @@ public class ClasspathFunctionResolver extends 
BaseFunctionResolver {
 Set classes = new HashSet<>();
 Set ret = new HashSet<>();
 for(ClassLoader cl : cls) {
-  for(Class c : ClassIndex.getAnnotated(Stellar.class, cl)) {
-LOG.debug("{}: Found class: {}", cl.getClass().getCanonicalName(), 
c.getCanonicalName());
-boolean isAssignable = StellarFunction.class.isAssignableFrom(c);
-boolean isFiltered = filterBuilder.apply(c.getCanonicalName());
-if( isAssignable && isFiltered ) {
-  String className = c.getName();
-  if(!classes.contains(className)) {
-LOG.debug("{}: Added class: {}", cl.getClass().getCanonicalName(), 
className);
-ret.add((Class) c);
-classes.add(className);
+  for(Class c : getStellarClasses(cl)) {
+try {
+  LOG.debug("{}: Found class: {}", cl.getClass().getCanonicalName(), 
c.getCanonicalName());
+  if (includeClass(c, filterBuilder)) {
+String className = c.getName();
+if (!classes.contains(className)) {
+  LOG.debug("{}: Added class: {}", 
cl.getClass().getCanonicalName(), className);
+  ret.add((Class) c);
+  classes.add(className);
+}
+  }
+}
+catch(Error le) {
+  //we have had some error loading a stellar function.  This could 
mean that
+  //the classpath is unstable (e.g. old copies of jars are on the 
classpath).
+  try {
+LOG.error("Skipping class " + c.getName() + ": " + le.getMessage()
++ ", please check that there are not old versions of 
stellar functions on the classpath.", le);
+  }
+  catch(Error ie) {
+//it's possible that getName() will throw an exception if the 
class is VERY malformed.
+LOG.error("Skipping class: " + le.getMessage()
++ ", please check that there are not old versions of 
stellar functions on the classpath.", le);
   }
 }
   }

[28/50] [abbrv] metron git commit: Add support for Vagrant Cachier plugin if present (simonellistonball via mmiklavc) closes apache/metron#993

[otto]

Add support for Vagrant Cachier plugin if present (simonellistonball via 
mmiklavc) closes apache/metron#993


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/53124d97
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/53124d97
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/53124d97

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 53124d97d09baafcf6d84eb291f6a3828289ec7c
Parents: eb5b2d4
Author: simonellistonball 
Authored: Fri Apr 13 09:30:54 2018 -0600
Committer: Michael Miklavcic 
Committed: Fri Apr 13 09:30:54 2018 -0600

--
 metron-deployment/development/README.md|  5 +
 metron-deployment/development/centos6/Vagrantfile  | 11 +++
 metron-deployment/development/ubuntu14/Vagrantfile | 11 +++
 3 files changed, 27 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/53124d97/metron-deployment/development/README.md
--
diff --git a/metron-deployment/development/README.md 
b/metron-deployment/development/README.md
index bc99809..2a04e5f 100644
--- a/metron-deployment/development/README.md
+++ b/metron-deployment/development/README.md
@@ -22,3 +22,8 @@ This directory contains environments useful for Metron 
developers.  These enviro
 * Metron running on CentOS 6
 * Metron running on Ubuntu 14
 * Fastcapa
+
+
+## Vagrant Cachier recommendations
+
+The development boxes are designed to be spun up and destroyed on a regular 
basis as part of the development cycle. In order to avoid the overhead of 
re-downloading many of the heavy platform dependencies, Vagrant can use the 
[vagrant-cachier](http://fgrehm.viewdocs.io/vagrant-cachier/) plugin to store 
package caches between builds. If the plugin has been installed to your vagrant 
it will be used, and packages will be cached in ~/.vagrant/cache.

http://git-wip-us.apache.org/repos/asf/metron/blob/53124d97/metron-deployment/development/centos6/Vagrantfile
--
diff --git a/metron-deployment/development/centos6/Vagrantfile 
b/metron-deployment/development/centos6/Vagrantfile
index 101a2dd..d0b7051 100644
--- a/metron-deployment/development/centos6/Vagrantfile
+++ b/metron-deployment/development/centos6/Vagrantfile
@@ -60,6 +60,17 @@ Vagrant.configure(2) do |config|
   config.hostmanager.enabled = true
   config.hostmanager.manage_host = true
 
+  # enable vagrant cachier if present
+  if Vagrant.has_plugin?("vagrant-cachier")
+config.cache.enable :yum
+config.cache.scope = :box
+
+config.cache.synced_folder_opts = {
+  type: :nfs,
+  mount_options: ['rw', 'vers=3', 'tcp', 'nolock']
+}
+  end
+
   # host definition
   hosts.each_with_index do |host, index|
 config.vm.define host[:hostname] do |node|

http://git-wip-us.apache.org/repos/asf/metron/blob/53124d97/metron-deployment/development/ubuntu14/Vagrantfile
--
diff --git a/metron-deployment/development/ubuntu14/Vagrantfile 
b/metron-deployment/development/ubuntu14/Vagrantfile
index 01c0d17..cfa3cdf 100644
--- a/metron-deployment/development/ubuntu14/Vagrantfile
+++ b/metron-deployment/development/ubuntu14/Vagrantfile
@@ -60,6 +60,17 @@ Vagrant.configure(2) do |config|
   config.hostmanager.enabled = true
   config.hostmanager.manage_host = true
 
+  # enable vagrant cachier if present
+  if Vagrant.has_plugin?("vagrant-cachier")
+config.cache.enable :apt
+config.cache.scope = :box
+
+config.cache.synced_folder_opts = {
+  type: :nfs,
+  mount_options: ['rw', 'vers=3', 'tcp', 'nolock']
+}
+  end
+
   # host definition
   hosts.each_with_index do |host, index|
 config.vm.define host[:hostname] do |node|

[29/50] [abbrv] metron git commit: METRON-1347: Indexing Topology should fail tuples without a source.type (cstella via mmiklavc) closes apache/metron#863

[otto]

METRON-1347: Indexing Topology should fail tuples without a source.type 
(cstella via mmiklavc) closes apache/metron#863


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bfe90ef1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bfe90ef1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bfe90ef1

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: bfe90ef1e579be53a14d9fd0e4dc19fc6a81baf0
Parents: 53124d9
Author: cstella 
Authored: Fri Apr 13 11:17:00 2018 -0600
Committer: Michael Miklavcic 
Committed: Fri Apr 13 11:17:00 2018 -0600

--
 .../bolt/BulkMessageWriterBoltTest.java | 25 ++
 metron-platform/metron-indexing/README.md   |  6 +++
 .../writer/bolt/BulkMessageWriterBolt.java  | 51 ++--
 3 files changed, 68 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/bfe90ef1/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
--
diff --git 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
index 308638e..dedf5e6 100644
--- 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
+++ 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
@@ -118,6 +118,31 @@ public class BulkMessageWriterBoltTest extends 
BaseEnrichmentBoltTest {
   private MessageGetStrategy messageGetStrategy;
 
   @Test
+  public void testSensorTypeMissing() throws Exception {
+BulkMessageWriterBolt bulkMessageWriterBolt = new 
BulkMessageWriterBolt("zookeeperUrl")
+
.withBulkMessageWriter(bulkMessageWriter).withMessageGetter(MessageGetters.JSON_FROM_FIELD.name())
+.withMessageGetterField("message");
+bulkMessageWriterBolt.setCuratorFramework(client);
+bulkMessageWriterBolt.setZKCache(cache);
+
bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(sensorType,
+new FileInputStream(sampleSensorIndexingConfigPath));
+
+bulkMessageWriterBolt.declareOutputFields(declarer);
+verify(declarer, times(1)).declareStream(eq("error"), argThat(
+new FieldsMatcher("message")));
+Map stormConf = new HashMap();
+bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector);
+BulkWriterComponent component = 
mock(BulkWriterComponent.class);
+bulkMessageWriterBolt.setWriterComponent(component);
+verify(bulkMessageWriter, 
times(1)).init(eq(stormConf),any(TopologyContext.class), 
any(WriterConfiguration.class));
+JSONObject message = (JSONObject) new 
JSONParser().parse(sampleMessageString);
+message.remove("source.type");
+when(tuple.getValueByField("message")).thenReturn(message);
+bulkMessageWriterBolt.execute(tuple);
+verify(component, times(1)).error(eq("null"), any(), any(), any());
+  }
+
+  @Test
   public void testFlushOnBatchSize() throws Exception {
 BulkMessageWriterBolt bulkMessageWriterBolt = new 
BulkMessageWriterBolt("zookeeperUrl")
 
.withBulkMessageWriter(bulkMessageWriter).withMessageGetter(MessageGetters.JSON_FROM_FIELD.name())

http://git-wip-us.apache.org/repos/asf/metron/blob/bfe90ef1/metron-platform/metron-indexing/README.md
--
diff --git a/metron-platform/metron-indexing/README.md 
b/metron-platform/metron-indexing/README.md
index d351d7c..f4a4501 100644
--- a/metron-platform/metron-indexing/README.md
+++ b/metron-platform/metron-indexing/README.md
@@ -32,6 +32,12 @@ Indices are written in batch and the batch size and batch 
timeout are specified
 [Sensor Indexing Configuration](#sensor-indexing-configuration) via the 
`batchSize` and `batchTimeout` parameters.
 These configs are variable by sensor type.
 
+## Minimal Assumptions for Message Structure
+
+At minimum, a message should have a `source.type` field.
+Without this field, the message tuple will be failed and not written
+with an appropriate error indicated in the Storm UI and logs.
+
 ## Indexing Architecture
 
 ![Architecture](indexing_arch.png)

http://git-wip-us.apache.org/repos/asf/metron/blob/bfe90ef1/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
--
diff --git 

[33/50] [abbrv] metron git commit: METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
--
diff --git 
a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml 
b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
deleted file mode 100644
index fd7ceff..000
--- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
+++ /dev/null
@@ -1,590 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "enrichment"
-config:
-topology.workers: ${enrichment.workers}
-topology.acker.executors: ${enrichment.acker.executors}
-topology.worker.childopts: ${topology.worker.childopts}
-topology.auto-credentials: ${topology.auto-credentials}
-topology.max.spout.pending: ${topology.max.spout.pending}
-
-components:
-
-# Enrichment
--   id: "stellarEnrichmentAdapter"
-className: 
"org.apache.metron.enrichment.adapters.stellar.StellarAdapter"
-configMethods:
--   name: "ofType"
-args:
-- "ENRICHMENT"
-
-# Any kafka props for the producer go here.
--   id: "kafkaWriterProps"
-className: "java.util.HashMap"
-configMethods:
-  -   name: "put"
-  args:
-  - "security.protocol"
-  - "${kafka.security.protocol}"
-
--   id: "stellarEnrichment"
-className: "org.apache.metron.enrichment.configuration.Enrichment"
-constructorArgs:
--   "stellar"
--   ref: "stellarEnrichmentAdapter"
-
--   id: "geoEnrichmentAdapter"
-className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
--   id: "geoEnrichment"
-className: "org.apache.metron.enrichment.configuration.Enrichment"
-constructorArgs:
--   "geo"
--   ref: "geoEnrichmentAdapter"
--   id: "hostEnrichmentAdapter"
-className: 
"org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
-constructorArgs:
-- '${enrichment.host.known_hosts}'
--   id: "hostEnrichment"
-className: "org.apache.metron.enrichment.configuration.Enrichment"
-constructorArgs:
--   "host"
--   ref: "hostEnrichmentAdapter"
-
--   id: "simpleHBaseEnrichmentConfig"
-className: 
"org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseConfig"
-configMethods:
--   name: "withProviderImpl"
-args:
-- "${hbase.provider.impl}"
--   name: "withHBaseTable"
-args:
-- "${enrichment.simple.hbase.table}"
--   name: "withHBaseCF"
-args:
-- "${enrichment.simple.hbase.cf}"
--   id: "simpleHBaseEnrichmentAdapter"
-className: 
"org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseAdapter"
-configMethods:
-   -name: "withConfig"
-args:
-- ref: "simpleHBaseEnrichmentConfig"
--   id: "simpleHBaseEnrichment"
-className: "org.apache.metron.enrichment.configuration.Enrichment"
-constructorArgs:
-  -   "hbaseEnrichment"
-  -   ref: "simpleHBaseEnrichmentAdapter"
--   id: "enrichments"
-className: "java.util.ArrayList"
-configMethods:
--   name: "add"
-args:
-- ref: "geoEnrichment"
--   name: "add"
-args:
-- ref: "hostEnrichment"
--   name: "add"
-args:
-- ref: "simpleHBaseEnrichment"
--   name: "add"
-args:
-- ref: "stellarEnrichment"
-
-#enrichment error
--   id: "enrichmentErrorKafkaWriter"
-className: "org.apache.metron.writer.kafka.KafkaWriter"
-configMethods:
--   name: "withTopic"
-args:
-- "${enrichment.error.topic}"
--   name: "withZkQuorum"
-args:
-- "${kafka.zk}"
-

[20/50] [abbrv] metron git commit: METRON-1505 Intermittent Profiler Integration Test Failure (nickwallen) closes apache/metron#977

[otto]

METRON-1505 Intermittent Profiler Integration Test Failure (nickwallen) closes 
apache/metron#977


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/46bc63db
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/46bc63db
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/46bc63db

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 46bc63dbcfe9f0ddabfd4821958962a2dac9378e
Parents: ab4f8e6
Author: nickwallen 
Authored: Sat Apr 7 11:28:01 2018 -0400
Committer: nickallen 
Committed: Sat Apr 7 11:28:01 2018 -0400

--
 .../profiler/DefaultMessageDistributor.java |  54 +++-
 .../src/main/flux/profiler/remote.yaml  |   2 +
 .../profiler/bolt/ProfileBuilderBolt.java   | 149 +++---
 .../profiler/bolt/ProfileSplitterBolt.java  |   1 -
 .../config/zookeeper/percentiles/profiler.json  |  12 -
 .../processing-time-test/profiler.json  |  11 +
 .../zookeeper/readme-example-1/profiler.json|  17 --
 .../zookeeper/readme-example-2/profiler.json|  18 --
 .../zookeeper/readme-example-3/profiler.json|  11 -
 .../zookeeper/readme-example-4/profiler.json|  11 -
 .../profiler/bolt/ProfileBuilderBoltTest.java   | 130 +++--
 .../integration/ProfilerIntegrationTest.java| 274 +--
 .../configuration/profiler/ProfileConfig.java   |  49 ++--
 .../ZKConfigurationsCacheIntegrationTest.java   |   4 +-
 .../org/apache/metron/hbase/bolt/HBaseBolt.java |  22 +-
 15 files changed, 319 insertions(+), 446 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/46bc63db/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
--
diff --git 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
index ea5126f..70f4228 100644
--- 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
+++ 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
@@ -25,6 +25,7 @@ import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.RemovalListener;
 import com.google.common.cache.RemovalNotification;
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.metron.common.configuration.profiler.ProfileConfig;
 import org.apache.metron.stellar.dsl.Context;
 import org.json.simple.JSONObject;
@@ -33,7 +34,6 @@ import org.slf4j.LoggerFactory;
 
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
@@ -74,7 +74,7 @@ public class DefaultMessageDistributor implements 
MessageDistributor {
* messages.  Once it has not received messages for a period of time, it is
* moved to the expired cache.
*/
-  private transient Cache activeCache;
+  private transient Cache activeCache;
 
   /**
* A cache of expired profiles.
@@ -85,7 +85,7 @@ public class DefaultMessageDistributor implements 
MessageDistributor {
* can flush the state of the expired profile.  If the client does not flush
* the expired profiles, this state will be lost forever.
*/
-  private transient Cache expiredCache;
+  private transient Cache expiredCache;
 
   /**
* Create a new message distributor.
@@ -222,7 +222,7 @@ public class DefaultMessageDistributor implements 
MessageDistributor {
* @param cache The cache to flush.
* @return The measurements captured when flushing the profiles.
*/
-  private List flushCache(Cache 
cache) {
+  private List flushCache(Cache 
cache) {
 
 List measurements = new ArrayList<>();
 for(ProfileBuilder profileBuilder: cache.asMap().values()) {
@@ -262,11 +262,19 @@ public class DefaultMessageDistributor implements 
MessageDistributor {
   /**
* Builds the key that is used to lookup the {@link ProfileBuilder} within 
the cache.
*
+   * The cache key is built using the hash codes of the profile and entity 
name.  If the profile
+   * definition is ever changed, the same cache entry will not be reused.  
This ensures that no
+   * state can be carried over from the old definition into the new, which 
might result in an
+   * invalid profile measurement.
+   *
* @param profile The profile 

[42/50] [abbrv] metron git commit: METRON-1539: Specialized RENAME field transformer closes apache/incubator-metron#1002

[otto]

METRON-1539: Specialized RENAME field transformer closes 
apache/incubator-metron#1002


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2b4f0b84
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2b4f0b84
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2b4f0b84

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 2b4f0b84062d65f9400421d66ec3b7d6d093bebf
Parents: 1c5435c
Author: cstella 
Authored: Wed Apr 25 11:49:56 2018 -0400
Committer: cstella 
Committed: Wed Apr 25 11:49:56 2018 -0400

--
 .../common/configuration/FieldTransformer.java  |  4 +-
 .../transformation/FieldTransformations.java|  1 +
 .../transformation/RenameTransformation.java| 55 +++
 .../transformation/FieldTransformationTest.java | 17 +---
 .../RenameTransformationTest.java   | 99 
 metron-platform/metron-parsers/README.md| 25 -
 6 files changed, 183 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
index df80691..43ce9d8 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
@@ -94,7 +94,9 @@ public class FieldTransformer implements Serializable {
 
   if (output == null || output.isEmpty()) {
 if (input == null || input.isEmpty()) {
-  throw new IllegalStateException("You must specify an input field if 
you want to leave the output fields empty");
+  //both are empty, so let's set them both to null
+  output = null;
+  input = null;
 } else {
   output = input;
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
index a905123..95ff390 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
@@ -25,6 +25,7 @@ public enum FieldTransformations {
   ,REMOVE(new RemoveTransformation())
   ,STELLAR(new StellarTransformation())
   ,SELECT(new SelectTransformation())
+  ,RENAME(new RenameTransformation())
   ;
   FieldTransformation mapping;
   FieldTransformations(FieldTransformation mapping) {

http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java
new file mode 100644
index 000..f8b9374
--- /dev/null
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package 

[23/50] [abbrv] metron git commit: METRON-1518 Build Failure When Using Profile HDP-2.5.0.0 (nickwallen) closes apache/metron#986

[otto]

METRON-1518 Build Failure When Using Profile HDP-2.5.0.0 (nickwallen) closes 
apache/metron#986


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ed50d48b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ed50d48b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ed50d48b

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ed50d48bb47cf3f301884f6e18fe4efc8c1b91f1
Parents: a8b555d
Author: nickwallen 
Authored: Tue Apr 10 17:16:20 2018 -0400
Committer: nickallen 
Committed: Tue Apr 10 17:16:20 2018 -0400

--
 .../profiler/bolt/ProfileBuilderBolt.java   | 51 +---
 1 file changed, 11 insertions(+), 40 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/ed50d48b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
--
diff --git 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
index fb3d2d0..ca02b58 100644
--- 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
+++ 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
@@ -42,13 +42,11 @@ import 
org.apache.metron.stellar.common.utils.ConversionUtils;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.zookeeper.SimpleEventListener;
 import org.apache.metron.zookeeper.ZKCache;
-import org.apache.storm.StormTimer;
 import org.apache.storm.task.OutputCollector;
 import org.apache.storm.task.TopologyContext;
 import org.apache.storm.topology.OutputFieldsDeclarer;
 import org.apache.storm.topology.base.BaseWindowedBolt;
 import org.apache.storm.tuple.Tuple;
-import org.apache.storm.utils.Utils;
 import org.apache.storm.windowing.TupleWindow;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
@@ -59,9 +57,9 @@ import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
-import java.util.function.Supplier;
 
 import static java.lang.String.format;
 import static 
org.apache.metron.profiler.bolt.ProfileSplitterBolt.ENTITY_TUPLE_FIELD;
@@ -155,8 +153,8 @@ public class ProfileBuilderBolt extends BaseWindowedBolt 
implements Reloadable {
   private FlushSignal activeFlushSignal;
 
   /**
-   * A timer that flushes expired profiles on a regular interval. The expired 
profiles
-   * are flushed on a separate thread.
+   * An executor that flushes expired profiles at a regular interval on a 
separate
+   * thread.
*
* Flushing expired profiles ensures that any profiles that stop 
receiving messages
* for an extended period of time will continue to be flushed.
@@ -164,7 +162,7 @@ public class ProfileBuilderBolt extends BaseWindowedBolt 
implements Reloadable {
* This introduces concurrency issues as the bolt is no longer single 
threaded. Due
* to this, all access to the {@code MessageDistributor} needs to be 
protected.
*/
-  private StormTimer expiredFlushTimer;
+  private transient ScheduledExecutorService flushExpiredExecutor;
 
   public ProfileBuilderBolt() {
 this.emitters = new ArrayList<>();
@@ -202,7 +200,7 @@ public class ProfileBuilderBolt extends BaseWindowedBolt 
implements Reloadable {
 this.configurations = new ProfilerConfigurations();
 this.activeFlushSignal = new 
FixedFrequencyFlushSignal(periodDurationMillis);
 setupZookeeper();
-startExpiredFlushTimer();
+startFlushingExpiredProfiles();
   }
 
   @Override
@@ -210,7 +208,7 @@ public class ProfileBuilderBolt extends BaseWindowedBolt 
implements Reloadable {
 try {
   zookeeperCache.close();
   zookeeperClient.close();
-  expiredFlushTimer.close();
+  flushExpiredExecutor.shutdown();
 
 } catch(Throwable e) {
   LOG.error("Exception when cleaning up", e);
@@ -421,39 +419,12 @@ public class ProfileBuilderBolt extends BaseWindowedBolt 
implements Reloadable {
   }
 
   /**
-   * Converts milliseconds to seconds and handles an ugly cast.
-   *
-   * @param millis Duration in milliseconds.
-   * @return Duration in seconds.
-   */
-  private int toSeconds(long millis) {
-return (int) TimeUnit.MILLISECONDS.toSeconds(millis);
-  }
-
-  /**
-   * Creates a timer that regularly flushes expired profiles on a separate 
thread.
-   */
-  private void startExpiredFlushTimer() {

[13/50] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json
new file mode 100644
index 000..037f1c6
--- /dev/null
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json
@@ -0,0 +1,88 @@
+{ "create" : { "_id": "all-metron-index", "_type": "index-pattern" } }
+{"title":"*_index_*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\"
 
:true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",
 
\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false,
 
\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scri
 

[44/50] [abbrv] metron git commit: METRON-1545 Upgrade Spring and Spring Boot (merrimanr) closes apache/metron#1008

[otto]

METRON-1545 Upgrade Spring and Spring Boot (merrimanr) closes apache/metron#1008


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ac056381
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ac056381
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ac056381

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ac05638160288b32535b986adbeb8f14e594e740
Parents: 3bb926d
Author: merrimanr 
Authored: Fri May 4 16:22:17 2018 -0500
Committer: merrimanr 
Committed: Fri May 4 16:22:17 2018 -0500

--
 dependencies_with_url.csv   | 56 +++-
 metron-interface/metron-rest/README.md  |  2 +
 metron-interface/metron-rest/pom.xml| 17 --
 .../metron/rest/MetronRestApplication.java  |  7 +++
 .../apache/metron/rest/MetronRestConstants.java |  2 +
 .../metron/rest/config/JpaConfiguration.java|  6 +--
 .../metron/rest/config/WebSecurityConfig.java   | 19 +--
 .../src/main/resources/application-test.yml |  2 +-
 .../metron-rest/src/main/scripts/metron-rest.sh |  2 +-
 9 files changed, 97 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/ac056381/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 1e73eb1..df3bcd2 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -4,6 +4,7 @@ org.objenesis:objenesis:jar:1.2:compile,Apache 
v2,http://objenesis.org/
 org.objenesis:objenesis:jar:2.1:compile,Apache v2,http://objenesis.org/
 org.ow2.asm:asm:jar:4.1:compile,BSD,http://asm.ow2.org/
 org.ow2.asm:asm:jar:5.0.3:compile,BSD,http://asm.ow2.org/
+org.ow2.asm:asm:jar:5.0.4:compile,BSD,http://asm.ow2.org/
 
org.reflections:reflections:jar:0.9.10:compile,BSD,https://github.com/ronmamo/reflections
 org.javassist:javassist:jar:3.19.0-GA:compile,Apache 
v2,https://github.com/jboss-javassist/javassist
 org.javassist:javassist:jar:3.17.1-GA:compile,Apache 
v2,https://github.com/jboss-javassist/javassist
@@ -23,6 +24,7 @@ com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD 
license,http://code.
 com.google.protobuf:protobuf-java:jar:2.6.1:compile,New BSD 
license,http://code.google.com/p/protobuf
 com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/
 com.jayway.jsonpath:json-path:jar:2.3.0:compile,Apache 
v2,https://github.com/json-path/JsonPath
+com.jayway.jsonpath:json-path:jar:2.4.0:compile,Apache 
v2,https://github.com/json-path/JsonPath
 net.minidev:accessors-smart:jar:1.2:compile,Apache 
v2,https://github.com/netplex/json-smart-v2
 net.minidev:json-smart:jar:2.3:compile,Apache 
v2,https://github.com/netplex/json-smart-v2
 com.maxmind.db:maxmind-db:jar:1.2.1:compile,CC-BY-SA 
3.0,https://github.com/maxmind/MaxMind-DB
@@ -34,10 +36,12 @@ 
it.unimi.dsi:fastutil:jar:7.0.6:compile,ASLv2,https://github.com/vigna/fastutil
 javassist:javassist:jar:3.12.1.GA:compile,Apache v2,http://www.javassist.org/
 javax.activation:activation:jar:1.1:compile,Common Development and 
Distribution License (CDDL) 
v1.0,http://java.sun.com/products/javabeans/jaf/index.jsp
 javax.annotation:jsr250-api:jar:1.0:compile,COMMON DEVELOPMENT AND 
DISTRIBUTION LICENSE (CDDL) Version 
1.0,http://jcp.org/aboutJava/communityprocess/final/jsr250/index.html
+javax.annotation:javax.annotation-api:jar:1.3.2:compile,CDDL 
1.1,https://github.com/javaee/javax.annotation/
 javax.mail:mail:jar:1.4:compile,Common Development and Distribution License 
(CDDL) v1.0,https://glassfish.dev.java.net/javaee5/mail/
 
javax.servlet:javax.servlet-api:jar:3.1.0:compile,CDDL,http://servlet-spec.java.net
 javax.xml.bind:jaxb-api:jar:2.2.11:compile,CDDL,http://jaxb.java.net/
 javax.xml.bind:jaxb-api:jar:2.2.2:compile,CDDL,https://jaxb.dev.java.net/
+javax.xml.bind:jaxb-api:jar:2.3.0:compile,CDDL,https://jaxb.dev.java.net/
 javax.xml.stream:stax-api:jar:1.0-2:compile,COMMON DEVELOPMENT AND 
DISTRIBUTION LICENSE (CDDL) Version 
1.0,https://docs.oracle.com/javase/7/docs/api/javax/xml/stream/package-summary.html
 jline:jline:jar:0.9.94:compile,BSD,http://jline.sourceforge.net
 junit:junit:jar:4.12:compile,Eclipse Public License 1.0,http://junit.org
@@ -87,15 +91,18 @@ 
org.slf4j:slf4j-api:jar:1.7.5:compile,MIT,http://www.slf4j.org
 org.slf4j:slf4j-api:jar:1.7.6:compile,MIT,http://www.slf4j.org
 org.slf4j:slf4j-api:jar:1.7.7:compile,MIT,http://www.slf4j.org
 org.slf4j:slf4j-api:jar:1.7.21:compile,MIT,http://www.slf4j.org
+org.slf4j:slf4j-api:jar:1.7.25:compile,MIT,http://www.slf4j.org
 org.slf4j:slf4j-log4j12:jar:1.6.1:compile,MIT,http://www.slf4j.org
 org.slf4j:slf4j-log4j12:jar:1.7.10:compile,MIT,http://www.slf4j.org
 

[34/50] [abbrv] metron git commit: METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984

[otto]

METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari 
(nickwallen) closes apache/metron#984


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/82212ba8
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/82212ba8
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/82212ba8

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 82212ba818a0ef3b92505e8d144487c69a8d4a44
Parents: 3fcbf8b
Author: nickwallen 
Authored: Tue Apr 17 09:43:16 2018 -0400
Committer: nickallen 
Committed: Tue Apr 17 09:43:16 2018 -0400

--
 .../packaging/ambari/metron-mpack/pom.xml   |   7 +-
 .../configuration/metron-enrichment-env.xml | 130 +++-
 .../package/scripts/enrichment_commands.py  |  20 +-
 .../package/scripts/enrichment_master.py|  12 +-
 .../package/scripts/params/params_linux.py  |  19 +-
 .../enrichment-splitjoin.properties.j2  |  63 ++
 .../templates/enrichment-unified.properties.j2  |  60 ++
 .../METRON/CURRENT/themes/metron_theme.json | 151 -
 .../docker/rpm-docker/SPECS/metron.spec |   5 +-
 .../main/config/enrichment-splitjoin.properties |  63 ++
 .../config/enrichment-splitjoin.properties.j2   |  63 ++
 .../main/config/enrichment-unified.properties   |  69 +++
 .../config/enrichment-unified.properties.j2 |  60 ++
 .../src/main/config/enrichment.properties   |  64 --
 .../src/main/config/enrichment.properties.j2|  63 --
 .../main/flux/enrichment/remote-splitjoin.yaml  | 590 +++
 .../main/flux/enrichment/remote-unified.yaml|  71 ++-
 .../src/main/flux/enrichment/remote.yaml| 590 ---
 .../main/scripts/start_enrichment_topology.sh   |  16 +-
 .../integration/EnrichmentIntegrationTest.java  |  61 +-
 .../UnifiedEnrichmentIntegrationTest.java   |  71 +++
 21 files changed, 1433 insertions(+), 815 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/pom.xml
--
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml 
b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 491e8dd..1843eb7 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -110,7 +110,8 @@
 
 
${basedir}/../../../../metron-platform/metron-enrichment/src/main/config
 
-
enrichment.properties.j2
+
enrichment-splitjoin.properties.j2
+
enrichment-unified.properties.j2
 
 false
 
@@ -171,8 +172,10 @@
 
 
${basedir}/src/main/resources/common-services/METRON/CURRENT/package/templates
 
-enrichment.properties.j2
+
enrichment-unified.properties.j2
+
enrichment-splitjoin.properties.j2
 elasticsearch.properties.j2
+hdfs.properties.j2
 
 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
index 9737660..81b135c 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
@@ -18,6 +18,10 @@
   limitations under the License.
 -->
 
+
+  
   
 geoip_url
 
http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz
@@ -33,6 +37,10 @@
   content
 
   
+
+  
   
 enrichment_kafka_start
 Enrichment Topology Spout Offset
@@ -81,6 +89,10 @@
 indexing
 Threat Intel Error Topic
   
+
+  
   
 enrichment_hbase_table
 enrichment

[21/50] [abbrv] metron git commit: METRON-1504: Enriching missing values does not match the semantics between the new enrichment topology and old closes apache/incubator-metron#976

[otto]

METRON-1504: Enriching missing values does not match the semantics between the 
new enrichment topology and old closes apache/incubator-metron#976


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3ba9ae25
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3ba9ae25
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3ba9ae25

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3ba9ae25126bc393e3b9307f4ffe63ac66a909f4
Parents: 46bc63d
Author: cstella 
Authored: Mon Apr 9 11:47:51 2018 -0400
Committer: cstella 
Committed: Mon Apr 9 11:47:51 2018 -0400

--
 .../enrichment/parallel/ParallelEnricher.java   |  3 +
 .../integration/EnrichmentIntegrationTest.java  | 12 +++-
 .../parallel/ParallelEnricherTest.java  | 59 +++-
 3 files changed, 70 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
--
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
index 6ddb892..b10c148 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
@@ -170,6 +170,9 @@ public class ParallelEnricher {
 for(Object o : m.keySet()) {
   String field = (String) o;
   Object value = m.get(o);
+  if(value == null) {
+continue;
+  }
   CacheKey cacheKey = new CacheKey(field, value, config);
   String prefix = adapter.getOutputPrefix(cacheKey);
   Supplier supplier = () -> {

http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
--
diff --git 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index 267ca62..3c55c95 100644
--- 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -95,7 +95,15 @@ public class EnrichmentIntegrationTest extends 
BaseIntegrationTest {
 
   private static List getInputMessages(String path){
 try{
-  return TestUtils.readSampleData(path);
+  List ret = TestUtils.readSampleData(path);
+  {
+//we want one of the fields without a destination IP to ensure that 
enrichments can function
+Map sansDestinationIp = JSONUtils.INSTANCE.load(new 
String(ret.get(ret.size() -1))
+   , 
JSONUtils.MAP_SUPPLIER);
+sansDestinationIp.remove(Constants.Fields.DST_ADDR.getName());
+ret.add(JSONUtils.INSTANCE.toJSONPretty(sansDestinationIp));
+  }
+  return ret;
 }catch(IOException ioe){
   return null;
 }
@@ -262,8 +270,6 @@ public class EnrichmentIntegrationTest extends 
BaseIntegrationTest {
 
 //ensure we always have a source ip and destination ip
 Assert.assertNotNull(jsonDoc.get(SRC_IP));
-Assert.assertNotNull(jsonDoc.get(DST_IP));
-
 Assert.assertNotNull(jsonDoc.get("ALL_CAPS"));
 Assert.assertNotNull(jsonDoc.get("map.blah"));
 Assert.assertNull(jsonDoc.get("map"));

http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
--
diff --git 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
index 4a4573b..d4fcdf4 100644
--- 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
+++ 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
@@ -79,7 +79,39 @@ public 

[24/50] [abbrv] metron git commit: METRON-1510 Update Metron website to include info about github update subscription (anandsubbu) closes apache/metron#981

[otto]

METRON-1510 Update Metron website to include info about github update 
subscription (anandsubbu) closes apache/metron#981


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/438893b7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/438893b7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/438893b7

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 438893b78c852e34d7edb965840500a78503d299
Parents: ed50d48
Author: anandsubbu 
Authored: Wed Apr 11 11:03:37 2018 +0530
Committer: anandsubbu 
Committed: Wed Apr 11 11:03:37 2018 +0530

--
 site/community/index.md | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/438893b7/site/community/index.md
--
diff --git a/site/community/index.md b/site/community/index.md
index 7c09b14..b222ba3 100644
--- a/site/community/index.md
+++ b/site/community/index.md
@@ -121,11 +121,15 @@ title: Apache Metron Community
  General & Public Discussion 
 [mailto:user-subscr...@metron.apache.org;>Subscribe]
 [mailto:user-unsubscr...@metron.apache.org;>Unsubscribe]
-[http://mail-archives.apache.org/mod_mbox/metron-user/;>Archives]
+[https://lists.apache.org/list.html?u...@metron.apache.org;>Archives]
  Code & Documentation Change 
 [mailto:dev-subscr...@metron.apache.org;>Subscribe]
 [mailto:dev-unsubscr...@metron.apache.org;>Unsubscribe]
-[http://mail-archives.apache.org/mod_mbox/metron-dev/;>Archives]
+[https://lists.apache.org/list.html?d...@metron.apache.org;>Archives]
+ Issues & Github updates 
+[mailto:issues-subscr...@metron.apache.org;>Subscribe]
+[mailto:issues-unsubscr...@metron.apache.org;>Unsubscribe]
+[https://lists.apache.org/list.html?iss...@metron.apache.org;>Archives]
 
 
 

[16/50] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
deleted file mode 100755
index 34df1e4..000
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
+++ /dev/null
@@ -1,198 +0,0 @@
-
-
-
-
-
-
-
-
-cluster_name
-metron
-Elasticsearch Cluster Name identifies your Elasticsearch 
subsystem
-
-
-masters_also_are_datanodes
-"false"
-ES Masters and Slaves cannot be installed on the same 
nodes.  Set this to "true" if you want the ES master nodes to serve as combined 
master/datanodes. Note: surround value in quotes.
-
-string
-
-
-
-zen_discovery_ping_unicast_hosts
-
-
-
-Unicast discovery list of hosts to act as gossip routers, 
comma-separated list with square brackets: [ eshost1, eshost2 ]
-
-
-index_number_of_shards
-4
-Set the number of shards (splits) of an index.  Changes 
are not effective after index creation. Usually set to 1 for single-node 
install.
-
-
-index_number_of_replicas
-2
-Set the number of replicas (copies in addition to the 
first) of an index. Usually set to 0 for single-node install.
-
-
-path_data
-"/opt/lmm/es_data"
-Comma-separated list of directories where to store index 
data allocated for each node: "/mnt/first","/mnt/second".  Number of paths 
should relate to number of shards, and preferably should be on separate 
physical volumes.
-
-
-http_cors_enabled
-"false"
-Enable or disable cross-origin resource sharing, i.e. 
whether a browser on another origin can do requests to Elasticsearch. Defaults 
to false.
-
-string
-
-
-
-http_port
-9200-9300
-Set a custom port to listen for HTTP traffic
-
-
-transport_tcp_port
-9300-9400
-Set a custom port for the node to node 
communication
-
-
-
-discovery_zen_ping_timeout
-3s
-Wait for ping responses for master discovery
-
-
-discovery_zen_fd_ping_interval
-15s
-Wait for ping for cluster discovery
-
-
-discovery_zen_fd_ping_timeout
-60s
-Wait for ping for cluster discovery
-
-
-discovery_zen_fd_ping_retries
-5
-Number of ping retries before blacklisting
-
-
-
-gateway_recover_after_data_nodes
-3
-Recover as long as this many data or master nodes have 
joined the cluster.
-
-
-recover_after_time
-15m
-recover_after_time
-
-
-expected_data_nodes
-0
-expected_data_nodes
-
-  
-
-index_merge_scheduler_max_thread_count
-5
-index.merge.scheduler.max_thread_count
-
-
-indices_memory_index_store_throttle_type
-none
-index_store_throttle_type
-
-
-index_refresh_interval
-1s
-index refresh interval
-
-
-index_translog_flush_threshold_size
-5g
-index_translog_flush_threshold_size
-
-
-indices_memory_index_buffer_size
-10%
-Percentage of heap used for write buffers
-
-
-bootstrap_memory_lock
-true
-The third option on Linux/Unix systems only, is to use 
mlockall to try to lock the process address space into RAM, preventing any 
Elasticsearch memory from being swapped out
-
-
-threadpool_bulk_queue_size
-3000
-It tells ES the number of  requests that can be queued 
for execution in the node when there is no thread available to execute a bulk 
request
-
-
-threadpool_index_queue_size
-1000
-It tells ES the number of  requests that can be queued 
for execution in the node when there is no thread available to execute index 
request
-
-
-indices_cluster_send_refresh_mapping
-false
-In order to make the index request more efficient, we 
have set this property on our data nodes
-
-
-indices_fielddata_cache_size
-25%
-You need to keep in mind that not setting this value 
properly can cause:Facet searches and sorting to have very poor performance:The 
ES node to run out of memory if 

[19/50] [abbrv] metron git commit: METRON-1449 Set Zookeeper URL for Stellar Running in Zeppelin Notebook (nickwallen) closes apache/metron#931

[otto]

METRON-1449 Set Zookeeper URL for Stellar Running in Zeppelin Notebook 
(nickwallen) closes apache/metron#931


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ab4f8e65
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ab4f8e65
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ab4f8e65

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ab4f8e65e13aaea3a10491290a7411b3b6dc955e
Parents: 0ab39a3
Author: nickwallen 
Authored: Fri Apr 6 16:40:01 2018 -0400
Committer: nickallen 
Committed: Fri Apr 6 16:40:01 2018 -0400

--
 metron-platform/metron-management/pom.xml   |   1 -
 .../shell/DefaultStellarShellExecutor.java  |  32 +++---
 .../shell/DefaultStellarShellExecutorTest.java  |  11 ++
 metron-stellar/stellar-zeppelin/README.md   |  80 --
 metron-stellar/stellar-zeppelin/pom.xml |  12 +++
 .../stellar/zeppelin/StellarInterpreter.java|  95 ++---
 .../zeppelin/StellarInterpreterProperty.java|  79 ++
 .../StellarInterpreterPropertyTest.java |  62 +++
 .../zeppelin/StellarInterpreterTest.java|  60 ++-
 .../integration/ConfigUploadComponent.java  |  82 +++
 .../StellarInterpreterIntegrationTest.java  | 104 +++
 11 files changed, 501 insertions(+), 117 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-platform/metron-management/pom.xml
--
diff --git a/metron-platform/metron-management/pom.xml 
b/metron-platform/metron-management/pom.xml
index c185662..962fd46 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -183,7 +183,6 @@
 
 
 
-
 
 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
index 3f2c495..781a0cf 100644
--- 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
@@ -52,6 +52,7 @@ import java.io.ByteArrayInputStream;
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -271,10 +272,7 @@ public class DefaultStellarShellExecutor implements 
StellarShellExecutor {
   globals = (Map) capability.get();
 
 } else {
-  // if it does not exist, create it.  this creates the global config for 
the current stellar executor
-  // session only.  this does not change the global config maintained 
externally in zookeeper
-  globals = new HashMap<>();
-  getContext().addCapability(GLOBAL_CONFIG, () -> globals);
+  throw new IllegalStateException("'GLOBAL_CONFIG' is missing");
 }
 
 return globals;
@@ -344,21 +342,25 @@ public class DefaultStellarShellExecutor implements 
StellarShellExecutor {
* @param zkClient An optional Zookeeper client.
*/
   private Context createContext(Properties properties, 
Optional zkClient) throws Exception {
+Context.Builder contextBuilder = new Context.Builder();
+Map globals;
+if (zkClient.isPresent()) {
 
-Context.Builder contextBuilder = new Context.Builder()
-.with(SHELL_VARIABLES, () -> variables)
-.with(STELLAR_CONFIG, () -> properties);
+  // fetch globals from zookeeper
+  globals = fetchGlobalConfig(zkClient.get());
+  contextBuilder.with(ZOOKEEPER_CLIENT, () -> zkClient.get());
 
-// load global configuration from zookeeper
-if (zkClient.isPresent()) {
-  Map global = fetchGlobalConfig(zkClient.get());
-  contextBuilder
-  .with(GLOBAL_CONFIG, () -> global)
-  .with(ZOOKEEPER_CLIENT, () -> zkClient.get())
-  .with(STELLAR_CONFIG, () -> getStellarConfig(global, 
properties));
+} else {
+
+  // use empty globals to allow a user to '%define' their own
+  globals = new HashMap<>();
 }
 
-return contextBuilder.build();
+return contextBuilder
+.with(SHELL_VARIABLES, () -> 

[27/50] [abbrv] metron git commit: METRON-1521: JSONMapParser is no longer serializable closes apache/incubator-metron#991

[otto]

METRON-1521: JSONMapParser is no longer serializable closes 
apache/incubator-metron#991


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/eb5b2d42
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/eb5b2d42
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/eb5b2d42

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: eb5b2d42b83901c05c15e00b9c3c5bce5dd72ab9
Parents: ea6992f
Author: cstella 
Authored: Thu Apr 12 09:33:12 2018 -0400
Committer: cstella 
Committed: Thu Apr 12 09:33:12 2018 -0400

--
 .../java/org/apache/metron/parsers/json/JSONMapParser.java | 4 ++--
 .../org/apache/metron/parsers/integration/ParserDriver.java| 6 +-
 2 files changed, 7 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/eb5b2d42/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
--
diff --git 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
index bddf35d..f5d67f9 100644
--- 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
+++ 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
@@ -89,8 +89,7 @@ public class JSONMapParser extends BasicParser {
   public static final String JSONP_QUERY = "jsonpQuery";
 
   private MapStrategy mapStrategy = MapStrategy.DROP;
-  private TypeRef>> typeRef = new 
TypeRef>>() {
-  };
+  private transient TypeRef>> typeRef = null;
   private String jsonpQuery = null;
 
 
@@ -99,6 +98,7 @@ public class JSONMapParser extends BasicParser {
 String strategyStr = (String) config.getOrDefault(MAP_STRATEGY_CONFIG, 
MapStrategy.DROP.name());
 mapStrategy = MapStrategy.valueOf(strategyStr);
 if (config.containsKey(JSONP_QUERY)) {
+  typeRef = new TypeRef>>() { };
   jsonpQuery = (String) config.get(JSONP_QUERY);
   Configuration.setDefaults(new Configuration.Defaults() {
 

http://git-wip-us.apache.org/repos/asf/metron/blob/eb5b2d42/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
--
diff --git 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
index deb0217..b03ea80 100644
--- 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
+++ 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
@@ -18,6 +18,7 @@
 package org.apache.metron.parsers.integration;
 
 import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang.SerializationUtils;
 import org.apache.metron.common.configuration.ConfigurationsUtils;
 import org.apache.metron.common.configuration.FieldValidator;
 import org.apache.metron.common.configuration.ParserConfigurations;
@@ -42,6 +43,7 @@ import org.mockito.Matchers;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -54,7 +56,7 @@ import static org.mockito.Mockito.when;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class ParserDriver {
+public class ParserDriver implements Serializable {
   private static final Logger LOG = LoggerFactory.getLogger(ParserBolt.class);
   public static class CollectingWriter implements MessageWriter{
 List output;
@@ -151,6 +153,8 @@ public class ParserDriver {
 
   public ProcessorResult> run(List in) {
 ShimParserBolt bolt = new ShimParserBolt(new ArrayList<>());
+byte[] b = SerializationUtils.serialize(bolt);
+ShimParserBolt b2 = (ShimParserBolt) SerializationUtils.deserialize(b);
 OutputCollector collector = mock(OutputCollector.class);
 bolt.prepare(null, null, collector);
 for(byte[] record : in) {

[12/50] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
new file mode 100755
index 000..f0903ac
--- /dev/null
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
@@ -0,0 +1,95 @@
+#!/usr/bin/python
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from elasticsearch import Elasticsearch
+from elasticsearch.helpers import bulk
+import cPickle as pickle
+import argparse, sys, os.path
+import errno
+import os
+
+
+class DashboardIndex(object):
+
+def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, 
**kwargs):
+"""
+:arg host: hostname of the node (default: localhost)
+:arg port: port to use (integer, default: 9200)
+:arg url_prefix: optional url prefix for elasticsearch
+:arg timeout: default timeout in seconds (float, default: 10)
+"""
+self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': 
url_prefix, 'timeout':timeout}])
+
+def get(self):
+"""
+Get .kibana index from Elasticsearch
+"""
+dotkibana = self.es.search(index='.kibana', size = 100)
+return dotkibana['hits']['hits']
+
+def load(self,filespec):
+"""
+Save Index data on local filesystem
+:args filespec: path/filename for saved file
+"""
+data=[]
+with open(filespec,'rb') as fp:
+data = pickle.load(fp)
+return data
+
+def save(self,filename,data):
+"""
+Save Index data on local filesystem
+:args filespec: path/filename for saved file
+"""
+with open(filename,'wb') as fp:
+pickle.dump(data,fp)
+
+def put(self,data):
+"""
+Bulk write data to Elasticsearch
+:args data: data to be written (note: index name is specified in data)
+"""
+bulk(self.es,data)
+
+def main(self,args):
+
+if args.save:
+print("running save with host:%s on port %d, filespec: %s" % 
(args.hostname, args.port, args.filespec))
+self.save(filename=args.filespec,data=di.get())
+else:
+"""
+Loads Kibana Dashboard definition from disk and replaces .kibana 
on index
+:args filespec: path/filename for saved file
+"""
+if not os.path.isfile(args.filespec):
+raise IOError(
+errno.ENOENT, os.strerror(errno.ENOENT), args.filespec)
+self.es.indices.delete(index='.kibana', ignore=[400, 404])
+self.put(data=di.load(filespec=args.filespec))
+
+if __name__ == '__main__':
+
+parser = argparse.ArgumentParser()
+parser.add_argument("hostname", help="ES Hostname or IP", type=str)
+parser.add_argument("port", help="ES Port", type=int)
+parser.add_argument("filespec", help="file to be pushed from or saved to", 
type=str)
+parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will 
be read and saved to filespec",action="store_true")
+args = parser.parse_args()
+di = DashboardIndex(host=args.hostname,port=args.port)
+di.main(args)

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
 

[11/50] [abbrv] metron git commit: METRON-1501 Parser messages that fail to validate are dropped silently (cestella via justinleet) closes apache/metron#972

[otto]

METRON-1501 Parser messages that fail to validate are dropped silently 
(cestella via justinleet) closes apache/metron#972


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0d847cf5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0d847cf5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0d847cf5

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0d847cf5f91dc0d3b3b6838eb4b4de4aa2cf2fec
Parents: 19b237d
Author: cestella 
Authored: Tue Apr 3 10:29:19 2018 -0400
Committer: leet 
Committed: Tue Apr 3 10:29:19 2018 -0400

--
 metron-platform/metron-parsers/README.md| 29 +++-
 .../apache/metron/parsers/bolt/ParserBolt.java  | 17 
 .../metron/parsers/bolt/ParserBoltTest.java | 16 ---
 3 files changed, 51 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/0d847cf5/metron-platform/metron-parsers/README.md
--
diff --git a/metron-platform/metron-parsers/README.md 
b/metron-platform/metron-parsers/README.md
index 3d9fdfe..6b9d62e 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -45,7 +45,34 @@ There are two general types types of parsers:
   * `ERROR` : Throw an error when a multidimensional map is encountered
 * `jsonpQuery` : A [JSON Path](#json_path) query string. If present, the 
result of the JSON Path query should be a list of messages. This is useful if 
you have a JSON document which contains a list or array of messages embedded in 
it, and you do not have another means of splitting the message.
 * A field called `timestamp` is expected to exist and, if it does not, 
then current time is inserted.  
-
+
+## Parser Error Routing
+
+Currently, we have a few mechanisms for either deferring processing of
+messages or marking messages as invalid.
+
+### Invalidation Errors
+
+There are two reasons a message will be marked as invalid:
+* Fail [global validation](../metron-common#validation-framework)
+* Fail the parser's validate function (generally that means to not have a 
`timestamp` field or a `original_string` field.
+
+Those messages which are marked as invalid are sent to the error queue
+with an indication that they are invalid in the error message.
+
+### Parser Errors
+
+Errors, which are defined as unexpected exceptions happening during the
+parse, are sent along to the error queue with a message indicating that
+there was an error in parse along with a stacktrace.  This is to
+distinguish from the invalid messages.
+ 
+## Filtered
+
+One can also filter a message by specifying a `filterClassName` in the
+parser config.  Filtered messages are just dropped rather than passed
+through.
+   
 ## Parser Architecture
 
 ![Architecture](parser_arch.png)

http://git-wip-us.apache.org/repos/asf/metron/blob/0d847cf5/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
--
diff --git 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index 6fc4ed7..e996f14 100644
--- 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -193,23 +193,28 @@ public class ParserBolt extends ConfiguredParserBolt 
implements Serializable {
 message.put(Constants.GUID, UUID.randomUUID().toString());
   }
 
-  if (parser.validate(message) && (filter == null || 
filter.emitTuple(message, stellarContext))) {
-numWritten++;
-List failedValidators = 
getFailedValidators(message, fieldValidations);
-if(failedValidators.size() > 0) {
+  if (filter == null || filter.emitTuple(message, stellarContext)) {
+boolean isInvalid = !parser.validate(message);
+List failedValidators = null;
+if(!isInvalid) {
+  failedValidators = getFailedValidators(message, 
fieldValidations);
+  isInvalid = !failedValidators.isEmpty();
+}
+if( isInvalid) {
   MetronError error = new MetronError()
   .withErrorType(Constants.ErrorType.PARSER_INVALID)
   .withSensorType(getSensorType())
   .addRawMessage(message);
-  Set errorFields = failedValidators.stream()
+  Set errorFields = failedValidators == 
null?null:failedValidators.stream()
   

[30/50] [abbrv] metron git commit: METRON-1519 Indexing Error Topic Property Not Displayed in MPack (nickwallen) closes apache/metron#987

[otto]

METRON-1519 Indexing Error Topic Property Not Displayed in MPack (nickwallen) 
closes apache/metron#987


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f8b7c585
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f8b7c585
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f8b7c585

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: f8b7c5852310a088bd15ca5d21ba4f98b51521be
Parents: bfe90ef
Author: nickwallen 
Authored: Fri Apr 13 14:47:17 2018 -0400
Committer: nickallen 
Committed: Fri Apr 13 14:47:17 2018 -0400

--
 .../common-services/METRON/CURRENT/themes/metron_theme.json| 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/f8b7c585/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 234b551..364b3ef 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -514,6 +514,10 @@
   "subsection-name": "subsection-indexing-kafka"
 },
 {
+  "config": "metron-indexing-env/indexing_error_topic",
+  "subsection-name": "subsection-indexing-kafka"
+},
+{
   "config": "metron-indexing-env/update_hbase_table",
   "subsection-name": "subsection-indexing-update"
 },
@@ -553,7 +557,6 @@
   "config": 
"metron-indexing-env/batch_indexing_topology_max_spout_pending",
   "subsection-name": "subsection-indexing-hdfs"
 },
-
 {
   "config": "metron-indexing-env/ra_indexing_kafka_spout_parallelism",
   "subsection-name": "subsection-indexing-storm"
@@ -562,7 +565,6 @@
   "config": 
"metron-indexing-env/batch_indexing_kafka_spout_parallelism",
   "subsection-name": "subsection-indexing-hdfs"
 },
-
 {
   "config": "metron-indexing-env/ra_indexing_writer_parallelism",
   "subsection-name": "subsection-indexing-storm"

[26/50] [abbrv] metron git commit: METRON-1516 Support for Ansible 2.5.0 (ottobackwards) closes apache/metron#989

[otto]

METRON-1516 Support for Ansible 2.5.0 (ottobackwards) closes apache/metron#989


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ea6992fd
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ea6992fd
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ea6992fd

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ea6992fd943c552c41565c7a320804cb58b733bd
Parents: 62d1a1b
Author: ottobackwards 
Authored: Thu Apr 12 06:15:38 2018 -0400
Committer: otto 
Committed: Thu Apr 12 06:15:38 2018 -0400

--
 metron-deployment/amazon-ec2/README.md  | 2 +-
 metron-deployment/amazon-ec2/playbook.yml   | 4 ++--
 metron-deployment/ansible/playbooks/metron_full_install.yml | 4 ++--
 metron-deployment/development/centos6/README.md | 2 +-
 metron-deployment/development/ubuntu14/README.md| 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/ea6992fd/metron-deployment/amazon-ec2/README.md
--
diff --git a/metron-deployment/amazon-ec2/README.md 
b/metron-deployment/amazon-ec2/README.md
index bc259ec..73a3d70 100644
--- a/metron-deployment/amazon-ec2/README.md
+++ b/metron-deployment/amazon-ec2/README.md
@@ -38,7 +38,7 @@ Getting Started
 
 The host used to deploy Apache Metron will need the following software tools 
installed.  The following versions are known to work as of the time of this 
writing, but by no means are these the only working versions.
 
-  - Ansible 2.0.0.2 or 2.2.2.0
+  - Ansible 2.0.0.2, 2.2.2.0, or 2.5.0
   - Python 2.7.11
   - Maven 3.3.9  
 

http://git-wip-us.apache.org/repos/asf/metron/blob/ea6992fd/metron-deployment/amazon-ec2/playbook.yml
--
diff --git a/metron-deployment/amazon-ec2/playbook.yml 
b/metron-deployment/amazon-ec2/playbook.yml
index de64490..470a181 100644
--- a/metron-deployment/amazon-ec2/playbook.yml
+++ b/metron-deployment/amazon-ec2/playbook.yml
@@ -23,8 +23,8 @@
 - conf/defaults.yml
   pre_tasks:
 - name: Verify Ansible Version
-  fail: msg="Metron Requires Ansible 2.0.0.2 or 2.2.2.0, current version 
is {{ ansible_version }}"
-  when: "ansible_version.full | version_compare('2.2.2.0', '!=') and 
ansible_version.full | version_compare('2.0.0.2', '!=')"
+  fail: msg="Metron Requires Ansible 2.0.0.2, 2.2.2.0 or 2.5.0, current 
version is {{ ansible_version }}"
+  when: "ansible_version.full | version_compare('2.2.2.0', '!=') and 
ansible_version.full | version_compare('2.0.0.2', '!=') and 
ansible_version.full | version_compare('2.5.0', '!=')"
   tasks:
 - include: tasks/create-keypair.yml
 - include: tasks/create-vpc.yml

http://git-wip-us.apache.org/repos/asf/metron/blob/ea6992fd/metron-deployment/ansible/playbooks/metron_full_install.yml
--
diff --git a/metron-deployment/ansible/playbooks/metron_full_install.yml 
b/metron-deployment/ansible/playbooks/metron_full_install.yml
index b517671..099d810 100644
--- a/metron-deployment/ansible/playbooks/metron_full_install.yml
+++ b/metron-deployment/ansible/playbooks/metron_full_install.yml
@@ -18,8 +18,8 @@
 - hosts: all
   pre_tasks:
 - name: Verify Ansible Version
-  fail: msg="Metron Requires Ansible 2.0.0.2 or 2.2.2.0, current version 
is {{ ansible_version }}"
-  when: "ansible_version.full | version_compare('2.2.2.0', '!=') and 
ansible_version.full | version_compare('2.0.0.2', '!=')"
+  fail: msg="Metron Requires Ansible 2.0.0.2, 2.2.2.0, or 2.5.0, current 
version is {{ ansible_version }}"
+  when: "ansible_version.full | version_compare('2.2.2.0', '!=') and 
ansible_version.full | version_compare('2.0.0.2', '!=') and 
ansible_version.full | version_compare('2.5.0', '!=')"
 
 - include: metron_build.yml
   tags:

http://git-wip-us.apache.org/repos/asf/metron/blob/ea6992fd/metron-deployment/development/centos6/README.md
--
diff --git a/metron-deployment/development/centos6/README.md 
b/metron-deployment/development/centos6/README.md
index bd8553c..ec85be3 100644
--- a/metron-deployment/development/centos6/README.md
+++ b/metron-deployment/development/centos6/README.md
@@ -29,7 +29,7 @@ Getting Started
 
 The computer used to deploy Apache Metron will need to have the following 
components installed.
 
- - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0)
+ - [Ansible](https://github.com/ansible/ansible) (2.0.0.2, 2.2.2.0, or 2.5.0)
  - [Docker](https://www.docker.com/community-edition)
  - 

[22/50] [abbrv] metron git commit: METRON-1465:Support for Elasticsearch X-pack (wardbekker via mmiklavc) closes apache/metron#946

[otto]

METRON-1465:Support for Elasticsearch X-pack (wardbekker via mmiklavc) closes 
apache/metron#946


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a8b555dc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a8b555dc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a8b555dc

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: a8b555dcc9f548d7b91789a46d9435b4d8b17581
Parents: 3ba9ae2
Author: wardbekker 
Authored: Mon Apr 9 13:14:13 2018 -0600
Committer: Michael Miklavcic 
Committed: Mon Apr 9 13:14:13 2018 -0600

--
 metron-deployment/Kerberos-manual-setup.md  | 209 +++
 .../roles/metron-builder/tasks/build-debs.yml   |   2 +-
 .../roles/metron-builder/tasks/build-rpms.yml   |   2 +-
 .../METRON/CURRENT/configuration/metron-env.xml |   2 -
 .../metron-rest/src/main/scripts/metron-rest.sh |   9 +
 .../src/main/config/zookeeper/global.json   |   5 +-
 .../apache/metron/common/utils/HDFSUtils.java   |  59 ++
 .../metron/common/utils/ReflectionUtils.java|  66 +-
 .../elasticsearch/dao/ElasticsearchDao.java |  33 ++-
 .../elasticsearch/utils/ElasticsearchUtils.java | 107 --
 .../writer/ElasticsearchWriter.java |   8 +-
 .../scripts/start_elasticsearch_topology.sh |   8 +-
 .../writer/ElasticsearchWriterTest.java |  19 +-
 .../stellar/common/utils/ConversionUtils.java   |  19 +-
 14 files changed, 486 insertions(+), 62 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/a8b555dc/metron-deployment/Kerberos-manual-setup.md
--
diff --git a/metron-deployment/Kerberos-manual-setup.md 
b/metron-deployment/Kerberos-manual-setup.md
index 47a63d8..456703a 100644
--- a/metron-deployment/Kerberos-manual-setup.md
+++ b/metron-deployment/Kerberos-manual-setup.md
@@ -30,6 +30,7 @@ This document provides instructions for kerberizing Metron's 
Vagrant-based devel
 * [Start Metron](#start-metron)
 * [Push Data](#push-data)
 * [More Information](#more-information)
+* [Elasticseach X-Pack](#x-pack)
 
 Setup
 -
@@ -533,3 +534,211 @@ In order to correct this, you should:
 ### References
 
 * 
[https://github.com/apache/storm/blob/master/SECURITY.md](https://github.com/apache/storm/blob/master/SECURITY.md)
+
+X-Pack
+--
+
+First, stop the random_access_indexing topology through the Storm UI or from 
the CLI, e.g.
+
+```
+storm kill random_access_indexing
+```
+
+Here are instructions for enabling X-Pack with Elasticsearch and Kibana: 
https://www.elastic.co/guide/en/x-pack/5.6/installing-xpack.html
+
+You need to be sure to add the appropriate username and password for 
Elasticsearch and Kibana to enable external connections from Metron components. 
e.g. the following will create a user "transport_client_user" with password 
"changeme" and "superuser" credentials.
+
+```
+sudo /usr/share/elasticsearch/bin/x-pack/users useradd transport_client_user 
-p changeme -r superuser
+```
+
+Once you've picked a password to connect to ES, you need to upload a 1-line 
file to HDFS with that password in it. Metron will use this file to securely 
read the password in order to connect to ES securely.
+
+Here is an example using "changeme" as the password
+
+```
+echo changeme > /tmp/xpack-password
+sudo -u hdfs hdfs dfs -mkdir /apps/metron/elasticsearch/
+sudo -u hdfs hdfs dfs -put /tmp/xpack-password /apps/metron/elasticsearch/
+sudo -u hdfs hdfs dfs -chown metron:metron 
/apps/metron/elasticsearch/xpack-password
+```
+
+New settings have been added to configure the Elasticsearch client. By default 
the client will run as the normal ES prebuilt transport client. If you enable 
X-Pack you should set the es.client.class as shown below.
+
+Add the es settings to global.json
+
+```
+/usr/metron/0.4.3/config/zookeeper/global.json ->
+
+  "es.client.settings" : {
+  "es.client.class" : 
"org.elasticsearch.xpack.client.PreBuiltXPackTransportClient",
+  "es.xpack.username" : "transport_client_user",
+  "es.xpack.password.file" : "/apps/metron/elasticsearch/xpack-password"
+  }
+```
+
+Submit the update to Zookeeper
+
+```
+$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i METRON_HOME/config/zookeeper/ 
-z $ZOOKEEPER
+```
+
+The last step before restarting the topology is to create a custom X-Pack 
shaded and relocated jar. This is up to you because of licensing restrictions, 
but here is a sample Maven pom file that should help.
+
+```
+
+
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+org.elasticsearch
+

[39/50] [abbrv] metron git commit: METRON-1511 Unable to Serialize Profiler Configuration (nickwallen) closes apache/metron#982

[otto]

METRON-1511 Unable to Serialize Profiler Configuration (nickwallen) closes 
apache/metron#982


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b5bf9a98
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b5bf9a98
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b5bf9a98

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: b5bf9a98725f866a7fee6470a8e763d17cc69ffd
Parents: a41611b
Author: nickwallen 
Authored: Mon Apr 23 09:36:06 2018 -0400
Committer: nickallen 
Committed: Mon Apr 23 09:36:06 2018 -0400

--
 .../configuration/profiler/ProfileConfig.java   |  57 --
 .../profiler/ProfileResultExpressions.java  |   4 +-
 .../profiler/ProfileTriageExpressions.java  |   8 ++
 .../configuration/profiler/ProfilerConfig.java  |  81 --
 .../profiler/ProfileConfigTest.java | 102 ++---
 .../profiler/ProfilerConfigTest.java| 109 +--
 6 files changed, 310 insertions(+), 51 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/b5bf9a98/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
index f5b46e6..f2272c3 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java
@@ -18,12 +18,15 @@
 package org.apache.metron.common.configuration.profiler;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import org.apache.commons.lang.builder.EqualsBuilder;
 import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.metron.common.utils.JSONUtils;
 
+import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -264,15 +267,47 @@ public class ProfileConfig implements Serializable {
 
   @Override
   public String toString() {
-return "ProfileConfig{" +
-"profile='" + profile + '\'' +
-", foreach='" + foreach + '\'' +
-", onlyif='" + onlyif + '\'' +
-", init=" + init +
-", update=" + update +
-", groupBy=" + groupBy +
-", result=" + result +
-", expires=" + expires +
-'}';
+return new ToStringBuilder(this)
+.append("profile", profile)
+.append("foreach", foreach)
+.append("onlyif", onlyif)
+.append("init", init)
+.append("update", update)
+.append("groupBy", groupBy)
+.append("result", result)
+.append("expires", expires)
+.toString();
+  }
+
+  /**
+   * Deserialize a {@link ProfileConfig}.
+   *
+   * @param bytes Raw bytes containing a UTF-8 JSON String.
+   * @return The Profile definition.
+   * @throws IOException
+   */
+  public static ProfileConfig fromBytes(byte[] bytes) throws IOException {
+return JSONUtils.INSTANCE.load(new String(bytes), ProfileConfig.class);
+  }
+
+  /**
+   * Deserialize a {@link ProfileConfig}.
+   *
+   * @param json A String containing JSON.
+   * @return The Profile definition.
+   * @throws IOException
+   */
+  public static ProfileConfig fromJSON(String json) throws IOException {
+return JSONUtils.INSTANCE.load(json, ProfileConfig.class);
+  }
+
+  /**
+   * Serialize the profile definition to a JSON string.
+   *
+   * @return The Profiler configuration serialized as a JSON string.
+   * @throws JsonProcessingException
+   */
+  public String toJSON() throws JsonProcessingException {
+return JSONUtils.INSTANCE.toJSON(this, true);
   }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/b5bf9a98/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java
index 82af223..5bcec72 100644
--- 

[15/50] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
deleted file mode 100644
index 037f1c6..000
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json
+++ /dev/null
@@ -1,88 +0,0 @@
-{ "create" : { "_id": "all-metron-index", "_type": "index-pattern" } }
-{"title":"*_index_*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\"
 
:true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\",
 
\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false,
 
\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scri
 

[07/50] [abbrv] metron git commit: METRON-590 Enable Use of Event Time in Profiler (nickwallen) closes apache/metron#965

[otto]

METRON-590 Enable Use of Event Time in Profiler (nickwallen) closes 
apache/metron#965


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3083b471
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3083b471
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3083b471

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3083b471fe912bc74d27017834e6c80ff177680e
Parents: 46ad9d9
Author: nickwallen 
Authored: Tue Mar 20 16:00:20 2018 -0400
Committer: nickallen 
Committed: Tue Mar 20 16:00:20 2018 -0400

--
 .../client/stellar/ProfilerFunctions.java   |  14 +-
 .../profiler/DefaultMessageDistributor.java | 207 +++-
 .../metron/profiler/DefaultProfileBuilder.java  | 110 ++--
 .../metron/profiler/MessageDistributor.java |  48 +-
 .../apache/metron/profiler/MessageRoute.java|  19 +-
 .../apache/metron/profiler/MessageRouter.java   |  11 +-
 .../apache/metron/profiler/ProfileBuilder.java  |  34 +-
 .../metron/profiler/ProfileMeasurement.java |   6 +-
 .../metron/profiler/StandAloneProfiler.java | 100 +++-
 .../org/apache/metron/profiler/clock/Clock.java |  18 +-
 .../metron/profiler/clock/ClockFactory.java |  38 ++
 .../profiler/clock/DefaultClockFactory.java |  57 ++
 .../metron/profiler/clock/EventTimeClock.java   |  72 +++
 .../metron/profiler/clock/FixedClock.java   |  39 +-
 .../profiler/clock/FixedClockFactory.java   |  44 ++
 .../apache/metron/profiler/clock/WallClock.java |  17 +-
 .../profiler/DefaultMessageDistributorTest.java | 171 +-
 .../profiler/DefaultProfileBuilderTest.java | 119 +++--
 .../metron/profiler/ProfilePeriodTest.java  |   1 -
 .../metron/profiler/StandAloneProfilerTest.java | 255 +
 .../profiler/clock/DefaultClockFactoryTest.java |  75 +++
 .../profiler/clock/EventTimeClockTest.java  | 115 +
 .../metron/profiler/clock/WallClockTest.java|  54 ++
 metron-analytics/metron-profiler/README.md  |  98 +++-
 .../src/main/config/profiler.properties |  14 +-
 .../src/main/flux/profiler/remote.yaml  |  42 +-
 .../profiler/bolt/DestinationHandler.java   |  56 --
 .../bolt/FixedFrequencyFlushSignal.java | 126 +
 .../metron/profiler/bolt/FlushSignal.java   |  51 ++
 .../profiler/bolt/HBaseDestinationHandler.java  |  58 ---
 .../metron/profiler/bolt/HBaseEmitter.java  |  63 +++
 .../profiler/bolt/KafkaDestinationHandler.java  | 110 
 .../metron/profiler/bolt/KafkaEmitter.java  | 114 
 .../metron/profiler/bolt/ManualFlushSignal.java |  54 ++
 .../profiler/bolt/ProfileBuilderBolt.java   | 374 +++---
 .../bolt/ProfileMeasurementEmitter.java |  59 +++
 .../profiler/bolt/ProfileSplitterBolt.java  | 132 -
 .../zookeeper/event-time-test/profiler.json |  12 +
 .../bolt/FixedFrequencyFlushSignalTest.java |  71 +++
 .../bolt/KafkaDestinationHandlerTest.java   | 203 
 .../metron/profiler/bolt/KafkaEmitterTest.java  | 208 
 .../profiler/bolt/ProfileBuilderBoltTest.java   | 516 +++
 .../profiler/bolt/ProfileHBaseMapperTest.java   |   6 +-
 .../profiler/bolt/ProfileSplitterBoltTest.java  | 288 +--
 .../profiler/integration/MessageBuilder.java|  75 +++
 .../integration/ProfilerIntegrationTest.java| 235 ++---
 .../configuration/metron-profiler-env.xml   |  77 ++-
 .../package/scripts/params/params_linux.py  |   7 +
 .../package/templates/profiler.properties.j2|  15 +-
 .../METRON/CURRENT/themes/metron_theme.json | 118 -
 .../configuration/profiler/ProfileConfig.java   |  53 ++
 .../configuration/profiler/ProfilerConfig.java  |  48 +-
 .../apache/metron/common/utils/JSONUtils.java   |  11 +-
 .../configurations/ProfilerUpdater.java |   1 +
 .../profiler/ProfileConfigTest.java |   5 +-
 .../profiler/ProfilerConfigTest.java| 120 +
 .../integration/components/KafkaComponent.java  |  39 +-
 57 files changed, 3987 insertions(+), 1096 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/3083b471/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java
--
diff --git 
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java
 
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java
index 64c1e2e..d6afe1d 100644
--- 
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java
+++ 

[04/50] [abbrv] metron git commit: METRON-590 Enable Use of Event Time in Profiler (nickwallen) closes apache/metron#965

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/3083b471/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index ceb9e4e..ccce022 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -287,6 +287,8 @@ profiler_input_topic = 
config['configurations']['metron-enrichment-env']['enrich
 profiler_kafka_start = 
config['configurations']['metron-profiler-env']['profiler_kafka_start']
 profiler_period_duration = 
config['configurations']['metron-profiler-env']['profiler_period_duration']
 profiler_period_units = 
config['configurations']['metron-profiler-env']['profiler_period_units']
+profiler_window_duration = 
config['configurations']['metron-profiler-env']['profiler_window_duration']
+profiler_window_units = 
config['configurations']['metron-profiler-env']['profiler_window_units']
 profiler_ttl = config['configurations']['metron-profiler-env']['profiler_ttl']
 profiler_ttl_units = 
config['configurations']['metron-profiler-env']['profiler_ttl_units']
 profiler_hbase_batch = 
config['configurations']['metron-profiler-env']['profiler_hbase_batch']
@@ -302,6 +304,11 @@ profiler_hbase_acl_configured_flag_file = 
status_params.profiler_hbase_acl_confi
 if not len(profiler_topology_worker_childopts) == 0:
 profiler_topology_worker_childopts += ' '
 profiler_topology_worker_childopts += 
config['configurations']['metron-profiler-env']['profiler_topology_worker_childopts']
+profiler_max_routes_per_bolt=config['configurations']['metron-profiler-env']['profiler_max_routes_per_bolt']
+profiler_window_lag=config['configurations']['metron-profiler-env']['profiler_window_lag']
+profiler_window_lag_units=config['configurations']['metron-profiler-env']['profiler_window_lag_units']
+profiler_topology_message_timeout_secs=config['configurations']['metron-profiler-env']['profiler_topology_message_timeout_secs']
+profiler_topology_max_spout_pending=config['configurations']['metron-profiler-env']['profiler_topology_max_spout_pending']
 
 # Indexing
 ra_indexing_kafka_start = 
config['configurations']['metron-indexing-env']['ra_indexing_kafka_start']

http://git-wip-us.apache.org/repos/asf/metron/blob/3083b471/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2
index 06fd209..fabdaa7 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2
@@ -22,6 +22,10 @@
 
 topology.worker.childopts={{profiler_topology_worker_childopts}}
 topology.auto-credentials={{topology_auto_credentials}}
+profiler.workers={{profiler_topology_workers}}
+profiler.executors={{profiler_acker_executors}}
+topology.message.timeout.secs={{profiler_topology_message_timeout_secs}}
+topology.max.spout.pending={{profiler_topology_max_spout_pending}}
 
 # Profiler #
 
@@ -29,10 +33,16 @@ profiler.input.topic={{enrichment_output_topic}}
 profiler.output.topic={{enrichment_input_topic}}
 profiler.period.duration={{profiler_period_duration}}
 profiler.period.duration.units={{profiler_period_units}}
-profiler.workers={{profiler_topology_workers}}
-profiler.executors={{profiler_acker_executors}}
+profiler.window.duration={{profiler_window_duration}}
+profiler.window.duration.units={{profiler_window_units}}
 profiler.ttl={{profiler_ttl}}
 profiler.ttl.units={{profiler_ttl_units}}
+profiler.window.lag={{profiler_window_lag}}
+profiler.window.lag.units={{profiler_window_lag_units}}
+profiler.max.routes.per.bolt={{profiler_max_routes_per_bolt}}
+
+# HBase #
+
 profiler.hbase.salt.divisor=1000
 profiler.hbase.table={{profiler_hbase_table}}
 profiler.hbase.column.family={{profiler_hbase_cf}}
@@ -43,6 +53,5 @@ 
profiler.hbase.flush.interval.seconds={{profiler_hbase_flush_interval}}
 
 

[47/50] [abbrv] metron git commit: METRON-1461 MIN MAX stellar function should take a stats or list object and return min/max (MohanDV via nickwallen) closes apache/metron#942

[otto]

METRON-1461 MIN MAX stellar function should take a stats or list object and 
return min/max (MohanDV via nickwallen) closes apache/metron#942


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d7edce97
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d7edce97
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d7edce97

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: d7edce974341a6fca0e8e87b11baed5ad0d5d0c1
Parents: ca4644b
Author: MohanDV 
Authored: Wed May 9 16:33:32 2018 -0400
Committer: nickallen 
Committed: Wed May 9 16:33:32 2018 -0400

--
 .../metron/statistics/StatisticsProvider.java   |  6 +--
 .../metron/stellar/dsl/functions/Ordinal.java   | 36 +++
 .../stellar/dsl/functions/OrdinalFunctions.java | 48 +---
 .../dsl/functions/OrdinalFunctionsTest.java | 41 -
 4 files changed, 110 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/d7edce97/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java
--
diff --git 
a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java
 
b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java
index 860aa4e..e737484 100644
--- 
a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java
+++ 
b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java
@@ -20,14 +20,14 @@
 package org.apache.metron.statistics;
 
 
+import org.apache.metron.stellar.dsl.functions.Ordinal;
+
 /**
  * Provides statistical functions.
  */
-public interface StatisticsProvider {
+public interface StatisticsProvider extends Ordinal{
   void addValue(double value);
   long getCount();
-  double getMin();
-  double getMax();
   double getMean();
   double getSum();
   double getVariance();

http://git-wip-us.apache.org/repos/asf/metron/blob/d7edce97/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java
new file mode 100644
index 000..d3bd9ce
--- /dev/null
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.dsl.functions;
+
+/**
+ * Interface that provides the statistical function get max and min from the 
implementing object.
+ */
+public interface Ordinal {
+
+/**
+ * get the min value
+ * @return min value
+ */
+double getMin();
+
+/**
+ * get the max value
+ * @return max value
+ */
+double getMax();
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/d7edce97/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
index 49e9369..6ac9ff5 100644
--- 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
@@ -22,12 +22,10 @@ import com.google.common.collect.Iterables;
 import org.apache.metron.stellar.common.utils.ConversionUtils;
 import 

[08/50] [abbrv] metron git commit: METRON-1491: The indexing topology restart logic is wrong (cstella via mmiklavc) closes apache/metron#964

[otto]

METRON-1491: The indexing topology restart logic is wrong (cstella via 
mmiklavc) closes apache/metron#964


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/5ed9631a
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/5ed9631a
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/5ed9631a

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 5ed9631a2936ec60d0ea6557ca4396cffdadc688
Parents: 3083b47
Author: cstella 
Authored: Tue Mar 20 16:08:02 2018 -0600
Committer: Michael Miklavcic 
Committed: Tue Mar 20 16:08:02 2018 -0600

--
 .../package/scripts/indexing_commands.py| 43 
 1 file changed, 35 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/5ed9631a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 4c862f0..fd78119 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -181,7 +181,7 @@ class IndexingCommands:
 def start_batch_indexing_topology(self, env):
 Logger.info('Starting ' + self.__batch_indexing_topology)
 
-if not self.is_topology_active(env):
+if not self.is_batch_topology_active(env):
 if self.__params.security_enabled:
 metron_security.kinit(self.__params.kinit_path_local,
   self.__params.metron_keytab_path,
@@ -200,7 +200,7 @@ class IndexingCommands:
 def start_random_access_indexing_topology(self, env):
 Logger.info('Starting ' + self.__random_access_indexing_topology)
 
-if not self.is_topology_active(env):
+if not self.is_random_access_topology_active(env):
 if self.__params.security_enabled:
 metron_security.kinit(self.__params.kinit_path_local,
   self.__params.metron_keytab_path,
@@ -263,21 +263,48 @@ class IndexingCommands:
 
 def restart_indexing_topology(self, env):
 Logger.info('Restarting the indexing topologies')
-self.stop_indexing_topology(env)
+self.restart_batch_indexing_topology(env)
+self.restart_random_access_indexing_topology(env)
+
+def restart_batch_indexing_topology(self, env):
+Logger.info('Restarting the batch indexing topology')
+self.stop_batch_indexing_topology(env)
+
+# Wait for old topology to be cleaned up by Storm, before starting 
again.
+retries = 0
+topology_active = self.is_batch_topology_active(env)
+while topology_active and retries < 3:
+Logger.info('Existing batch topology still active. Will wait and 
retry')
+time.sleep(10)
+retries += 1
+topology_active = self.is_batch_topology_active(env)
+
+if not topology_active:
+Logger.info('Waiting for storm kill to complete')
+time.sleep(30)
+self.start_batch_indexing_topology(env)
+Logger.info('Done restarting the batch indexing topology')
+else:
+Logger.warning('Retries exhausted. Existing topology not cleaned 
up.  Aborting topology start.')
+
+def restart_random_access_indexing_topology(self, env):
+Logger.info('Restarting the random access indexing topology')
+self.stop_random_access_indexing_topology(env)
 
 # Wait for old topology to be cleaned up by Storm, before starting 
again.
 retries = 0
-topology_active = self.is_topology_active(env)
-while self.is_topology_active(env) and retries < 3:
-Logger.info('Existing topology still active. Will wait and retry')
+topology_active = self.is_random_access_topology_active(env)
+while topology_active and retries < 3:
+Logger.info('Existing random access topology still active. Will 
wait and retry')
 time.sleep(10)
 retries += 1
+topology_active = self.is_random_access_topology_active(env)
 
 if not topology_active:
 Logger.info('Waiting for storm kill to complete')
 time.sleep(30)
-

[02/50] [abbrv] metron git commit: METRON-1483: Create a tool to monitor performance of the topologies closes apache/incubator-metron#958

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/46ad9d93/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java
--
diff --git 
a/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java
 
b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java
new file mode 100644
index 000..5d5c240
--- /dev/null
+++ 
b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.performance.sampler;
+
+import java.util.Random;
+
+public class UnbiasedSampler implements Sampler {
+
+  @Override
+  public int sample(Random rng, int limit) {
+return rng.nextInt(limit);
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/46ad9d93/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java
--
diff --git 
a/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java
 
b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java
new file mode 100644
index 000..c13f236
--- /dev/null
+++ 
b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.performance.util;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public enum KafkaUtil {
+  INSTANCE;
+
+  public List getTopicPartition(KafkaConsumer 
consumer, String topic) {
+
+List partitions = consumer.partitionsFor(topic);
+List ret = new ArrayList<>(partitions.size());
+for(PartitionInfo par : partitions) {
+  ret.add(new TopicPartition(topic, par.partition()));
+}
+return ret;
+  }
+
+  public Map getKafkaOffsetMap(KafkaConsumer 
consumer, String topic ) {
+Map ret = new HashMap<>();
+if(!consumer.subscription().contains(topic)) {
+  consumer.subscribe(Collections.singletonList(topic));
+}
+consumer.poll(0);
+List partitions = getTopicPartition(consumer, topic);
+consumer.seekToEnd(partitions);
+for(TopicPartition par : partitions) {
+  ret.put(par.partition(), consumer.position(par)-1);
+}
+return ret;
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/46ad9d93/metron-contrib/metron-performance/src/main/scripts/load_tool.sh
--
diff --git a/metron-contrib/metron-performance/src/main/scripts/load_tool.sh 
b/metron-contrib/metron-performance/src/main/scripts/load_tool.sh
new file mode 100755
index 000..1b56a6e
--- /dev/null
+++ b/metron-contrib/metron-performance/src/main/scripts/load_tool.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license 

[14/50] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
deleted file mode 100755
index f0903ac..000
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/python
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-from elasticsearch import Elasticsearch
-from elasticsearch.helpers import bulk
-import cPickle as pickle
-import argparse, sys, os.path
-import errno
-import os
-
-
-class DashboardIndex(object):
-
-def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, 
**kwargs):
-"""
-:arg host: hostname of the node (default: localhost)
-:arg port: port to use (integer, default: 9200)
-:arg url_prefix: optional url prefix for elasticsearch
-:arg timeout: default timeout in seconds (float, default: 10)
-"""
-self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': 
url_prefix, 'timeout':timeout}])
-
-def get(self):
-"""
-Get .kibana index from Elasticsearch
-"""
-dotkibana = self.es.search(index='.kibana', size = 100)
-return dotkibana['hits']['hits']
-
-def load(self,filespec):
-"""
-Save Index data on local filesystem
-:args filespec: path/filename for saved file
-"""
-data=[]
-with open(filespec,'rb') as fp:
-data = pickle.load(fp)
-return data
-
-def save(self,filename,data):
-"""
-Save Index data on local filesystem
-:args filespec: path/filename for saved file
-"""
-with open(filename,'wb') as fp:
-pickle.dump(data,fp)
-
-def put(self,data):
-"""
-Bulk write data to Elasticsearch
-:args data: data to be written (note: index name is specified in data)
-"""
-bulk(self.es,data)
-
-def main(self,args):
-
-if args.save:
-print("running save with host:%s on port %d, filespec: %s" % 
(args.hostname, args.port, args.filespec))
-self.save(filename=args.filespec,data=di.get())
-else:
-"""
-Loads Kibana Dashboard definition from disk and replaces .kibana 
on index
-:args filespec: path/filename for saved file
-"""
-if not os.path.isfile(args.filespec):
-raise IOError(
-errno.ENOENT, os.strerror(errno.ENOENT), args.filespec)
-self.es.indices.delete(index='.kibana', ignore=[400, 404])
-self.put(data=di.load(filespec=args.filespec))
-
-if __name__ == '__main__':
-
-parser = argparse.ArgumentParser()
-parser.add_argument("hostname", help="ES Hostname or IP", type=str)
-parser.add_argument("port", help="ES Port", type=int)
-parser.add_argument("filespec", help="file to be pushed from or saved to", 
type=str)
-parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will 
be read and saved to filespec",action="store_true")
-args = parser.parse_args()
-di = DashboardIndex(host=args.hostname,port=args.port)
-di.main(args)

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
 

[50/50] [abbrv] metron git commit: METRON-1551 Profiler Should Not Use Java Serialization (nickwallen) closes apache/metron#1012

[otto]

METRON-1551 Profiler Should Not Use Java Serialization (nickwallen) closes 
apache/metron#1012


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9ce4ba5a
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9ce4ba5a
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9ce4ba5a

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 9ce4ba5a9c8febc3a3ed6992f73f99396d7248a9
Parents: b9453aa
Author: nickwallen 
Authored: Mon May 14 17:00:34 2018 -0400
Committer: nickallen 
Committed: Mon May 14 17:00:34 2018 -0400

--
 .../metron/profiler/ProfileMeasurement.java |   3 +-
 .../apache/metron/profiler/ProfilePeriod.java   |   3 +-
 .../metron/profiler/ProfileMeasurementTest.java | 108 +++
 .../metron/profiler/ProfilePeriodTest.java  |  49 +
 metron-analytics/metron-profiler/README.md  |  62 +++
 .../src/main/config/profiler.properties |  13 +++
 .../src/main/flux/profiler/remote.yaml  |   3 +
 .../zookeeper/profile-with-stats/profiler.json  |  12 +++
 .../integration/ProfilerIntegrationTest.java|  83 +-
 .../statistics/sampling/UniformSampler.java |   5 +-
 .../package/templates/profiler.properties.j2|  13 +++
 .../configuration/profiler/ProfileResult.java   |   4 +-
 .../profiler/ProfileResultExpressions.java  |   4 +-
 .../profiler/ProfileTriageExpressions.java  |   3 +-
 .../configuration/profiler/ProfilerConfig.java  |  12 +--
 .../apache/metron/common/utils/SerDeUtils.java  |   7 +-
 .../profiler/ProfilerConfigTest.java|  85 ++-
 .../stellar/common/utils/BloomFilter.java   |   8 +-
 .../metron/stellar/common/utils/SerDeUtils.java |  16 +--
 .../common/utils/StellarProcessorUtils.java | 101 +++--
 20 files changed, 537 insertions(+), 57 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/9ce4ba5a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
--
diff --git 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
index f6cc286..4737c3d 100644
--- 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
+++ 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
@@ -22,6 +22,7 @@ package org.apache.metron.profiler;
 
 import org.apache.metron.common.configuration.profiler.ProfileConfig;
 
+import java.io.Serializable;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -33,7 +34,7 @@ import java.util.concurrent.TimeUnit;
  * A profile contains many individual {@link ProfileMeasurement} values 
captured over a
  * period of time.  These values in aggregate form a time series.
  */
-public class ProfileMeasurement {
+public class ProfileMeasurement implements Serializable {
 
   /**
* The name of the profile that this measurement is associated with.

http://git-wip-us.apache.org/repos/asf/metron/blob/9ce4ba5a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
--
diff --git 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
index c2d8b21..cbb8275 100644
--- 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
+++ 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
@@ -20,6 +20,7 @@
 
 package org.apache.metron.profiler;
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -33,7 +34,7 @@ import static java.lang.String.format;
  * The Profiler captures a ProfileMeasurement once every ProfilePeriod.  There 
can be
  * multiple ProfilePeriods every hour.
  */
-public class ProfilePeriod {
+public class ProfilePeriod implements Serializable {
 
   /**
* A monotonically increasing number identifying the period.  The first 
period is 0

http://git-wip-us.apache.org/repos/asf/metron/blob/9ce4ba5a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileMeasurementTest.java
--
diff --git 

[17/50] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
--
diff --git 
a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
 
b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
new file mode 100755
index 000..24f2306
--- /dev/null
+++ 
b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+
+def yamlify_variables(var) :
+  if isinstance(var, type(True)):
+return str(var).lower()
+  else:
+return var
+
+# server configurations
+config = Script.get_config()
+
+masters_also_are_datanodes = 
config['configurations']['elastic-site']['masters_also_are_datanodes']
+elastic_home = config['configurations']['elastic-sysconfig']['elastic_home']
+data_dir = config['configurations']['elastic-sysconfig']['data_dir']
+work_dir = config['configurations']['elastic-sysconfig']['work_dir']
+conf_dir = config['configurations']['elastic-sysconfig']['conf_dir']
+heap_size = config['configurations']['elastic-sysconfig']['heap_size']
+max_open_files = 
config['configurations']['elastic-sysconfig']['max_open_files']
+max_map_count = config['configurations']['elastic-sysconfig']['max_map_count']
+
+elastic_user = config['configurations']['elastic-env']['elastic_user']
+elastic_group = config['configurations']['elastic-env']['elastic_group']
+log_dir = config['configurations']['elastic-env']['elastic_log_dir']
+pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
+
+hostname = config['hostname']
+java64_home = config['hostLevelParams']['java_home']
+elastic_env_sh_template = config['configurations']['elastic-env']['content']
+sysconfig_template = config['configurations']['elastic-sysconfig']['content']
+
+cluster_name = config['configurations']['elastic-site']['cluster_name']
+zen_discovery_ping_unicast_hosts = 
config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts']
+
+path_data = config['configurations']['elastic-site']['path_data']
+http_cors_enabled = 
config['configurations']['elastic-site']['http_cors_enabled']
+http_port = config['configurations']['elastic-site']['http_port']
+transport_tcp_port = 
config['configurations']['elastic-site']['transport_tcp_port']
+
+recover_after_time = 
config['configurations']['elastic-site']['recover_after_time']
+gateway_recover_after_data_nodes = 
config['configurations']['elastic-site']['gateway_recover_after_data_nodes']
+expected_data_nodes = 
config['configurations']['elastic-site']['expected_data_nodes']
+index_merge_scheduler_max_thread_count = 
config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count']
+index_translog_flush_threshold_size = 
config['configurations']['elastic-site']['index_translog_flush_threshold_size']
+index_refresh_interval = 
config['configurations']['elastic-site']['index_refresh_interval']
+indices_memory_index_store_throttle_type = 
config['configurations']['elastic-site']['indices_memory_index_store_throttle_type']
+index_number_of_shards = 
config['configurations']['elastic-site']['index_number_of_shards']
+index_number_of_replicas = 
config['configurations']['elastic-site']['index_number_of_replicas']
+indices_memory_index_buffer_size = 
config['configurations']['elastic-site']['indices_memory_index_buffer_size']
+bootstrap_memory_lock = 
yamlify_variables(config['configurations']['elastic-site']['bootstrap_memory_lock'])
+threadpool_bulk_queue_size = 
config['configurations']['elastic-site']['threadpool_bulk_queue_size']
+cluster_routing_allocation_node_concurrent_recoveries = 
config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries']
+cluster_routing_allocation_disk_watermark_low = 

[10/50] [abbrv] metron git commit: METRON-1497 Rest endpoint '/api/v1/search/search' needs to handle null when elastic search response return null for getAggregations (MohanDV via justinleet) closes a

[otto]

METRON-1497 Rest endpoint '/api/v1/search/search' needs to handle null when 
elastic search response return null for getAggregations (MohanDV via 
justinleet) closes apache/metron#968


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/19b237de
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/19b237de
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/19b237de

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 19b237de27eed37ee87eca6d8e2198083fdb88d7
Parents: 9e95d4b
Author: MohanDV 
Authored: Tue Apr 3 10:25:52 2018 -0400
Committer: leet 
Committed: Tue Apr 3 10:25:52 2018 -0400

--
 .../apache/metron/elasticsearch/dao/ElasticsearchDao.java | 10 ++
 1 file changed, 6 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/19b237de/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
--
diff --git 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index 9bb109d..26e5731 100644
--- 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -608,10 +608,12 @@ public class ElasticsearchDao implements IndexDao {
 Map> fieldCounts = new HashMap<>();
 for (String field: fields) {
   Map valueCounts = new HashMap<>();
-  Aggregation aggregation = 
aggregations.get(getFacetAggregationName(field));
-  if (aggregation instanceof Terms) {
-Terms terms = (Terms) aggregation;
-terms.getBuckets().stream().forEach(bucket -> 
valueCounts.put(formatKey(bucket.getKey(), commonColumnMetadata.get(field)), 
bucket.getDocCount()));
+  if(aggregations != null ){
+Aggregation aggregation = 
aggregations.get(getFacetAggregationName(field));
+if (aggregation instanceof Terms) {
+  Terms terms = (Terms) aggregation;
+  terms.getBuckets().stream().forEach(bucket -> 
valueCounts.put(formatKey(bucket.getKey(), commonColumnMetadata.get(field)), 
bucket.getDocCount()));
+}
   }
   fieldCounts.put(field, valueCounts);
 }

[25/50] [abbrv] metron git commit: METRON-1494 Profiler Emits Messages to Kafka When Not Needed (nickwallen) closes apache/metron#967

[otto]

METRON-1494 Profiler Emits Messages to Kafka When Not Needed (nickwallen) 
closes apache/metron#967


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/62d1a1bf
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/62d1a1bf
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/62d1a1bf

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 62d1a1bf7e8b9b3ee2f260c358719ea5080c9045
Parents: 438893b
Author: nickwallen 
Authored: Wed Apr 11 17:57:09 2018 -0400
Committer: nickallen 
Committed: Wed Apr 11 17:57:09 2018 -0400

--
 .../metron/profiler/DefaultProfileBuilder.java  |   5 +
 .../bolt/FixedFrequencyFlushSignal.java |  13 +-
 .../metron/profiler/bolt/HBaseEmitter.java  |  12 +-
 .../metron/profiler/bolt/KafkaEmitter.java  |  78 +--
 .../profiler/bolt/ProfileSplitterBolt.java  |   5 +
 .../metron/profiler/bolt/HBaseEmitterTest.java  | 120 +++
 .../metron/profiler/bolt/KafkaEmitterTest.java  | 201 +--
 7 files changed, 358 insertions(+), 76 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
--
diff --git 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
index 4b564c9..66034ac 100644
--- 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
+++ 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
@@ -124,8 +124,13 @@ public class DefaultProfileBuilder implements 
ProfileBuilder, Serializable {
*/
   @Override
   public void apply(JSONObject message, long timestamp) {
+LOG.debug("Applying message to profile; profile={}, entity={}, 
timestamp={}",
+profileName, entity, timestamp);
+
 try {
   if (!isInitialized()) {
+LOG.debug("Initializing profile; profile={}, entity={}, timestamp={}",
+profileName, entity, timestamp);
 
 // execute each 'init' expression
 assign(definition.getInit(), message, "init");

http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
--
diff --git 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
index b9f57dd..8c0a0b1 100644
--- 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
+++ 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
@@ -94,7 +94,8 @@ public class FixedFrequencyFlushSignal implements FlushSignal 
{
 
   // set the next time to flush
   flushTime = currentTime + flushFrequency;
-  LOG.debug("Setting flush time; flushTime={}, currentTime={}, 
flushFreq={}",
+  LOG.debug("Setting flush time; '{}' ms until flush; flushTime={}, 
currentTime={}, flushFreq={}",
+  timeToNextFlush(),
   flushTime,
   currentTime,
   flushFrequency);
@@ -112,7 +113,7 @@ public class FixedFrequencyFlushSignal implements 
FlushSignal {
 boolean flush = currentTime > flushTime;
 LOG.debug("Flush={}, '{}' ms until flush; currentTime={}, flushTime={}",
 flush,
-flush ? 0 : (flushTime-currentTime),
+timeToNextFlush(),
 currentTime,
 flushTime);
 
@@ -123,4 +124,12 @@ public class FixedFrequencyFlushSignal implements 
FlushSignal {
   public long currentTimeMillis() {
 return currentTime;
   }
+
+  /**
+   * Returns the number of milliseconds to the next flush.
+   * @return The time left until the next flush.
+   */
+  private long timeToNextFlush() {
+return Math.max(0, flushTime - currentTime);
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java
--
diff --git 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java
 

[49/50] [abbrv] metron git commit: METRON-1549: Add empty object test to WriterBoltIntegrationTest implementation (mmiklavc via mmiklavc) closes apache/metron#1009

[otto]

METRON-1549: Add empty object test to WriterBoltIntegrationTest implementation 
(mmiklavc via mmiklavc) closes apache/metron#1009


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b9453aab
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b9453aab
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b9453aab

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: b9453aabd781c7c67258d9506af176fbcab85be1
Parents: a17c1ad
Author: mmiklavc 
Authored: Fri May 11 12:04:01 2018 -0600
Committer: Michael Miklavcic 
Committed: Fri May 11 12:04:01 2018 -0600

--
 .../integration/WriterBoltIntegrationTest.java  | 315 ++-
 1 file changed, 231 insertions(+), 84 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/b9453aab/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
--
diff --git 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
index cde08bc..d565147 100644
--- 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
+++ 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
@@ -17,40 +17,56 @@
  */
 package org.apache.metron.writers.integration;
 
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
 import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import javax.annotation.Nullable;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.configuration.SensorParserConfig;
-import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.common.field.validation.FieldValidation;
 import org.apache.metron.common.utils.JSONUtils;
 import 
org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
-import org.apache.metron.integration.*;
+import org.apache.metron.integration.BaseIntegrationTest;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.Processor;
+import org.apache.metron.integration.ProcessorResult;
+import org.apache.metron.integration.ReadinessState;
 import org.apache.metron.integration.components.KafkaComponent;
-import org.apache.metron.integration.processors.KafkaMessageSet;
 import org.apache.metron.integration.components.ZKServerComponent;
+import org.apache.metron.integration.processors.KafkaMessageSet;
 import org.apache.metron.integration.processors.KafkaProcessor;
-import org.apache.metron.parsers.csv.CSVParser;
 import 
org.apache.metron.parsers.integration.components.ParserTopologyComponent;
-import org.apache.metron.test.utils.UnitTestHelper;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.stellar.dsl.Context;
 import org.json.simple.JSONObject;
-import org.json.simple.parser.ParseException;
 import org.junit.Assert;
 import org.junit.Test;
 
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.util.*;
-
 public class WriterBoltIntegrationTest extends BaseIntegrationTest {
+  private ZKServerComponent zkServerComponent;
+  private KafkaComponent kafkaComponent;
+  private ConfigUploadComponent configUploadComponent;
+  private ParserTopologyComponent parserTopologyComponent;
 
-  public static class MockValidator implements FieldValidation{
+  public static class MockValidator implements FieldValidation {
 
 @Override
 public boolean isValid(Map input, Map 
validationConfig, Map globalConfig, Context context) {
-  if(input.get("action").equals("invalid")) {
+  if (input.get("action").equals("invalid")) {
 return false;
   }
   return true;
@@ -60,6 +76,7 @@ public class WriterBoltIntegrationTest extends 
BaseIntegrationTest {
 public void initialize(Map validationConfig, Map globalConfig) {
 }
   }
+
   /**
* {
*   "fieldValidations" : [
@@ -68,7 +85,7 @@ public class WriterBoltIntegrationTest extends 

[35/50] [abbrv] metron git commit: METRON-1527: Remove dead test file sitting in source folder (mmiklavc via mmiklavc) closes apache/metron#994

[otto]

METRON-1527: Remove dead test file sitting in source folder (mmiklavc via 
mmiklavc) closes apache/metron#994


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08252f59
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08252f59
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08252f59

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 08252f59b2df1f4ef149b899c944f6a8a0aa049e
Parents: 82212ba
Author: mmiklavc 
Authored: Tue Apr 17 10:55:08 2018 -0600
Committer: Michael Miklavcic 
Committed: Tue Apr 17 10:55:08 2018 -0600

--
 .../org/apache/metron/common/writer/test.json   | 31 
 1 file changed, 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/08252f59/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/test.json
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/test.json
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/test.json
deleted file mode 100644
index 023cd63..000
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/test.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
-  "index": "bro",
-  "batchSize": 1,
-  "enrichment" : {
-"fieldMap":
-{
-  "geo": ["ip_dst_addr", "ip_src_addr"],
-  "host": ["host"]
-}
-  },
-  "threatIntel" : {
-"fieldMap":
-{
-  "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
-},
-"fieldToTypeMap":
-{
-  "ip_src_addr" : ["malicious_ip"],
-  "ip_dst_addr" : ["malicious_ip"]
-},
-"triageConfig" : {
-  "riskLevelRules" : [
-{
-  "rule" : "ip_src_addr == '31.24.30.31'",
-  "score" : 10
-}
-  ],
-  "aggregator" : "MAX"
-}
-  }
-}

[05/50] [abbrv] metron git commit: METRON-590 Enable Use of Event Time in Profiler (nickwallen) closes apache/metron#965

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/3083b471/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignalTest.java
--
diff --git 
a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignalTest.java
 
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignalTest.java
new file mode 100644
index 000..b8949c5
--- /dev/null
+++ 
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignalTest.java
@@ -0,0 +1,71 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.bolt;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests the {@code FixedFrequencyFlushSignal} class.
+ */
+public class FixedFrequencyFlushSignalTest {
+
+  @Test
+  public void testSignalFlush() {
+
+FixedFrequencyFlushSignal signal = new FixedFrequencyFlushSignal(1000);
+
+// not time to flush yet
+assertFalse(signal.isTimeToFlush());
+
+// advance time
+signal.update(5000);
+
+// not time to flush yet
+assertFalse(signal.isTimeToFlush());
+
+// advance time
+signal.update(7000);
+
+// time to flush
+assertTrue(signal.isTimeToFlush());
+  }
+
+  @Test
+  public void testOutOfOrderTimestamps() {
+FixedFrequencyFlushSignal signal = new FixedFrequencyFlushSignal(1000);
+
+// advance time, out-of-order
+signal.update(5000);
+signal.update(1000);
+signal.update(7000);
+signal.update(3000);
+
+// need to flush @ 5000 + 1000 = 6000. if anything > 6000 (even 
out-of-order), then it should signal a flush
+assertTrue(signal.isTimeToFlush());
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testNegativeFrequency() {
+new FixedFrequencyFlushSignal(-1000);
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/3083b471/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/KafkaDestinationHandlerTest.java
--
diff --git 
a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/KafkaDestinationHandlerTest.java
 
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/KafkaDestinationHandlerTest.java
deleted file mode 100644
index c3f2584..000
--- 
a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/KafkaDestinationHandlerTest.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- *
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- */
-
-package org.apache.metron.profiler.bolt;
-
-import com.google.common.collect.ImmutableMap;
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.metron.common.configuration.profiler.ProfileConfig;
-import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.profiler.ProfileMeasurement;
-import org.apache.metron.statistics.OnlineStatisticsProvider;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Values;
-import org.json.simple.JSONObject;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Mockito;
-
-import 

[41/50] [abbrv] metron git commit: METRON-1520: Add caching for stellar field transformations closes apache/incubator-metron#990

[otto]

METRON-1520: Add caching for stellar field transformations closes 
apache/incubator-metron#990


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1c5435cc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1c5435cc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1c5435cc

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 1c5435ccbe96c03e59c6a18d681da43561769dba
Parents: 37e3fd3
Author: cstella 
Authored: Wed Apr 25 11:48:44 2018 -0400
Committer: cstella 
Committed: Wed Apr 25 11:48:44 2018 -0400

--
 metron-platform/Performance-tuning-guide.md |  13 ++
 .../configuration/SensorParserConfig.java   |  15 ++
 .../transformation/StellarTransformation.java   |   3 +-
 .../StellarTransformationTest.java  |  30 
 metron-platform/metron-parsers/README.md|  13 ++
 .../apache/metron/parsers/bolt/ParserBolt.java  |  15 +-
 .../stellar/common/CachingStellarProcessor.java | 144 +++
 .../org/apache/metron/stellar/dsl/Context.java  |  43 +-
 .../common/CachingStellarProcessorTest.java | 104 ++
 9 files changed, 371 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/1c5435cc/metron-platform/Performance-tuning-guide.md
--
diff --git a/metron-platform/Performance-tuning-guide.md 
b/metron-platform/Performance-tuning-guide.md
index e2d1ae2..c2d19d6 100644
--- a/metron-platform/Performance-tuning-guide.md
+++ b/metron-platform/Performance-tuning-guide.md
@@ -60,6 +60,19 @@ parallelism will leave you with idle consumers since Kafka 
limits the max number
 important because Kafka has certain ordering guarantees for message delivery 
per partition that would not be possible if more than
 one consumer in a given consumer group were able to read from that partition.
 
+## Sensor Topology Tuning Suggestions
+
+If you are using stellar field transformations in your sensors, by default, 
stellar expressions
+are not cached.  Sensors that use stellar field transformations by see a 
performance
+boost by turning on caching via setting the `cacheConfig`
+[property](metron-parsers#parser_configuration).
+This is beneficial if your transformations:
+
+* Are complex (e.g. `ENRICHMENT_GET` calls or other high latency calls)
+* All Yield the same results for the same inputs ( caching is either off or 
applied to all transformations)
+  * If any of your transformations are non-deterministic, caching should not 
be used as it will result in the likelihood of incorrect results being returned.
+
+
 ## Component Tuning Levers
 
 ### High Level Overview

http://git-wip-us.apache.org/repos/asf/metron/blob/1c5435cc/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
index 2d0ccd8..d347481 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
@@ -45,11 +45,26 @@ public class SensorParserConfig implements Serializable {
   private Integer parserNumTasks = 1;
   private Integer errorWriterParallelism = 1;
   private Integer errorWriterNumTasks = 1;
+  private Map cacheConfig = new HashMap<>();
   private Map spoutConfig = new HashMap<>();
   private String securityProtocol = null;
   private Map stormConfig = new HashMap<>();
 
   /**
+   * Cache config for stellar field transformations.
+   * * stellar.cache.maxSize - The maximum number of elements in the cache.
+   * * stellar.cache.maxTimeRetain - The maximum amount of time an element is 
kept in the cache (in minutes).
+   * @return
+   */
+  public Map getCacheConfig() {
+return cacheConfig;
+  }
+
+  public void setCacheConfig(Map cacheConfig) {
+this.cacheConfig = cacheConfig;
+  }
+
+  /**
* Return the number of workers for the topology.  This property will be 
used for the parser unless overridden on the CLI.
* @return
*/

http://git-wip-us.apache.org/repos/asf/metron/blob/1c5435cc/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java
--
diff --git 

[01/50] [abbrv] metron git commit: METRON-1487 Define Performance Benchmarks for Enrichment Topology (nickwallen) closes apache/metron#961

[otto]

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1090-stellar-assignment 610540ef0 -> 9ce4ba5a9


METRON-1487 Define Performance Benchmarks for Enrichment Topology (nickwallen) 
closes apache/metron#961


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e3eeec38
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e3eeec38
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e3eeec38

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: e3eeec38a66e1f10c296c110d47bc8bc3e995629
Parents: 52dd9fb
Author: nickwallen 
Authored: Sat Mar 17 09:22:15 2018 -0400
Committer: nickallen 
Committed: Sat Mar 17 09:22:15 2018 -0400

--
 metron-platform/Performance-tuning-guide.md |   2 +
 .../metron-enrichment/Performance.md| 514 +++
 2 files changed, 516 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/e3eeec38/metron-platform/Performance-tuning-guide.md
--
diff --git a/metron-platform/Performance-tuning-guide.md 
b/metron-platform/Performance-tuning-guide.md
index bc8801b..7d79ace 100644
--- a/metron-platform/Performance-tuning-guide.md
+++ b/metron-platform/Performance-tuning-guide.md
@@ -422,6 +422,7 @@ modifying the options outlined above, increasing the poll 
timeout, or both.
 
 ## Reference
 
+* [Enrichment Performance](metron-enrichment/Performance.md)
 * http://storm.apache.org/releases/1.0.1/flux.html
 * 
https://stackoverflow.com/questions/17257448/what-is-the-task-in-storm-parallelism
 * 
http://storm.apache.org/releases/current/Understanding-the-parallelism-of-a-Storm-topology.html
@@ -429,3 +430,4 @@ modifying the options outlined above, increasing the poll 
timeout, or both.
 * 
https://www.confluent.io/blog/how-to-choose-the-number-of-topicspartitions-in-a-kafka-cluster/
 * 
https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.1/bk_storm-component-guide/content/storm-kafkaspout-perf.html
 
+

http://git-wip-us.apache.org/repos/asf/metron/blob/e3eeec38/metron-platform/metron-enrichment/Performance.md
--
diff --git a/metron-platform/metron-enrichment/Performance.md 
b/metron-platform/metron-enrichment/Performance.md
new file mode 100644
index 000..4016a0d
--- /dev/null
+++ b/metron-platform/metron-enrichment/Performance.md
@@ -0,0 +1,514 @@
+
+
+# Enrichment Performance
+
+This guide defines a set of benchmarks used to measure the performance of the 
Enrichment topology.  The guide also provides detailed steps on how to execute 
those benchmarks along with advice for tuning the Unified Enrichment topology.
+
+* [Benchmarks](#benchmarks)
+* [Benchmark Execution](#benchmark-execution)
+* [Performance Tuning](#performance-tuning)
+* [Benchmark Results](#benchmark-results)
+
+## Benchmarks
+
+The following section describes a set of enrichments that will be used to 
benchmark the performance of the Enrichment topology.
+
+* [Geo IP Enrichment](#geo-ip-enrichment)
+* [HBase Enrichment](#hbase-enrichment)
+* [Stellar Enrichment](#stellar-enrichment)
+
+### Geo IP Enrichment
+
+This benchmark measures the performance of executing a Geo IP enrichment.  
Given a valid IP address the enrichment will append detailed location 
information for that IP.  The location information is sourced from an external 
Geo IP data source like [Maxmind](https://github.com/maxmind/GeoIP2-java).
+
+ Configuration
+
+Adding the following Stellar expression to the Enrichment topology 
configuration will define a Geo IP enrichment.
+```
+geo := GEO_GET(ip_dst_addr)
+```
+
+After the enrichment process completes, the  telemetry message will contain a 
set of fields with location information for the given IP address.
+```
+{
+   "ip_dst_addr":"151.101.129.140",
+   ...
+   "geo.city":"San Francisco",
+   "geo.country":"US",
+   "geo.dmaCode":"807",
+   "geo.latitude":"37.7697",
+   "geo.location_point":"37.7697,-122.3933",
+   "geo.locID":"5391959",
+   "geo.longitude":"-122.3933",
+   "geo.postalCode":"94107",
+ }
+```
+
+### HBase Enrichment
+
+This benchmark measures the performance of executing an enrichment that 
retrieves data from an external HBase table. This type of enrichment is useful 
for enriching telemetry from an Asset Database or other source of relatively 
static data.
+
+ Configuration
+
+Adding the following Stellar expression to the Enrichment topology 
configuration will define an Hbase enrichment.  This looks up the 'ip_dst_addr' 
within an HBase table 'top-1m' and returns a hostname.
+```
+top1m := ENRICHMENT_GET('top-1m', ip_dst_addr, 'top-1m', 't')
+```
+
+After the telemetry has been enriched, it will contain the host and IP 

[09/50] [abbrv] metron git commit: METRON-1500 Enhance 'prepare-commit' to Support Feature Branches (nickwallen) closes apache/metron#971

[otto]

METRON-1500 Enhance 'prepare-commit' to Support Feature Branches (nickwallen) 
closes apache/metron#971


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9e95d4b6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9e95d4b6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9e95d4b6

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 9e95d4b61410c8033f0a4ea51e831566d5d933d3
Parents: 5ed9631
Author: nickwallen 
Authored: Fri Mar 23 12:23:34 2018 -0400
Committer: nickallen 
Committed: Fri Mar 23 12:23:34 2018 -0400

--
 dev-utilities/committer-utils/prepare-commit | 27 ++-
 1 file changed, 22 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/9e95d4b6/dev-utilities/committer-utils/prepare-commit
--
diff --git a/dev-utilities/committer-utils/prepare-commit 
b/dev-utilities/committer-utils/prepare-commit
index 67116cb..ae8d7ab 100755
--- a/dev-utilities/committer-utils/prepare-commit
+++ b/dev-utilities/committer-utils/prepare-commit
@@ -19,9 +19,9 @@
 # not likely to change
 METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron.git;
 
BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git;
-BASE_BRANCH=master
 CONFIG_FILE=~/.metron-prepare-commit
 GITHUB_REMOTE="origin"
+BASE_BRANCH=master
 
 # does a config file already exist?
 if [ -f $CONFIG_FILE ]; then
@@ -114,6 +114,11 @@ if [ ! -d "$WORK" ]; then
   read -p "  origin repo [$ORIGIN]: " INPUT
   [ -n "$INPUT" ] && ORIGIN=$INPUT
 
+  # what branch did the PR get submitted against?  could be a feature branch
+  BASE_BRANCH=`curl -s 
https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/$PR | python -c 
'import sys, json; print json.load(sys.stdin)["base"]["ref"]'`
+  read -p "  base branch to merge into [$BASE_BRANCH]: " INPUT
+  [ -n "$INPUT" ] && BASE_BRANCH=$INPUT
+
   # clone the repository and fetch updates
   mkdir -p $WORK
   git clone $ORIGIN $WORK
@@ -125,11 +130,23 @@ if [ ! -d "$WORK" ]; then
 
   # fetch any changes from upstream
   git remote add upstream $UPSTREAM
-  git fetch upstream $BASE_BRANCH
+  if git fetch upstream "$BASE_BRANCH"; then
+
+if [ $BASE_BRANCH = "master" ]; then
+  # merge any changes from upstream
+  git checkout $BASE_BRANCH
+  git merge upstream/$BASE_BRANCH
 
-  # merge any changes from upstream
-  git checkout $BASE_BRANCH
-  git merge upstream/$BASE_BRANCH
+else
+  # create a local branch from the remote feature branch
+  git checkout -B $BASE_BRANCH upstream/$BASE_BRANCH
+
+fi
+
+  else
+# unable to fetch the base branch
+exit $?
+  fi
 
 else
 

[48/50] [abbrv] metron git commit: METRON-1541 Mvn clean results in git status having deleted files. (justinleet via nickwallen) closes apache/metron#1003

[otto]

METRON-1541 Mvn clean results in git status having deleted files. (justinleet 
via nickwallen) closes apache/metron#1003


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a17c1adf
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a17c1adf
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a17c1adf

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: a17c1adfd4cca63a49ef542c21cef90b18c74be0
Parents: d7edce9
Author: justinleet 
Authored: Wed May 9 16:47:29 2018 -0400
Committer: nickallen 
Committed: Wed May 9 16:47:29 2018 -0400

--
 metron-deployment/packaging/ambari/.gitignore   |  2 +
 .../enrichment-splitjoin.properties.j2  | 63 
 .../templates/enrichment-unified.properties.j2  | 60 ---
 3 files changed, 2 insertions(+), 123 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/a17c1adf/metron-deployment/packaging/ambari/.gitignore
--
diff --git a/metron-deployment/packaging/ambari/.gitignore 
b/metron-deployment/packaging/ambari/.gitignore
index ca2e75c..2f93166 100644
--- a/metron-deployment/packaging/ambari/.gitignore
+++ b/metron-deployment/packaging/ambari/.gitignore
@@ -3,3 +3,5 @@ archive.zip
 elasticsearch.properties.j2
 hdfs.properties.j2
 enrichment.properties.j2
+enrichment-splitjoin.properties.j2
+enrichment-unified.properties.j2

http://git-wip-us.apache.org/repos/asf/metron/blob/a17c1adf/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
deleted file mode 100644
index a0b21c9..000
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
+++ /dev/null
@@ -1,63 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-# Storm #
-enrichment.workers={{enrichment_workers}}
-enrichment.acker.executors={{enrichment_acker_executors}}
-topology.worker.childopts={{enrichment_topology_worker_childopts}}
-topology.auto-credentials={{topology_auto_credentials}}
-topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
-
-# Kafka #
-kafka.zk={{zookeeper_quorum}}
-kafka.broker={{kafka_brokers}}
-kafka.security.protocol={{kafka_security_protocol}}
-
-# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
-kafka.start={{enrichment_kafka_start}}
-
-enrichment.input.topic={{enrichment_input_topic}}
-enrichment.output.topic={{enrichment_output_topic}}
-enrichment.error.topic={{enrichment_error_topic}}
-threat.intel.error.topic={{threatintel_error_topic}}
-
-# JoinBolt #
-enrichment.join.cache.size={{enrichment_join_cache_size}}
-threat.intel.join.cache.size={{threatintel_join_cache_size}}
-
-# Enrichment #
-hbase.provider.impl={{enrichment_hbase_provider_impl}}
-enrichment.simple.hbase.table={{enrichment_hbase_table}}
-enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
-enrichment.host.known_hosts={{enrichment_host_known_hosts}}
-
-# Threat Intel #
-threat.intel.tracker.table={{threatintel_hbase_table}}
-threat.intel.tracker.cf={{threatintel_hbase_cf}}
-threat.intel.simple.hbase.table={{threatintel_hbase_table}}
-threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
-
-# Parallelism #
-kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}}
-enrichment.split.parallelism={{enrichment_split_parallelism}}
-enrichment.stellar.parallelism={{enrichment_stellar_parallelism}}

[18/50] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943

[otto]

METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) 
closes apache/metron#943


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0ab39a32
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0ab39a32
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0ab39a32

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0ab39a32b61104812a6a9e69e999e7053e3e7e90
Parents: 0d847cf
Author: mmiklavc 
Authored: Thu Apr 5 09:07:48 2018 -0600
Committer: Michael Miklavcic 
Committed: Thu Apr 5 09:07:48 2018 -0600

--
 .../roles/ambari_master/defaults/main.yml   |   2 +
 .../ambari_master/tasks/elasticsearch_mpack.yml |  26 ++
 .../ansible/roles/ambari_master/tasks/main.yml  |   3 +-
 .../roles/ambari_master/tasks/metron_mpack.yml  |  26 ++
 .../ansible/roles/ambari_master/tasks/mpack.yml |  26 --
 .../roles/load_web_templates/tasks/main.yml |   2 +-
 .../manual-install/Manual_Install_CentOS6.md|   4 +-
 metron-deployment/packaging/ambari/README.md|  25 +-
 .../ambari/elasticsearch-mpack/README.md|  62 +
 .../ambari/elasticsearch-mpack/pom.xml  |  95 +++
 .../src/main/assemblies/elasticsearch-mpack.xml |  43 +++
 .../ELASTICSEARCH/5.6.2/metainfo.xml|  29 ++
 .../ELASTICSEARCH/5.6.2/repos/repoinfo.xml  |  45 
 .../addon-services/KIBANA/5.6.2/metainfo.xml|  30 +++
 .../KIBANA/5.6.2/quicklinks/quicklinks.json |  27 ++
 .../KIBANA/5.6.2/repos/repoinfo.xml |  60 +
 .../5.6.2/configuration/elastic-env.xml |  86 ++
 .../5.6.2/configuration/elastic-jvm-options.xml | 144 ++
 .../5.6.2/configuration/elastic-site.xml| 198 ++
 .../5.6.2/configuration/elastic-sysconfig.xml   |  97 +++
 .../5.6.2/configuration/elastic-systemd.xml |  30 +++
 .../ELASTICSEARCH/5.6.2/metainfo.xml|  97 +++
 .../5.6.2/package/scripts/elastic_commands.py   | 266 +++
 .../5.6.2/package/scripts/elastic_master.py |  72 +
 .../5.6.2/package/scripts/elastic_slave.py  |  71 +
 .../5.6.2/package/scripts/params.py | 108 
 .../5.6.2/package/scripts/properties_config.py  |  34 +++
 .../5.6.2/package/scripts/service_check.py  | 114 
 .../5.6.2/package/scripts/status_params.py  |  27 ++
 .../templates/elasticsearch.master.yaml.j2  |  77 ++
 .../templates/elasticsearch.slave.yaml.j2   |  78 ++
 .../templates/elasticsearch_limits.conf.j2  |  20 ++
 .../5.6.2/quicklinks/quicklinks.json|  43 +++
 .../ELASTICSEARCH/5.6.2/role_command_order.json |   8 +
 .../KIBANA/5.6.2/configuration/kibana-env.xml   |  72 +
 .../KIBANA/5.6.2/configuration/kibana-site.xml  | 113 
 .../common-services/KIBANA/5.6.2/metainfo.xml   |  84 ++
 .../KIBANA/5.6.2/package/scripts/common.py  |  56 
 .../5.6.2/package/scripts/kibana_master.py  |  81 ++
 .../KIBANA/5.6.2/package/scripts/params.py  |  50 
 .../KIBANA/5.6.2/quicklinks/quicklinks.json |  28 ++
 .../src/main/resources/mpack.json   |  76 ++
 .../packaging/ambari/metron-mpack/README.md |  20 +-
 .../src/main/assemblies/metron-mpack.xml|  14 -
 .../ELASTICSEARCH/5.6.2/metainfo.xml|  29 --
 .../ELASTICSEARCH/5.6.2/repos/repoinfo.xml  |  45 
 .../addon-services/KIBANA/5.6.2/metainfo.xml|  30 ---
 .../KIBANA/5.6.2/quicklinks/quicklinks.json |  27 --
 .../KIBANA/5.6.2/repos/repoinfo.xml |  60 -
 .../5.6.2/configuration/elastic-env.xml |  86 --
 .../5.6.2/configuration/elastic-jvm-options.xml | 144 --
 .../5.6.2/configuration/elastic-site.xml| 198 --
 .../5.6.2/configuration/elastic-sysconfig.xml   |  97 ---
 .../5.6.2/configuration/elastic-systemd.xml |  30 ---
 .../ELASTICSEARCH/5.6.2/metainfo.xml|  97 ---
 .../5.6.2/package/scripts/elastic_commands.py   | 266 ---
 .../5.6.2/package/scripts/elastic_master.py |  72 -
 .../5.6.2/package/scripts/elastic_slave.py  |  71 -
 .../5.6.2/package/scripts/params.py | 108 
 .../5.6.2/package/scripts/properties_config.py  |  34 ---
 .../5.6.2/package/scripts/service_check.py  | 114 
 .../5.6.2/package/scripts/status_params.py  |  27 --
 .../templates/elasticsearch.master.yaml.j2  |  77 --
 .../templates/elasticsearch.slave.yaml.j2   |  78 --
 .../templates/elasticsearch_limits.conf.j2  |  20 --
 .../5.6.2/quicklinks/quicklinks.json|  43 ---
 .../ELASTICSEARCH/5.6.2/role_command_order.json |   8 -
 .../KIBANA/5.6.2/configuration/kibana-env.xml   |  72 -
 .../KIBANA/5.6.2/configuration/kibana-site.xml  | 113