from:"jonzeolla"

metron git commit: METRON-999 Add virtualization support checks to platform-info.sh (JonZeolla via jonzeolla) closes apache/metron#618

2017-06-28 Thread jonzeolla

Repository: metron
Updated Branches:
  refs/heads/master d52f574f8 -> 095be23dc


METRON-999 Add virtualization support checks to platform-info.sh (JonZeolla via 
jonzeolla) closes apache/metron#618


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/095be23d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/095be23d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/095be23d

Branch: refs/heads/master
Commit: 095be23dce0f818a98a1fc587053d133ca542856
Parents: d52f574
Author: JonZeolla <zeo...@gmail.com>
Authored: Wed Jun 28 14:04:39 2017 -0400
Committer: jonzeolla <jonzeo...@apache.org>
Committed: Wed Jun 28 14:04:39 2017 -0400

--
 metron-deployment/scripts/platform-info.sh | 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/095be23d/metron-deployment/scripts/platform-info.sh
--
diff --git a/metron-deployment/scripts/platform-info.sh 
b/metron-deployment/scripts/platform-info.sh
index a1dce34..a24098d 100755
--- a/metron-deployment/scripts/platform-info.sh
+++ b/metron-deployment/scripts/platform-info.sh
@@ -89,6 +89,11 @@ case "${OSTYPE}" in
 cat /proc/cpuinfo | grep -i cores | cut -d: -f2 | awk '{corecount+=$1} END 
{print "Total cores: " corecount}'
 echo "Disk information:"
 df -h | grep "^/"
+if [[ $(egrep '(vmx|svm)' /proc/cpuinfo) ]]; then
+  echo "This CPU appears to support virtualization"
+else
+  echo "This CPU may not support virtualization"
+fi
 ;;
   darwin*)
 sysctl hw.memsize | awk '{print "Total System Memory = " $2/1048576 " MB"}'
@@ -98,6 +103,11 @@ case "${OSTYPE}" in
 sysctl machdep.cpu | grep 'machdep.cpu.core_count' | cut -d: -f2 | cut 
-d\@ -f2 | awk '{print "Total cores:" $0}'
 echo "Disk information:"
 df -h | grep "^/"
+if [[ $(sysctl kern.hv_support | awk -F' ' '{print $2}') == "1" && 
$(sysctl -a | grep machdep.cpu.features | grep VMX) ]]; then
+  echo "This CPU appears to support virtualization"
+else
+  echo "This CPU may not support virtualization"
+fi
 ;;
   *)
 echo "Unable to detect system resources for ${OSTYPE}"

[2/2] metron git commit: METRON-508 Expand Elasticsearch templates to support the standard bro logs (JonZeolla) closes apache/metron#586

2017-07-05 Thread jonzeolla

METRON-508 Expand Elasticsearch templates to support the standard bro logs 
(JonZeolla) closes apache/metron#586


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/cc7bbc94
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/cc7bbc94
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/cc7bbc94

Branch: refs/heads/master
Commit: cc7bbc94870f0e0d5026c14ac710ca8882d0589b
Parents: ab87bdf
Author: JonZeolla <zeo...@gmail.com>
Authored: Tue Jul 4 21:52:00 2017 -0400
Committer: JonZeolla <jonzeo...@apache.org>
Committed: Tue Jul 4 21:52:00 2017 -0400

--
 .../CURRENT/package/files/bro_index.template| 793 ++-
 .../sample/data/bro/parsed/BroExampleParsed |  17 +
 .../main/sample/data/bro/raw/BroExampleOutput   |  17 +
 .../apache/metron/parsers/bro/JSONCleaner.java  |   2 +-
 .../metron/parsers/bro/BasicBroParserTest.java  | 982 ++-
 5 files changed, 1742 insertions(+), 69 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/cc7bbc94/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
index 7521af7..18c5d9b 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -122,76 +122,166 @@
   }
   ],
   "properties": {
+/*
+ * WARNING
+ *
+ * Because Metron inserts many distinct bro records into a single 
ElasticSearch index
+ * by default, it encounters field collisions due to field name reuse 
across bro logs.
+ *
+ * Be careful when modifying this file to not unintentionally affect 
other logs.
+ * For instance, the "version" field exists in the HTTP, SSL, and SSH 
logs.  If you
+ * were to only consider the SSH log, you would set the type to 
integer, but because
+ * in the SSL and HTTP logs version is a string, we must set the type 
to string.
+ */
+/*
+ * Metron-specific fields
+ */
+"source:type": {
+  "type": "string",
+  "index": "not_analyzed"
+},
+/*
+ * Widely-used Bro fields (potentially renamed during Metron ingest)
+ */
 "timestamp": {
   "type": "date",
   "format": "epoch_millis"
 },
-"source:type": {
+"uid": {
   "type": "string",
   "index": "not_analyzed"
 },
-"ip_dst_addr": {
+"ip_src_addr": {
   "type": "ip"
 },
-"ip_dst_port": {
+"ip_src_port": {
   "type": "integer"
 },
-"ip_src_addr": {
+"ip_dst_addr": {
   "type": "ip"
 },
-"ip_src_port": {
+"ip_dst_port": {
   "type": "integer"
 },
-"status_code": {
+/*
+ * HTTP log support
+ * 
https://www.bro.org/sphinx/scripts/base/protocols/http/main.bro.html#type-HTTP::Info
+ *
+ * Notable Fields
+ *   Field: password
+ *   Notes: Field exists in the HTTP and FTP logs
+ *
+ *   Field: capture_password
+ *   Notes: Field exists in the HTTP and FTP logs
+ *
+ *   Field: trans_depth
+ *   Notes: Field exists in the HTTP and SMTP logs
+ *
+ *   Field: user_agent
+ *   Notes: Field exists in the HTTP and SMTP logs
+ *
+ *   Field: version
+ *   Notes: Field exists in the HTTP, SSL, and SSH logs
+ *
+ *   Field: host
+ *   Notes: Field exists in the HTTP and Software logs
+ *
+ *   Field: username
+ *   Notes: Field exists in the HTTP and RADIUS logs
+ */
+"trans_depth": {
   "type": "integer"
 },

[1/2] metron git commit: METRON-508 Expand Elasticsearch templates to support the standard bro logs (JonZeolla) closes apache/metron#586

2017-07-05 Thread jonzeolla

Repository: metron
Updated Branches:
  refs/heads/master ab87bdf67 -> cc7bbc948


http://git-wip-us.apache.org/repos/asf/metron/blob/cc7bbc94/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
--
diff --git 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
index 7b09746..aa60d1f 100644
--- 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
+++ 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.metron.parsers.bro;
 
+import org.adrianwalker.multilinestring.Multiline;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.log4j.Level;
 import org.apache.metron.test.utils.UnitTestHelper;
@@ -66,43 +67,92 @@ public class BasicBroParserTest {
}
}
 
+   /**
+* {
+* "ts":1449511228.474,
+* "uid":"CFgSLp4HgsGqXnNjZi",
+* "id.orig_h":"104.130.172.191",
+* "id.orig_p":33893,
+* "id.resp_h":"69.20.0.164",
+* "id.resp_p":53,
+* "proto":"udp",
+* "trans_id":3514,
+* "rcode":3,
+* "rcode_name":"NXDOMAIN",
+* "AA":false,
+* "TC":false,
+* "RD":false,
+* "RA":false,
+* "Z":0,
+* "rejected":false,
+* "sensor":"cloudbro",
+* "type":"dns"
+* }
+*/
+   @Multiline
+   public final static String unwrappedBroMessage;
+
@Test
public void testUnwrappedBroMessage() throws ParseException {
-String rawMessage = 
"{\"timestamp\":1449511228.474,\"uid\":\"CFgSLp4HgsGqXnNjZi\",\"source_ip\":\"104.130.172.191\",\"source_port\":33893,\"dest_ip\":\"69.20.0.164\",\"dest_port\":53,\"proto\":\"udp\",\"trans_id\":3514,\"rcode\":3,\"rcode_name\":\"NXDOMAIN\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"rejected\":false,\"sensor\":\"cloudbro\",\"type\":\"dns\"}";
-
-JSONObject rawJson = (JSONObject)jsonParser.parse(rawMessage);
-
-JSONObject broJson = broParser.parse(rawMessage.getBytes()).get(0);
+JSONObject rawJson = (JSONObject)jsonParser.parse(unwrappedBroMessage);
+JSONObject broJson = 
broParser.parse(unwrappedBroMessage.getBytes()).get(0);
 
-   String expectedBroTimestamp = "1449511228.474";
+   String expectedBroTimestamp = "1449511228.474";
Assert.assertEquals(broJson.get("bro_timestamp"), expectedBroTimestamp);
 String expectedTimestamp = "1449511228474";
-   
Assert.assertEquals(broJson.get("timestamp").toString(), expectedTimestamp);
- 
Assert.assertEquals(broJson.get("ip_src_addr").toString(), 
rawJson.get("source_ip").toString());
- 
Assert.assertEquals(broJson.get("ip_dst_addr").toString(), 
rawJson.get("dest_ip").toString());
- Assert.assertEquals(broJson.get("ip_src_port"), 
rawJson.get("source_port"));
-Assert.assertEquals(broJson.get("ip_dst_port"), 
rawJson.get("dest_port"));
+   Assert.assertEquals(broJson.get("timestamp").toString(), 
expectedTimestamp);
+
+   Assert.assertEquals(broJson.get("ip_src_addr").toString(), 
rawJson.get("id.orig_h").toString());
+   Assert.assertEquals(broJson.get("ip_dst_addr").toString(), 
rawJson.get("id.resp_h").toString());
+   Assert.assertEquals(broJson.get("ip_src_port"), 
rawJson.get("id.orig_p"));
+Assert.assertEquals(broJson.get("ip_dst_port"), 
rawJson.get("id.resp_p"));
 Assert.assertEquals(broJson.get("uid").toString(), 
rawJson.get("uid").toString());
 Assert.assertEquals(broJson.get("trans_id").toString(), 
rawJson.get("trans_id").toString());
 Assert.assertEquals(broJson.get("sensor").toString(), 
rawJson.get("sensor").toString());
-Assert.assertEquals(broJson.get("protocol").toString(), 
rawJson.get("type").toString());
+Assert.assertEquals(broJson.get("type").toString(), 
rawJson.get("type").toString());
 Assert.assertEquals(broJson.get("rcode").toString(), 
rawJson.get("rcode").toString());
 Assert.assertEquals(broJson.get("rcode_name").toString(), 
rawJson.get("rcode_name").toString());
-   
Assert.assertTrue(broJson.get("original_string").toString().startsWith("DNS"));
+
+   
Assert.assertTrue(broJson.get("original_string").toString().startsWith("DNS"));
 }
 
+/**
+* {
+* "http": {
+*  "ts":1402307733.473,
+*  "uid":"CTo78A11g7CYbbOHvj",
+*  "id.orig_h":"192.249.113.37",
+*  "id.orig_p":58808,
+*  "id.resp_h":"72.163.4.161",
+*

metron git commit: METRON-959 Trivial documentation formatting bugs (JonZeolla) closes apache/metron#591

2017-05-25 Thread jonzeolla

Repository: metron
Updated Branches:
  refs/heads/master 7ef929ae4 -> 08f95dc8f


METRON-959 Trivial documentation formatting bugs (JonZeolla) closes 
apache/metron#591


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08f95dc8
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08f95dc8
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08f95dc8

Branch: refs/heads/master
Commit: 08f95dc8f5588dc9d50e204410ca3d09f84f3c8b
Parents: 7ef929a
Author: JonZeolla <zeo...@gmail.com>
Authored: Thu May 25 18:15:41 2017 -0400
Committer: jonzeolla <jonzeo...@apache.org>
Committed: Thu May 25 18:15:41 2017 -0400

--
 metron-analytics/metron-maas-service/README.md |  6 +++---
 metron-platform/metron-indexing/README.md  |  4 ++--
 metron-platform/metron-management/README.md|  2 +-
 metron-platform/metron-parsers/README.md   | 20 ++--
 site-book/README.md|  8 
 5 files changed, 20 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/08f95dc8/metron-analytics/metron-maas-service/README.md
--
diff --git a/metron-analytics/metron-maas-service/README.md 
b/metron-analytics/metron-maas-service/README.md
index 6b1663b..65f1a01 100644
--- a/metron-analytics/metron-maas-service/README.md
+++ b/metron-analytics/metron-maas-service/README.md
@@ -1,6 +1,6 @@
 # Model Management Infrastructure
 
-##Introduction
+## Introduction
 
 One of the main features envisioned and requested is the ability to augment 
the threat intelligence and enrichment processes with insights derived from 
machine learning or statistical models.  The challenges with this sort of 
infrastructure are
 * Applying the model may be sufficiently computationally/resource intensive 
that we need to support scaling via load balancing, which will require service 
discovery and management.
@@ -12,7 +12,7 @@ To support a high throughput environment that is manageable, 
it is evident that
 * Deployment should happen using Yarn to manage resources
 * Clients should have new model endpoints pushed to them
 
-##Architecture
+## Architecture
 
 ![Architecture](maas_arch.png)
 
@@ -95,7 +95,7 @@ usage: ModelSubmission
  -zr,--zk_root  Zookeeper Root
 ```
 
-##Stellar Integration
+## Stellar Integration
 
 Two Stellar functions have been added to provide the ability to call out to 
models deployed via Model as a Service.
 One aimed at recovering a load balanced endpoint of a deployed model given the 
name and, optionally, the version.

http://git-wip-us.apache.org/repos/asf/metron/blob/08f95dc8/metron-platform/metron-indexing/README.md
--
diff --git a/metron-platform/metron-indexing/README.md 
b/metron-platform/metron-indexing/README.md
index a536d87..0cf3a66 100644
--- a/metron-platform/metron-indexing/README.md
+++ b/metron-platform/metron-indexing/README.md
@@ -1,4 +1,4 @@
-#Indexing
+# Indexing
 
 ## Introduction
 
@@ -26,7 +26,7 @@ and sent to
 
 By default, errors during indexing are sent back into the `indexing` kafka 
queue so that they can be indexed and archived.
 
-##Sensor Indexing Configuration
+## Sensor Indexing Configuration
 The sensor specific configuration is intended to configure the
 indexing used for a given sensor type (e.g. `snort`).  
 

http://git-wip-us.apache.org/repos/asf/metron/blob/08f95dc8/metron-platform/metron-management/README.md
--
diff --git a/metron-platform/metron-management/README.md 
b/metron-platform/metron-management/README.md
index 0cfcac5..b920460 100644
--- a/metron-platform/metron-management/README.md
+++ b/metron-platform/metron-management/README.md
@@ -1,4 +1,4 @@
-#Stellar REPL Management Utilities
+# Stellar REPL Management Utilities
 
 In order to augment the functionality of the Stellar REPL, a few
 management functions surrounding the management of the configurations

http://git-wip-us.apache.org/repos/asf/metron/blob/08f95dc8/metron-platform/metron-parsers/README.md
--
diff --git a/metron-platform/metron-parsers/README.md 
b/metron-platform/metron-parsers/README.md
index cc45834..97e8e5f 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -1,4 +1,4 @@
-#Parsers
+# Parsers
 
 ## Introduction
 
@@ -32,7 +32,7 @@ topology in kafka.  Errors are collected with the context of 
the error
 `error` queue.  Invalid messages as determined by global validation
 functions are also treated as errors and sent to an `error` queue. 
  
-##Message Format
+## Message Format
 
 All

metron git commit: METRON-1088 Upgrade bro to 2.5.2 (JonZeolla) closes apache/metron#844

2017-11-22 Thread jonzeolla

Repository: metron
Updated Branches:
  refs/heads/master 8022f2c8c -> 59fe1b453


METRON-1088 Upgrade bro to 2.5.2 (JonZeolla) closes apache/metron#844


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/59fe1b45
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/59fe1b45
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/59fe1b45

Branch: refs/heads/master
Commit: 59fe1b453279bf5c7df627ea656c762b3a98e777
Parents: 8022f2c
Author: JonZeolla <zeo...@gmail.com>
Authored: Wed Nov 22 20:37:38 2017 -0500
Committer: JonZeolla <jonzeo...@apache.org>
Committed: Wed Nov 22 20:37:38 2017 -0500

--
 .../inventory/full-dev-platform/group_vars/all  |   2 +-
 .../inventory/quick-dev-platform/group_vars/all |   2 +-
 .../CURRENT/package/files/bro_index.template| 472 ++-
 .../playbooks/docker_probe_install.yml  |   2 +-
 metron-deployment/roles/bro/tasks/bro.yml   |   3 +
 .../roles/bro/tasks/dependencies.yml|  11 +
 .../roles/bro/tasks/metron-bro-plugin-kafka.yml |   3 +
 metron-deployment/roles/bro/vars/main.yml   |   2 +-
 .../sample/data/bro/parsed/BroExampleParsed |   4 +
 .../main/sample/data/bro/raw/BroExampleOutput   |   4 +
 .../metron/parsers/bro/BasicBroParserTest.java  | 226 +
 11 files changed, 711 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/59fe1b45/metron-deployment/inventory/full-dev-platform/group_vars/all
--
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all 
b/metron-deployment/inventory/full-dev-platform/group_vars/all
index 9aa04ab..08e405b 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -42,7 +42,7 @@ enrichment_hbase_table: enrichment
 # metron
 metron_version: 0.4.2
 metron_directory: /usr/metron/{{ metron_version }}
-bro_version: "2.4.2"
+bro_version: "2.5.2"
 fixbuf_version: "1.7.1"
 yaf_version: "2.8.0"
 daq_version: "2.0.6-1"

http://git-wip-us.apache.org/repos/asf/metron/blob/59fe1b45/metron-deployment/inventory/quick-dev-platform/group_vars/all
--
diff --git a/metron-deployment/inventory/quick-dev-platform/group_vars/all 
b/metron-deployment/inventory/quick-dev-platform/group_vars/all
index 28f235d..d2d8590 100644
--- a/metron-deployment/inventory/quick-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/quick-dev-platform/group_vars/all
@@ -41,7 +41,7 @@ enrichment_hbase_table: enrichment
 # metron
 metron_version: 0.4.2
 metron_directory: /usr/metron/{{ metron_version }}
-bro_version: "2.4.2"
+bro_version: "2.5.2"
 fixbuf_version: "1.7.1"
 yaf_version: "2.8.0"
 daq_version: "2.0.6-1"

http://git-wip-us.apache.org/repos/asf/metron/blob/59fe1b45/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
index 3a68d75..b0103f2 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -118,7 +118,7 @@
   },
   "match": "threat:triage:rules:*:name",
   "match_mapping_type": "*"
-  }
+}
   }
   ],
   "properties": {
@@ -171,6 +171,12 @@
  * 
https://www.bro.org/sphinx/scripts/base/protocols/http/main.bro.html#type-HTTP::Info
  *
  * Notable Fields
+ *   Field: method
+ *   Notes: Field exists in the HTTP and SIP logs
+ *
+ *   Field: uri
+ *   Notes: Field exists in the HTTP and SIP logs
+ *
  *   Field: password
  *   Notes: Field exists in the HTTP and FTP logs
  *
@@ -178,19 +184,31 @@
  *   Notes: Field exists in the HTTP and FTP logs
  *
  *   Field: trans_depth
- *   Notes: Field exists in the HTTP and SMTP logs
+ *   Notes: Field exists in the HTTP, SMTP, and SIP logs
  *

metron git commit: METRON-1309 Change metron-deployment to pull the plugin from apache/metron-bro-plugin-kafka (JonZeolla) closes apache/metron#837

2017-11-12 Thread jonzeolla

Repository: metron
Updated Branches:
  refs/heads/master d5b9a96b1 -> c4c930f7c


METRON-1309 Change metron-deployment to pull the plugin from 
apache/metron-bro-plugin-kafka (JonZeolla) closes apache/metron#837


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c4c930f7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c4c930f7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c4c930f7

Branch: refs/heads/master
Commit: c4c930f7c5ca16972a9edf6c072c67d7b5ac2316
Parents: d5b9a96
Author: JonZeolla <zeo...@gmail.com>
Authored: Sun Nov 12 08:13:36 2017 -0500
Committer: jonzeolla <jonzeo...@apache.org>
Committed: Sun Nov 12 08:13:36 2017 -0500

--
 .../manual-install/Manual_Install_CentOS6.md|   6 +-
 .../roles/bro/tasks/bro-plugin-kafka.yml|  44 
 metron-deployment/roles/bro/tasks/main.yml  |   2 +-
 .../roles/bro/tasks/metron-bro-plugin-kafka.yml |  47 
 metron-sensors/README.md|   2 +-
 metron-sensors/bro-plugin-kafka/.gitignore  |  31 ---
 metron-sensors/bro-plugin-kafka/CHANGES |  16 --
 metron-sensors/bro-plugin-kafka/CMakeLists.txt  |  43 
 metron-sensors/bro-plugin-kafka/COPYING | 201 ---
 metron-sensors/bro-plugin-kafka/MAINTAINER  |  18 --
 metron-sensors/bro-plugin-kafka/Makefile|  44 
 metron-sensors/bro-plugin-kafka/README.md   | 254 ---
 metron-sensors/bro-plugin-kafka/VERSION |  18 --
 .../bro-plugin-kafka/cmake/FindLibRDKafka.cmake |  51 
 .../bro-plugin-kafka/cmake/FindOpenSSL.cmake|  74 --
 metron-sensors/bro-plugin-kafka/configure   | 130 --
 .../bro-plugin-kafka/configure.plugin   |  43 
 .../scripts/Bro/Kafka/__load__.bro  |  21 --
 .../scripts/Bro/Kafka/logs-to-kafka.bro |  45 
 .../bro-plugin-kafka/scripts/__load__.bro   |  25 --
 .../bro-plugin-kafka/scripts/init.bro   |  28 --
 .../bro-plugin-kafka/src/KafkaWriter.cc | 241 --
 .../bro-plugin-kafka/src/KafkaWriter.h  |  82 --
 metron-sensors/bro-plugin-kafka/src/Plugin.cc   |  36 ---
 metron-sensors/bro-plugin-kafka/src/Plugin.h|  35 ---
 .../bro-plugin-kafka/src/TaggedJSON.cc  |  43 
 .../bro-plugin-kafka/src/TaggedJSON.h   |  50 
 metron-sensors/bro-plugin-kafka/src/kafka.bif   |  24 --
 .../bro-plugin-kafka/src/kafka_const.bif|  20 --
 metron-sensors/bro-plugin-kafka/tests/Makefile  |  19 --
 .../bro-plugin-kafka/tests/Scripts/get-bro-env  |  36 ---
 metron-sensors/bro-plugin-kafka/tests/btest.cfg |  36 ---
 .../tests/kafka/show-plugin.bro |  19 --
 pom.xml |   1 -
 34 files changed, 52 insertions(+), 1733 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/c4c930f7/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md
--
diff --git 
a/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md 
b/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md
index aeb75eb..e078a72 100644
--- a/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md
+++ b/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md
@@ -638,15 +638,15 @@ Edit crontab with `# crontab -e` and add:
 
 bro-kafka:
 ```
-# cp -r /root/metron/metron-sensors/bro-plugin-kafka /tmp
-# cd /tmp/bro-plugin-kafka
+# git clone https://github.com/apache/metron-bro-plugin-kafka 
/tmp/metron-bro-plugin-kafka
+# cd /tmp/metron-bro-plugin-kafka
 # rm -rf build/
 # ./configure --bro-dist=/tmp/bro-2.4.2 
--install-root=/usr/local/bro/lib/bro/plugins/ --with-librdkafka=/usr/local
 # make -j4
 # make install
 ```
 
-Configure bro-kafka plugin:
+Configure metron-bro-plugin-kafka:
 ```
 # cat << EOF >> /usr/local/bro/share/bro/site/local.bro
 @load Bro/Kafka/logs-to-kafka.bro

http://git-wip-us.apache.org/repos/asf/metron/blob/c4c930f7/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml
--
diff --git a/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml 
b/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml
deleted file mode 100644
index fe57ef2..000
--- a/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may n

metron-bro-plugin-kafka git commit: METRON-1303 Reorganize the metron-bro-plugin-kafka (JonZeolla) closes apache/metron-bro-plugin-kafka#1

2017-11-14 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Branches:
  refs/heads/master 25e40be1c -> c035becc1


METRON-1303 Reorganize the metron-bro-plugin-kafka (JonZeolla) closes 
apache/metron-bro-plugin-kafka#1


Project: http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/commit/c035becc
Tree: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/tree/c035becc
Diff: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/diff/c035becc

Branch: refs/heads/master
Commit: c035becc16cfa5f0b37ecdc6c09beb360955d4c0
Parents: 25e40be
Author: JonZeolla <zeo...@gmail.com>
Authored: Tue Nov 14 07:35:00 2017 -0500
Committer: jonzeolla <jonzeo...@apache.org>
Committed: Tue Nov 14 07:35:00 2017 -0500

--
 CMakeLists.txt  |  2 +-
 README.md   | 29 -
 src/Plugin.cc   |  6 +++---
 src/Plugin.h|  2 +-
 tests/kafka/show-plugin.bro |  2 +-
 5 files changed, 18 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/c035becc/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 30bf3b5..2bc8d4b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@ find_package(OpenSSL)
 
 if (LIBRDKAFKA_FOUND AND OPENSSL_FOUND)
   include_directories(BEFORE ${LibRDKafka_INCLUDE_DIR} ${OpenSSL_INCLUDE_DIR})
-  bro_plugin_begin(BRO KAFKA)
+  bro_plugin_begin(APACHE KAFKA)
   bro_plugin_cc(src/KafkaWriter.cc)
   bro_plugin_cc(src/Plugin.cc)
   bro_plugin_cc(src/TaggedJSON.cc)

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/c035becc/README.md
--
diff --git a/README.md b/README.md
index 427988b..2901e18 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,8 @@
-Logging Bro Output to Kafka
-===
+# Logging Bro Output to Kafka
 
 A Bro log writer that sends logging output to Kafka.  This provides a 
convenient means for tools in the Hadoop ecosystem, such as Storm, Spark, and 
others, to process the data generated by Bro.
 
-Installation
-
+## Installation
 
 1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native 
client library for Kafka.  This plugin has been tested against the latest 
release of librdkafka, which at the time of this writing is v0.9.4.  
 
@@ -29,12 +27,11 @@ Installation
 1. Run the following command to ensure that the plugin was installed 
successfully.
 
 ```
-$ bro -N Bro::Kafka
-Bro::Kafka - Writes logs to Kafka (dynamic, version 0.1)
+$ bro -N Apache::Kafka
+Apache::Kafka - Writes logs to Kafka (dynamic, version 0.1)
 ```
 
-Activation
---
+## Activation
 
 The following examples highlight different ways that the plugin can be used.  
Simply add the Bro script language to your `local.bro` file (for example, 
`/usr/share/bro/site/local.bro`) as shown to demonstrate the example.
 
@@ -46,7 +43,7 @@ The goal in this example is to send all HTTP and DNS records 
to a Kafka topic na
  * Defining `logs_to_send` will ensure that only HTTP and DNS records are sent.
 
 ```
-@load Bro/Kafka/logs-to-kafka.bro
+@load Apache/Kafka/logs-to-kafka.bro
 redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);
 redef Kafka::topic_name = "bro";
 redef Kafka::kafka_conf = table(
@@ -63,7 +60,7 @@ It is also possible to send each log stream to a uniquely 
named topic.  The goal
  * Each log writer accepts a separate configuration table.
 
 ```
-@load Bro/Kafka/logs-to-kafka.bro
+@load Apache/Kafka/logs-to-kafka.bro
 redef Kafka::topic_name = "";
 redef Kafka::tag_json = T;
 
@@ -101,7 +98,7 @@ You may want to configure bro to filter log messages with 
certain characteristic
  * If the log message contains a 128 byte long source or destination IP 
address, the log is not sent to kafka.
 
 ```
-@load Bro/Kafka/logs-to-kafka.bro
+@load Apache/Kafka/logs-to-kafka.bro
 redef Kafka::topic_name = "bro";
 redef Kafka::tag_json = T;
 
@@ -141,11 +138,10 @@ event bro_init() =-5
 
  Notes
  * `logs_to_send` is mutually exclusive with `$pred`, thus for each log you 
want to set `$pred` on, you must individually setup a `Log::add_filter` and 
refrain from including that log in `logs_to_send`.
- * You can also filter IPv6 logs from within your Metron cluster [using 
Stellar](../../metron-stellar/stellar-common#IS_IP).  In that case, you 
wouldn't apply a predicate in your bro configuration, and instead Stellar would 
filter the logs out before they were processed by the enrichment layer of 
Metron.
+ * You can also filter IPv6 logs from within your Metron cluster [using 
Stellar](https:

metron git commit: METRON-1313 Update metron-deployment to use bro-pkg to install the kafka plugin (JonZeolla) closes apache/metron#847

2017-12-07 Thread jonzeolla

Repository: metron
Updated Branches:
  refs/heads/master 43bbab8f5 -> 2e78df67c


METRON-1313 Update metron-deployment to use bro-pkg to install the kafka plugin 
(JonZeolla) closes apache/metron#847


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2e78df67
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2e78df67
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2e78df67

Branch: refs/heads/master
Commit: 2e78df67c12a6fcad726551128e9753ad36d5ee9
Parents: 43bbab8
Author: JonZeolla <zeo...@gmail.com>
Authored: Thu Dec 7 16:21:38 2017 -0500
Committer: jonzeolla <jonzeo...@apache.org>
Committed: Thu Dec 7 16:21:38 2017 -0500

--
 .../roles/bro/tasks/dependencies.yml| 16 +-
 .../roles/bro/tasks/metron-bro-plugin-kafka.yml | 33 ++--
 metron-deployment/roles/bro/vars/main.yml   | 10 +-
 3 files changed, 32 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/2e78df67/metron-deployment/roles/bro/tasks/dependencies.yml
--
diff --git a/metron-deployment/roles/bro/tasks/dependencies.yml 
b/metron-deployment/roles/bro/tasks/dependencies.yml
index a74557e..3cd3bae 100644
--- a/metron-deployment/roles/bro/tasks/dependencies.yml
+++ b/metron-deployment/roles/bro/tasks/dependencies.yml
@@ -15,6 +15,11 @@
 #  limitations under the License.
 #
 ---
+- name: Install yum repositories
+  yum: name={{ item }} update_cache=yes
+  with_items:
+- centos-release-scl
+
 - name: Install prerequisites
   yum: name={{ item }}
   with_items:
@@ -33,17 +38,10 @@
 - perl
 - crontabs
 - net-tools
-- centos-release-scl
-  register: result
-  until: result.rc == 0
-  retries: 5
-  delay: 10
-
-- name: Install additional prerequisites
-  yum: name={{ item }}
-  with_items:
 - devtoolset-4-gcc
 - devtoolset-4-gcc-c++
+- python27
+- rh-git29
   register: result
   until: result.rc == 0
   retries: 5

http://git-wip-us.apache.org/repos/asf/metron/blob/2e78df67/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml
--
diff --git a/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml 
b/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml
index f4575b3..7043387 100644
--- a/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml
+++ b/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml
@@ -15,36 +15,35 @@
 #  limitations under the License.
 #
 ---
-- name: Clone metron-bro-plugin-kafka into /tmp
-  git:
-repo: https://github.com/apache/metron-bro-plugin-kafka
-version: master
-dest: /tmp/metron-bro-plugin-kafka
+- name: Install bro-pkg
+  environment:
+LD_LIBRARY_PATH: "{{ python27_lib }}"
+  command: "{{ python27_bin }}/pip install bro-pkg"
+
+- name: Configure bro-pkg
+  environment:
+PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
+LD_LIBRARY_PATH: "{{ python27_lib }}"
+  command: "{{ python27_bin }}/bro-pkg autoconfig"
 
-- name: Compile and install the plugin
-  shell: "{{ item }}"
+- name: Install the metron-bro-plugin-kafka package
   environment:
+PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
+LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}"
 CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
 CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
-  args:
-chdir: "/tmp/metron-bro-plugin-kafka"
-creates: "{{ bro_home }}/lib/bro/plugins/BRO_KAFKA"
-  with_items:
-- rm -rf build/
-- "./configure --bro-dist=/tmp/bro-{{ bro_version }} --install-root={{ 
bro_home }}/lib/bro/plugins/ --with-librdkafka={{ librdkafka_home }}"
-- make
-- make install
+  command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka 
--force --version {{ metron_bro_plugin_kafka_version }}"
 
 - name: Configure bro-kafka plugin
   lineinfile:
 dest: "{{ bro_home }}/share/bro/site/local.bro"
 line: "{{ item }}"
   with_items:
-- "@load Bro/Kafka/logs-to-kafka.bro"
+- "@load packages"
 - "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);"
 - "redef Kafka::topic_name = \"{{ bro_topic }}\";"
 - "redef Kafka::tag_json = T;"
 - "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] = \"{{ 
kafka_broker_url }}\");"
 
 - name: Deploy bro configuration changes
-  shell: "{{ bro_home }}/bin/broctl deploy"
+  shell: "{{ bro_bin }}/broctl deploy"

http://git-wip-us.apache.org/r

metron-bro-plugin-kafka git commit: METRON-1329 Simplify metron-bro-plugin-kafka package loading (JonZeolla) closes apache/metron-bro-plugin-kafka#4

2017-12-07 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Branches:
  refs/heads/master 4db999e82 -> b145bf01f


METRON-1329 Simplify metron-bro-plugin-kafka package loading (JonZeolla) closes 
apache/metron-bro-plugin-kafka#4


Project: http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/commit/b145bf01
Tree: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/tree/b145bf01
Diff: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/diff/b145bf01

Branch: refs/heads/master
Commit: b145bf01f888591cbdf39ea663b34ae02254e0ea
Parents: 4db999e
Author: JonZeolla <zeo...@gmail.com>
Authored: Thu Dec 7 15:10:41 2017 -0500
Committer: jonzeolla <jonzeo...@apache.org>
Committed: Thu Dec 7 15:10:41 2017 -0500

--
 README.md  |  8 ++---
 bro-pkg.meta   |  2 +-
 scripts/Apache/Kafka/__load__.bro  | 21 ++
 scripts/Apache/Kafka/logs-to-kafka.bro | 36 +++
 scripts/Bro/Kafka/__load__.bro | 21 --
 scripts/Bro/Kafka/logs-to-kafka.bro| 45 -
 scripts/init.bro   |  1 +
 7 files changed, 63 insertions(+), 71 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/b145bf01/README.md
--
diff --git a/README.md b/README.md
index ea8d7da..e5d61e6 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ The goal in this example is to send all HTTP and DNS records 
to a Kafka topic na
  * Defining `logs_to_send` will ensure that only HTTP and DNS records are sent.
 
 ```
-@load metron-bro-plugin-kafka/Bro/Kafka
+@load packages/metron-bro-plugin-kafka/Apache/Kafka
 redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);
 redef Kafka::topic_name = "bro";
 redef Kafka::kafka_conf = table(
@@ -60,7 +60,7 @@ It is also possible to send each log stream to a uniquely 
named topic.  The goal
  * Each log writer accepts a separate configuration table.
 
 ```
-@load metron-bro-plugin-kafka/Bro/Kafka
+@load packages/metron-bro-plugin-kafka/Apache/Kafka
 redef Kafka::topic_name = "";
 redef Kafka::tag_json = T;
 
@@ -98,7 +98,7 @@ You may want to configure bro to filter log messages with 
certain characteristic
  * If the log message contains a 128 byte long source or destination IP 
address, the log is not sent to kafka.
 
 ```
-@load metron-bro-plugin-kafka/Bro/Kafka
+@load packages/metron-bro-plugin-kafka/Apache/Kafka
 redef Kafka::topic_name = "bro";
 redef Kafka::tag_json = T;
 
@@ -237,7 +237,7 @@ ${KAFKA_HOME}/kafka-broker/bin/kafka-acls.sh --authorizer 
kafka.security.auth.Si
 
 The following is how the `${BRO_HOME}/share/bro/site/local.bro` looks:
 ```
-@load metron-bro-plugin-kafka/Bro/Kafka
+@load packages/metron-bro-plugin-kafka/Apache/Kafka
 redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);
 redef Kafka::topic_name = "bro";
 redef Kafka::tag_json = T;

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/b145bf01/bro-pkg.meta
--
diff --git a/bro-pkg.meta b/bro-pkg.meta
index a31fdaf..fbed55e 100644
--- a/bro-pkg.meta
+++ b/bro-pkg.meta
@@ -1,7 +1,7 @@
 [package]
 description = A Bro log writer plugin that sends logging output to Kafka.
 tags = log writer, bro plugin, kafka
-script_dir = scripts
+script_dir = build/scripts/Apache/Kafka
 build_command = ./configure --bro-dist=%(bro_dist)s 
--with-librdkafka=%(LIBRDKAFKA_ROOT)s && make
 test_command = ( cd tests && btest -d )
 plugin_dir = build

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/b145bf01/scripts/Apache/Kafka/__load__.bro
--
diff --git a/scripts/Apache/Kafka/__load__.bro 
b/scripts/Apache/Kafka/__load__.bro
new file mode 100644
index 000..2a1efa8
--- /dev/null
+++ b/scripts/Apache/Kafka/__load__.bro
@@ -0,0 +1,21 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the Li

[metron-bro-plugin-kafka] Git Push Summary

2017-12-07 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Tags:  refs/tags/0.1 [created] 2eb7f4c9a

metron git commit: METRON-1308 Fix Metron Documentation (JonZeolla) closes apache/metron#836

2017-12-05 Thread jonzeolla

Repository: metron
Updated Branches:
  refs/heads/master e9729269d -> 252142259


METRON-1308 Fix Metron Documentation (JonZeolla) closes apache/metron#836


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/25214225
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/25214225
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/25214225

Branch: refs/heads/master
Commit: 25214225923e16b21d9fdf4ae7f522738aee2649
Parents: e972926
Author: JonZeolla <zeo...@gmail.com>
Authored: Tue Dec 5 10:18:16 2017 -0500
Committer: JonZeolla <jonzeo...@apache.org>
Committed: Tue Dec 5 10:18:16 2017 -0500

--
 metron-stellar/stellar-common/README.md | 42 ++--
 .../dsl/functions/DataStructureFunctions.java   |  2 +-
 .../dsl/functions/FunctionalFunctions.java  |  4 +-
 .../stellar/dsl/functions/SetFunctions.java |  4 +-
 4 files changed, 26 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/25214225/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md 
b/metron-stellar/stellar-common/README.md
index fd8b5d8..e5b7dac 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -153,9 +153,9 @@ Where:
 | [ `FILL_RIGHT`](#fill_right) 
  |
 | [ `FILTER`](#filter) 
  |
 | [ `FLOOR`](#floor)   
  |
+| [ `FORMAT`](#format) 
  |
 | [ `FUZZY_LANGS`](#fuzzy_langs)   
  |
 | [ `FUZZY_SCORE`](#fuzzy_score)   
  |
-| [ `FORMAT`](#format) 
  |
 | [ `GEO_GET`](#geo_get)   
  |
 | [ `GEOHASH_CENTROID`](#geohash_centroid) 
  |
 | [ `GEOHASH_DIST`](#geohash_dist) 
  |
@@ -168,10 +168,10 @@ Where:
 | [ `GET_LAST`](#get_last) 
  |
 | [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings)   
  |
 | [ `HASH`](#hash) 
  |
+| [ `HLLP_ADD`](../../metron-analytics/metron-statistics#hllp_add) 
  |
 | [ 
`HLLP_CARDINALITY`](../../metron-analytics/metron-statistics#hllp_cardinality)  
 |
 | [ `HLLP_INIT`](../../metron-analytics/metron-statistics#hllp_init)   
  |
 | [ `HLLP_MERGE`](../../metron-analytics/metron-statistics#hllp_merge) 
  |
-| [ `HLLP_OFFER`](../../metron-analytics/metron-statistics#hllp_offer) 
  |
 | [ `IN_SUBNET`](#in_subnet)   
  |
 | [ `IS_DATE`](#is_date)   
  |
 | [ `IS_ENCODING`](#is_encoding)   
  |
@@ -180,7 +180,7 @@ Where:
 | [ `IS_EMPTY`](#is_empty) 
  |
 | [ `IS_INTEGER`](#is_integer) 
  |
 | [ `IS_IP`](#is_ip)   
  |
-| [ `IS_NAN`](#is_nan) 
|
+| [ `IS_NAN`](#is_nan) 
  |
 | [ `IS_URL`](#is_url) 
  |
 | [ `JOIN`](#join) 
  |
 | [ `KAFKA_GET`](#kafka_get)   
  |
@@ -196,7 +196,7 @@ Where:
 | [ `MAAS_MODEL_APPLY`](#maas_model_apply) 
  |
 | [ `MAP`](#map)   
  |
 | [ `MAP_EXISTS`](#map_exists) 
  |
-| [ `MAP

metron git commit: METRON-1468 Add support for apache/metron-bro-plugin-kafka to prepare-commit (JonZeolla) closes apache/metron#948

2018-03-09 Thread jonzeolla

Repository: metron
Updated Branches:
  refs/heads/master 8bc7732e4 -> 03a4aa384


METRON-1468 Add support for apache/metron-bro-plugin-kafka to prepare-commit 
(JonZeolla) closes apache/metron#948


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/03a4aa38
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/03a4aa38
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/03a4aa38

Branch: refs/heads/master
Commit: 03a4aa3846773065d51086167a28fb4c37d7b2a9
Parents: 8bc7732
Author: JonZeolla <zeo...@gmail.com>
Authored: Fri Mar 9 09:05:03 2018 -0500
Committer: JonZeolla <jonzeo...@apache.org>
Committed: Fri Mar 9 09:05:03 2018 -0500

--
 dev-utilities/committer-utils/README.md  | 75 ---
 dev-utilities/committer-utils/prepare-commit | 50 +++
 2 files changed, 75 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/03a4aa38/dev-utilities/committer-utils/README.md
--
diff --git a/dev-utilities/committer-utils/README.md 
b/dev-utilities/committer-utils/README.md
index 60e1d51..ac428cc 100644
--- a/dev-utilities/committer-utils/README.md
+++ b/dev-utilities/committer-utils/README.md
@@ -22,7 +22,7 @@ This project contains tools to assist Apache Metron project 
committers.
 
 ## Prepare Commit
 
-This script automates the process of merging a pull request into 
`apache/master`.  The script will prompt for the pull request number.  Most of 
the remaining information is automatically extracted from Github or the Apache 
JIRA.
+This script automates the process of merging a pull request into 
`apache/metron` or `apache/metron-bro-plugin-kafka`.  The script will prompt 
for the pull request number.  Most of the remaining information is 
automatically extracted from Github or the Apache JIRA.
 
 When prompted the `[value in brackets]` is used by default.  To accept the 
default, simply press `enter`.  If you would like to change the default, type 
it in and hit `enter` when done.
 
@@ -33,36 +33,39 @@ In the following example, I enter the pull request number 
when prompted.   Using
 The first time the script is run, you will be prompted for additional 
information including your Apache username, Apache email, and Github username.  
These values are persisted in `~/.metron-prepare-commit`.  Subsequent 
executions of the script will retrieve these values, rather than prompting you 
again for them.
 
 ```
-$ prepare-commit
-  your github username []: nickwallen
-  your apache userid []: nickallen
-  your apache email [nickal...@apache.org]:
+$ ./prepare-commit
+  your github username []: jonzeolla
+  your apache userid []: jonzeolla
+  your apache email [jonzeo...@apache.org]:
 ```
 
-1. Enter the Github pull request number.
+1. Select a repository and enter a GitHub pull request number.
 
 ```
-  pull request: 897
-  local working directory [/Users/nallen/tmp/metron-pr897]:
+[1] metron
+[2] metron-bro-plugin-kafka
+  which repo? [1]: 1
+  pull request: 946
+  local working directory [/Users/jzeolla/tmp/metron-pr946]:
   origin repo [https://github.com/apache/metron]:
-
-Cloning into '/Users/nallen/tmp/metron-pr897'...
-remote: Counting objects: 36277, done.
-remote: Compressing objects: 100% (108/108), done.
-remote: Total 36277 (delta 38), reused 54 (delta 20), pack-reused 36138
-Receiving objects: 100% (36277/36277), 57.85 MiB | 7.36 MiB/s, done.
-Resolving deltas: 100% (13653/13653), done.
+Cloning into '/Users/jzeolla/tmp/metron-pr946'...
+remote: Counting objects: 37861, done.
+remote: Compressing objects: 100% (71/71), done.
+remote: Total 37861 (delta 27), reused 47 (delta 4), pack-reused 37757
+Receiving objects: 100% (37861/37861), 58.18 MiB | 4.38 MiB/s, done.
+Resolving deltas: 100% (14439/14439), done.
 From https://git-wip-us.apache.org/repos/asf/metron
  * branch  master -> FETCH_HEAD
  * [new branch]master -> upstream/master
 Already on 'master'
 Your branch is up to date with 'origin/master'.
 Already up to date.
-remote: Counting objects: 5, done.
-remote: Total 5 (delta 3), reused 3 (delta 3), pack-reused 2
-Unpacking objects: 100% (5/5), done.
+remote: Counting objects: 82, done.
+remote: Compressing objects: 100% (22/22), done.
+remote: Total 82 (delta 28), reused 48 (delta 28), pack-reused 26
+Unpacking objects: 100% (82/82), done.
 From https://github.com/apache/metron
- * [new ref]   refs/pull/897/head -> pr-897
+ * [new ref]   refs/pull/946/head -> pr-946
 ```
 
 1. Enter cont

metron-bro-plugin-kafka git commit: METRON-1324 Increment metron-bro-plugin-kafka version (JonZeolla) closes apache/metron-bro-plugin-kafka#7

2018-03-09 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Branches:
  refs/heads/master b145bf01f -> 4c04fbe13


METRON-1324 Increment metron-bro-plugin-kafka version (JonZeolla) closes 
apache/metron-bro-plugin-kafka#7


Project: http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/commit/4c04fbe1
Tree: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/tree/4c04fbe1
Diff: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/diff/4c04fbe1

Branch: refs/heads/master
Commit: 4c04fbe13263662ef5f4bc842a50d69558f14dde
Parents: b145bf0
Author: JonZeolla <zeo...@gmail.com>
Authored: Fri Mar 9 09:10:12 2018 -0500
Committer: jonzeolla <jonzeo...@apache.org>
Committed: Fri Mar 9 09:10:12 2018 -0500

--
 README.md   | 2 +-
 VERSION | 2 +-
 bro-pkg.meta| 2 +-
 src/Plugin.cc   | 2 +-
 tests/Baseline/kafka.show-plugin/output | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/4c04fbe1/README.md
--
diff --git a/README.md b/README.md
index e5d61e6..0c67347 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,7 @@ A Bro log writer that sends logging output to Kafka.  This 
provides a convenient
 
 ```
 $ bro -N Apache::Kafka
-Apache::Kafka - Writes logs to Kafka (dynamic, version 0.1)
+Apache::Kafka - Writes logs to Kafka (dynamic, version 0.2)
 ```
 
 ## Activation

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/4c04fbe1/VERSION
--
diff --git a/VERSION b/VERSION
index 204e5ca..e59aa6f 100644
--- a/VERSION
+++ b/VERSION
@@ -15,4 +15,4 @@
 #  limitations under the License.
 #
 
-0.1
+0.2

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/4c04fbe1/bro-pkg.meta
--
diff --git a/bro-pkg.meta b/bro-pkg.meta
index fbed55e..3d0cfa0 100644
--- a/bro-pkg.meta
+++ b/bro-pkg.meta
@@ -6,7 +6,7 @@ build_command = ./configure --bro-dist=%(bro_dist)s 
--with-librdkafka=%(LIBRDKAF
 test_command = ( cd tests && btest -d )
 plugin_dir = build
 config_files = scripts/init.bro
-version = 0.1
+version = 0.2
 depends =
   bro >=2.5.0
   bro-pkg >=1.2

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/4c04fbe1/src/Plugin.cc
--
diff --git a/src/Plugin.cc b/src/Plugin.cc
index f2f63a5..ed0cb39 100644
--- a/src/Plugin.cc
+++ b/src/Plugin.cc
@@ -31,6 +31,6 @@ plugin::Configuration Plugin::Configure()
 config.name = "Apache::Kafka";
 config.description = "Writes logs to Kafka";
 config.version.major = 0;
-config.version.minor = 1;
+config.version.minor = 2;
 return config;
 }

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/4c04fbe1/tests/Baseline/kafka.show-plugin/output
--
diff --git a/tests/Baseline/kafka.show-plugin/output 
b/tests/Baseline/kafka.show-plugin/output
index d8b215d..1094ebb 100644
--- a/tests/Baseline/kafka.show-plugin/output
+++ b/tests/Baseline/kafka.show-plugin/output
@@ -1,4 +1,4 @@
-Apache::Kafka - Writes logs to Kafka (dynamic, version 0.1)
+Apache::Kafka - Writes logs to Kafka (dynamic, version 0.2)
 [Writer] KafkaWriter (Log::WRITER_KAFKAWRITER)
 [Constant] Kafka::kafka_conf
 [Constant] Kafka::topic_name

metron-bro-plugin-kafka git commit: METRON-1800 Increment metron-bro-plugin-kafka version (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#10

2018-10-11 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Branches:
  refs/heads/master 38f45e2bb -> d1a09b6a5


METRON-1800 Increment metron-bro-plugin-kafka version (JonZeolla via jonzeolla) 
closes apache/metron-bro-plugin-kafka#10


Project: http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/commit/d1a09b6a
Tree: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/tree/d1a09b6a
Diff: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/diff/d1a09b6a

Branch: refs/heads/master
Commit: d1a09b6a50f20e5fa6cf5c758eea8b0d39ce65be
Parents: 38f45e2
Author: JonZeolla 
Authored: Thu Oct 11 07:20:32 2018 -0400
Committer: jonzeolla 
Committed: Thu Oct 11 07:20:32 2018 -0400

--
 README.md   | 2 +-
 VERSION | 2 +-
 bro-pkg.meta| 2 +-
 src/Plugin.cc   | 2 +-
 tests/Baseline/kafka.show-plugin/output | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/d1a09b6a/README.md
--
diff --git a/README.md b/README.md
index bc2faf4..91b6e1b 100644
--- a/README.md
+++ b/README.md
@@ -35,7 +35,7 @@ This software is a part of the [Apache 
Metron](http://metron.apache.org/) projec
 
 ```
 $ bro -N Apache::Kafka
-Apache::Kafka - Writes logs to Kafka (dynamic, version 0.2)
+Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3)
 ```
 
 ## Activation

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/d1a09b6a/VERSION
--
diff --git a/VERSION b/VERSION
index e59aa6f..c30be1e 100644
--- a/VERSION
+++ b/VERSION
@@ -15,4 +15,4 @@
 #  limitations under the License.
 #
 
-0.2
+0.3

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/d1a09b6a/bro-pkg.meta
--
diff --git a/bro-pkg.meta b/bro-pkg.meta
index 3d0cfa0..095ed5b 100644
--- a/bro-pkg.meta
+++ b/bro-pkg.meta
@@ -6,7 +6,7 @@ build_command = ./configure --bro-dist=%(bro_dist)s 
--with-librdkafka=%(LIBRDKAF
 test_command = ( cd tests && btest -d )
 plugin_dir = build
 config_files = scripts/init.bro
-version = 0.2
+version = 0.3
 depends =
   bro >=2.5.0
   bro-pkg >=1.2

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/d1a09b6a/src/Plugin.cc
--
diff --git a/src/Plugin.cc b/src/Plugin.cc
index ed0cb39..a981d22 100644
--- a/src/Plugin.cc
+++ b/src/Plugin.cc
@@ -31,6 +31,6 @@ plugin::Configuration Plugin::Configure()
 config.name = "Apache::Kafka";
 config.description = "Writes logs to Kafka";
 config.version.major = 0;
-config.version.minor = 2;
+config.version.minor = 3;
 return config;
 }

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/d1a09b6a/tests/Baseline/kafka.show-plugin/output
--
diff --git a/tests/Baseline/kafka.show-plugin/output 
b/tests/Baseline/kafka.show-plugin/output
index 1094ebb..814421c 100644
--- a/tests/Baseline/kafka.show-plugin/output
+++ b/tests/Baseline/kafka.show-plugin/output
@@ -1,4 +1,4 @@
-Apache::Kafka - Writes logs to Kafka (dynamic, version 0.2)
+Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3)
 [Writer] KafkaWriter (Log::WRITER_KAFKAWRITER)
 [Constant] Kafka::kafka_conf
 [Constant] Kafka::topic_name

metron git commit: METRON-1821 Align prepare-release-candidate with documentation (JonZeolla via jonzeolla) closes apache/metron#1236

2018-10-15 Thread jonzeolla

Repository: metron
Updated Branches:
  refs/heads/master 90c5e1d28 -> 647896604


METRON-1821 Align prepare-release-candidate with documentation (JonZeolla via 
jonzeolla) closes apache/metron#1236


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/64789660
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/64789660
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/64789660

Branch: refs/heads/master
Commit: 647896604f938ca195bf25885651f9be3e897c71
Parents: 90c5e1d
Author: JonZeolla 
Authored: Mon Oct 15 09:42:44 2018 -0400
Committer: jonzeolla 
Committed: Mon Oct 15 09:42:44 2018 -0400

--
 .../release-utils/prepare-release-candidate | 23 +---
 1 file changed, 10 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/64789660/dev-utilities/release-utils/prepare-release-candidate
--
diff --git a/dev-utilities/release-utils/prepare-release-candidate 
b/dev-utilities/release-utils/prepare-release-candidate
index bedee9b..635285c 100755
--- a/dev-utilities/release-utils/prepare-release-candidate
+++ b/dev-utilities/release-utils/prepare-release-candidate
@@ -33,9 +33,10 @@ 
BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_N
 
 DEV_REPO="https://dist.apache.org/repos/dist/dev/metron;
 RELEASE_REPO=" https://dist.apache.org/repos/dist/release/metron;
-PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git;
+PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_NAME}.git;
 
 RC_PREFIX=rc
+UPPER_RC_PREFIX="$(tr '[:lower:]' '[:upper:]' <<< ${RC_PREFIX})"
 TAG_POSTFIX="-release"
 
 CONFIG_FILE=~/.metron-prepare-release-candidate
@@ -223,33 +224,29 @@ if [ "${PRACTICE_RUN}" = true ]; then
   printf "This is a practice run. Not running \n" "${CAPITAL_REPO}" "$VERSION"
 else
   printf "Pushing branch %s_%s\n" "${CAPITAL_REPO}" "$VERSION"
-  git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}"
+  git push --set-upstream origin "${CAPITAL_REPO}_${VERSION}"
 fi
 
 # Create directory for release artifacts
 if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
-  ART_DIR="$WORKDIR/dev/metron/${VERSION}-RC${RC_NUM}"
+  ART_DIR="$WORKDIR/dev/metron/${VERSION}-${UPPER_RC_PREFIX}${RC_NUM}"
 else
   # We're using a sub module, so put it in it's own directory.
-  ART_DIR="$WORKDIR/dev/metron/${CHOSEN_REPO}/${VERSION}-RC${RC_NUM}"
+  
ART_DIR="$WORKDIR/dev/metron/${CHOSEN_REPO}/${VERSION}-${UPPER_RC_PREFIX}${RC_NUM}"
 fi
 mkdir -p "$ART_DIR"
 
 # Setup various parameters we need for the release artifacts
 if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
-  CORE_PREFIX="apache-metron-"
+  CORE_PREFIX="apache-metron_"
   ARTIFACT_PREFIX="${CORE_PREFIX}"
   TAG_VERSION="${CURRENT_VERSION}"
   TAG="${CORE_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
 elif [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then
-  PLUGIN_PREFIX="apache-metron-bro-plugin-kafka_"
-  ARTIFACT_PREFIX="${PLUGIN_PREFIX}"
+  BRO_PLUGIN_PREFIX="apache-metron-bro-plugin-kafka_"
+  ARTIFACT_PREFIX="${BRO_PLUGIN_PREFIX}"
   TAG_VERSION="${CURRENT_VERSION}"
-  TAG="${PLUGIN_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
-  # Handle special tag case from prior release
-  if [ "${TAG_VERSION}" = "0.1" ]; then
-TAG="0.1"
-  fi
+  TAG="${BRO_PLUGIN_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
 else
   # If we ever add new modules, add them as needed.
   printf "Unrecognized module: %s\n" "${CHOSEN_REPO}"
@@ -324,7 +321,7 @@ if [ "${PRACTICE_RUN}" = true ]; then
   printf "\n" ${COMMIT_DIR}
   printf "\n" "${CHOSEN_REPO}" 
"${COMMIT_DIR}"
 else
-  printf "Adding artifacts for Metron ${VERSION}-RC${RC_NUM} to dev SVN\n"
+  printf "Adding artifacts for Metron ${VERSION}-${UPPER_RC_PREFIX}${RC_NUM} 
to dev SVN\n"
   # Metron goes in the root of the dir, submodules go in folder
   if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
 cd "$WORKDIR/dev/metron/"

metron-bro-plugin-kafka git commit: METRON-1818 Remove config_files from bro-pkg.meta (JonZeolla) closes apache/metron-bro-plugin-kafka#11

2018-10-16 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Branches:
  refs/heads/master d1a09b6a5 -> 0fd459454


METRON-1818 Remove config_files from bro-pkg.meta (JonZeolla) closes 
apache/metron-bro-plugin-kafka#11


Project: http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/commit/0fd45945
Tree: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/tree/0fd45945
Diff: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/diff/0fd45945

Branch: refs/heads/master
Commit: 0fd45945490a50960cb9b841e1a9c943759156f9
Parents: d1a09b6
Author: JonZeolla 
Authored: Tue Oct 16 12:47:50 2018 -0400
Committer: jonzeolla 
Committed: Tue Oct 16 12:47:50 2018 -0400

--
 bro-pkg.meta | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/0fd45945/bro-pkg.meta
--
diff --git a/bro-pkg.meta b/bro-pkg.meta
index 095ed5b..8bde2cc 100644
--- a/bro-pkg.meta
+++ b/bro-pkg.meta
@@ -5,7 +5,6 @@ script_dir = build/scripts/Apache/Kafka
 build_command = ./configure --bro-dist=%(bro_dist)s 
--with-librdkafka=%(LIBRDKAFKA_ROOT)s && make
 test_command = ( cd tests && btest -d )
 plugin_dir = build
-config_files = scripts/init.bro
 version = 0.3
 depends =
   bro >=2.5.0

metron-bro-plugin-kafka git commit: METRON-1828 Improve bro plugin contributing documentation (JonZeolla) closes apache/metron-bro-plugin-kafka#14

2018-10-16 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Branches:
  refs/heads/master 0fd459454 -> 0883c53e0


METRON-1828 Improve bro plugin contributing documentation (JonZeolla) closes 
apache/metron-bro-plugin-kafka#14


Project: http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/commit/0883c53e
Tree: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/tree/0883c53e
Diff: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/diff/0883c53e

Branch: refs/heads/master
Commit: 0883c53e09214c9533fc7dc96c9f33cc1a1202ec
Parents: 0fd4594
Author: JonZeolla 
Authored: Tue Oct 16 12:48:32 2018 -0400
Committer: jonzeolla 
Committed: Tue Oct 16 12:48:32 2018 -0400

--
 .github/PULL_REQUEST_TEMPLATE.md | 26 ++
 README.md|  6 ++
 2 files changed, 32 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/0883c53e/.github/PULL_REQUEST_TEMPLATE.md
--
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 000..c001a7b
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,26 @@
+## Contributor Comments
+[Please place any comments here.  A description of the problem/enhancement, 
how to reproduce the issue, your testing methodology, etc.]
+
+
+## Pull Request Checklist
+
+Thank you for submitting a contribution to Apache Metron's Bro kafka writer 
plugin.
+
+In order to streamline the review of the contribution we ask you follow these 
guidelines and ask you to double check the following:
+
+### For all changes:
+- [ ] Is there a JIRA ticket associated with this PR? If not one needs to be 
created at [Metron 
Jira](https://issues.apache.org/jira/browse/METRON/?selectedTab=com.atlassian.jira.jira-projects-plugin:summary-panel).
+- [ ] Does your PR title start with METRON- where  is the JIRA number 
you are trying to resolve? Pay particular attention to the hyphen "-" character.
+- [ ] Has your PR been rebased against the latest commit within the target 
branch (typically master)?
+
+### For code changes:
+- [ ] Have you included steps to reproduce the behavior or problem that is 
being changed or addressed?
+- [ ] Have you included steps or a guide to how the change may be verified and 
tested manually?
+- [ ] Have you ensured that the full suite of tests and checks have been 
executed via:
+  ```
+  bro-pkg test $GITHUB_USERNAME/metron-bro-plugin-kafka --version $BRANCH
+  ```
+- [ ] Have you written or updated unit tests and or integration tests to 
verify your changes?
+- [ ] If adding new dependencies to the code, are these dependencies licensed 
in a way that is compatible for inclusion under [ASF 
2.0](http://www.apache.org/legal/resolved.html#category-a)?
+- [ ] Have you verified the basic functionality of the build by building and 
running locally with Apache Metron's [Vagrant full-dev 
environment](https://github.com/apache/metron/tree/master/metron-deployment/development/centos6)
 or the equivalent?
+

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/0883c53e/README.md
--
diff --git a/README.md b/README.md
index 91b6e1b..90d8444 100644
--- a/README.md
+++ b/README.md
@@ -8,6 +8,7 @@ This software is a part of the [Apache 
Metron](http://metron.apache.org/) projec
 * [Activation](#activation)
 * [Settings](#settings)
 * [Kerberos](#kerberos)
+* [Contributing](#contributing)
 
 ## Installation
 
@@ -262,3 +263,8 @@ redef Kafka::kafka_conf = table( ["metadata.broker.list"] = 
"node1:6667"
, ["sasl.kerberos.principal"] = 
"met...@example.com"
);
 ```
+
+## Contributing
+
+If you are interested in contributing to this plugin, please see the Apache 
Metron 
[CONTRIBUTING.md](https://github.com/apache/metron/blob/master/CONTRIBUTING.md).
+

metron git commit: METRON-1826 Update librdkafka and devtoolset (JonZeolla via jonzeolla) closes apache/metron#1238

2018-10-22 Thread jonzeolla

Repository: metron
Updated Branches:
  refs/heads/master 739782ab7 -> 0ba180262


METRON-1826 Update librdkafka and devtoolset (JonZeolla via jonzeolla) closes 
apache/metron#1238


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0ba18026
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0ba18026
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0ba18026

Branch: refs/heads/master
Commit: 0ba18026222f004316b96633780509c71934121e
Parents: 739782a
Author: JonZeolla 
Authored: Mon Oct 22 19:48:56 2018 -0400
Committer: jonzeolla 
Committed: Mon Oct 22 19:48:56 2018 -0400

--
 .../ansible/roles/bro/tasks/bro.yml |  4 +--
 .../ansible/roles/bro/tasks/dependencies.yml|  4 +--
 .../roles/bro/tasks/metron-bro-plugin-kafka.yml |  4 +--
 .../ansible/roles/librdkafka/defaults/main.yml  |  4 +--
 metron-deployment/development/centos6/README.md | 26 ++--
 .../development/ubuntu14/README.md  | 26 ++--
 6 files changed, 56 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/bro/tasks/bro.yml
--
diff --git a/metron-deployment/ansible/roles/bro/tasks/bro.yml 
b/metron-deployment/ansible/roles/bro/tasks/bro.yml
index 222ef0e..9ba3ffa 100644
--- a/metron-deployment/ansible/roles/bro/tasks/bro.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/bro.yml
@@ -30,8 +30,8 @@
 - name: Compile and Install bro
   shell: "{{ item }}"
   environment:
-CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
-CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
+CXX: /opt/rh/devtoolset-6/root/usr/bin/g++
+CC: /opt/rh/devtoolset-6/root/usr/bin/gcc
   args:
 chdir: "/tmp/bro-{{ bro_version }}"
 creates: "{{ bro_home }}/bin/bro"

http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
--
diff --git a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml 
b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
index 3cd3bae..f8b1e28 100644
--- a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
@@ -38,8 +38,8 @@
 - perl
 - crontabs
 - net-tools
-- devtoolset-4-gcc
-- devtoolset-4-gcc-c++
+- devtoolset-6-gcc
+- devtoolset-6-gcc-c++
 - python27
 - rh-git29
   register: result

http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
--
diff --git 
a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml 
b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
index 7043387..ec7af97 100644
--- a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
@@ -30,8 +30,8 @@
   environment:
 PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
 LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}"
-CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
-CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
+CXX: /opt/rh/devtoolset-6/root/usr/bin/g++
+CC: /opt/rh/devtoolset-6/root/usr/bin/gcc
   command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka 
--force --version {{ metron_bro_plugin_kafka_version }}"
 
 - name: Configure bro-kafka plugin

http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
--
diff --git a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml 
b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
index 063c22f..6e25cb3 100644
--- a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
+++ b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
@@ -15,6 +15,6 @@
 #  limitations under the License.
 #
 ---
-librdkafka_version: 0.9.4
-librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz
+librdkafka_version: 0.11.5
+librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz
 librdkafka_home: /usr/local

http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/development/centos6/README.md
--
diff --git a/metron-deployment/development/centos6/README.md 
b/metron-deployment/development/centos6/README.md
index e873452..99ec967 100

metron-bro-plugin-kafka git commit: METRON-1865 Fix metron-bro-plugin-kafka tests (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#16

2018-11-09 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Branches:
  refs/heads/master 0883c53e0 -> 279a2970b


METRON-1865 Fix metron-bro-plugin-kafka tests (JonZeolla via jonzeolla) closes 
apache/metron-bro-plugin-kafka#16


Project: http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/commit/279a2970
Tree: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/tree/279a2970
Diff: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/diff/279a2970

Branch: refs/heads/master
Commit: 279a2970b6a7b8fd7d91036b0b07798d03dbda71
Parents: 0883c53
Author: JonZeolla 
Authored: Fri Nov 9 20:17:12 2018 -0500
Committer: jonzeolla 
Committed: Fri Nov 9 20:17:12 2018 -0500

--
 tests/Baseline/kafka.show-plugin/output | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/279a2970/tests/Baseline/kafka.show-plugin/output
--
diff --git a/tests/Baseline/kafka.show-plugin/output 
b/tests/Baseline/kafka.show-plugin/output
index 814421c..b9ebb05 100644
--- a/tests/Baseline/kafka.show-plugin/output
+++ b/tests/Baseline/kafka.show-plugin/output
@@ -4,5 +4,6 @@ Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3)
 [Constant] Kafka::topic_name
 [Constant] Kafka::max_wait_on_shutdown
 [Constant] Kafka::tag_json
+[Constant] Kafka::json_timestamps
 [Constant] Kafka::debug

metron-bro-plugin-kafka git commit: METRON-1866 Improve metron-bro-plugin-kafka documentation (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#17 [Forced Update!]

2018-11-10 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Branches:
  refs/heads/master 37f05f8f4 -> 8d0400fef (forced update)


METRON-1866 Improve metron-bro-plugin-kafka documentation (JonZeolla via 
jonzeolla) closes apache/metron-bro-plugin-kafka#17


Project: http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/commit/8d0400fe
Tree: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/tree/8d0400fe
Diff: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/diff/8d0400fe

Branch: refs/heads/master
Commit: 8d0400fef9488908e46e4a0922e01d19f52b04c2
Parents: 772788d
Author: JonZeolla 
Authored: Sat Nov 10 14:29:03 2018 -0500
Committer: Jon Zeolla 
Committed: Sat Nov 10 14:52:35 2018 -0500

--
 README.md | 98 +-
 1 file changed, 76 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/8d0400fe/README.md
--
diff --git a/README.md b/README.md
index ae6b260..ee21a7e 100644
--- a/README.md
+++ b/README.md
@@ -12,24 +12,79 @@ This software is a part of the [Apache 
Metron](http://metron.apache.org/) projec
 
 ## Installation
 
+### `bro-pkg` Installation
+
+`bro-pkg` is the preferred mechanism for installing this plugin, as it will 
dynamically retrieve, build, test, and load the plugin.  Note, that you will 
still need to [activate](#activation) and configure the plugin after your 
installation.
+
+1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native 
client library for Kafka.  This plugin has been tested against the latest 
release of librdkafka, which at the time of this writing is v0.9.4.
+
+In order to use this plugin within a kerberized Kafka environment, you 
will also need `libsasl2` installed and will need to pass `--enable-sasl` to 
the `configure` script.
+
+```
+$ curl -L https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz | 
tar xvz
+$ cd librdkafka-0.9.4/
+$ ./configure --enable-sasl
+$ make
+$ sudo make install
+```
+
+1. Configure `bro-pkg` by following the quickstart guide 
[here](https://bro-package-manager.readthedocs.io/en/stable/quickstart.html).
+
+1. Install the plugin using `bro-pkg install`.
+
+```
+$ bro-pkg install apache/metron-bro-plugin-kafka --version master
+The following packages will be INSTALLED:
+  bro/apache/metron-bro-plugin-kafka (master)
+
+Verify the following REQUIRED external dependencies:
+(Ensure their installation on all relevant systems before proceeding):
+  from bro/apache/metron-bro-plugin-kafka (master):
+librdkafka ~0.9.4
+
+Proceed? [Y/n]
+bro/apache/metron-bro-plugin-kafka asks for LIBRDKAFKA_ROOT (Path to 
librdkafka installation tree) ? [/usr/local/lib]
+Saved answers to config file: /home/jonzeolla/.bro-pkg/config
+Running unit tests for "bro/apache/metron-bro-plugin-kafka"
+all 10 tests successful
+
+
+Installing "bro/apache/metron-bro-plugin-kafka"
+Installed "bro/apache/metron-bro-plugin-kafka" (master)
+Loaded "bro/apache/metron-bro-plugin-kafka"
+```
+
+1. Run the following command to ensure that the plugin was installed 
successfully.
+
+```
+$ bro -N Apache::Kafka
+Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3)
+```
+
+### Manual Installation
+
+Manually installing the plugin should only occur in situations where 
installing and configuring `bro-pkg` is not reasonable, such as in a docker 
container.  If you are running bro in an environment where you do not have 
Internet connectivity, investigate 
[bundles](https://bro-package-manager.readthedocs.io/en/stable/bro-pkg.html#bundle)
 or creating an internal [package 
source](https://bro-package-manager.readthedocs.io/en/stable/source.html).
+
+These instructions could also be helpful if you were interested in 
distributing this as a package (such as a deb or rpm).
+
 1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native 
client library for Kafka.  This plugin has been tested against the latest 
release of librdkafka, which at the time of this writing is v0.9.4.  
 
 In order to use this plugin within a kerberized Kafka environment, you 
will also need `libsasl2` installed and will need to pass `--enable-sasl` to 
the `configure` script.
 
 ```
-curl -L https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz | tar 
xvz
-cd librdkafka-0.9.4/
-./configure --enable-sasl
-make
-sudo make install
+$ curl -L https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz | 
tar xvz
+$ cd librdkafka-0.9.4/
+$ ./configure --enabl

metron-bro-plugin-kafka git commit: METRON-1866 Improve metron-bro-plugin-kafka documentation (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#17

2018-11-10 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Branches:
  refs/heads/master 772788d1a -> 37f05f8f4


METRON-1866 Improve metron-bro-plugin-kafka documentation (JonZeolla via 
jonzeolla) closes apache/metron-bro-plugin-kafka#17


Project: http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/commit/37f05f8f
Tree: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/tree/37f05f8f
Diff: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/diff/37f05f8f

Branch: refs/heads/master
Commit: 37f05f8f4aee431ef61eed19444665ea344d337c
Parents: 772788d
Author: JonZeolla 
Authored: Sat Nov 10 14:29:03 2018 -0500
Committer: jonzeolla 
Committed: Sat Nov 10 14:29:03 2018 -0500

--
 README.md | 116 -
 1 file changed, 98 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/37f05f8f/README.md
--
diff --git a/README.md b/README.md
index ae6b260..595fff4 100644
--- a/README.md
+++ b/README.md
@@ -12,24 +12,79 @@ This software is a part of the [Apache 
Metron](http://metron.apache.org/) projec
 
 ## Installation
 
+### `bro-pkg` Installation
+
+`bro-pkg` is the preferred mechanism for installing this plugin, as it will 
dynamically retrieve, build, test, and load the plugin.  Note, that you will 
still need to [activate](#activation) and configure the plugin after your 
installation.
+
+1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native 
client library for Kafka.  This plugin has been tested against the latest 
release of librdkafka, which at the time of this writing is v0.9.4.
+
+In order to use this plugin within a kerberized Kafka environment, you 
will also need `libsasl2` installed and will need to pass `--enable-sasl` to 
the `configure` script.
+
+```
+$ curl -L https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz | 
tar xvz
+$ cd librdkafka-0.9.4/
+$ ./configure --enable-sasl
+$ make
+$ sudo make install
+```
+
+1. Configure `bro-pkg` by following the quickstart guide 
[here](https://bro-package-manager.readthedocs.io/en/stable/quickstart.html).
+
+1. Install the plugin using `bro-pkg install`.
+
+```
+$ bro-pkg install apache/metron-bro-plugin-kafka --version master
+The following packages will be INSTALLED:
+  bro/apache/metron-bro-plugin-kafka (master)
+
+Verify the following REQUIRED external dependencies:
+(Ensure their installation on all relevant systems before proceeding):
+  from bro/apache/metron-bro-plugin-kafka (master):
+librdkafka ~0.9.4
+
+Proceed? [Y/n]
+bro/apache/metron-bro-plugin-kafka asks for LIBRDKAFKA_ROOT (Path to 
librdkafka installation tree) ? [/usr/local/lib]
+Saved answers to config file: /home/jonzeolla/.bro-pkg/config
+Running unit tests for "bro/apache/metron-bro-plugin-kafka"
+all 10 tests successful
+
+
+Installing "bro/apache/metron-bro-plugin-kafka"
+Installed "bro/apache/metron-bro-plugin-kafka" (master)
+Loaded "bro/apache/metron-bro-plugin-kafka"
+```
+
+1. Run the following command to ensure that the plugin was installed 
successfully.
+
+```
+$ bro -N Apache::Kafka
+Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3)
+```
+
+### Manual Installation
+
+Manually installing the plugin should only occur in situations where 
installing and configuring `bro-pkg` is not reasonable, such as in a docker 
container.  If you are running bro in an environment where you do not have 
Internet connectivity, investigate 
[bundles](https://bro-package-manager.readthedocs.io/en/stable/bro-pkg.html#bundle)
 or creating an internal [package 
source](https://bro-package-manager.readthedocs.io/en/stable/source.html).
+
+These instructions could also be helpful if you were interested in 
distributing this as a package (such as a deb or rpm).
+
 1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native 
client library for Kafka.  This plugin has been tested against the latest 
release of librdkafka, which at the time of this writing is v0.9.4.  
 
 In order to use this plugin within a kerberized Kafka environment, you 
will also need `libsasl2` installed and will need to pass `--enable-sasl` to 
the `configure` script.
 
 ```
-curl -L https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz | tar 
xvz
-cd librdkafka-0.9.4/
-./configure --enable-sasl
-make
-sudo make install
+$ curl -L https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz | 
tar xvz
+$ cd librdkafka-0.9.4/
+$ ./configure --enable-sasl
+$ make
+

metron-bro-plugin-kafka git commit: METRON-1827 Update librdkafka in metron-bro-plugin-kafka (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#13

2018-11-14 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Branches:
  refs/heads/master 8d0400fef -> 177430ec5


METRON-1827 Update librdkafka in metron-bro-plugin-kafka (JonZeolla via 
jonzeolla) closes apache/metron-bro-plugin-kafka#13


Project: http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/commit/177430ec
Tree: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/tree/177430ec
Diff: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/diff/177430ec

Branch: refs/heads/master
Commit: 177430ec58bae9ca2c812ff2247dcba0a109f6b8
Parents: 8d0400f
Author: JonZeolla 
Authored: Wed Nov 14 09:09:38 2018 -0500
Committer: jonzeolla 
Committed: Wed Nov 14 09:09:38 2018 -0500

--
 README.md| 16 
 bro-pkg.meta |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/177430ec/README.md
--
diff --git a/README.md b/README.md
index ee21a7e..7cc2c46 100644
--- a/README.md
+++ b/README.md
@@ -16,13 +16,13 @@ This software is a part of the [Apache 
Metron](http://metron.apache.org/) projec
 
 `bro-pkg` is the preferred mechanism for installing this plugin, as it will 
dynamically retrieve, build, test, and load the plugin.  Note, that you will 
still need to [activate](#activation) and configure the plugin after your 
installation.
 
-1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native 
client library for Kafka.  This plugin has been tested against the latest 
release of librdkafka, which at the time of this writing is v0.9.4.
+1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native 
client library for Kafka.  This plugin has been tested against the latest 
release of librdkafka, which at the time of this writing is v0.11.5.
 
 In order to use this plugin within a kerberized Kafka environment, you 
will also need `libsasl2` installed and will need to pass `--enable-sasl` to 
the `configure` script.
 
 ```
-$ curl -L https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz | 
tar xvz
-$ cd librdkafka-0.9.4/
+$ curl -L https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz | 
tar xvz
+$ cd librdkafka-0.11.5/
 $ ./configure --enable-sasl
 $ make
 $ sudo make install
@@ -40,7 +40,7 @@ This software is a part of the [Apache 
Metron](http://metron.apache.org/) projec
 Verify the following REQUIRED external dependencies:
 (Ensure their installation on all relevant systems before proceeding):
   from bro/apache/metron-bro-plugin-kafka (master):
-librdkafka ~0.9.4
+librdkafka ~0.11.5
 
 Proceed? [Y/n]
 bro/apache/metron-bro-plugin-kafka asks for LIBRDKAFKA_ROOT (Path to 
librdkafka installation tree) ? [/usr/local/lib]
@@ -67,13 +67,13 @@ Manually installing the plugin should only occur in 
situations where installing
 
 These instructions could also be helpful if you were interested in 
distributing this as a package (such as a deb or rpm).
 
-1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native 
client library for Kafka.  This plugin has been tested against the latest 
release of librdkafka, which at the time of this writing is v0.9.4.  
+1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native 
client library for Kafka.  This plugin has been tested against the latest 
release of librdkafka, which at the time of this writing is v0.11.5.
 
 In order to use this plugin within a kerberized Kafka environment, you 
will also need `libsasl2` installed and will need to pass `--enable-sasl` to 
the `configure` script.
 
 ```
-$ curl -L https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz | 
tar xvz
-$ cd librdkafka-0.9.4/
+$ curl -L https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz | 
tar xvz
+$ cd librdkafka-0.11.5/
 $ ./configure --enable-sasl
 $ make
 $ sudo make install
@@ -268,7 +268,7 @@ redef Kafka::topic_name = "bro";
 The global configuration settings for Kafka.  These values are passed through
 directly to librdkafka.  Any valid librdkafka settings can be defined in this
 table.  The full set of valid librdkafka settings are available
-[here](https://github.com/edenhill/librdkafka/blob/v0.9.4/CONFIGURATION.md).
+[here](https://github.com/edenhill/librdkafka/blob/v0.11.5/CONFIGURATION.md).
 
 ```
 redef Kafka::kafka_conf = table(

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/177430ec/bro-pkg.meta
--
diff --git a/bro-pkg.meta b/bro-pkg.meta
index 8bde2cc..86e2334 100644
--- a/bro-pkg.meta
+++ b/bro-pkg.meta
@@ -10,6 +10

[metron-bro-plugin-kafka] Git Push Summary

2018-10-10 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Tags:  refs/tags/0.2 [created] 1dfc5239f

metron git commit: METRON-1758 Add support for Ansible 2.6 in dev (JonZeolla via jonzeolla) closes apache/metron#1179

2018-10-01 Thread jonzeolla

Repository: metron
Updated Branches:
  refs/heads/master fbabfcad4 -> 1723a0eae


METRON-1758 Add support for Ansible 2.6 in dev (JonZeolla via jonzeolla) closes 
apache/metron#1179


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1723a0ea
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1723a0ea
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1723a0ea

Branch: refs/heads/master
Commit: 1723a0eae57adcfc4a2efe2422f52fbed4c237e7
Parents: fbabfca
Author: JonZeolla 
Authored: Mon Oct 1 08:41:25 2018 -0400
Committer: jonzeolla 
Committed: Mon Oct 1 08:41:25 2018 -0400

--
 metron-deployment/amazon-ec2/README.md  |  2 +-
 metron-deployment/amazon-ec2/ansible.cfg|  1 +
 metron-deployment/amazon-ec2/playbook.yml   |  4 --
 metron-deployment/ansible.cfg   | 23 --
 .../0_minimum_ansible_version.py| 45 
 .../ansible/playbooks/metron_full_install.yml   | 16 +++
 .../ambari_common/tasks/ambari-repo-ubuntu.yml  |  3 +-
 .../ansible/roles/ambari_common/tasks/main.yml  | 13 +++---
 .../roles/ambari_common/tasks/nodejs.yml|  6 ++-
 .../ansible/roles/enable-swap/tasks/main.yml|  3 +-
 .../ansible/roles/java_jdk/tasks/main.yml   |  6 +--
 .../ansible/roles/metron-builder/tasks/main.yml |  6 +--
 .../roles/metron-packages/tasks/main.yml| 12 ++
 metron-deployment/development/centos6/README.md |  5 +--
 .../development/centos6/Vagrantfile |  2 +-
 .../development/centos6/ansible.cfg |  2 +-
 .../development/centos6/ansible/playbook.yml|  2 +-
 .../development/fastcapa/centos-7.1/ansible.cfg |  2 +-
 .../development/fastcapa/centos-7.4/ansible.cfg |  2 +-
 .../development/ubuntu14/README.md  |  5 +--
 .../development/ubuntu14/Vagrantfile|  2 +-
 .../development/ubuntu14/ansible.cfg|  2 +-
 .../development/ubuntu14/ansible/playbook.yml   |  2 +-
 23 files changed, 84 insertions(+), 82 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/amazon-ec2/README.md
--
diff --git a/metron-deployment/amazon-ec2/README.md 
b/metron-deployment/amazon-ec2/README.md
index b2efc9e..ed1c73b 100644
--- a/metron-deployment/amazon-ec2/README.md
+++ b/metron-deployment/amazon-ec2/README.md
@@ -38,7 +38,7 @@ Getting Started
 
 The host used to deploy Apache Metron will need the following software tools 
installed.  The following versions are known to work as of the time of this 
writing, but by no means are these the only working versions.
 
-  - Ansible 2.0.0.2, 2.2.2.0, or 2.5.0
+  - Ansible 2.4.0+
   - Python 2.7.11
   - Maven 3.3.9  
 

http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/amazon-ec2/ansible.cfg
--
diff --git a/metron-deployment/amazon-ec2/ansible.cfg 
b/metron-deployment/amazon-ec2/ansible.cfg
index 0fe3343..4291cc8 100644
--- a/metron-deployment/amazon-ec2/ansible.cfg
+++ b/metron-deployment/amazon-ec2/ansible.cfg
@@ -22,6 +22,7 @@ pipelining = True
 remote_user = centos
 forks = 20
 log_path = ./ansible.log
+callback_plugins = ../ansible/callback_plugins
 
 # fix for "ssh throws 'unix domain socket too long' " problem
 #[ssh_connection]

http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/amazon-ec2/playbook.yml
--
diff --git a/metron-deployment/amazon-ec2/playbook.yml 
b/metron-deployment/amazon-ec2/playbook.yml
index cb1e326..4e8880a 100644
--- a/metron-deployment/amazon-ec2/playbook.yml
+++ b/metron-deployment/amazon-ec2/playbook.yml
@@ -21,10 +21,6 @@
 - hosts: localhost
   vars_files:
 - conf/defaults.yml
-  pre_tasks:
-- name: Verify Ansible Version
-  fail: msg="Metron Requires Ansible 2.0.0.2, 2.2.2.0 or 2.5.0, current 
version is {{ ansible_version }}"
-  when: "ansible_version.full | version_compare('2.2.2.0', '!=') and 
ansible_version.full | version_compare('2.0.0.2', '!=') and 
ansible_version.full | version_compare('2.5.0', '!=')"
   tasks:
 - include: tasks/create-keypair.yml
 - include: tasks/create-vpc.yml

http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible.cfg
--
diff --git a/metron-deployment/ansible.cfg b/metron-deployment/ansible.cfg
deleted file mode 100644
index 9b3916b..000
--- a/metron-deployment/ansible.cfg
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.

metron-bro-plugin-kafka git commit: METRON-1885 Remove version from bro plugin btest (JonZeolla) closes apache/metron-bro-plugin-kafka#19

2018-11-21 Thread jonzeolla

Repository: metron-bro-plugin-kafka
Updated Branches:
  refs/heads/master 177430ec5 -> 667922f66


METRON-1885 Remove version from bro plugin btest (JonZeolla) closes 
apache/metron-bro-plugin-kafka#19


Project: http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/commit/667922f6
Tree: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/tree/667922f6
Diff: 
http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/diff/667922f6

Branch: refs/heads/master
Commit: 667922f6607bd625e2f32e91ae5b6c61bcaaa04f
Parents: 177430e
Author: JonZeolla 
Authored: Wed Nov 21 14:16:19 2018 -0500
Committer: jonzeolla 
Committed: Wed Nov 21 14:16:19 2018 -0500

--
 tests/Baseline/kafka.show-plugin/output | 2 +-
 tests/kafka/show-plugin.bro | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/667922f6/tests/Baseline/kafka.show-plugin/output
--
diff --git a/tests/Baseline/kafka.show-plugin/output 
b/tests/Baseline/kafka.show-plugin/output
index b9ebb05..e6ad77a 100644
--- a/tests/Baseline/kafka.show-plugin/output
+++ b/tests/Baseline/kafka.show-plugin/output
@@ -1,4 +1,4 @@
-Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3)
+Apache::Kafka - Writes logs to Kafka (dynamic)
 [Writer] KafkaWriter (Log::WRITER_KAFKAWRITER)
 [Constant] Kafka::kafka_conf
 [Constant] Kafka::topic_name

http://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka/blob/667922f6/tests/kafka/show-plugin.bro
--
diff --git a/tests/kafka/show-plugin.bro b/tests/kafka/show-plugin.bro
index 1eb3dde..ba065cc 100644
--- a/tests/kafka/show-plugin.bro
+++ b/tests/kafka/show-plugin.bro
@@ -15,5 +15,5 @@
 #  limitations under the License.
 #
 
-# @TEST-EXEC: bro -NN Apache::Kafka >output
+# @TEST-EXEC: bro -NN Apache::Kafka | sed 's/, version.*)/)/' > output
 # @TEST-EXEC: btest-diff output

[metron] branch master updated: METRON-1932 Update ES and Kibana to 5.6.14 (JonZeolla) closes apache/metron#1296

2018-12-18 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git


The following commit(s) were added to refs/heads/master by this push:
 new e4d793a  METRON-1932 Update ES and Kibana to 5.6.14 (JonZeolla) closes 
apache/metron#1296
e4d793a is described below

commit e4d793a55a56a01fa101c8da77d92f99325ba539
Author: JonZeolla 
AuthorDate: Tue Dec 18 15:30:09 2018 -0500

METRON-1932 Update ES and Kibana to 5.6.14 (JonZeolla) closes 
apache/metron#1296
---
 dependencies_with_url.csv  | 25 ++-
 metron-deployment/Kerberos-manual-setup.md |  2 +-
 .../ansible/roles/ambari_master/defaults/main.yml  |  2 +-
 metron-deployment/packaging/ambari/README.md   | 28 +++---
 .../ELASTICSEARCH/{5.6.2 => 5.6.14}/metainfo.xml   |  4 ++--
 .../{5.6.2 => 5.6.14}/repos/repoinfo.xml   |  0
 .../KIBANA/{5.6.2 => 5.6.14}/metainfo.xml  |  4 ++--
 .../{5.6.2 => 5.6.14}/quicklinks/quicklinks.json   |  0
 .../KIBANA/{5.6.2 => 5.6.14}/repos/repoinfo.xml|  0
 .../configuration/elastic-env.xml  |  0
 .../configuration/elastic-jvm-options.xml  |  0
 .../configuration/elastic-site.xml |  0
 .../configuration/elastic-sysconfig.xml|  0
 .../configuration/elastic-systemd.xml  |  0
 .../ELASTICSEARCH/{5.6.2 => 5.6.14}/metainfo.xml   |  8 +++
 .../package/scripts/elastic_commands.py|  0
 .../package/scripts/elastic_master.py  |  0
 .../package/scripts/elastic_slave.py   |  0
 .../{5.6.2 => 5.6.14}/package/scripts/params.py|  0
 .../package/scripts/properties_config.py   |  0
 .../package/scripts/service_check.py   |  0
 .../package/scripts/status_params.py   |  0
 .../package/templates/elasticsearch.master.yaml.j2 |  0
 .../package/templates/elasticsearch.slave.yaml.j2  |  0
 .../package/templates/elasticsearch_limits.conf.j2 |  0
 .../{5.6.2 => 5.6.14}/quicklinks/quicklinks.json   |  0
 .../{5.6.2 => 5.6.14}/role_command_order.json  |  0
 .../{5.6.2 => 5.6.14}/configuration/kibana-env.xml |  0
 .../configuration/kibana-site.xml  |  0
 .../KIBANA/{5.6.2 => 5.6.14}/metainfo.xml  |  8 +++
 .../{5.6.2 => 5.6.14}/package/scripts/common.py|  0
 .../package/scripts/kibana_master.py   |  0
 .../{5.6.2 => 5.6.14}/package/scripts/params.py|  0
 .../{5.6.2 => 5.6.14}/quicklinks/quicklinks.json   |  0
 .../src/main/resources/mpack.json  |  6 ++---
 .../packaging/ambari/metron-mpack/README.md|  4 ++--
 .../scripts/dashboard/dashboard-bulkload.json  |  2 +-
 .../metron-mpack/src/main/resources/mpack.json |  4 ++--
 metron-platform/metron-elasticsearch/README.md | 10 
 .../elasticsearch/client/ElasticsearchClient.java  |  2 +-
 pom.xml|  2 +-
 41 files changed, 57 insertions(+), 54 deletions(-)

diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 0a63922..5462a12 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -434,16 +434,16 @@ 
io.netty:netty-resolver:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
 io.netty:netty-transport:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
 
joda-time:joda-time:jar:2.9.5:compile,ASLv2,https://github.com/JodaOrg/joda-time
 net.sf.jopt-simple:jopt-simple:jar:5.0.2:compile,The MIT 
License,http://jopt-simple.sourceforge.net
-org.elasticsearch.client:elasticsearch-rest-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
-org.elasticsearch.client:transport:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
-org.elasticsearch:elasticsearch:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch.client:elasticsearch-rest-client:jar:5.6.14:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch.client:transport:jar:5.6.14:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.elasticsearch:elasticsearch:jar:5.6.14:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
 
org.elasticsearch:jna:jar:4.4.0-1:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
-org.elasticsearch.plugin:lang-mustache-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
-org.elasticsearch.plugin:parent-join-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
-org.elasticsearch.plugin:percolator-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
-org.elasticsearch.plugin:reindex-client:jar:5.6.2:compile,ASLv2,htt

[metron-bro-plugin-kafka] branch master updated: METRON-1990 Bro plugin docker should exit nonzero if it encounters issues (JonZeolla) closes apache/metron-bro-plugin-kafka#28

2019-02-25 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
 new 2655f71  METRON-1990 Bro plugin docker should exit nonzero if it 
encounters issues (JonZeolla) closes apache/metron-bro-plugin-kafka#28
2655f71 is described below

commit 2655f71694bde2e12e61fa9522a41a452161d845
Author: JonZeolla 
AuthorDate: Mon Feb 25 08:14:38 2019 -0500

METRON-1990 Bro plugin docker should exit nonzero if it encounters issues 
(JonZeolla) closes apache/metron-bro-plugin-kafka#28
---
 docker/in_docker_scripts/process_data_file.sh  |  5 -
 docker/run_end_to_end.sh   |  3 +++
 docker/scripts/docker_run_bro_container.sh | 10 +-
 docker/scripts/docker_run_consume_bro_kafka.sh |  3 +++
 docker/scripts/docker_run_create_bro_topic_in_kafka.sh |  3 +++
 docker/scripts/docker_run_get_offset_bro_kafka.sh  |  3 +++
 6 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/docker/in_docker_scripts/process_data_file.sh 
b/docker/in_docker_scripts/process_data_file.sh
index 021c5fd..81819a2 100755
--- a/docker/in_docker_scripts/process_data_file.sh
+++ b/docker/in_docker_scripts/process_data_file.sh
@@ -74,5 +74,8 @@ if [ ! -d /root/data ]; then
   exit 1
 fi
 cd /root/test_output/"${OUTPUT_DIRECTORY_NAME}" || exit 1
-find /root/data -type f -name "${PCAP_FILE_NAME}" -exec echo "processing" '{}' 
\; -exec bro -r '{}' /usr/local/bro/share/bro/site/local.bro -C \;
+find /root/data -type f -name "${PCAP_FILE_NAME}" -print0 | xargs -0 bro -r {} 
/usr/local/bro/share/bro/site/local.bro -C
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
 echo "done with ${PCAP_FILE_NAME}"
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index 950dc60..6baf679 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -131,6 +131,9 @@ fi
 # Download the pcaps
 bash "${SCRIPT_DIR}"/download_sample_pcaps.sh --data-path="${DATA_PATH}"
 
+# By not catching $? here we are accepting that a failed pcap download will not
+# exit the script
+
 mkdir "${TEST_OUTPUT_PATH}" || exit 1
 
 # Run the bro container and optionally the passed script _IN_ the container
diff --git a/docker/scripts/docker_run_bro_container.sh 
b/docker/scripts/docker_run_bro_container.sh
index 1e31b25..aba5d5f 100755
--- a/docker/scripts/docker_run_bro_container.sh
+++ b/docker/scripts/docker_run_bro_container.sh
@@ -137,13 +137,13 @@ echo "==="
 declare -a DOCKER_CMD_BASE
 DOCKER_CMD="bash"
 DOCKER_CMD_BASE[0]="docker run -d -t --name ${CONTAINER_NAME} --network 
${NETWORK_NAME} "
-DOCKER_CMD_BASE[2]="-v \"${OUR_SCRIPTS_PATH}:/root/built_in_scripts\" "
-DOCKER_CMD_BASE[3]="-v \"${BRO_PLUGIN_PATH}:/root/code\" "
-DOCKER_CMD_BASE[4]="-v \"${TEST_OUTPUT_PATH}:/root/test_output\" "
-OFFSET=5
+DOCKER_CMD_BASE[1]="-v \"${OUR_SCRIPTS_PATH}:/root/built_in_scripts\" "
+DOCKER_CMD_BASE[2]="-v \"${BRO_PLUGIN_PATH}:/root/code\" "
+DOCKER_CMD_BASE[3]="-v \"${TEST_OUTPUT_PATH}:/root/test_output\" "
+OFFSET=4
 if [[ -n "$SCRIPTS_PATH" ]]; then
   DOCKER_CMD_BASE[$OFFSET]="-v \"${SCRIPTS_PATH}:/root/scripts\" "
-  OFFSET=6
+  OFFSET=5
 fi
 
 if [[ -n "$DATA_PATH" ]]; then
diff --git a/docker/scripts/docker_run_consume_bro_kafka.sh 
b/docker/scripts/docker_run_consume_bro_kafka.sh
index eea740a..5a7bbcf 100755
--- a/docker/scripts/docker_run_consume_bro_kafka.sh
+++ b/docker/scripts/docker_run_consume_bro_kafka.sh
@@ -83,4 +83,7 @@ done
 
 docker run --rm --network "${NETWORK_NAME}" ches/kafka \
   kafka-console-consumer.sh --topic bro --offset "${OFFSET}" --partition 0 
--bootstrap-server kafka:9092 --timeout-ms 1000
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
 
diff --git a/docker/scripts/docker_run_create_bro_topic_in_kafka.sh 
b/docker/scripts/docker_run_create_bro_topic_in_kafka.sh
index cb00e2c..f4c84ab 100755
--- a/docker/scripts/docker_run_create_bro_topic_in_kafka.sh
+++ b/docker/scripts/docker_run_create_bro_topic_in_kafka.sh
@@ -71,4 +71,7 @@ echo "==="
 
 docker run --rm --network "${NETWORK_NAME}" ches/kafka \
   kafka-topics.sh --create --topic bro --replication-factor 1 --partitions 1 
--zookeeper zookeeper:2181
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi
 
diff --git a/docker/scripts/docker_run_get_offset_bro_kafka.sh 
b/docker/scripts/docker_run_get_offset_bro_kafka.sh
index d3b6621..eafd08f 100755
--- a/docker/scripts/docker_run_get_offset_bro_kafka.sh
+++ b/docker/scripts/docker_run_get_offset_bro_kafka.sh
@@ -72,4 +72,7 @@ done
 
 docker run --rm --network "${NETWORK_NAME}" ches/kafka \
   kafka-run-class.sh kafka.tools.GetOffsetShell --topic bro --broker-list 
kafka:9092
+rc=$?; if [[ ${rc} != 0 ]]; then
+  exit ${rc}
+fi

[metron-bro-plugin-kafka] branch METRON-2045 created (now dfc7884)

2019-03-22 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a change to branch METRON-2045
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git.


  at dfc7884  METRON-2045: Pass a version argument to the bro plugin docker 
scripts

This branch includes the following new commits:

 new dfc7884  METRON-2045: Pass a version argument to the bro plugin docker 
scripts

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[metron-bro-plugin-kafka] 01/01: METRON-2045: Pass a version argument to the bro plugin docker scripts

2019-03-22 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch METRON-2045
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git

commit dfc78843b9c827d8bca474e1f27894e5d8c585f2
Author: Jon Zeolla 
AuthorDate: Fri Mar 22 12:23:29 2019 -0400

METRON-2045: Pass a version argument to the bro plugin docker scripts
---
 docker/README.md  |  2 +-
 docker/in_docker_scripts/build_bro_plugin.sh  | 59 ++-
 docker/run_end_to_end.sh  | 30 +---
 docker/scripts/docker_execute_build_bro_plugin.sh | 23 +++--
 docker/scripts/docker_execute_shell.sh|  2 +-
 5 files changed, 103 insertions(+), 13 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index a965d8b..47295cf 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -50,7 +50,7 @@ testing scripts to be added to a pull request, and 
subsequently to a test suite.
 └── wait_for_zk.sh
 ```
 
-- `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the plugin.
+- `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the provided 
version of the plugin.
 - `configure_bro_plugin.sh`: Configures the plugin for the kafka container, 
and routes all traffic types.
 - `process_data_file.sh`: Runs `bro -r` on the passed file
 - `wait-for-it.sh`: Waits for a port to be open, so we know something is 
available.
diff --git a/docker/in_docker_scripts/build_bro_plugin.sh 
b/docker/in_docker_scripts/build_bro_plugin.sh
index 46277bd..064fe91 100755
--- a/docker/in_docker_scripts/build_bro_plugin.sh
+++ b/docker/in_docker_scripts/build_bro_plugin.sh
@@ -18,16 +18,73 @@
 #
 
 shopt -s nocasematch
+shopt -s globstar nullglob
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
 
 #
 # Runs bro-pkg to build and install the plugin
 #
 
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "--plugin-version[REQUIRED] The plugin version."
+  echo "-h/--help   Usage information."
+  echo " "
+  echo " "
+}
+
+PLUGIN_VERSION=
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # PLUGIN_VERSION
+  #
+  #   --plugin-version
+  #
+--plugin-version=*)
+  PLUGIN_VERSION="${i#*=}"
+  shift # past argument=value
+;;
+
+  #
+  # -h/--help
+  #
+-h | --help)
+  help
+  exit 0
+  shift # past argument with no value
+;;
+
+  #
+  # Unknown option
+  #
+*)
+  UNKNOWN_OPTION="${i#*=}"
+  echo "Error: unknown option: $UNKNOWN_OPTION"
+  help
+;;
+  esac
+done
+
+if [[ -z "${PLUGIN_VERSION}" ]]; then
+  echo "PLUGIN_VERSION must be passed"
+  exit 1
+fi
+
+echo "PLUGIN_VERSION = ${PLUGIN_VERSION}"
+
 cd /root || exit 1
 
 echo ""
 
-bro-pkg install code --force
+bro-pkg install code --version "${PLUGIN_VERSION}" --force
 rc=$?; if [[ ${rc} != 0 ]]; then
   echo "ERROR running bro-pkg install ${rc}"
   exit ${rc}
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index 6dfd146..0c94e74 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -29,6 +29,7 @@ function help {
   echo "--skip-docker-build [OPTIONAL] Skip build of bro 
docker machine."
   echo "--data-path [OPTIONAL] The pcap data path. 
Default: ./data"
   echo "--kafka-topic   [OPTIONAL] The kafka topic to 
consume from. Default: bro"
+  echo "--plugin-version[OPTIONAL] The plugin version. 
Default: the current branch name"
   echo "-h/--help   Usage information."
   echo " "
   echo "COMPATABILITY"
@@ -53,6 +54,7 @@ DATE=$(date)
 LOG_DATE=${DATE// /_}
 TEST_OUTPUT_PATH="${ROOT_DIR}/test_output/"${LOG_DATE//:/_}
 KAFKA_TOPIC="bro"
+PLUGIN_VERSION=$(cd "${ROOT_DIR}" && git rev-parse --symbolic-full-name 
--abbrev-ref HEAD)
 
 # Handle command line options
 for i in "$@"; do
@@ -66,6 +68,7 @@ for i in "$@"; do
   SKIP_REBUILD_BRO=true
   shift # past argument
 ;;
+
   #
   # DATA_PATH
   #
@@ -73,6 +76,7 @@ for i in "$@"; do
   DATA_PATH="${i#*=}"
   shift # past argument=value
 ;;
+
   #
   # KAFKA_TOPIC
   #
@@ -82,6 +86,17 @@ for i in "$@"; do
   KAFKA_TOPIC="${i#*=}"
   shift # past argument=value
 ;;
+
+  #
+  # PLUGIN_VERSION
+  #
+  #   --plugin-version
+  #
+--plugin-version=*)
+  PLUGIN_VERSION="${i#*=}"
+  shift # past argument=value
+;;
+
   #
   # -h/--help
   #
@@ -96,9 +111,10 @@ done
 EXTRA_ARGS="$*&

[metron-bro-plugin-kafka] branch master updated: METRON-1910 bro plugin segfaults on src/KafkaWriter.cc:72 (JonZeolla) closes apache/metron-bro-plugin-kafka#20

2019-02-13 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
 new b360b85  METRON-1910 bro plugin segfaults on src/KafkaWriter.cc:72 
(JonZeolla) closes apache/metron-bro-plugin-kafka#20
b360b85 is described below

commit b360b85e00a8d6b8db9c790c44a767a54c81eb2b
Author: JonZeolla 
AuthorDate: Wed Feb 13 09:51:29 2019 -0500

METRON-1910 bro plugin segfaults on src/KafkaWriter.cc:72 (JonZeolla) 
closes apache/metron-bro-plugin-kafka#20
---
 src/KafkaWriter.cc | 39 ++-
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/src/KafkaWriter.cc b/src/KafkaWriter.cc
index 79b5aa0..1d4a28a 100644
--- a/src/KafkaWriter.cc
+++ b/src/KafkaWriter.cc
@@ -20,15 +20,18 @@
 using namespace logging;
 using namespace writer;
 
+// The Constructor is called once for each log filter that uses this log 
writer.
 KafkaWriter::KafkaWriter(WriterFrontend* frontend):
 WriterBackend(frontend),
 formatter(NULL),
 producer(NULL),
 topic(NULL)
 {
-  // need thread-local copies of all user-defined settings coming from
-  // bro scripting land.  accessing these is not thread-safe and 'DoInit'
-  // is potentially accessed from multiple threads.
+  /**
+   * We need thread-local copies of all user-defined settings coming from bro
+   * scripting land.  accessing these is not thread-safe and 'DoInit' is
+   * potentially accessed from multiple threads.
+   */
 
   // tag_json - thread local copy
   tag_json = BifConst::Kafka::tag_json;
@@ -67,16 +70,13 @@ KafkaWriter::KafkaWriter(WriterFrontend* frontend):
 
 KafkaWriter::~KafkaWriter()
 {
-
-// Cleanup all the things
-delete topic;
-delete producer;
-delete formatter;
-delete conf;
-delete topic_conf;
-
+  // Cleanup must happen in DoFinish, not in the destructor
 }
 
+/**
+ * DoInit is called once for each call to the constructor, but in a separate
+ * thread
+ */
 bool KafkaWriter::DoInit(const WriterInfo& info, int num_fields, const 
threading::Field* const* fields)
 {
 // Timeformat object, default to TS_EPOCH
@@ -87,11 +87,13 @@ bool KafkaWriter::DoInit(const WriterInfo& info, int 
num_fields, const threading
 topic_name = info.path;
 }
 
-// format timestamps
-// NOTE: This string comparision implementation is currently the necessary
-// way to do it, as there isn't a way to pass the Bro enum into C++ enum.
-// This makes the user interface consistent with the existing Bro Logging
-// configuration for the ASCII log output.
+/**
+ * Format the timestamps
+ * NOTE: This string comparision implementation is currently the necessary
+ * way to do it, as there isn't a way to pass the Bro enum into C++ enum.
+ * This makes the user interface consistent with the existing Bro Logging
+ * configuration for the ASCII log output.
+ */
 if ( strcmp(json_timestamps.c_str(), "JSON::TS_EPOCH") == 0 ) {
   tf = threading::formatter::JSON::TS_EPOCH;
 }
@@ -177,7 +179,8 @@ bool KafkaWriter::DoInit(const WriterInfo& info, int 
num_fields, const threading
 /**
  * Writer-specific method called just before the threading system is
  * going to shutdown. It is assumed that once this messages returns,
- * the thread can be safely terminated.
+ * the thread can be safely terminated. As such, all resources created must be
+ * removed here.
  */
 bool KafkaWriter::DoFinish(double network_time)
 {
@@ -202,6 +205,8 @@ bool KafkaWriter::DoFinish(double network_time)
 delete topic;
 delete producer;
 delete formatter;
+delete conf;
+delete topic_conf;
 
 return success;
 }

[metron-bro-plugin-kafka] branch master updated: METRON-1992 Support sending a log to multiple topics (JonZeolla) closes apache/metron-bro-plugin-kafka#23

2019-02-14 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
 new d96568e  METRON-1992 Support sending a log to multiple topics 
(JonZeolla) closes apache/metron-bro-plugin-kafka#23
d96568e is described below

commit d96568e6df21b3f091ffc3ad3fd7f93d7cb16788
Author: JonZeolla 
AuthorDate: Thu Feb 14 09:25:46 2019 -0500

METRON-1992 Support sending a log to multiple topics (JonZeolla) closes 
apache/metron-bro-plugin-kafka#23
---
 README.md  | 41 +++--
 src/KafkaWriter.cc | 18 +++---
 src/KafkaWriter.h  |  2 ++
 3 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 7cc2c46..72436e9 100644
--- a/README.md
+++ b/README.md
@@ -114,7 +114,7 @@ redef Kafka::kafka_conf = table(
 
 ### Example 2 - Send all active logs
 
-This plugin has the ability send all active logs to kafka with the following 
configuration.
+This plugin has the ability send all active logs to the "bro" kafka topic with 
the following configuration.
 
 ```
 @load packages/metron-bro-plugin-kafka/Apache/Kafka
@@ -138,7 +138,7 @@ redef Kafka::kafka_conf = table(
 );
 ```
 
-### Example 4 - Send logs to unique topics
+### Example 4 - Send each bro log to a unique topic
 
 It is also possible to send each log stream to a uniquely named topic.  The 
goal in this example is to send all HTTP records to a Kafka topic named `http` 
and all DNS records to a separate Kafka topic named `dns`.
  * The `topic_name` value must be set to an empty string.
@@ -228,6 +228,43 @@ event bro_init() =-10
  * You can also filter IPv6 logs from within your Metron cluster [using 
Stellar](https://github.com/apache/metron/tree/master/metron-stellar/stellar-common#is_ip).
  In that case, you wouldn't apply a predicate in your bro configuration, and 
instead Stellar would filter the logs out before they were processed by the 
enrichment layer of Metron.
  * It is also possible to use the `is_v6_subnet()` bro function in your 
predicate, as of their [2.5 
release](https://www.bro.org/sphinx-git/install/release-notes.html#bro-2-5), 
however the above example should work on [bro 
2.4](https://www.bro.org/sphinx-git/install/release-notes.html#bro-2-4) and 
newer, which has been the focus of the kafka plugin.
 
+### Example 6 - Sending a log to multiple topics
+
+You are able to send a single bro log to multiple different kafka topics in 
the same kafka cluster by overriding the default topic (configured with 
`Kafka::topic_name`) by creating a custom bro `Log::Filter`.  In this example, 
the DHCP, RADIUS, and DNS logs are sent to the "bro" topic; the RADIUS log is 
duplicated to the "shew_bro_radius" topic; and the DHCP log is duplicated to 
the "shew_bro_dhcp" topic.
+
+```
+@load packages/metron-bro-plugin-kafka/Apache/Kafka
+redef Kafka::logs_to_send = set(DHCP::LOG, RADIUS::LOG, DNS::LOG);
+redef Kafka::topic_name = "bro";
+redef Kafka::kafka_conf = table(
+["metadata.broker.list"] = 
"server1.example.com:9092,server2.example.com:9092"
+);
+redef Kafka::tag_json = T;
+
+event bro_init() =-10
+{
+# Send RADIUS to the shew_bro_radius topic
+local shew_radius_filter: Log::Filter = [
+$name = "kafka-radius-shew",
+$writer = Log::WRITER_KAFKAWRITER,
+$path = "shew_bro_radius"
+$config = table(["topic_name"] = "shew_bro_radius")
+];
+Log::add_filter(RADIUS::LOG, shew_radius_filter);
+
+# Send DHCP to the shew_bro_dhcp topic
+local shew_dhcp_filter: Log::Filter = [
+$name = "kafka-dhcp-shew",
+$writer = Log::WRITER_KAFKAWRITER,
+$path = "shew_bro_dhcp"
+$config = table(["topic_name"] = "shew_bro_dhcp")
+];
+Log::add_filter(DHCP::LOG, shew_dhcp_filter);
+}
+```
+
+_Note_:  Because `Kafka::tag_json` is set to True in this example, the value 
of `$path` is used as the tag for each `Log::Filter`. If you were to add a log 
filter with the same `$path` as an existing filter, Bro will append "-N", where 
N is an integer starting at 2, to the end of the log path so that each filter 
has its own unique log path. For instance, the second instance of `conn` would 
become `conn-2`.
+
 ## Settings
 
 ### `logs_to_send`
diff --git a/src/KafkaWriter.cc b/src/KafkaWriter.cc
index 1d4a28a..563ef74 100644
--- a/src/KafkaWriter.cc
+++ b/src/KafkaWriter.cc
@@ -73,6 +73,15 @@ KafkaWriter::~KafkaWriter()
   // Cleanup must happen in DoFinish, not in the destructor
 }
 
+string KafkaWriter::GetConfigValue(const WriterInfo& info, const string name) 
const
+{
+map::const_iterator it = 
info.config.find(name.c_str());
+if (it == info.c

[metron-bro-plugin-kafka] branch master updated: METRON-2000 Fix bro plugin docker line counting for BRO_COUNT (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#24

2019-02-14 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
 new bfc9cbb  METRON-2000 Fix bro plugin docker line counting for BRO_COUNT 
(JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#24
bfc9cbb is described below

commit bfc9cbbdc97c3a12c59e9d9786bd7e3996a196f5
Author: JonZeolla 
AuthorDate: Thu Feb 14 09:27:45 2019 -0500

METRON-2000 Fix bro plugin docker line counting for BRO_COUNT (JonZeolla 
via jonzeolla) closes apache/metron-bro-plugin-kafka#24
---
 docker/in_docker_scripts/configure_bro_plugin.sh | 6 ++
 docker/scripts/split_kakfa_output_by_log.sh  | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/docker/in_docker_scripts/configure_bro_plugin.sh 
b/docker/in_docker_scripts/configure_bro_plugin.sh
index 74551c2..c292504 100755
--- a/docker/in_docker_scripts/configure_bro_plugin.sh
+++ b/docker/in_docker_scripts/configure_bro_plugin.sh
@@ -37,5 +37,11 @@ echo "Configuring kafka plugin"
   echo "redef Software::asset_tracking = ALL_HOSTS;"
 } >> /usr/local/bro/share/bro/site/local.bro
 
+# Load "known-devices-and-hostnames.bro" which is necessary in bro 2.5.5 to
+# create the log Known::DEVICES_LOG
 sed -i '86 a @load policy/protocols/dhcp/known-devices-and-hostnames.bro' 
/usr/local/bro/share/bro/site/local.bro
 
+# Comment out the load statement for "log-hostcerts-only.bro" in bro 2.5.5's
+# default local.bro in order to log all certificates to x509.log
+sed -i 's%^@load protocols/ssl/log-hostcerts-only%#&%' 
/usr/local/bro/share/bro/site/local.bro
+
diff --git a/docker/scripts/split_kakfa_output_by_log.sh 
b/docker/scripts/split_kakfa_output_by_log.sh
index 71ec82b..74d55e3 100755
--- a/docker/scripts/split_kakfa_output_by_log.sh
+++ b/docker/scripts/split_kakfa_output_by_log.sh
@@ -97,7 +97,7 @@ do
   grep {\""${BASE_LOG_FILE_NAME}"\": "${LOG_DIRECTORY}"/kafka-output.log > 
"${LOG_DIRECTORY}"/"${BASE_LOG_FILE_NAME}".kafka.log
 
   KAKFA_COUNT=$(cat "${LOG_DIRECTORY}/${BASE_LOG_FILE_NAME}.kafka.log" | 
wc -l)
-  BRO_COUNT=$(grep -v "#" "${log}" | wc -l)
+  BRO_COUNT=$(grep -v "^#" "${log}" | wc -l)
 
   echo "${BASE_LOG_FILE_NAME},${BRO_COUNT},${KAKFA_COUNT}" >> 
"${RESULTS_FILE}"
 fi

[metron-bro-plugin-kafka] branch master updated: METRON-2045 Pass a version argument to the bro plugin docker scripts (JonZeolla) closes apache/metron-bro-plugin-kafka#35

2019-04-29 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
 new 43c9166  METRON-2045 Pass a version argument to the bro plugin docker 
scripts (JonZeolla) closes apache/metron-bro-plugin-kafka#35
43c9166 is described below

commit 43c9166787649e4ac2ab295a1baba94d54903651
Author: JonZeolla 
AuthorDate: Mon Apr 29 08:24:12 2019 -0400

METRON-2045 Pass a version argument to the bro plugin docker scripts 
(JonZeolla) closes apache/metron-bro-plugin-kafka#35
---
 docker/README.md  | 12 +++--
 docker/in_docker_scripts/build_bro_plugin.sh  | 59 ++-
 docker/run_end_to_end.sh  | 30 +---
 docker/scripts/docker_execute_build_bro_plugin.sh | 23 +++--
 docker/scripts/docker_execute_shell.sh|  2 +-
 5 files changed, 110 insertions(+), 16 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index bde7b5e..eac2919 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -50,7 +50,7 @@ testing scripts to be added to a pull request, and 
subsequently to a test suite.
 └── wait_for_zk.sh
 ```
 
-- `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the plugin.
+- `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the provided 
version of the plugin.
 - `configure_bro_plugin.sh`: Configures the plugin for the kafka container, 
and routes all traffic types.
 - `process_data_file.sh`: Runs `bro -r` on the passed file
 - `wait-for-it.sh`: Waits for a port to be open, so we know something is 
available.
@@ -294,7 +294,11 @@ Other scripts may then be used to do your testing, for 
example running:
 # `run_end_to_end.sh`
 ## Parameters
 ```bash
---skip-docker-build[OPTIONAL] Skip build of bro docker machine.
---data-path[OPTIONAL] The pcap data path. Default: ./data
---kafka-topic  [OPTIONAL] The kafka topic name to use. 
Default: bro
+--skip-docker-build [OPTIONAL] Skip build of bro docker machine.
+--data-path [OPTIONAL] The pcap data path. Default: ./data
+--kafka-topic   [OPTIONAL] The kafka topic name to use. 
Default: bro
+--plugin-version[OPTIONAL] The plugin version. Default: the 
current branch name
 ```
+
+> NOTE: The provided `--plugin-version` is passed to the [`bro-pkg 
install`](https://docs.zeek.org/projects/package-manager/en/stable/bro-pkg.html#install-command)
 command within the container, which allows you to specify a version tag, 
branch name, or commit hash.  However, that tag, branch, or commit *must* be 
available in the currently checked out plugin repository.
+
diff --git a/docker/in_docker_scripts/build_bro_plugin.sh 
b/docker/in_docker_scripts/build_bro_plugin.sh
index 46277bd..064fe91 100755
--- a/docker/in_docker_scripts/build_bro_plugin.sh
+++ b/docker/in_docker_scripts/build_bro_plugin.sh
@@ -18,16 +18,73 @@
 #
 
 shopt -s nocasematch
+shopt -s globstar nullglob
+shopt -s nocasematch
+set -u # nounset
+set -e # errexit
+set -E # errtrap
+set -o pipefail
 
 #
 # Runs bro-pkg to build and install the plugin
 #
 
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "--plugin-version[REQUIRED] The plugin version."
+  echo "-h/--help   Usage information."
+  echo " "
+  echo " "
+}
+
+PLUGIN_VERSION=
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # PLUGIN_VERSION
+  #
+  #   --plugin-version
+  #
+--plugin-version=*)
+  PLUGIN_VERSION="${i#*=}"
+  shift # past argument=value
+;;
+
+  #
+  # -h/--help
+  #
+-h | --help)
+  help
+  exit 0
+  shift # past argument with no value
+;;
+
+  #
+  # Unknown option
+  #
+*)
+  UNKNOWN_OPTION="${i#*=}"
+  echo "Error: unknown option: $UNKNOWN_OPTION"
+  help
+;;
+  esac
+done
+
+if [[ -z "${PLUGIN_VERSION}" ]]; then
+  echo "PLUGIN_VERSION must be passed"
+  exit 1
+fi
+
+echo "PLUGIN_VERSION = ${PLUGIN_VERSION}"
+
 cd /root || exit 1
 
 echo ""
 
-bro-pkg install code --force
+bro-pkg install code --version "${PLUGIN_VERSION}" --force
 rc=$?; if [[ ${rc} != 0 ]]; then
   echo "ERROR running bro-pkg install ${rc}"
   exit ${rc}
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index 4c61560..3ec0145 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -29,6 +29,7 @@ function help {
   echo "--skip-docker-build [OPTIONAL] Skip build of bro 
docker machine."
   echo "--data-path [OPTIONAL] The pcap data path

[metron-bro-plugin-kafka] branch METRON-2045 deleted (was dfc7884)

2019-04-23 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a change to branch METRON-2045
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git.


 was dfc7884  METRON-2045: Pass a version argument to the bro plugin docker 
scripts

The revisions that were on this branch are still contained in
other references; therefore, this change does not discard any commits
from the repository.

[metron-bro-plugin-kafka] branch master updated: METRON-2347 [BRO-PLUGIN-KAFKA] Use docker compose in end to end tests (JonZeolla) closes apache/metron-bro-plugin-kafka#42

2020-04-29 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
 new 9f2cb11  METRON-2347 [BRO-PLUGIN-KAFKA] Use docker compose in end to 
end tests (JonZeolla) closes apache/metron-bro-plugin-kafka#42
9f2cb11 is described below

commit 9f2cb11c10722ee19eb57d213bb4af180ea4f12b
Author: JonZeolla 
AuthorDate: Wed Apr 29 14:36:35 2020 -0400

METRON-2347 [BRO-PLUGIN-KAFKA] Use docker compose in end to end tests 
(JonZeolla) closes apache/metron-bro-plugin-kafka#42
---
 NOTICE |   4 +-
 docker/README.md   | 219 +++--
 .../{bro-localbuild-container => bro}/.screenrc|   0
 .../{bro-localbuild-container => bro}/Dockerfile   |  24 ++-
 docker/containers/bro/Makefile |   2 +
 docker/containers/bro/requirements-to-freeze.txt   |   1 +
 docker/containers/bro/requirements.txt |   7 +
 .../kafka/Dockerfile}  |  22 +--
 .../zookeeper/Dockerfile}  |  22 +--
 docker/docker-compose.yml  |  35 
 docker/finish_end_to_end.sh|   7 +-
 docker/in_docker_scripts/wait-for-it.sh| 162 ---
 docker/run_end_to_end.sh   |  83 +++-
 docker/scripts/build_container.sh  | 113 ---
 docker/scripts/cleanup_docker.sh   |  91 -
 docker/scripts/create_docker_network.sh|  79 
 docker/scripts/destroy_docker_network.sh   |  76 ---
 docker/scripts/docker_execute_build_bro_plugin.sh  |   6 +-
 .../scripts/docker_execute_configure_bro_plugin.sh |   4 +-
 sh => docker_execute_create_topic_in_kafka.sh} |  22 +--
 docker/scripts/docker_execute_process_data_file.sh |   6 +-
 docker/scripts/docker_execute_shell.sh |   4 +-
 docker/scripts/docker_run_bro_container.sh | 168 
 docker/scripts/docker_run_consume_kafka.sh |   6 +-
 docker/scripts/docker_run_get_offset_kafka.sh  |   6 +-
 docker/scripts/docker_run_kafka_container.sh   |  82 
 docker/scripts/docker_run_wait_for_kafka.sh|  82 
 docker/scripts/docker_run_wait_for_zookeeper.sh|  82 
 docker/scripts/docker_run_zookeeper_container.sh   |  82 
 docker/scripts/stop_container.sh   |  85 
 30 files changed, 190 insertions(+), 1392 deletions(-)

diff --git a/NOTICE b/NOTICE
index 7f966de..accd03e 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,7 +1,5 @@
Apache Metron
-   Copyright 2015-2018 The Apache Software Foundation
+   Copyright 2015-2020 The Apache Software Foundation
 
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
-
-   This product bundles wait-for-it.sh, which is available under a "MIT 
Software License" license.  For details, see 
https://github.com/vishnubob/wait-for-it
diff --git a/docker/README.md b/docker/README.md
index c82e247..6fa1972 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -25,14 +25,18 @@ testing scripts to be added to a pull request, and 
subsequently to a test suite.
 
 ```bash
 ├── containers
-│   └── bro-localbuild-container
+│   └── bro
+│   └── kafka
+│   └── zookeeper
 ├── data
 ├── in_docker_scripts
 ├── scripts
 └── test_output
 ```
 - `containers`: The parent of all of the containers that this project defines. 
 We use several containers, not all of them ours.
-  - `bro-localbuild-container`: The docker container directory for our bro 
container, used for building bro, the librdkafka, and our plugin, as well as 
running bro.
+  - `bro`: The directory for our bro container, used for building bro, the 
librdkafka, and our plugin, as well as running bro.
+  - `kafka`: The directory for our kafka container.
+  - `zookeeper`: The directory for our zookeeper container.
 - `data`: The default path for pcap data to be used in tests.
 - `in_docker_scripts`: This directory is mapped to the bro docker container as 
/root/built_in_scripts.  These represent the library of scripts we provide to 
be run in the docker container.
 - `scripts`: These are the scripts that are run on the host for creating the 
docker bits, running containers, running or executing commands against 
containers ( such as executing one of the built_in_scripts ), and cleaning up 
resources.
@@ -45,43 +49,27 @@ testing scripts to be added to a pull request, and 
subsequently to a test suite.
 ├── build_bro_plugin.sh
 ├── configure_bro_plugin.sh
 ├── process_data_file.sh
-├── wait-for-it.sh
-├── wait_for_kafka.sh
-└── wait_for_zk.sh
 ```
 
 - `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the provided 
version of the plugin.
 - `conf

[metron-bro-plugin-kafka] branch master updated: METRON-2349 [BRO-PLUGIN-KAFKA] Fix hard coded topic_name in e2e tests (JonZeolla) closes apache/metron-bro-plugin-kafka#43

2020-04-30 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
 new cecf443  METRON-2349 [BRO-PLUGIN-KAFKA] Fix hard coded topic_name in 
e2e tests (JonZeolla) closes apache/metron-bro-plugin-kafka#43
cecf443 is described below

commit cecf443d0e0aeabe303f9fd056ed0f1c8ce602c7
Author: JonZeolla 
AuthorDate: Thu Apr 30 07:44:31 2020 -0400

METRON-2349 [BRO-PLUGIN-KAFKA] Fix hard coded topic_name in e2e tests 
(JonZeolla) closes apache/metron-bro-plugin-kafka#43
---
 docker/README.md   |  4 ++
 docker/in_docker_scripts/configure_bro_plugin.sh   | 44 +-
 docker/run_end_to_end.sh   |  2 +-
 .../scripts/docker_execute_configure_bro_plugin.sh | 20 +++---
 4 files changed, 63 insertions(+), 7 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index 6fa1972..3653181 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -53,6 +53,10 @@ testing scripts to be added to a pull request, and 
subsequently to a test suite.
 
 - `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the provided 
version of the plugin.
 - `configure_bro_plugin.sh`: Configures the plugin for the kafka container, 
and routes all traffic types.
+  ## Parameters
+  ```bash
+  --kafka-topic  [OPTIONAL] The kafka topic to configure. 
Default: bro"
+  ```
 - `process_data_file.sh`: Runs `bro -r` on the passed file
 
 
diff --git a/docker/in_docker_scripts/configure_bro_plugin.sh 
b/docker/in_docker_scripts/configure_bro_plugin.sh
index c292504..2b3235a 100755
--- a/docker/in_docker_scripts/configure_bro_plugin.sh
+++ b/docker/in_docker_scripts/configure_bro_plugin.sh
@@ -25,11 +25,53 @@ shopt -s nocasematch
 # Configures the plugin for all the traffic types
 #
 
+function help {
+  echo " "
+  echo "usage: ${0}"
+  echo "--kafka-topic  [OPTIONAL] The kafka topic to 
configure. Default: bro"
+  echo "-h/--help  Usage information."
+  echo " "
+  echo " "
+}
+
+KAFKA_TOPIC=bro
+
+# Handle command line options
+for i in "$@"; do
+  case $i in
+  #
+  # KAFKA_TOPIC
+  #
+  #   --kafka-topic
+  #
+--kafka-topic=*)
+  KAFKA_TOPIC="${i#*=}"
+  shift # past argument=value
+;;
+  #
+  # -h/--help
+  #
+-h | --help)
+  help
+  exit 0
+  shift # past argument with no value
+;;
+  #
+  # Unknown option
+  #
+*)
+  UNKNOWN_OPTION="${i#*=}"
+  echo "Error: unknown option: $UNKNOWN_OPTION"
+  help
+;;
+  esac
+done
+
 echo "Configuring kafka plugin"
 {
   echo "@load packages"
   echo "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG, Conn::LOG, 
DPD::LOG, FTP::LOG, Files::LOG, Known::CERTS_LOG, SMTP::LOG, SSL::LOG, 
Weird::LOG, Notice::LOG, DHCP::LOG, SSH::LOG, Software::LOG, RADIUS::LOG, 
X509::LOG, Known::DEVICES_LOG, RFB::LOG, Stats::LOG, CaptureLoss::LOG, 
SIP::LOG);"
-  echo "redef Kafka::topic_name = \"bro\";"
+  echo "redef Kafka::topic_name = \"${KAFKA_TOPIC}\";"
   echo "redef Kafka::tag_json = T;"
   echo "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] = 
\"kafka:9092\");"
   echo "redef Kafka::logs_to_exclude = set(Conn::LOG, DHCP::LOG);"
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index 1fcd553..ec5f7b0 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -198,7 +198,7 @@ rc=$?; if [[ ${rc} != 0 ]]; then
 fi
 
 # Configure the bro plugin
-bash "${SCRIPT_DIR}"/docker_execute_configure_bro_plugin.sh
+bash "${SCRIPT_DIR}"/docker_execute_configure_bro_plugin.sh 
--kafka-topic="${KAFKA_TOPIC}"
 rc=$?; if [[ ${rc} != 0 ]]; then
   echo "ERROR> FAILED TO CONFIGURE PLUGIN.  CHECK LOGS  ${rc}"
   exit ${rc}
diff --git a/docker/scripts/docker_execute_configure_bro_plugin.sh 
b/docker/scripts/docker_execute_configure_bro_plugin.sh
index 69162c3..ee62a58 100755
--- a/docker/scripts/docker_execute_configure_bro_plugin.sh
+++ b/docker/scripts/docker_execute_configure_bro_plugin.sh
@@ -31,12 +31,14 @@ function help {
   echo " "
   echo "usage: ${0}"
   echo "--container-name[OPTIONAL] The Docker container 
name. Default: metron-bro-plugin-kafka_bro_1"
+  echo "--kafka-topic   [OPTIONAL] The kafka topic to 
create. Default: bro"
   echo "-h/--help   Usage information."
   echo " "
   echo " "
 }
 
 CONTAINER_NAME=metron-bro-plugin-kafka_bro_1
+KAFKA_TOPIC=bro
 
 # Handle command line options
 f

[metron-bro-plugin-kafka] branch master updated: METRON-1768 Adjust versioning of metron-bro-plugin-kafka to be x.y.z (JonZeolla) closes apache/metron-bro-plugin-kafka#8

2020-05-05 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
 new 5166fee  METRON-1768 Adjust versioning of metron-bro-plugin-kafka to 
be x.y.z (JonZeolla) closes apache/metron-bro-plugin-kafka#8
5166fee is described below

commit 5166fee4200438e43ba65efbc23e5c22eff1bcc1
Author: JonZeolla 
AuthorDate: Tue May 5 20:26:17 2020 -0400

METRON-1768 Adjust versioning of metron-bro-plugin-kafka to be x.y.z 
(JonZeolla) closes apache/metron-bro-plugin-kafka#8
---
 README.md | 4 ++--
 VERSION   | 2 +-
 zkg.meta  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 7ffbac0..041da4f 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,7 @@ This software is a part of the [Apache 
Metron](https://metron.apache.org/) proje
 
 ```
 $ zeek -N Apache::Kafka
-Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3)
+Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3.0)
 ```
 
 ### Manual Installation
@@ -91,7 +91,7 @@ These instructions could also be helpful if you were 
interested in distributing
 
 ```
 $ zeek -N Apache::Kafka
-Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3)
+Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3.0)
 ```
 
 ## Activation
diff --git a/VERSION b/VERSION
index c30be1e..b6a0a05 100644
--- a/VERSION
+++ b/VERSION
@@ -15,4 +15,4 @@
 #  limitations under the License.
 #
 
-0.3
+0.3.0
diff --git a/zkg.meta b/zkg.meta
index 679daa3..d384a82 100644
--- a/zkg.meta
+++ b/zkg.meta
@@ -5,7 +5,7 @@ script_dir = build/scripts/Apache/Kafka
 build_command = ./configure --zeek-dist=%(zeek_dist)s 
--with-librdkafka=%(LIBRDKAFKA_ROOT)s && make
 test_command = ( cd tests && btest -d )
 plugin_dir = build
-version = 0.3
+version = 0.3.0
 depends =
   zeek >=3.0.0
   zkg >=2.0

[metron-bro-plugin-kafka] branch master updated: METRON-2270 Update to reflect bro project rename (JonZeolla) closes apache/metron-bro-plugin-kafka#44

2020-05-05 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
 new 587e9da  METRON-2270 Update to reflect bro project rename (JonZeolla) 
closes apache/metron-bro-plugin-kafka#44
587e9da is described below

commit 587e9dac9b915f7d0c1116a4432447b5d92e
Author: JonZeolla 
AuthorDate: Tue May 5 20:07:34 2020 -0400

METRON-2270 Update to reflect bro project rename (JonZeolla) closes 
apache/metron-bro-plugin-kafka#44
---
 CMakeLists.txt |  28 ++---
 Makefile   |   2 +-
 README.md  | 132 +++--
 bro-pkg.meta   |  15 ---
 configure  | 122 +++
 configure.plugin   |   9 +-
 .../release-utils/metron-bro-kafka-rc-check|  34 +++---
 docker/README.md   |  75 ++--
 docker/containers/bro/requirements-to-freeze.txt   |   1 -
 docker/containers/{bro => zeek}/.screenrc  |   0
 docker/containers/{bro => zeek}/Dockerfile |  44 ---
 docker/containers/{bro => zeek}/Makefile   |   0
 docker/containers/zeek/requirements-to-freeze.txt  |   1 +
 docker/containers/{bro => zeek}/requirements.txt   |   4 +-
 docker/docker-compose.yml  |  10 +-
 .../{build_bro_plugin.sh => build_plugin.sh}   |  41 ---
 ...configure_bro_plugin.sh => configure_plugin.sh} |  20 ++--
 docker/in_docker_scripts/process_data_file.sh  |   2 +-
 docker/run_end_to_end.sh   |  38 +++---
 docker/scripts/analyze_results.sh  |   6 +-
 ...ro_plugin.sh => docker_execute_build_plugin.sh} |  12 +-
 ...lugin.sh => docker_execute_configure_plugin.sh} |  16 +--
 sh => docker_execute_configure_zeek_plugin.sh} |  16 +--
 .../docker_execute_create_topic_in_kafka.sh|   4 +-
 docker/scripts/docker_execute_process_data_file.sh |   6 +-
 docker/scripts/docker_execute_shell.sh |   4 +-
 docker/scripts/docker_run_consume_kafka.sh |   4 +-
 docker/scripts/docker_run_get_offset_kafka.sh  |   4 +-
 docker/scripts/download_sample_pcaps.sh|   4 +-
 docker/scripts/split_kafka_output_by_log.sh|  10 +-
 .../Apache/Kafka/{__load__.bro => __load__.zeek}   |   2 +-
 .../{logs-to-kafka.bro => logs-to-kafka.zeek}  |   4 +-
 scripts/{__load__.bro => __load__.zeek}|  13 +-
 scripts/{init.bro => init.zeek}|   6 +-
 src/KafkaWriter.cc |  14 +--
 src/KafkaWriter.h  |   5 +-
 src/Plugin.h   |   4 +-
 src/TaggedJSON.h   |   5 +-
 tests/.gitignore   |   2 +
 tests/Baseline/kafka.resolved-topic-default/output |   2 +-
 .../Scripts/diff-remove-timestamps |  22 ++--
 tests/Scripts/get-bro-env  |  36 --
 tests/Scripts/get-zeek-env |  51 
 tests/btest.cfg|  14 +--
 ...-l2e-no-overlap.bro => l2s-l2e-no-overlap.zeek} |   2 +-
 .../{l2s-set-l2e-set.bro => l2s-set-l2e-set.zeek}  |   2 +-
 ...2s-set-l2e-unset.bro => l2s-set-l2e-unset.zeek} |   2 +-
 ...2s-unset-l2e-set.bro => l2s-unset-l2e-set.zeek} |   2 +-
 ...nset-l2e-unset.bro => l2s-unset-l2e-unset.zeek} |   2 +-
 ...topic-config.bro => resolved-topic-config.zeek} |   2 +-
 ...pic-default.bro => resolved-topic-default.zeek} |   2 +-
 ...bro => resolved-topic-override-and-config.zeek} |   4 +-
 ...-only.bro => resolved-topic-override-only.zeek} |   4 +-
 ...e-set.bro => send-all-active-logs-l2e-set.zeek} |   2 +-
 ...set.bro => send-all-active-logs-l2e-unset.zeek} |   2 +-
 ...o => send-all-active-logs-l2s-set-l2e-set.zeek} |   2 +-
 ...=> send-all-active-logs-l2s-set-l2e-unset.zeek} |   2 +-
 tests/kafka/{show-plugin.bro => show-plugin.zeek}  |   2 +-
 tests/random.seed  |  21 
 zkg.meta   |  15 +++
 60 files changed, 528 insertions(+), 384 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 18b8a14..5d8613a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -15,25 +15,25 @@
 #  limitations under the License.
 #
 
-cmake_minimum_required(VERSION 2.8)
-project(Plugin)
-include(BroPlugin)
+cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+project(ZeekPlugin_Kafka)
+include(ZeekPlugin)
 find_package(LibRDKafka)
 find_package(OpenSSL)
 
 if (LIBRDKAFKA_FOUND AND OPENSSL_FOUND)
   include_directories(BEFORE ${LibRDKafka_INCLUD

[metron-bro-plugin-kafka] branch master updated: METRON-1817 Remove plugin dependency on zeek_dist (JonZeolla) closes apache/metron-bro-plugin-kafka#12

2020-05-06 Thread jonzeolla

This is an automated email from the ASF dual-hosted git repository.

jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git


The following commit(s) were added to refs/heads/master by this push:
 new 57b6660  METRON-1817 Remove plugin dependency on zeek_dist (JonZeolla) 
closes apache/metron-bro-plugin-kafka#12
57b6660 is described below

commit 57b6660950a7a306a50e04c517a8d4dee5684576
Author: JonZeolla 
AuthorDate: Wed May 6 09:54:32 2020 -0400

METRON-1817 Remove plugin dependency on zeek_dist (JonZeolla) closes 
apache/metron-bro-plugin-kafka#12
---
 README.md | 2 +-
 zkg.meta  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 041da4f..054ca04 100644
--- a/README.md
+++ b/README.md
@@ -82,7 +82,7 @@ These instructions could also be helpful if you were 
interested in distributing
 1. Build the plugin using the following commands.
 
 ```
-$ ./configure --zeek-dist=$zeek_dist --with-librdkafka=$librdkafka_root
+$ ./configure --with-librdkafka=$librdkafka_root
 $ make
 $ sudo make install
 ```
diff --git a/zkg.meta b/zkg.meta
index d384a82..2b1325a 100644
--- a/zkg.meta
+++ b/zkg.meta
@@ -2,7 +2,7 @@
 description = A Zeek log writer plugin that sends logging output to Kafka.
 tags = log writer, zeek plugin, kafka
 script_dir = build/scripts/Apache/Kafka
-build_command = ./configure --zeek-dist=%(zeek_dist)s 
--with-librdkafka=%(LIBRDKAFKA_ROOT)s && make
+build_command = ./configure --with-librdkafka=%(LIBRDKAFKA_ROOT)s && make
 test_command = ( cd tests && btest -d )
 plugin_dir = build
 version = 0.3.0

39 matches

Mail list logo