from:"cestella"

[4/4] metron git commit: Merge branch 'master' into feature/METRON-1554-pcap-query-panel

2018-07-26 Thread cestella

Merge branch 'master' into feature/METRON-1554-pcap-query-panel


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7dff4def
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7dff4def
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7dff4def

Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 7dff4def23a7c839ea599a120ab6d4cf5b9bbbef
Parents: 6c90724 ccdbeff
Author: cstella 
Authored: Thu Jul 26 12:29:04 2018 -0400
Committer: cstella 
Committed: Thu Jul 26 12:29:04 2018 -0400

--
 .../docker/rpm-docker/SPECS/metron.spec |   1 +
 .../e2e/utils/clean_metron_update_table.ts  |  18 ++
 .../metron-alerts/e2e/utils/e2e_util.ts |  20 +-
 .../alert-filters.component.spec.ts |  18 ++
 .../alert-filters/alert-filters.component.ts|  20 +-
 .../table-view/table-view.component.spec.ts |  18 ++
 .../tree-view/tree-view.component.spec.ts   |  18 ++
 .../alerts/meta-alerts/meta-alerts.module.ts|  18 ++
 .../date-picker/date-picker.component.spec.ts   |  18 ++
 .../shared/date-picker/date-picker.component.ts |  18 ++
 .../shared/date-picker/date-picker.module.ts|  18 ++
 .../alert-severity-hexagon.directive.spec.ts|  18 ++
 .../app/shared/group-by/group-by.component.ts   |  18 ++
 .../metron-alerts/src/app/shared/index.ts   |  18 ++
 .../src/app/shared/pipes/map-keys.pipe.spec.ts  |  18 ++
 .../src/app/shared/pipes/map-keys.pipe.ts   |  18 ++
 .../src/app/shared/pipes/time-lapse.pipe.ts |  18 ++
 .../time-range/time-range.component.spec.ts |  18 ++
 .../app/shared/time-range/time-range.module.ts  |  18 ++
 metron-interface/metron-alerts/src/polyfills.ts |  18 ++
 .../metron-config/e2e/utils/e2e_util.ts |  18 ++
 .../src/app/app.config.interface.ts |  18 ++
 .../metron-config/src/app/app.config.ts |  18 ++
 .../sensor-grok/sensor-grok.component.ts|  18 ++
 .../sensor-parser-config-readonly/index.ts  |  18 ++
 .../app/sensors/sensor-parser-config/index.ts   |  18 ++
 .../shared/ace-editor/ace-editor.component.ts   |  18 ++
 .../app/shared/ace-editor/ace-editor.module.ts  |  18 ++
 .../advanced-config-form.module.ts  |  18 ++
 .../src/app/shared/metron-modal/index.ts|  18 ++
 .../shared/metron-table/metron-sorter/index.ts  |  18 ++
 .../metron-sorter.component.spec.ts |  18 ++
 .../metron-sorter/metron-sorter.component.ts|  18 ++
 .../shared/metron-table/metron-table.module.ts  |  18 ++
 .../src/app/shared/multiple-input/index.ts  |  18 ++
 .../multiple-input/multiple-input.module.ts |  18 ++
 .../src/app/shared/number-spinner/index.ts  |  18 ++
 .../number-spinner/number-spinner.module.ts |  18 ++
 .../shared/sample-data/sample-data.module.ts|  18 ++
 .../src/environments/environment.prod.ts|  18 ++
 .../src/environments/environment.ts |  18 ++
 metron-interface/metron-rest/README.md  |  12 +-
 .../parsed/jsonMapExampleParsed |   6 +
 .../raw/jsonMapExampleOutput|   3 +
 metron-platform/metron-parsers/README.md|   5 +
 .../zookeeper/parsers/jsonMapWrappedQuery.json  |   5 +
 .../metron/parsers/json/JSONMapParser.java  |  40 +++-
 .../JSONMapWrappedQueryIntegrationTest.java |  37 
 .../integration/ParserIntegrationTest.java  |  22 +-
 .../json/JSONMapParserWrappedQueryTest.java | 199 +++
 pom.xml |  19 +-
 51 files changed, 1044 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/7dff4def/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--

http://git-wip-us.apache.org/repos/asf/metron/blob/7dff4def/metron-interface/metron-rest/README.md
--
diff --cc metron-interface/metron-rest/README.md
index 2d9a535,caa9c78..68ec559
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@@ -253,15 -242,8 +253,15 @@@ Request and Response objects are JSON f
  | [ `GET /api/v1/metaalert/add/alert`](#get-apiv1metaalertaddalert)|
  | [ `GET /api/v1/metaalert/remove/alert`](#get-apiv1metaalertremovealert)|
  | [ `GET 
/api/v1/metaalert/update/status/{guid}/{status}`](#get-apiv1metaalertupdatestatusguidstatus)|
 +| [ `POST /api/v1/pcap/fixed`](#post-apiv1pcapfixed)|
 +| [ `POST /api/v1/pcap/query`](#post-apiv1pcapquery)|
 +| [ `GET /api/v1/pcap`](#get-apiv1pcap)|
 +| [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)|
 +| [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)|
 +| [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)|
 +| [ `GET

[1/4] metron git commit: METRON-1606 Add a wrap to incoming messages in the metron json parser (ottobackwards) closes apache/metron#1054

2018-07-26 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1554-pcap-query-panel 6c90724d8 -> 7dff4def2


METRON-1606 Add a wrap to incoming messages in the metron json 
parser (ottobackwards) closes apache/metron#1054


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6bac842d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6bac842d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6bac842d

Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 6bac842d7ad2819c56412837d8befb603ef15f35
Parents: aceca46
Author: ottobackwards 
Authored: Fri Jul 20 17:44:35 2018 -0400
Committer: otto 
Committed: Fri Jul 20 17:44:35 2018 -0400

--
 .../docker/rpm-docker/SPECS/metron.spec |   1 +
 .../parsed/jsonMapExampleParsed |   6 +
 .../raw/jsonMapExampleOutput|   3 +
 metron-platform/metron-parsers/README.md|   5 +
 .../zookeeper/parsers/jsonMapWrappedQuery.json  |   5 +
 .../metron/parsers/json/JSONMapParser.java  |  40 +++-
 .../JSONMapWrappedQueryIntegrationTest.java |  37 
 .../integration/ParserIntegrationTest.java  |  22 +-
 .../json/JSONMapParserWrappedQueryTest.java | 199 +++
 9 files changed, 299 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/6bac842d/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec 
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 4b88fd0..b308908 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -151,6 +151,7 @@ This package installs the Metron Parser files
 %{metron_home}/config/zookeeper/parsers/bro.json
 %{metron_home}/config/zookeeper/parsers/jsonMap.json
 %{metron_home}/config/zookeeper/parsers/jsonMapQuery.json
+%{metron_home}/config/zookeeper/parsers/jsonMapWrappedQuery.json
 %{metron_home}/config/zookeeper/parsers/snort.json
 %{metron_home}/config/zookeeper/parsers/squid.json
 %{metron_home}/config/zookeeper/parsers/websphere.json

http://git-wip-us.apache.org/repos/asf/metron/blob/6bac842d/metron-platform/metron-integration-test/src/main/sample/data/jsonMapWrappedQuery/parsed/jsonMapExampleParsed
--
diff --git 
a/metron-platform/metron-integration-test/src/main/sample/data/jsonMapWrappedQuery/parsed/jsonMapExampleParsed
 
b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapWrappedQuery/parsed/jsonMapExampleParsed
new file mode 100644
index 000..c6aac78
--- /dev/null
+++ 
b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapWrappedQuery/parsed/jsonMapExampleParsed
@@ -0,0 +1,6 @@
+{ "string" : "foo", "number" : 1, "ignored" : [ "blah" ], "original_string":"{ 
\"string\" : \"foo\", \"number\" : 1, \"ignored\" : [ \"blah\" ] 
}","timestamp":1, 
"source.type":"jsonMapWrappedQuery","guid":"this-is-random-uuid-will-be-36-chars"
 }
+{ "number" : 4 , "original_string" : "{ \"number\" : 4 }", 
"source.type":"jsonMapWrappedQuery","timestamp":1,"guid":"this-is-random-uuid-will-be-36-chars"}
+{ "string" : "bar", "number" : 2, "ignored" : [ "blah" ], "original_string":"{ 
\"string\" : \"bar\", \"number\" : 2, \"ignored\" : [ \"blah\" ] 
}","timestamp":1, 
"source.type":"jsonMapWrappedQuery","guid":"this-is-random-uuid-will-be-36-chars"
 }
+{ "number" : 5 , "original_string" : "{ \"number\" : 5 }", 
"source.type":"jsonMapWrappedQuery","timestamp":1,"guid":"this-is-random-uuid-will-be-36-chars"}
+{ "string" : "baz", "number" : 3, "ignored" : [ "blah" ], "original_string":"{ 
\"string\" : \"baz\", \"number\" : 3, \"ignored\" : [ \"blah\" ] 
}","timestamp":1, 
"source.type":"jsonMapWrappedQuery","guid":"this-is-random-uuid-will-be-36-chars"
 }
+{ "number" : 6 , "original_string" : "{ \"number\" : 6 }", 
"source.type":"jsonMapWrappedQuery","timestamp":1,"guid":"this-is-random-uuid-will-be-36-chars"}

http://git-wip-us.apache.org/repos/asf/metron/blob/6bac842d/metron-platform/metron-integration-test/src/main/sample/data/jsonMapWrappedQuery/raw/jsonMapExampleOutput
--
diff --git 
a/metron-platform/metron-integration-test/src/main/sample/data/jsonMapWrappedQuery/raw/jsonMapExampleOutput
 
b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapWrappedQuery/raw/jsonMapExampleOutput
new file mode 100644
index 000..f142a55
--- /dev/null
+++

[2/4] metron git commit: METRON-1694: Clean up Metron REST docs closes apache/incubator-metron#1131

2018-07-26 Thread cestella

METRON-1694: Clean up Metron REST docs closes apache/incubator-metron#1131


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/44a6f6f1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/44a6f6f1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/44a6f6f1

Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 44a6f6f11d8d750f176f53d6032e46c90f88d9e4
Parents: 6bac842
Author: JonZeolla 
Authored: Thu Jul 26 09:43:04 2018 -0400
Committer: cstella 
Committed: Thu Jul 26 09:43:04 2018 -0400

--
 metron-interface/metron-rest/README.md | 13 ++---
 1 file changed, 6 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/44a6f6f1/metron-interface/metron-rest/README.md
--
diff --git a/metron-interface/metron-rest/README.md 
b/metron-interface/metron-rest/README.md
index 44594f7..caa9c78 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -216,7 +216,7 @@ Request and Response objects are JSON formatted.  The JSON 
schemas are available
 
 ||
 | -- |
-| [ `POST /api/v1/alerts/ui/escalate`](#get-apiv1alertsuiescalate)|
+| [ `POST /api/v1/alerts/ui/escalate`](#post-apiv1alertsuiescalate)|
 | [ `GET /api/v1/alerts/ui/settings`](#get-apiv1alertsuisettings)|
 | [ `GET /api/v1/alerts/ui/settings/all`](#get-apiv1alertsuisettingsall)|
 | [ `DELETE /api/v1/alerts/ui/settings`](#delete-apiv1alertsuisettings)|
@@ -242,9 +242,8 @@ Request and Response objects are JSON formatted.  The JSON 
schemas are available
 | [ `GET /api/v1/metaalert/add/alert`](#get-apiv1metaalertaddalert)|
 | [ `GET /api/v1/metaalert/remove/alert`](#get-apiv1metaalertremovealert)|
 | [ `GET 
/api/v1/metaalert/update/status/{guid}/{status}`](#get-apiv1metaalertupdatestatusguidstatus)|
-| [ `GET /api/v1/search/search`](#get-apiv1searchsearch)|
-| [ `POST /api/v1/search/search`](#get-apiv1searchsearch)|
-| [ `POST /api/v1/search/group`](#get-apiv1searchgroup)|
+| [ `POST /api/v1/search/search`](#post-apiv1searchsearch)|
+| [ `POST /api/v1/search/group`](#post-apiv1searchgroup)|
 | [ `GET /api/v1/search/findOne`](#get-apiv1searchfindone)|
 | [ `GET /api/v1/search/column/metadata`](#get-apiv1searchcolumnmetadata)|
 | [ `GET /api/v1/sensor/enrichment/config`](#get-apiv1sensorenrichmentconfig)|
@@ -293,7 +292,7 @@ Request and Response objects are JSON formatted.  The JSON 
schemas are available
 | [ `GET /api/v1/storm/{name}`](#get-apiv1stormname)|
 | [ `GET /api/v1/storm/supervisors`](#get-apiv1stormsupervisors)|
 | [ `PATCH /api/v1/update/patch`](#patch-apiv1updatepatch)|
-| [ `PUT /api/v1/update/replace`](#patch-apiv1updatereplace)|
+| [ `PUT /api/v1/update/replace`](#put-apiv1updatereplace)|
 | [ `GET /api/v1/user`](#get-apiv1user)|
 
 ### `POST /api/v1/alerts/ui/escalate`
@@ -832,8 +831,8 @@ Request and Response objects are JSON formatted.  The JSON 
schemas are available
  }
 ```
   * Returns:
-* 200 - nothing
-* 404 - document not found
+* 200 - Nothing
+* 404 - Document not found
 
 ### `PUT /api/v1/update/replace`
   * Description: Replace a document

[3/4] metron git commit: METRON-1687: Upgrade the rat plugin to 0.13-SNAPSHOT closes apache/incubator-metron#1126

2018-07-26 Thread cestella

METRON-1687: Upgrade the rat plugin to 0.13-SNAPSHOT closes 
apache/incubator-metron#1126


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ccdbeff5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ccdbeff5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ccdbeff5

Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: ccdbeff5076553382091d4b9423ed48ccdba10ee
Parents: 44a6f6f
Author: cstella 
Authored: Thu Jul 26 09:44:59 2018 -0400
Committer: cstella 
Committed: Thu Jul 26 09:44:59 2018 -0400

--
 .../e2e/utils/clean_metron_update_table.ts  | 18 ++
 .../metron-alerts/e2e/utils/e2e_util.ts | 20 +++-
 .../alert-filters.component.spec.ts | 18 ++
 .../alert-filters/alert-filters.component.ts| 20 +++-
 .../table-view/table-view.component.spec.ts | 18 ++
 .../tree-view/tree-view.component.spec.ts   | 18 ++
 .../alerts/meta-alerts/meta-alerts.module.ts| 18 ++
 .../date-picker/date-picker.component.spec.ts   | 18 ++
 .../shared/date-picker/date-picker.component.ts | 18 ++
 .../shared/date-picker/date-picker.module.ts| 18 ++
 .../alert-severity-hexagon.directive.spec.ts| 18 ++
 .../app/shared/group-by/group-by.component.ts   | 18 ++
 .../metron-alerts/src/app/shared/index.ts   | 18 ++
 .../src/app/shared/pipes/map-keys.pipe.spec.ts  | 18 ++
 .../src/app/shared/pipes/map-keys.pipe.ts   | 18 ++
 .../src/app/shared/pipes/time-lapse.pipe.ts | 18 ++
 .../time-range/time-range.component.spec.ts | 18 ++
 .../app/shared/time-range/time-range.module.ts  | 18 ++
 metron-interface/metron-alerts/src/polyfills.ts | 18 ++
 .../metron-config/e2e/utils/e2e_util.ts | 18 ++
 .../src/app/app.config.interface.ts | 18 ++
 .../metron-config/src/app/app.config.ts | 18 ++
 .../sensor-grok/sensor-grok.component.ts| 18 ++
 .../sensor-parser-config-readonly/index.ts  | 18 ++
 .../app/sensors/sensor-parser-config/index.ts   | 18 ++
 .../shared/ace-editor/ace-editor.component.ts   | 18 ++
 .../app/shared/ace-editor/ace-editor.module.ts  | 18 ++
 .../advanced-config-form.module.ts  | 18 ++
 .../src/app/shared/metron-modal/index.ts| 18 ++
 .../shared/metron-table/metron-sorter/index.ts  | 18 ++
 .../metron-sorter.component.spec.ts | 18 ++
 .../metron-sorter/metron-sorter.component.ts| 18 ++
 .../shared/metron-table/metron-table.module.ts  | 18 ++
 .../src/app/shared/multiple-input/index.ts  | 18 ++
 .../multiple-input/multiple-input.module.ts | 18 ++
 .../src/app/shared/number-spinner/index.ts  | 18 ++
 .../number-spinner/number-spinner.module.ts | 18 ++
 .../shared/sample-data/sample-data.module.ts| 18 ++
 .../src/environments/environment.prod.ts| 18 ++
 .../src/environments/environment.ts | 18 ++
 pom.xml | 19 +--
 41 files changed, 739 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/ccdbeff5/metron-interface/metron-alerts/e2e/utils/clean_metron_update_table.ts
--
diff --git 
a/metron-interface/metron-alerts/e2e/utils/clean_metron_update_table.ts 
b/metron-interface/metron-alerts/e2e/utils/clean_metron_update_table.ts
index c6e99df..e4da2be 100644
--- a/metron-interface/metron-alerts/e2e/utils/clean_metron_update_table.ts
+++ b/metron-interface/metron-alerts/e2e/utils/clean_metron_update_table.ts
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT

[metron] Git Push Summary

2018-07-26 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/METRON-1554-pcap-query-panel [deleted] 2acb0f2db

metron git commit: METRON-1694: Clean up Metron REST docs closes apache/incubator-metron#1131

2018-07-26 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 6bac842d7 -> 44a6f6f11


METRON-1694: Clean up Metron REST docs closes apache/incubator-metron#1131


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/44a6f6f1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/44a6f6f1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/44a6f6f1

Branch: refs/heads/master
Commit: 44a6f6f11d8d750f176f53d6032e46c90f88d9e4
Parents: 6bac842
Author: JonZeolla 
Authored: Thu Jul 26 09:43:04 2018 -0400
Committer: cstella 
Committed: Thu Jul 26 09:43:04 2018 -0400

--
 metron-interface/metron-rest/README.md | 13 ++---
 1 file changed, 6 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/44a6f6f1/metron-interface/metron-rest/README.md
--
diff --git a/metron-interface/metron-rest/README.md 
b/metron-interface/metron-rest/README.md
index 44594f7..caa9c78 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -216,7 +216,7 @@ Request and Response objects are JSON formatted.  The JSON 
schemas are available
 
 ||
 | -- |
-| [ `POST /api/v1/alerts/ui/escalate`](#get-apiv1alertsuiescalate)|
+| [ `POST /api/v1/alerts/ui/escalate`](#post-apiv1alertsuiescalate)|
 | [ `GET /api/v1/alerts/ui/settings`](#get-apiv1alertsuisettings)|
 | [ `GET /api/v1/alerts/ui/settings/all`](#get-apiv1alertsuisettingsall)|
 | [ `DELETE /api/v1/alerts/ui/settings`](#delete-apiv1alertsuisettings)|
@@ -242,9 +242,8 @@ Request and Response objects are JSON formatted.  The JSON 
schemas are available
 | [ `GET /api/v1/metaalert/add/alert`](#get-apiv1metaalertaddalert)|
 | [ `GET /api/v1/metaalert/remove/alert`](#get-apiv1metaalertremovealert)|
 | [ `GET 
/api/v1/metaalert/update/status/{guid}/{status}`](#get-apiv1metaalertupdatestatusguidstatus)|
-| [ `GET /api/v1/search/search`](#get-apiv1searchsearch)|
-| [ `POST /api/v1/search/search`](#get-apiv1searchsearch)|
-| [ `POST /api/v1/search/group`](#get-apiv1searchgroup)|
+| [ `POST /api/v1/search/search`](#post-apiv1searchsearch)|
+| [ `POST /api/v1/search/group`](#post-apiv1searchgroup)|
 | [ `GET /api/v1/search/findOne`](#get-apiv1searchfindone)|
 | [ `GET /api/v1/search/column/metadata`](#get-apiv1searchcolumnmetadata)|
 | [ `GET /api/v1/sensor/enrichment/config`](#get-apiv1sensorenrichmentconfig)|
@@ -293,7 +292,7 @@ Request and Response objects are JSON formatted.  The JSON 
schemas are available
 | [ `GET /api/v1/storm/{name}`](#get-apiv1stormname)|
 | [ `GET /api/v1/storm/supervisors`](#get-apiv1stormsupervisors)|
 | [ `PATCH /api/v1/update/patch`](#patch-apiv1updatepatch)|
-| [ `PUT /api/v1/update/replace`](#patch-apiv1updatereplace)|
+| [ `PUT /api/v1/update/replace`](#put-apiv1updatereplace)|
 | [ `GET /api/v1/user`](#get-apiv1user)|
 
 ### `POST /api/v1/alerts/ui/escalate`
@@ -832,8 +831,8 @@ Request and Response objects are JSON formatted.  The JSON 
schemas are available
  }
 ```
   * Returns:
-* 200 - nothing
-* 404 - document not found
+* 200 - Nothing
+* 404 - Document not found
 
 ### `PUT /api/v1/update/replace`
   * Description: Replace a document

metron git commit: METRON-1659: The platform-info.sh should check for the vagrant hostmanager plugin closes apache/incubator-metron#1100

2018-07-16 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master ae915a02c -> a4bec95b9


METRON-1659: The platform-info.sh should check for the vagrant hostmanager 
plugin closes apache/incubator-metron#1100


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a4bec95b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a4bec95b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a4bec95b

Branch: refs/heads/master
Commit: a4bec95b9fc0b5b82c4462ff13636071b11ab1f8
Parents: ae915a0
Author: JonZeolla 
Authored: Mon Jul 16 14:02:36 2018 -0400
Committer: cstella 
Committed: Mon Jul 16 14:02:36 2018 -0400

--
 metron-deployment/scripts/platform-info.sh | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/a4bec95b/metron-deployment/scripts/platform-info.sh
--
diff --git a/metron-deployment/scripts/platform-info.sh 
b/metron-deployment/scripts/platform-info.sh
index fd2248a..456de72 100755
--- a/metron-deployment/scripts/platform-info.sh
+++ b/metron-deployment/scripts/platform-info.sh
@@ -51,6 +51,10 @@ ansible --version
 echo "--"
 vagrant --version
 
+# vagrant hostmanager plugin
+echo "--"
+vagrant plugin list | grep vagrant-hostmanager || echo "The Vagrant 
Hostmanager Plugin is NOT installed"
+
 # python
 echo "--"
 python --version 2>&1

metron git commit: METRON-1658: Upgrade bro to 2.5.4 closes apache/incubator-metron#1101

2018-07-16 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 75d6a72a3 -> ae915a02c


METRON-1658: Upgrade bro to 2.5.4 closes apache/incubator-metron#1101


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ae915a02
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ae915a02
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ae915a02

Branch: refs/heads/master
Commit: ae915a02ce90fb9b4c03166e7b57f8aa2a5d7890
Parents: 75d6a72
Author: JonZeolla 
Authored: Mon Jul 16 12:01:49 2018 -0400
Committer: cstella 
Committed: Mon Jul 16 12:01:49 2018 -0400

--
 metron-deployment/ansible/playbooks/docker_probe_install.yml   | 2 +-
 metron-deployment/ansible/roles/bro/vars/main.yml  | 2 +-
 .../development/centos6/ansible/inventory/group_vars/all   | 2 +-
 .../development/ubuntu14/ansible/inventory/group_vars/all  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/ae915a02/metron-deployment/ansible/playbooks/docker_probe_install.yml
--
diff --git a/metron-deployment/ansible/playbooks/docker_probe_install.yml 
b/metron-deployment/ansible/playbooks/docker_probe_install.yml
index 3de212e..5e5165e 100644
--- a/metron-deployment/ansible/playbooks/docker_probe_install.yml
+++ b/metron-deployment/ansible/playbooks/docker_probe_install.yml
@@ -32,7 +32,7 @@
   vars:
 metron_version: 0.5.1
 metron_directory: /usr/metron/{{ metron_version }}
-bro_version: "2.5.2"
+bro_version: "2.5.4"
 fixbuf_version: "1.7.1"
 yaf_version: "2.8.0"
 daq_version: "2.0.6-1"

http://git-wip-us.apache.org/repos/asf/metron/blob/ae915a02/metron-deployment/ansible/roles/bro/vars/main.yml
--
diff --git a/metron-deployment/ansible/roles/bro/vars/main.yml 
b/metron-deployment/ansible/roles/bro/vars/main.yml
index 11d6c31..9e6f221 100644
--- a/metron-deployment/ansible/roles/bro/vars/main.yml
+++ b/metron-deployment/ansible/roles/bro/vars/main.yml
@@ -17,7 +17,7 @@
 ---
 bro_home: /usr/local/bro
 bro_bin: "{{ bro_home }}/bin"
-bro_version: 2.5.2
+bro_version: 2.5.4
 bro_daemon_log: /var/log/bro.log
 bro_topic: bro
 

http://git-wip-us.apache.org/repos/asf/metron/blob/ae915a02/metron-deployment/development/centos6/ansible/inventory/group_vars/all
--
diff --git 
a/metron-deployment/development/centos6/ansible/inventory/group_vars/all 
b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
index 95792c3..192ed16 100644
--- a/metron-deployment/development/centos6/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
@@ -37,7 +37,7 @@ enrichment_hbase_table: enrichment
 # metron
 metron_version: 0.5.1
 metron_directory: /usr/metron/{{ metron_version }}
-bro_version: "2.5.2"
+bro_version: "2.5.4"
 fixbuf_version: "1.7.1"
 yaf_version: "2.8.0"
 daq_version: "2.0.6-1"

http://git-wip-us.apache.org/repos/asf/metron/blob/ae915a02/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
--
diff --git 
a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all 
b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
index 6276fc0..195972a 100644
--- a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
@@ -37,7 +37,7 @@ enrichment_hbase_table: enrichment
 # metron
 metron_version: 0.5.1
 metron_directory: /usr/metron/{{ metron_version }}
-bro_version: "2.5.2"
+bro_version: "2.5.4"
 fixbuf_version: "1.7.1"
 yaf_version: "2.8.0"
 daq_version: "2.0.6-1"

metron git commit: METRON-1236 Add start/stop/restart commands that execute successfully, when ambari agents run as non-root user closes apache/incubator-metron#1105

2018-07-16 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 4544da701 -> 75d6a72a3


METRON-1236 Add start/stop/restart commands that execute successfully, when 
ambari agents run as non-root user closes apache/incubator-metron#1105


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/75d6a72a
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/75d6a72a
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/75d6a72a

Branch: refs/heads/master
Commit: 75d6a72a3867d1f89dbec8dc059870482ac81e94
Parents: 4544da7
Author: Stefan Kupstaitis-Dunkler 
Authored: Mon Jul 16 11:58:24 2018 -0400
Committer: cstella 
Committed: Mon Jul 16 11:58:24 2018 -0400

--
 .../METRON/CURRENT/package/scripts/alerts_ui_commands.py| 9 ++---
 .../CURRENT/package/scripts/management_ui_commands.py   | 9 ++---
 2 files changed, 12 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/75d6a72a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/alerts_ui_commands.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/alerts_ui_commands.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/alerts_ui_commands.py
index 41cd375..f837043 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/alerts_ui_commands.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/alerts_ui_commands.py
@@ -38,7 +38,8 @@ class AlertsUICommands:
 :param env: Environment
 """
 Logger.info('Starting Alerts UI')
-Execute("service metron-alerts-ui start")
+start_cmd = ('service', 'metron-alerts-ui', 'start')
+Execute(start_cmd, sudo=True)
 Logger.info('Done starting Alerts UI')
 
 def stop_alerts_ui(self):
@@ -47,7 +48,8 @@ class AlertsUICommands:
 :param env: Environment
 """
 Logger.info('Stopping Alerts UI')
-Execute("service metron-alerts-ui stop")
+stop_cmd = ('service', 'metron-alerts-ui', 'stop')
+Execute(stop_cmd, sudo=True)
 Logger.info('Done stopping Alerts UI')
 
 def restart_alerts_ui(self, env):
@@ -56,7 +58,8 @@ class AlertsUICommands:
 :param env: Environment
 """
 Logger.info('Restarting the Alerts UI')
-Execute('service metron-alerts-ui restart')
+restart_cmd = ('service', 'metron-alerts-ui', 'restart')
+Execute(restart_cmd, sudo=True)
 Logger.info('Done restarting the Alerts UI')
 
 def status_alerts_ui(self, env):

http://git-wip-us.apache.org/repos/asf/metron/blob/75d6a72a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_commands.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_commands.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_commands.py
index 1f16fa4..4e81df6 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_commands.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_commands.py
@@ -40,7 +40,8 @@ class ManagementUICommands:
 :param env: Environment
 """
 Logger.info('Starting Management UI')
-Execute("service metron-management-ui start")
+start_cmd = ('service', 'metron-management-ui', 'start')
+Execute(start_cmd, sudo=True)
 Logger.info('Done starting Management UI')
 
 def stop_management_ui(self):
@@ -49,7 +50,8 @@ class ManagementUICommands:
 :param env: Environment
 """
 Logger.info('Stopping Management UI')
-Execute("service metron-management-ui stop")
+stop_cmd = ('service', 'metron-management-ui', 'stop')
+Execute(stop_cmd, sudo=True)
 Logger.info('Done stopping Management UI')
 
 def restart_management_ui(self, env):
@@ -58,7 +60,8 @@ class ManagementUICommands:
 :param env: Environment
 """
 Logger.info('Restarting the Management UI')
-Execute('service metron-management-ui restart')
+restart_cmd = ('service',

metron git commit: METRON-1670: Stellar WEEK_OF_YEAR test is locale sensitive closes apache/incubator-metron#1104

2018-07-16 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 28f4b5704 -> 4544da701


METRON-1670: Stellar WEEK_OF_YEAR test is locale sensitive closes 
apache/incubator-metron#1104


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4544da70
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4544da70
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4544da70

Branch: refs/heads/master
Commit: 4544da7010114af4332ff236949b12a96c457f39
Parents: 28f4b57
Author: Simon Elliston Ball 
Authored: Mon Jul 16 11:52:32 2018 -0400
Committer: cstella 
Committed: Mon Jul 16 11:52:32 2018 -0400

--
 .../apache/metron/stellar/dsl/functions/DateFunctionsTest.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/4544da70/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
--
diff --git 
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
 
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
index 0060694..1f1f4f4 100644
--- 
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
+++ 
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
@@ -182,7 +182,8 @@ public class DateFunctionsTest {
   @Test
   public void testWeekOfYear() {
 Object result = run("WEEK_OF_YEAR(epoch)");
-assertEquals(35, result);
+calendar.setTimeInMillis(AUG2016);
+assertEquals(calendar.get(Calendar.WEEK_OF_YEAR), result);
   }
 
   /**

[metron] Git Push Summary

2018-07-12 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1663-knoxsso [created] 28f4b5704

metron git commit: METRON-1643: Create a REGEX_ROUTING field transformation closes apache/incubator-metron#1083

2018-07-06 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 4eb9a8ebc -> f183d987a


METRON-1643: Create a REGEX_ROUTING field transformation closes 
apache/incubator-metron#1083


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f183d987
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f183d987
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f183d987

Branch: refs/heads/master
Commit: f183d987ab1c9856c585c45b0f9f549aadccb65f
Parents: 4eb9a8e
Author: cstella 
Authored: Fri Jul 6 11:40:54 2018 -0400
Committer: cstella 
Committed: Fri Jul 6 11:40:54 2018 -0400

--
 .../transformation/FieldTransformations.java|   1 +
 .../RegexSelectTransformation.java  |  97 +++
 .../RegexSelectTransformationTest.java  | 169 +++
 metron-platform/metron-parsers/README.md|  23 +++
 4 files changed, 290 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
index 95ff390..d80c1a4 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
@@ -26,6 +26,7 @@ public enum FieldTransformations {
   ,STELLAR(new StellarTransformation())
   ,SELECT(new SelectTransformation())
   ,RENAME(new RenameTransformation())
+  ,REGEX_SELECT(new RegexSelectTransformation())
   ;
   FieldTransformation mapping;
   FieldTransformations(FieldTransformation mapping) {

http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java
new file mode 100644
index 000..b8d1c07
--- /dev/null
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.field.transformation;
+
+import com.google.common.collect.Iterables;
+import org.apache.metron.stellar.common.utils.PatternCache;
+import org.apache.metron.stellar.dsl.Context;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+public class RegexSelectTransformation implements FieldTransformation {
+  @Override
+  public Map map( Map input
+, List outputField
+, LinkedHashMap 
fieldMappingConfig
+, Context context
+, Map... sensorConfig
+) {
+String outField = null;
+if(!(outputField == null || outputField.isEmpty())) {
+  outField = outputField.get(0);
+}
+String inVal = null;
+if(!(input == null || input.isEmpty() || input.size() > 1)) {
+  Object inValObj = Iterables.getFirst(input.entrySet(), null).getValue();
+  if(inValObj != null) {
+inVal = inValObj.toString();
+  }
+}
+Map ret = new HashMap<>(1);
+if(outField == null || inVal == null) {
+  //in the situation where

metron git commit: METRON-1647 Fix logging level score closes apache/incubator-metron#1089

2018-06-29 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 4519f7210 -> 55d46b0ec


METRON-1647 Fix logging level score closes apache/incubator-metron#1089


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/55d46b0e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/55d46b0e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/55d46b0e

Branch: refs/heads/master
Commit: 55d46b0ec630fc926a6deeafaa079ae3841a3677
Parents: 4519f72
Author: Pravin Rahangdale 
Authored: Fri Jun 29 09:22:43 2018 -0400
Committer: cstella 
Committed: Fri Jun 29 09:22:43 2018 -0400

--
 .../main/java/org/apache/metron/maas/service/runner/Runner.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/55d46b0e/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
--
diff --git 
a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
 
b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
index 8f0b9e5..4103d3d 100644
--- 
a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
+++ 
b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/service/runner/Runner.java
@@ -177,7 +177,7 @@ public class Runner {
 String hostname = RunnerOptions.HOSTNAME.get(cli);
 CuratorFramework client = null;
 
-LOG.error("Running script " + script);
+LOG.info("Running script " + script);
 LOG.info("Local Directory Contents");
 for(File f : new File(".").listFiles()) {
   LOG.info("  " + f.getName());

metron git commit: METRON-1621: Sorting alerts table by score closes apache/incubator-metron#1088

2018-06-29 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master c4c790dbe -> 4519f7210


METRON-1621: Sorting alerts table by score closes apache/incubator-metron#1088


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4519f721
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4519f721
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4519f721

Branch: refs/heads/master
Commit: 4519f72104b294c2f0eb8135181a3fa68eee9088
Parents: c4c790d
Author: tiborm 
Authored: Fri Jun 29 09:17:27 2018 -0400
Committer: cstella 
Committed: Fri Jun 29 09:17:27 2018 -0400

--
 .../alert-filters/alert-filters.e2e-spec.ts |  2 +-
 .../e2e/alerts-list/alerts-list.po.ts   |  6 +-
 .../meta-alerts/meta-alert.e2e-spec.ts  |  2 +-
 .../table-view/table-view.e2e-spec.ts   | 90 
 .../e2e/alerts-list/table-view/table-view.po.ts | 26 ++
 .../alerts-list/tree-view/tree-view.e2e-spec.ts |  4 +-
 .../e2e/mock-data/alerts_ui_e2e_index.data  | 16 ++--
 .../metron-alerts/e2e/utils/e2e_util.ts | 62 --
 .../metron-alerts/protractor.conf.js|  1 +
 .../table-view/table-view.component.html| 10 +--
 10 files changed, 172 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/4519f721/metron-interface/metron-alerts/e2e/alerts-list/alert-filters/alert-filters.e2e-spec.ts
--
diff --git 
a/metron-interface/metron-alerts/e2e/alerts-list/alert-filters/alert-filters.e2e-spec.ts
 
b/metron-interface/metron-alerts/e2e/alerts-list/alert-filters/alert-filters.e2e-spec.ts
index d9caf7c..6783d3d 100644
--- 
a/metron-interface/metron-alerts/e2e/alerts-list/alert-filters/alert-filters.e2e-spec.ts
+++ 
b/metron-interface/metron-alerts/e2e/alerts-list/alert-filters/alert-filters.e2e-spec.ts
@@ -47,7 +47,7 @@ describe('Test spec for facet filters', function() {
   });
 
   it('should display facets data', async function() : Promise {
-let facetValues = [ 'enrichm...:country 3', 'ip_dst_addr 8', 'ip_src_addr 
2', 'source:type 1' ];
+let facetValues = [ 'enrichm...:country 3', 'ip_dst_addr 8', 'ip_src_addr 
6', 'source:type 1' ];
 
 await page.navgateToAlertList();
 expect(await page.getFacetsTitle()).toEqualBcoz('Filters', 'for Title as 
Filters');

http://git-wip-us.apache.org/repos/asf/metron/blob/4519f721/metron-interface/metron-alerts/e2e/alerts-list/alerts-list.po.ts
--
diff --git a/metron-interface/metron-alerts/e2e/alerts-list/alerts-list.po.ts 
b/metron-interface/metron-alerts/e2e/alerts-list/alerts-list.po.ts
index 13aeb27..389d218 100644
--- a/metron-interface/metron-alerts/e2e/alerts-list/alerts-list.po.ts
+++ b/metron-interface/metron-alerts/e2e/alerts-list/alerts-list.po.ts
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-import {browser, element, by, protractor} from 'protractor';
+import {browser, element, by, protractor, ElementArrayFinder} from 
'protractor';
 import * as moment from 'moment/moment';
 import {
   waitForElementVisibility, waitForElementPresence, waitForElementInVisibility,
@@ -446,10 +446,6 @@ export class MetronAlertsPage {
   .element(by.xpath('../..')).all(by.css('td a')).get(8).getText();
   }
 
-  sortTable(colName: string) {
-element.all(by.css('table thead 
th')).all(by.linkText(colName)).get(0).click();
-  }
-
   getCellValue(rowIndex: number, colIndex: number, previousText: string) {
 let cellElement = element.all(by.css('table tbody 
tr')).get(rowIndex).all(by.css('td')).get(colIndex);
 return this.waitForTextChange(cellElement, previousText).then(() => 
cellElement.getText());

http://git-wip-us.apache.org/repos/asf/metron/blob/4519f721/metron-interface/metron-alerts/e2e/alerts-list/meta-alerts/meta-alert.e2e-spec.ts
--
diff --git 
a/metron-interface/metron-alerts/e2e/alerts-list/meta-alerts/meta-alert.e2e-spec.ts
 
b/metron-interface/metron-alerts/e2e/alerts-list/meta-alerts/meta-alert.e2e-spec.ts
index 5425523..5bf7fd8 100644
--- 
a/metron-interface/metron-alerts/e2e/alerts-list/meta-alerts/meta-alert.e2e-spec.ts
+++ 
b/metron-interface/metron-alerts/e2e/alerts-list/meta-alerts/meta-alert.e2e-spec.ts
@@ -158,7 +158,7 @@ describe('Test spec for meta alerts workflow', function() {
   'source:type': '1',
   'ip_dst_addr': '7',
   'enrichm...:country': '3',
-  'ip_src_addr': '2'
+  'ip_src_addr': '4'
 };
 let alertsInMetaAlerts = [
   '82f8046d-d...03b17480dd',

http://git-wip-us.apache.org/repos/asf/metron/blob/4519f721/metron-interface/metron-alerts/e2e/alerts-list/table-view/table-view.e2e-spec.ts

metron git commit: METRON-1619: Stellar empty collections should be considered false in boolean expressions closes apache/incubator-metron#1064

2018-06-28 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 81282de28 -> c4c790dbe


METRON-1619: Stellar empty collections should be considered false in boolean 
expressions closes apache/incubator-metron#1064


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c4c790db
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c4c790db
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c4c790db

Branch: refs/heads/master
Commit: c4c790dbe4c481c8f19293ef9d6134539674d253
Parents: 81282de
Author: cstella 
Authored: Thu Jun 28 15:57:08 2018 -0400
Committer: cstella 
Committed: Thu Jun 28 15:57:08 2018 -0400

--
 metron-stellar/stellar-common/README.md | 14 
 .../metron/stellar/common/StellarCompiler.java  | 82 +---
 .../org/apache/metron/stellar/dsl/Token.java|  1 +
 .../stellar/dsl/functions/BasicStellarTest.java | 25 ++
 .../metron/stellar/dsl/functions/MatchTest.java | 35 +
 use-cases/forensic_clustering/README.md |  2 +-
 use-cases/geographic_login_outliers/README.md   |  5 +-
 use-cases/typosquat_detection/README.md |  2 +-
 8 files changed, 149 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/c4c790db/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md 
b/metron-stellar/stellar-common/README.md
index d8d09b4..0dd8e16 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -54,6 +54,20 @@ The Stellar language supports the following:
 * The ability to have parenthesis to make order of operations explicit
 * User defined functions, including Lambda expressions 
 
+### Boolean Expressions
+
+Variables may be used in boolean expressions and variables which are not
+explicitly boolean may be interpreted as booleans subject to the
+following rules:
+* Similar to python and javascript, empty collections (e.g. `[]`) will be
+  interpreted as `false`
+* Similar to python and javascript, missing variables will be
+  interpreted as `false`
+* Variables set to `null` will be interpreted as `false`
+
+Otherwise, boolean variables will be interpreted as their values
+reflect. 
+
 ### Stellar Language Keywords
 The following keywords need to be single quote escaped in order to be used in 
Stellar expressions:
 

http://git-wip-us.apache.org/repos/asf/metron/blob/c4c790db/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
index 72f0d0a..8a328a2 100644
--- 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
@@ -29,7 +29,10 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+
+import com.google.common.collect.Iterables;
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.metron.stellar.common.evaluators.ArithmeticEvaluator;
@@ -104,6 +107,52 @@ public class StellarCompiler extends StellarBaseListener {
   return tokenDeque;
 }
 
+/**
+ * When treating empty or missing values as false, we need to ensure we 
ONLY do so in a conditional context.
+ * @param tokenValueType
+ * @return
+ */
+private boolean isConditionalContext(Class tokenValueType) {
+  return tokenValueType != null && (
+   tokenValueType == BooleanArg.class
+|| tokenValueType == IfExpr.class
+|| tokenValueType == MatchClauseCheckExpr.class
+  );
+}
+
+/**
+ * Determine if a token and value is an empty list in the appropriate 
conditional context
+ * @param token
+ * @param value
+ * @return
+ */
+private boolean isEmptyList(Token token, Object value) {
+  if(value != null && isConditionalContext(token.getUnderlyingType())) {
+if (value instanceof Iterable) {
+  return Iterables.isEmpty((Iterable) value);
+} else if (value instanceof Map) {
+  return ((Map) value).isEmpty();
+}
+else {
+  return false;
+}
+  }else {
+return false;
+  }
+}
+
+/**
+ * Determine if a token is missing in a conditional context.
+ * @param token
+ * @return
+ */
+

metron git commit: METRON-1645: Check wether the Solr management pack is installed before configuring the solr principal name. closes apache/incubator-metron#1086

2018-06-28 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master e0fc47582 -> 3a6fb38e5


METRON-1645: Check wether the Solr management pack is installed before 
configuring the solr principal name. closes apache/incubator-metron#1086


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3a6fb38e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3a6fb38e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3a6fb38e

Branch: refs/heads/master
Commit: 3a6fb38e55c1b8b9c1fa9ec74ed233c68a18d076
Parents: e0fc475
Author: MohanDV 
Authored: Thu Jun 28 09:33:21 2018 -0400
Committer: cstella 
Committed: Thu Jun 28 09:33:21 2018 -0400

--
 .../METRON/CURRENT/package/scripts/params/params_linux.py   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/3a6fb38e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 07132a0..0ab9bcf 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -258,8 +258,9 @@ if security_enabled:
 kafka_keytab_path = config['configurations']['kafka-env']['kafka_keytab']
 
 nimbus_seeds = config['configurations']['storm-site']['nimbus.seeds']
-
-solr_principal_name = solr_principal_name.replace('_HOST', 
hostname_lowercase)
+# Check wether Solr mpack is installed
+if 'solr-config-env' in config['configurations']:
+solr_principal_name = solr_principal_name.replace('_HOST', 
hostname_lowercase)
 
 # Management UI
 metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", [hostname])[0]

[2/2] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr

2018-06-15 Thread cestella

Merge branch 'master' into feature/METRON-1416-upgrade-solr


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a89a72c3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a89a72c3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a89a72c3

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: a89a72c3b2ceb5066fd7a310636507ce274c66ad
Parents: 10271c4 b081e80
Author: cstella 
Authored: Fri Jun 15 20:32:18 2018 -0400
Committer: cstella 
Committed: Fri Jun 15 20:32:18 2018 -0400

--
 .../alert-details/alert-details.component.html  |  4 +--
 .../alert-details/alert-details.component.ts| 24 ++---
 .../alerts-list/alerts-list.component.html  |  4 +--
 .../alerts/alerts-list/alerts-list.component.ts | 13 ++
 .../src/app/alerts/alerts-list/query-builder.ts |  5 ++--
 .../table-view/table-view.component.html| 18 ++---
 .../table-view/table-view.component.ts  | 19 --
 .../tree-view/tree-view.component.html  | 16 ++--
 .../tree-view/tree-view.component.ts| 27 +---
 .../meta-alerts/meta-alerts.component.html  |  2 +-
 .../alerts/meta-alerts/meta-alerts.component.ts | 11 +---
 .../src/app/model/group-request.ts  |  2 +-
 .../src/app/service/global-config.service.ts| 16 +---
 .../metron-alerts/src/app/utils/constants.ts|  3 +--
 .../ElasticsearchMetaAlertIntegrationTest.java  |  5 ++--
 15 files changed, 120 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --cc 
metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index ffd7dd0,c939f04..39d47ac
--- 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++ 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@@ -29,8 -30,8 +30,9 @@@ import {Patch} from '../../model/patch'
  import {AlertComment} from './alert-comment';
  import {AuthenticationService} from '../../service/authentication.service';
  import {MetronDialogBox} from '../../shared/metron-dialog-box';
- import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
 -import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from 
'../../utils/constants';
 +import {CommentAddRemoveRequest} from 
"../../model/comment-add-remove-request";
++import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
+ import { GlobalConfigService } from '../../service/global-config.service';
  
  export enum AlertState {
NEW, OPEN, ESCALATE, DISMISS, RESOLVE

http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
--
diff --cc 
metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
index 98459db,4496e37..07f7015
--- 
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
+++ 
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
@@@ -36,10 -36,10 +36,10 @@@ import {AlertSearchDirective} from '../
  import {SearchResponse} from '../../model/search-response';
  import {ElasticsearchUtils} from '../../utils/elasticsearch-utils';
  import {Filter} from '../../model/filter';
- import {THREAT_SCORE_FIELD_NAME, TIMESTAMP_FIELD_NAME, ALL_TIME} from 
'../../utils/constants';
+ import {TIMESTAMP_FIELD_NAME, ALL_TIME} from '../../utils/constants';
  import {TableViewComponent} from './table-view/table-view.component';
  import {Pagination} from '../../model/pagination';
 -import {META_ALERTS_SENSOR_TYPE, META_ALERTS_INDEX} from 
'../../utils/constants';
 +import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
  import {MetaAlertService} from '../../service/meta-alert.service';
  import {Facets} from '../../model/facets';
  import { GlobalConfigService } from '../../service/global-config.service';

http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.html
--

http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.ts
--

http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/utils/constants.ts

metron git commit: METRON-1617: Make threat triage score function with dots as well as colons closes apache/incubator-metron#1062

2018-06-15 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 828ab7134 -> b081e80c0


METRON-1617: Make threat triage score function with dots as well as colons 
closes apache/incubator-metron#1062


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b081e80c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b081e80c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b081e80c

Branch: refs/heads/master
Commit: b081e80c07819fdaad0e5014790ddf02b2909c80
Parents: 828ab71
Author: cstella 
Authored: Fri Jun 15 19:59:13 2018 -0400
Committer: cstella 
Committed: Fri Jun 15 19:59:13 2018 -0400

--
 .../alert-details/alert-details.component.html  |  4 +--
 .../alert-details/alert-details.component.ts| 22 ++--
 .../alerts-list/alerts-list.component.html  |  4 +--
 .../alerts/alerts-list/alerts-list.component.ts | 13 ++
 .../src/app/alerts/alerts-list/query-builder.ts |  5 ++--
 .../table-view/table-view.component.html| 18 ++---
 .../table-view/table-view.component.ts  | 19 --
 .../tree-view/tree-view.component.html  | 16 ++--
 .../tree-view/tree-view.component.ts| 27 +---
 .../meta-alerts/meta-alerts.component.html  |  2 +-
 .../alerts/meta-alerts/meta-alerts.component.ts | 11 +---
 .../src/app/model/group-request.ts  |  2 +-
 .../src/app/service/global-config.service.ts| 16 +---
 .../metron-alerts/src/app/utils/constants.ts|  3 +--
 .../ElasticsearchMetaAlertIntegrationTest.java  |  2 ++
 15 files changed, 118 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/b081e80c/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.html
--
diff --git 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.html
 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.html
index 8b0efae..f8fdc1d 100644
--- 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.html
+++ 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.html
@@ -34,8 +34,8 @@
 
 
 
- 
- {{ 
alertSource['threat:triage:score'] }} 
+ 
+ {{ getScore(alertSource) }} 
 
 
  {{ 
(alertSource.name && alertSource.name.length > 0)? alertSource.name : alertId | 
centerEllipses:20 }} 

http://git-wip-us.apache.org/repos/asf/metron/blob/b081e80c/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --git 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index c8d0d7a..c939f04 100644
--- 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++ 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -18,6 +18,7 @@
 import { Component, OnInit } from '@angular/core';
 import {Router, ActivatedRoute} from '@angular/router';
 import * as moment from 'moment/moment';
+import {Observable, Subscription} from 'rxjs/Rx';
 
 import {SearchService} from '../../service/search.service';
 import {UpdateService} from '../../service/update.service';
@@ -30,6 +31,7 @@ import {AlertComment} from './alert-comment';
 import {AuthenticationService} from '../../service/authentication.service';
 import {MetronDialogBox} from '../../shared/metron-dialog-box';
 import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from 
'../../utils/constants';
+import { GlobalConfigService } from '../../service/global-config.service';
 
 export enum AlertState {
   NEW, OPEN, ESCALATE, DISMISS, RESOLVE
@@ -71,6 +73,9 @@ export class AlertDetailsComponent implements OnInit {
   alertFields: string[] = [];
   alertCommentStr = '';
   alertCommentsWrapper: AlertCommentWrapper[] = [];
+  globalConfig: {} = {};
+  globalConfigService: GlobalConfigService;
+  configSubscription: Subscription;
 
   constructor(private router: Router,
   private activatedRoute: ActivatedRoute,
@@ -78,8 +83,9 @@ export class AlertDetailsComponent implements OnInit {
   private updateService: UpdateService,
   private alertsService: AlertsService,

[2/3] metron git commit: METRON-1588 Migrate storm-kafka-client to 1.2.1 closes apache/incubator-metron#1039

2018-06-14 Thread cestella

METRON-1588 Migrate storm-kafka-client to 1.2.1 closes 
apache/incubator-metron#1039


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/828ab713
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/828ab713
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/828ab713

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 828ab71346b458b73cb14d42d68e7471ead7fa4e
Parents: 7757046
Author: Jungtaek Lim 
Authored: Thu Jun 14 12:07:05 2018 -0400
Committer: cstella 
Committed: Thu Jun 14 12:07:05 2018 -0400

--
 NOTICE  |   6 +
 dependencies_with_url.csv   |   4 +
 metron-platform/metron-api/pom.xml  |   4 -
 .../parsers/topology/ParserTopologyBuilder.java |   2 +-
 .../metron-storm-kafka-override/pom.xml |  83 -
 .../KafkaSpoutRetryExponentialBackoff.java  | 336 +++
 .../storm/kafka/spout/internal/Timer.java   |   9 +-
 metron-platform/metron-storm-kafka/pom.xml  |   5 -
 pom.xml |   4 +-
 9 files changed, 435 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/828ab713/NOTICE
--
diff --git a/NOTICE b/NOTICE
index ff6550b..5227768 100644
--- a/NOTICE
+++ b/NOTICE
@@ -10,3 +10,9 @@
This product includes software developed by Chef Software 
(https://www.chef.io)
Copyright (c) 2012-2015, Chef Software, Inc.
 
+   This includes derived works from the Apache Storm (ASLv2 licensed) project 
(https://github.com/apache/storm):
+   Copyright 2015 The Apache Software Foundation
+   The derived work is adapted from
+ org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java
+ org/apache/storm/kafka/spout/internal/Timer.java
+   and can be found in the org.apache.storm.kafka package

http://git-wip-us.apache.org/repos/asf/metron/blob/828ab713/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index df3bcd2..438ce3e 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -119,15 +119,19 @@ 
com.fasterxml.jackson.core:jackson-annotations:jar:2.2.3:compile,ASLv2,http://wi
 
com.fasterxml.jackson.core:jackson-annotations:jar:2.7.4:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-annotations:jar:2.8.3:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-annotations:jar:2.9.0:compile,ASLv2,http://github.com/FasterXML/jackson
+com.fasterxml.jackson.core:jackson-annotations:jar:2.9.4:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-core:jar:2.2.3:compile,ASLv2,http://wiki.fasterxml.com/JacksonHome
+com.fasterxml.jackson.core:jackson-core:jar:2.6.3:compile,ASLv2,https://github.com/FasterXML/jackson-core
 
com.fasterxml.jackson.core:jackson-core:jar:2.6.6:compile,ASLv2,https://github.com/FasterXML/jackson-core
 
com.fasterxml.jackson.core:jackson-core:jar:2.7.4:compile,ASLv2,https://github.com/FasterXML/jackson-core
 
com.fasterxml.jackson.core:jackson-core:jar:2.8.3:compile,ASLv2,https://github.com/FasterXML/jackson-core
+com.fasterxml.jackson.core:jackson-core:jar:2.9.4:compile,ASLv2,https://github.com/FasterXML/jackson-core
 
com.fasterxml.jackson.core:jackson-core:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-core
 
com.fasterxml.jackson.core:jackson-databind:jar:2.2.3:compile,ASLv2,http://wiki.fasterxml.com/JacksonHome
 
com.fasterxml.jackson.core:jackson-databind:jar:2.4.3:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-databind:jar:2.7.4:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-databind:jar:2.8.3:compile,ASLv2,http://github.com/FasterXML/jackson
+com.fasterxml.jackson.core:jackson-databind:jar:2.9.4:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-databind:jar:2.9.5:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar:2.6.6:compile,ASLv2,http://wiki.fasterxml.com/JacksonForCbor
 
com.fasterxml.jackson.dataformat:jackson-dataformat-smile:jar:2.6.6:compile,ASLv2,http://wiki.fasterxml.com/JacksonForSmile

http://git-wip-us.apache.org/repos/asf/metron/blob/828ab713/metron-platform/metron-api/pom.xml
--
diff --git a/metron-platform/metron-api/pom.xml 
b/metron-platform/metron-api/pom.xml
index e3bf12d..59b1622 100644
--- a/metron-platform/metron-api/pom.xml
+++ b/metron-platform/metron-api/pom.xml
@@ -31,10 +31,6 @@

[3/3] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr

2018-06-14 Thread cestella

Merge branch 'master' into feature/METRON-1416-upgrade-solr


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0e484279
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0e484279
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0e484279

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 0e4842799f8a3afa15074cb121bdbd205e50fb58
Parents: 4c4452e 828ab71
Author: cstella 
Authored: Thu Jun 14 12:18:43 2018 -0400
Committer: cstella 
Committed: Thu Jun 14 12:18:43 2018 -0400

--
 NOTICE  |   6 +
 dependencies_with_url.csv   |   4 +
 metron-platform/metron-api/pom.xml  |   4 -
 .../parsers/topology/ParserTopologyBuilder.java |   2 +-
 .../metron-storm-kafka-override/pom.xml |  83 -
 .../KafkaSpoutRetryExponentialBackoff.java  | 336 +++
 .../storm/kafka/spout/internal/Timer.java   |   9 +-
 metron-platform/metron-storm-kafka/pom.xml  |   5 -
 pom.xml |   4 +-
 site/documentation/index.md |  14 +-
 10 files changed, 442 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/0e484279/dependencies_with_url.csv
--

http://git-wip-us.apache.org/repos/asf/metron/blob/0e484279/pom.xml
--

[1/3] metron git commit: METRON-1612 Fix website download links (justinleet) closes apache/metron#1058

2018-06-14 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr 4c4452e59 -> 0e4842799


METRON-1612 Fix website download links (justinleet) closes apache/metron#1058


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/77570464
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/77570464
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/77570464

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 77570464b4db205c3a3b95fcc3a0e8f7f6a1c540
Parents: 32d132b
Author: justinleet 
Authored: Mon Jun 11 10:35:09 2018 -0400
Committer: leet 
Committed: Mon Jun 11 10:35:09 2018 -0400

--
 site/documentation/index.md | 14 +++---
 1 file changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/77570464/site/documentation/index.md
--
diff --git a/site/documentation/index.md b/site/documentation/index.md
index 585f656..f842a67 100644
--- a/site/documentation/index.md
+++ b/site/documentation/index.md
@@ -82,18 +82,18 @@ title: Apache Metron Documentation
 http://www.apache.org/dyn/closer.cgi/metron/0.5.0/apache-metron-0.5.0.tar.gz;
 target="new">
 apache-metron-0.5.0.tar.gz
 
- [ https://dist.apache.org/repos/dist/release/metron/0.5.0/apache-metron-0.5.0.tar.gz.asc;
 target="new">
+ [ https://www.apache.org/dist/metron/0.5.0/apache-metron-0.5.0.tar.gz.asc; 
target="new">
 PGP
  ]
- [ https://dist.apache.org/repos/dist/release/metron/0.5.0/apache-metron-0.5.0.tar.gz.sha1;
 target="new">
-SHA
+ [ https://www.apache.org/dist/metron/0.5.0/apache-metron-0.5.0.tar.gz.sha1; 
target="new">
+SHA1
  ]
- [ https://dist.apache.org/repos/dist/release/metron/0.5.0/apache-metron-0.5.0.tar.gz.sha512;
 target="new">
+ [ https://www.apache.org/dist/metron/0.5.0/apache-metron-0.5.0.tar.gz.sha256;
 target="new">
+SHA256
+ ]
+ [ https://www.apache.org/dist/metron/0.5.0/apache-metron-0.5.0.tar.gz.sha512;
 target="new">
 SHA512
  ]
- [ https://dist.apache.org/repos/dist/release/metron/0.5.0/apache-metron-0.5.0.tar.gz.md5;
 target="new">
-MD5
-  ]
  [  Book ]

metron git commit: METRON-1588 Migrate storm-kafka-client to 1.2.1 closes apache/incubator-metron#1039

2018-06-14 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 77570464b -> 828ab7134


METRON-1588 Migrate storm-kafka-client to 1.2.1 closes 
apache/incubator-metron#1039


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/828ab713
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/828ab713
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/828ab713

Branch: refs/heads/master
Commit: 828ab71346b458b73cb14d42d68e7471ead7fa4e
Parents: 7757046
Author: Jungtaek Lim 
Authored: Thu Jun 14 12:07:05 2018 -0400
Committer: cstella 
Committed: Thu Jun 14 12:07:05 2018 -0400

--
 NOTICE  |   6 +
 dependencies_with_url.csv   |   4 +
 metron-platform/metron-api/pom.xml  |   4 -
 .../parsers/topology/ParserTopologyBuilder.java |   2 +-
 .../metron-storm-kafka-override/pom.xml |  83 -
 .../KafkaSpoutRetryExponentialBackoff.java  | 336 +++
 .../storm/kafka/spout/internal/Timer.java   |   9 +-
 metron-platform/metron-storm-kafka/pom.xml  |   5 -
 pom.xml |   4 +-
 9 files changed, 435 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/828ab713/NOTICE
--
diff --git a/NOTICE b/NOTICE
index ff6550b..5227768 100644
--- a/NOTICE
+++ b/NOTICE
@@ -10,3 +10,9 @@
This product includes software developed by Chef Software 
(https://www.chef.io)
Copyright (c) 2012-2015, Chef Software, Inc.
 
+   This includes derived works from the Apache Storm (ASLv2 licensed) project 
(https://github.com/apache/storm):
+   Copyright 2015 The Apache Software Foundation
+   The derived work is adapted from
+ org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java
+ org/apache/storm/kafka/spout/internal/Timer.java
+   and can be found in the org.apache.storm.kafka package

http://git-wip-us.apache.org/repos/asf/metron/blob/828ab713/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index df3bcd2..438ce3e 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -119,15 +119,19 @@ 
com.fasterxml.jackson.core:jackson-annotations:jar:2.2.3:compile,ASLv2,http://wi
 
com.fasterxml.jackson.core:jackson-annotations:jar:2.7.4:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-annotations:jar:2.8.3:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-annotations:jar:2.9.0:compile,ASLv2,http://github.com/FasterXML/jackson
+com.fasterxml.jackson.core:jackson-annotations:jar:2.9.4:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-core:jar:2.2.3:compile,ASLv2,http://wiki.fasterxml.com/JacksonHome
+com.fasterxml.jackson.core:jackson-core:jar:2.6.3:compile,ASLv2,https://github.com/FasterXML/jackson-core
 
com.fasterxml.jackson.core:jackson-core:jar:2.6.6:compile,ASLv2,https://github.com/FasterXML/jackson-core
 
com.fasterxml.jackson.core:jackson-core:jar:2.7.4:compile,ASLv2,https://github.com/FasterXML/jackson-core
 
com.fasterxml.jackson.core:jackson-core:jar:2.8.3:compile,ASLv2,https://github.com/FasterXML/jackson-core
+com.fasterxml.jackson.core:jackson-core:jar:2.9.4:compile,ASLv2,https://github.com/FasterXML/jackson-core
 
com.fasterxml.jackson.core:jackson-core:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-core
 
com.fasterxml.jackson.core:jackson-databind:jar:2.2.3:compile,ASLv2,http://wiki.fasterxml.com/JacksonHome
 
com.fasterxml.jackson.core:jackson-databind:jar:2.4.3:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-databind:jar:2.7.4:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-databind:jar:2.8.3:compile,ASLv2,http://github.com/FasterXML/jackson
+com.fasterxml.jackson.core:jackson-databind:jar:2.9.4:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.core:jackson-databind:jar:2.9.5:compile,ASLv2,http://github.com/FasterXML/jackson
 
com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar:2.6.6:compile,ASLv2,http://wiki.fasterxml.com/JacksonForCbor
 
com.fasterxml.jackson.dataformat:jackson-dataformat-smile:jar:2.6.6:compile,ASLv2,http://wiki.fasterxml.com/JacksonForSmile

http://git-wip-us.apache.org/repos/asf/metron/blob/828ab713/metron-platform/metron-api/pom.xml
--
diff --git a/metron-platform/metron-api/pom.xml 
b/metron-platform/metron-api/pom.xml
index e3bf12d..59b1622 100644
--- a/metron-platform/metron-api/pom.xml
+++

metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr

2018-06-12 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr 476856192 -> 4c4452e59


Merge branch 'master' into feature/METRON-1416-upgrade-solr


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4c4452e5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4c4452e5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4c4452e5

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 4c4452e59c93306a049e6879f7b6fed2ee86c74e
Parents: 4768561
Author: cstella 
Authored: Tue Jun 12 21:40:01 2018 -0400
Committer: cstella 
Committed: Tue Jun 12 21:40:01 2018 -0400

--
 .../dao/ElasticsearchSearchDao.java |  30 +--
 .../ElasticsearchMetaAlertIntegrationTest.java  | 223 ++-
 2 files changed, 134 insertions(+), 119 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/4c4452e5/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
--
diff --git 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
index 5725534..5cd0a4d 100644
--- 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
+++ 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
@@ -443,22 +443,24 @@ public class ElasticsearchSearchDao implements SearchDao {
   private List getGroupResults(GroupRequest groupRequest, int 
index, Aggregations aggregations, Map commonColumnMetadata) {
 List groups = groupRequest.getGroups();
 String field = groups.get(index).getField();
-Terms terms = aggregations.get(getGroupByAggregationName(field));
 List searchResultGroups = new ArrayList<>();
-for(Bucket bucket: terms.getBuckets()) {
-  GroupResult groupResult = new GroupResult();
-  groupResult.setKey(formatKey(bucket.getKey(), 
commonColumnMetadata.get(field)));
-  groupResult.setTotal(bucket.getDocCount());
-  Optional scoreField = groupRequest.getScoreField();
-  if (scoreField.isPresent()) {
-Sum score = 
bucket.getAggregations().get(getSumAggregationName(scoreField.get()));
-groupResult.setScore(score.getValue());
-  }
-  if (index < groups.size() - 1) {
-groupResult.setGroupedBy(groups.get(index + 1).getField());
-groupResult.setGroupResults(getGroupResults(groupRequest, index + 1, 
bucket.getAggregations(), commonColumnMetadata));
+if(aggregations != null) {
+  Terms terms = aggregations.get(getGroupByAggregationName(field));
+  for (Bucket bucket : terms.getBuckets()) {
+GroupResult groupResult = new GroupResult();
+groupResult.setKey(formatKey(bucket.getKey(), 
commonColumnMetadata.get(field)));
+groupResult.setTotal(bucket.getDocCount());
+Optional scoreField = groupRequest.getScoreField();
+if (scoreField.isPresent()) {
+  Sum score = 
bucket.getAggregations().get(getSumAggregationName(scoreField.get()));
+  groupResult.setScore(score.getValue());
+}
+if (index < groups.size() - 1) {
+  groupResult.setGroupedBy(groups.get(index + 1).getField());
+  groupResult.setGroupResults(getGroupResults(groupRequest, index + 1, 
bucket.getAggregations(), commonColumnMetadata));
+}
+searchResultGroups.add(groupResult);
   }
-  searchResultGroups.add(groupResult);
 }
 return searchResultGroups;
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/4c4452e5/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
--
diff --git 
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
 
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
index 99f0490..224ec56 100644
--- 
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
+++ 
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
@@ -1,20 +1,21 @@
+
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for

[1/2] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr

2018-06-12 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr 0717cfc25 -> 476856192


Merge branch 'master' into feature/METRON-1416-upgrade-solr


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e7233b0f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e7233b0f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e7233b0f

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: e7233b0f82f372e6b1339de56a115fa7e4581f07
Parents: 4170887
Author: cstella 
Authored: Mon Jun 11 21:41:06 2018 -0400
Committer: cstella 
Committed: Tue Jun 12 09:46:46 2018 -0400

--
 .../org/apache/metron/common/Constants.java |  1 +
 .../elasticsearch/dao/ElasticsearchDao.java |  4 ++
 .../dao/ElasticsearchMetaAlertDao.java  | 30 +-
 .../dao/ElasticsearchMetaAlertUpdateDao.java|  4 +-
 .../indexing/dao/metaalert/MetaAlertConfig.java | 43 
 .../AbstractLuceneMetaAlertUpdateDao.java   |  3 ++
 .../dao/metaalert/MetaAlertIntegrationTest.java |  2 +-
 .../AbstractLuceneMetaAlertUpdateDaoTest.java   | 28 +
 .../metron/solr/dao/SolrMetaAlertDao.java   | 29 +
 .../metron/solr/dao/SolrMetaAlertSearchDao.java | 15 ---
 .../metron/solr/dao/SolrMetaAlertUpdateDao.java |  2 +-
 .../SolrMetaAlertIntegrationTest.java   | 28 +
 12 files changed, 129 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index f74660c..4a8bea2 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -28,6 +28,7 @@ public class Constants {
   public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000;
   public static final String SENSOR_TYPE = "source.type";
   public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field";
+  public static final String THREAT_SCORE_FIELD_PROPERTY = 
"threat.triage.score.field";
   public static final String ENRICHMENT_TOPIC = "enrichments";
   public static final String INDEXING_TOPIC = "indexing";
   public static final String ERROR_STREAM = "error";

http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
--
diff --git 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index eae0a39..3eb86ce 100644
--- 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -85,6 +85,10 @@ public class ElasticsearchDao implements IndexDao {
 //uninitialized.
   }
 
+  public AccessConfig getAccessConfig() {
+return accessConfig;
+  }
+
   @Override
   public synchronized void init(AccessConfig config) {
 if (this.client == null) {

http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
--
diff --git 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index f73a640..649077e 100644
--- 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -68,6 +68,7 @@ import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static org.apache.metron.common.Constants.GUID;
@@ -83,10 +84,8 @@ public class ElasticsearchMetaAlertDao implements 
MetaAlertDao {
   public static final String THREAT_TRIAGE_FIELD = 
MetaAlertConstants.THREAT_FIELD_DEFAULT

metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr

2018-06-11 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr 417088763 -> 0717cfc25


Merge branch 'master' into feature/METRON-1416-upgrade-solr


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0717cfc2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0717cfc2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0717cfc2

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 0717cfc25acc80b80121b424bed93db7014a1376
Parents: 4170887
Author: cstella 
Authored: Mon Jun 11 21:41:06 2018 -0400
Committer: cstella 
Committed: Mon Jun 11 21:41:06 2018 -0400

--
 .../org/apache/metron/common/Constants.java |  1 +
 .../elasticsearch/dao/ElasticsearchDao.java |  4 ++
 .../dao/ElasticsearchMetaAlertDao.java  | 30 +-
 .../dao/ElasticsearchMetaAlertUpdateDao.java|  4 +-
 .../indexing/dao/metaalert/MetaAlertConfig.java | 43 
 .../AbstractLuceneMetaAlertUpdateDao.java   |  3 ++
 .../dao/metaalert/MetaAlertIntegrationTest.java |  2 +-
 .../AbstractLuceneMetaAlertUpdateDaoTest.java   | 26 +---
 .../metron/solr/dao/SolrMetaAlertDao.java   | 29 +
 .../metron/solr/dao/SolrMetaAlertSearchDao.java | 15 ---
 .../metron/solr/dao/SolrMetaAlertUpdateDao.java |  2 +-
 .../SolrMetaAlertIntegrationTest.java   | 28 +
 12 files changed, 128 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index f74660c..4a8bea2 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -28,6 +28,7 @@ public class Constants {
   public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000;
   public static final String SENSOR_TYPE = "source.type";
   public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field";
+  public static final String THREAT_SCORE_FIELD_PROPERTY = 
"threat.triage.score.field";
   public static final String ENRICHMENT_TOPIC = "enrichments";
   public static final String INDEXING_TOPIC = "indexing";
   public static final String ERROR_STREAM = "error";

http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
--
diff --git 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index eae0a39..3eb86ce 100644
--- 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -85,6 +85,10 @@ public class ElasticsearchDao implements IndexDao {
 //uninitialized.
   }
 
+  public AccessConfig getAccessConfig() {
+return accessConfig;
+  }
+
   @Override
   public synchronized void init(AccessConfig config) {
 if (this.client == null) {

http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
--
diff --git 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index f73a640..649077e 100644
--- 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -68,6 +68,7 @@ import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static org.apache.metron.common.Constants.GUID;
@@ -83,10 +84,8 @@ public class ElasticsearchMetaAlertDao implements 
MetaAlertDao {
   public static final String THREAT_TRIAGE_FIELD = 
MetaAlertConstants.THREAT_FIELD_DEFAULT

[22/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-interface/metron-rest/index.html
--
diff --git a/site/current-book/metron-interface/metron-rest/index.html 
b/site/current-book/metron-interface/metron-rest/index.html
index f673ca6..eb378b3 100644
--- a/site/current-book/metron-interface/metron-rest/index.html
+++ b/site/current-book/metron-interface/metron-rest/index.html
@@ -1,253 +1,157 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Metron REST
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Metron REST
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Metron REST
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   



 
-  
-
-  
-  
-Deployment
-  
-  
-  
-
-  
-  
-Alerts
-
-  
-  
-
-  
-  
-Config
-
-  
-  
-
-Rest
-  
-

[19/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-platform/metron-data-management/index.html
--
diff --git 
a/site/current-book/metron-platform/metron-data-management/index.html 
b/site/current-book/metron-platform/metron-data-management/index.html
index df34389..dea600c 100644
--- a/site/current-book/metron-platform/metron-data-management/index.html
+++ b/site/current-book/metron-platform/metron-data-management/index.html
@@ -1,359 +1,190 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Resource Data Management
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Resource Data Management
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Resource Data Management
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   



 
-  
-
-  
-  
-Deployment
-  
-  
-  
-
-  
-  
-Alerts
-
-  
-  
-
-  
-  
-Config
-
-  
-  
-
-  
-  
-Rest
-
-

[15/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-platform/metron-management/index.html
--
diff --git a/site/current-book/metron-platform/metron-management/index.html 
b/site/current-book/metron-platform/metron-management/index.html
index cca14f1..f9ea1ce 100644
--- a/site/current-book/metron-platform/metron-management/index.html
+++ b/site/current-book/metron-platform/metron-management/index.html
@@ -1,1040 +1,644 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Stellar REPL Management Utilities
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Stellar REPL Management Utilities
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Stellar REPL Management Utilities
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   



 
-  
-
-  
-  
-Deployment
-  
-  
-  
-
-  
-  
-Alerts
-
-  
-  
-
-  
-  
-Config
-
-  
-  
-
-  
-  
-Rest
-
-

[31/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-deployment/amazon-ec2/index.html
--
diff --git a/site/current-book/metron-deployment/amazon-ec2/index.html 
b/site/current-book/metron-deployment/amazon-ec2/index.html
index 4e7e771..8bf69a8 100644
--- a/site/current-book/metron-deployment/amazon-ec2/index.html
+++ b/site/current-book/metron-deployment/amazon-ec2/index.html
@@ -1,536 +1,428 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Apache Metron on Amazon EC2
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Apache Metron on Amazon EC2
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Apache Metron on Amazon EC2
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   




   
-  
-
-  
-  
-Deployment
-
-  
-  
-
-  
-  
-Kerberos-ambari-setup
-
-  
-  
-
-  
-  
-Kerberos-manual-setup
-
-  
-  
-
-Amazon-ec2
-  
-

[14/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-platform/metron-parsers/index.html
--
diff --git a/site/current-book/metron-platform/metron-parsers/index.html 
b/site/current-book/metron-platform/metron-parsers/index.html
index 23955ac..807a24e 100644
--- a/site/current-book/metron-platform/metron-parsers/index.html
+++ b/site/current-book/metron-platform/metron-parsers/index.html
@@ -1,379 +1,211 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Parsers
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Parsers
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Parsers
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   



 
-  
-
-  
-  
-Deployment
-  
-  
-  
-
-  
-  
-Alerts
-
-  
-  
-
-  
-  
-Config
-
-  
-  
-
-  
-  
-Rest
-
-

[20/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-platform/metron-api/index.html
--
diff --git a/site/current-book/metron-platform/metron-api/index.html 
b/site/current-book/metron-platform/metron-api/index.html
index 690ab94..ddcaec5 100644
--- a/site/current-book/metron-platform/metron-api/index.html
+++ b/site/current-book/metron-platform/metron-api/index.html
@@ -1,346 +1,161 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Metron PCAP Service
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Metron PCAP Service
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Metron PCAP Service
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   



 
-  
-
-  
-  
-Deployment
-  
-  
-  
-
-  
-  
-Alerts
-
-  
-  
-
-  
-  
-Config
-
-  
-  
-
-  
-  
-Rest
-
-

[28/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html
--
diff --git 
a/site/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html
 
b/site/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html
index eeadfe6..5cf9775 100644
--- 
a/site/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html
+++ 
b/site/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html
@@ -1,387 +1,230 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   




   
-  
-
-  
-  
-Deployment
-
-  
-  
-
-  
-  
-Kerberos-ambari-setup
-
-  
-  
-
-  
-  
-Kerberos-manual-setup
-
-  
-

[21/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-platform/Performance-tuning-guide.html
--
diff --git a/site/current-book/metron-platform/Performance-tuning-guide.html 
b/site/current-book/metron-platform/Performance-tuning-guide.html
index 941e2d7..00d2907 100644
--- a/site/current-book/metron-platform/Performance-tuning-guide.html
+++ b/site/current-book/metron-platform/Performance-tuning-guide.html
@@ -1,383 +1,676 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Metron Performance Tuning Guide
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Metron Performance Tuning Guide
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Metron Performance Tuning Guide
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   



 
-  
-
-  
-  
-Deployment
-  
-  
-  
-
-  
-  
-Alerts
-
-  
-  
-
-  
-  
-Config
-
-  
-  
-
-  
-  
-Rest
-
-

[35/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-analytics/metron-statistics/HLLP.html
--
diff --git a/site/current-book/metron-analytics/metron-statistics/HLLP.html 
b/site/current-book/metron-analytics/metron-statistics/HLLP.html
index f56beba..fdbf631 100644
--- a/site/current-book/metron-analytics/metron-statistics/HLLP.html
+++ b/site/current-book/metron-analytics/metron-statistics/HLLP.html
@@ -1,234 +1,109 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  HyperLogLogPlus
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-HyperLogLogPlus
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+HyperLogLogPlus
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   

 
-  
-
-  
-  
-Analytics
-
-  
-  
-
-  
-  
-Maas-service
-
-  
-  
-
-  
-  
-Profiler
-
-  
-  
-
-  
-  
-Profiler-client
-
-   
   
-  
-
-  
-  
-Statistics
-
-  
-  
-
-HLLP
-  
-  
-
-  
-
-  
-  
-
-  
-  
-Docker
-
-

[13/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-platform/metron-parsers/parser-testing.html
--
diff --git 
a/site/current-book/metron-platform/metron-parsers/parser-testing.html 
b/site/current-book/metron-platform/metron-parsers/parser-testing.html
index 0edfd53..ff22013 100644
--- a/site/current-book/metron-platform/metron-parsers/parser-testing.html
+++ b/site/current-book/metron-platform/metron-parsers/parser-testing.html
@@ -1,356 +1,180 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Parser Contribution and Testing
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Parser Contribution and Testing
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Parser Contribution and Testing
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   



 
-  
-
-  
-  
-Deployment
-  
-  
-  
-
-  
-  
-Alerts
-
-  
-  
-
-  
-  
-Config
-
-  
-  
-
-  
-  
-Rest
-

[42/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/css/maven-base.css
--
diff --git a/site/current-book/css/maven-base.css 
b/site/current-book/css/maven-base.css
new file mode 100644
index 000..322efae
--- /dev/null
+++ b/site/current-book/css/maven-base.css
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+body {
+  margin: 0px;
+  padding: 0px;
+}
+table {
+  padding:0px;
+  width: 100%;
+  margin-left: -2px;
+  margin-right: -2px;
+}
+acronym {
+  cursor: help;
+  border-bottom: 1px dotted #feb;
+}
+table.bodyTable th, table.bodyTable td {
+  padding: 2px 4px 2px 4px;
+  vertical-align: top;
+}
+div.clear{
+  clear:both;
+  visibility: hidden;
+}
+div.clear hr{
+  display: none;
+}
+#bannerLeft, #bannerRight {
+  font-size: xx-large;
+  font-weight: bold;
+}
+#bannerLeft img, #bannerRight img {
+  margin: 0px;
+}
+.xleft, #bannerLeft img {
+  float:left;
+}
+.xright, #bannerRight {
+  float:right;
+}
+#banner {
+  padding: 0px;
+}
+#breadcrumbs {
+  padding: 3px 10px 3px 10px;
+}
+#leftColumn {
+ width: 170px;
+ float:left;
+ overflow: auto;
+}
+#bodyColumn {
+  margin-right: 1.5em;
+  margin-left: 197px;
+}
+#legend {
+  padding: 8px 0 8px 0;
+}
+#navcolumn {
+  padding: 8px 4px 0 8px;
+}
+#navcolumn h5 {
+  margin: 0;
+  padding: 0;
+  font-size: small;
+}
+#navcolumn ul {
+  margin: 0;
+  padding: 0;
+  font-size: small;
+}
+#navcolumn li {
+  list-style-type: none;
+  background-image: none;
+  background-repeat: no-repeat;
+  background-position: 0 0.4em;
+  padding-left: 16px;
+  list-style-position: outside;
+  line-height: 1.2em;
+  font-size: smaller;
+}
+#navcolumn li.expanded {
+  background-image: url(../images/expanded.gif);
+}
+#navcolumn li.collapsed {
+  background-image: url(../images/collapsed.gif);
+}
+#navcolumn li.none {
+  text-indent: -1em;
+  margin-left: 1em;
+}
+#poweredBy {
+  text-align: center;
+}
+#navcolumn img {
+  margin-top: 10px;
+  margin-bottom: 3px;
+}
+#poweredBy img {
+  display:block;
+  margin: 20px 0 20px 17px;
+}
+#search img {
+margin: 0px;
+display: block;
+}
+#search #q, #search #btnG {
+border: 1px solid #999;
+margin-bottom:10px;
+}
+#search form {
+margin: 0px;
+}
+#lastPublished {
+  font-size: x-small;
+}
+.navSection {
+  margin-bottom: 2px;
+  padding: 8px;
+}
+.navSectionHead {
+  font-weight: bold;
+  font-size: x-small;
+}
+.section {
+  padding: 4px;
+}
+#footer {
+  padding: 3px 10px 3px 10px;
+  font-size: x-small;
+}
+#breadcrumbs {
+  font-size: x-small;
+  margin: 0pt;
+}
+.source {
+  padding: 12px;
+  margin: 1em 7px 1em 7px;
+}
+.source pre {
+  margin: 0px;
+  padding: 0px;
+}
+#navcolumn img.imageLink, .imageLink {
+  padding-left: 0px;
+  padding-bottom: 0px;
+  padding-top: 0px;
+  padding-right: 2px;
+  border: 0px;
+  margin: 0px;
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/css/maven-theme.css
--
diff --git a/site/current-book/css/maven-theme.css 
b/site/current-book/css/maven-theme.css
new file mode 100644
index 000..4e2bdfb
--- /dev/null
+++ b/site/current-book/css/maven-theme.css
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+body {
+  padding: 0px 0px 10px 0px;
+}
+body, td, select, input, li{
+  font-family: Verdana, Helvetica, Arial,

[43/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/css/apache-maven-fluido-1.7.min.css
--
diff --git a/site/current-book/css/apache-maven-fluido-1.7.min.css 
b/site/current-book/css/apache-maven-fluido-1.7.min.css
new file mode 100644
index 000..b436baf
--- /dev/null
+++ b/site/current-book/css/apache-maven-fluido-1.7.min.css
@@ -0,0 +1,17 @@
+/*!
+ * Bootstrap v2.3.2
+ *
+ * Copyright 2013 Twitter, Inc
+ * Licensed under the Apache License v2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Designed and built with all the love in the world by @mdo and @fat.
+ 
*/.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;line-height:0;content:""}.clearfix:after{clear:both}.hide-text{font:0/0
 
a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio,canvas,video{display:inline-block;*display:inline;*zoom:1}audio:not([controls]){display:none}html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}a:focus{outline:thin
 dotted #333;outline:5px auto 
-webkit-focus-ring-color;outline-offset:-2px}a:hover,a:active{outline:0}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{width:auto\9;height:auto;max-width:100%;vertical-align:middle;border:0;-ms-interpolation-mode:bicubic}#map_canvas
 img,.google-maps img{max-widt
 
h:none}button,input,select,textarea{margin:0;font-size:100%;vertical-align:middle}button,input{*overflow:visible;line-height:normal}button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0}button,html
 
input[type="button"],input[type="reset"],input[type="submit"]{cursor:pointer;-webkit-appearance:button}label,select,button,input[type="button"],input[type="reset"],input[type="submit"],input[type="radio"],input[type="checkbox"]{cursor:pointer}input[type="search"]{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-appearance:textfield}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button{-webkit-appearance:none}textarea{overflow:auto;vertical-align:top}@media
 
print{*{color:#000!important;text-shadow:none!important;background:transparent!important;box-shadow:none!important}a,a:visited{text-decoration:underline}a[href]:after{content:"
 (" attr(href) ")"}abbr[title]:after{content:" (" attr(title) 
 ")"}.ir 
a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px
 solid 
#999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100%!important}@page{margin:.5cm}p,h2,h3{orphans:3;widows:3}h2,h3{page-break-after:avoid}}body{margin:0;font-family:"Helvetica
 
Neue",Helvetica,Arial,sans-serif;font-size:14px;line-height:20px;color:#333;background-color:#fff}a{color:#08c;text-decoration:none}a:hover,a:focus{color:#005580;text-decoration:underline}.img-rounded{-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.img-polaroid{padding:4px;background-color:#fff;border:1px
 solid #ccc;border:1px solid rgba(0,0,0,0.2);-webkit-box-shadow:0 1px 3px 
rgba(0,0,0,0.1);-moz-box-shadow:0 1px 3px rgba(0,0,0,0.1);box-shadow:0 1px 3px 
rgba(0,0,0,0.1)}.img-circle{-webkit-border-radius:500px;-moz-border-radius:500px;border-radius:500px}.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;line-hei
 
ght:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:20px}.container,.navbar-static-top
 .container,.navbar-fixed-top .container,.navbar-fixed-bottom 
.container{width:940px}.span12{width:940px}.span11{width:860px}.span10{width:780px}.span9{width:700px}.span8{width:620px}.span7{width:540px}.span6{width:460px}.span5{width:380px}.span4{width:300px}.span3{width:220px}.span2{width:140px}.span1{width:60px}.offset12{margin-left:980px}.offset11{margin-left:900px}.offset10{margin-left:820px}.offset9{margin-left:740px}.offset8{margin-left:660px}.offset7{margin-left:580px}.offset6{margin-left:500px}.offset5{margin-left:420px}.offset4{margin-left:340px}.offset3{margin-left:260px}.offset2{margin-left:180px}.offset1{margin-left:100px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid
 
[class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.127
 
659574468085%;*margin-left:2.074468085106383%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid
 [class*="span"]:first-child{margin-left:0}.row-fluid .controls-row 
[class*="span"]+[class*="span"]{margin-left:2.127659574468085%}.row-fluid

[39/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/js/apache-maven-fluido-1.7.min.js
--
diff --git a/site/current-book/js/apache-maven-fluido-1.7.min.js 
b/site/current-book/js/apache-maven-fluido-1.7.min.js
new file mode 100644
index 000..1324fb4
--- /dev/null
+++ b/site/current-book/js/apache-maven-fluido-1.7.min.js
@@ -0,0 +1,25 @@
+/*!
+ * jQuery JavaScript Library v1.11.2
+ * http://jquery.com/
+ *
+ * Includes Sizzle.js
+ * http://sizzlejs.com/
+ *
+ * Copyright 2005, 2014 jQuery Foundation, Inc. and other contributors
+ * Released under the MIT license
+ * http://jquery.org/license
+ *
+ * Date: 2014-12-17T15:27Z
+ */
+(function(b,a){if(typeof module==="object"& 
module.exports==="object"){module.exports=b.document?a(b,true):function(c){if(!c.document){throw
 new Error("jQuery requires a window with a document")}return 
a(c)}}else{a(b)}}(typeof window!=="undefined"?window:this,function(a5,av){var 
aP=[];var P=aP.slice;var az=aP.concat;var x=aP.push;var bU=aP.indexOf;var 
ac={};var y=ac.toString;var K=ac.hasOwnProperty;var D={};var 
ai="1.11.2",bI=function(e,i){return new 
bI.fn.init(e,i)},E=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,bS=/^-ms-/,aW=/-([\da-z])/gi,O=function(e,i){return
 
i.toUpperCase()};bI.fn=bI.prototype={jquery:ai,constructor:bI,selector:"",length:0,toArray:function(){return
 P.call(this)},get:function(e){return 
e!=null?(e<0?this[e+this.length]:this[e]):P.call(this)},pushStack:function(e){var
 
i=bI.merge(this.constructor(),e);i.prevObject=this;i.context=this.context;return
 i},each:function(i,e){return bI.each(this,i,e)},map:function(e){return 
this.pushStack(bI.map(this,function(b7,b6){retu
 rn e.call(b7,b6,b7)}))},slice:function(){return 
this.pushStack(P.apply(this,arguments))},first:function(){return 
this.eq(0)},last:function(){return this.eq(-1)},eq:function(b7){var 
e=this.length,b6=+b7+(b7<0?e:0);return 
this.pushStack(b6>=0&=0},isEmptyObject:function(i){var e;for(e 
in i){return false}return true},isPlainObject:function(b7){var 
i;if(!b7||bI.type(b7)!=="object"||b7.nodeType||bI.isWindow(b7)){return 
false}try{if(b7.constructor&&!K.call(b7,"constructor")&&!K.call(b7.constructor.prototype,"isPrototypeOf")){return
 false}}catch(b6){return false}if(D.ownLast){for(i in b7){return 
K.call(b7,i)}}for(i in b7){}return 
i===undefined||K.call(b7,i)},type:function(e){if(e==null){return e+""}return 
typeof e==="object"||typeof e==="function"?ac[y.call(e)]||"object":typeof 
e},globalEval:function(e){if(e&(e)){(a5.execScript||function(i){a5["eval"].call(a5,i)})(e)}},camelCase:function(e){return
 e.replace(bS,"ms-").replace(aW,O)},nodeN
 ame:function(i,e){return 
i.nodeName&()===e.toLowerCase()},each:function(ca,cb,b6){var
 
b9,b7=0,b8=ca.length,e=ad(ca);if(b6){if(e){for(;b70&&(i-1) in b6}var m=
+/*!
+ * Sizzle CSS Selector Engine v2.2.0-pre
+ * http://sizzlejs.com/
+ *
+ * Copyright 2008, 2014 jQuery Foundation, Inc. and other contributors
+ * Released under the MIT license
+ * http://jquery.org/license
+ *
+ * Date: 2014-12-16
+ */
+(function(de){var 
cy,dh,cn,cH,cK,ci,cW,dg,dm,cI,cX,cZ,cC,co,c8,c3,df,ce,cF,da="sizzle"+1*new 
Date(),cJ=de.document,di=0,c4=0,b9=cA(),c9=cA(),cG=cA(),cE=function(i,e){if(i===e){cX=true}return
 
0},cQ=1<<31,cO=({}).hasOwnProperty,dc=[],dd=dc.pop,cM=dc.push,b7=dc.push,cm=dc.slice,cd=function(dq,dp){var
 dn=0,e=dq.length;for(;dn+~]|"+cp+")"+cp+"
 *"),ct=new RegExp("="+cp+"*([^\\]'\"]*?)"+cp+"*\\]","g"),cS=new 
RegExp(ck),cU=new RegExp("^"+cL+"$"),c2={ID:new RegExp("^#("+b6+")"),CLASS:new 
RegExp("^\\.("+b6+")"),TAG:new RegExp("^("+b6.replace("w","w*")+")"),ATTR:new 
RegExp("^"+c6),PSEUDO:new RegExp("^"+ck),CHILD:new 
RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+cp+"*(even|odd|(([+-]|)(\\d*)n|)"+cp+"*(?:([+-]|)"+cp+"*(\\d+)|))"+cp+"*\\)|)","i"),bool:new
 RegExp("^(?:"+b8+")$","i"),needsContext:new 
RegExp("^"+cp+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+cp+"*((?:-\\d)?\\d*)"+cp+"*\\)|)(?=[^-]|$)","i")},cc=/^(?:input|select|textarea|button)$/i,cl=/^h\d$/i,cP=/^[^{]+\{\s*\[native
 \w/,cR=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,c1=/[+~]/,cN=/'|\\/g,cs=new 
RegExp("([\\da-f]{1,6}"+cp+"?|("+cp+")|.)","ig"),c5=function(e,dp,i){var 
dn="0x"+dp-65536;return 
dn!==dn||i?dp:dn<0?String.fromCharCode(dn+65536):String.fromCharCode(dn>>10|55296,dn&1023|56320)},dl=function(){cZ()};try{b7.apply((dc=cm.call(cJ.childNodes)),cJ
 
.childNodes);dc[cJ.childNodes.length].nodeType}catch(cD){b7={apply:dc.length?function(i,e){cM.apply(i,cm.call(e))}:function(dq,dp){var
 e=dq.length,dn=0;while((dq[e++]=dp[dn++])){}dq.length=e-1}}}function 
cw(dv,dn,dz,dB){var 
dA,ds,dt,dx,dy,dr,dq,e,dp,dw;if((dn?dn.ownerDocument||dn:cJ)!==cC){cZ(dn)}dn=dn||cC;dz=dz||[];dx=dn.nodeType;if(typeof
 dv!=="string"||!dv||dx!==1&!==9&!==11){return 
dz}if(!dB&){if(dx!==11&&(dA=cR.exec(dv))){if((dt=dA[1])){if(dx===9){ds=dn.getElementById(dt);if(ds&){if(ds.id===dt){dz.push(ds);return

[46/50] [abbrv] metron git commit: METRON-1572 Enhance KAFKA_PUT function (nickwallen) closes apache/metron#1024

2018-06-11 Thread cestella

METRON-1572 Enhance KAFKA_PUT function (nickwallen) closes apache/metron#1024


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/40796c06
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/40796c06
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/40796c06

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 40796c06ad96ae45dd853925fbae8c26509f6c2f
Parents: ae1d3eb
Author: nickwallen 
Authored: Fri Jun 8 08:39:20 2018 -0400
Committer: nickallen 
Committed: Fri Jun 8 08:39:20 2018 -0400

--
 .../metron/management/KafkaFunctions.java   | 87 +++-
 .../KafkaFunctionsIntegrationTest.java  | 21 +
 2 files changed, 89 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/40796c06/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java
--
diff --git 
a/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java
 
b/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java
index 316e19d..a0c92eb 100644
--- 
a/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java
+++ 
b/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java
@@ -22,6 +22,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.serialization.StringDeserializer;
@@ -37,7 +38,7 @@ import org.slf4j.LoggerFactory;
 
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -46,6 +47,9 @@ import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import static java.lang.String.format;
 import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG;
@@ -332,18 +336,26 @@ public class KafkaFunctions {
   /**
* KAFKA_PUT
*
-   * Sends messages to a Kafka topic.
+   * Sends messages to a Kafka topic.
*
-   * Example: Put two messages on the topic 'topic'.
+   * Example: Put two messages on the topic 'topic'.
+   * 
+   *  {@code
*  KAFKA_PUT('topic', ["message1", "message2"])
+   *  }
+   * 
*
-   * Example: Put a message on a topic and also define an alternative Kafka 
broker.
+   * Example: Put a message on a topic and also define an alternative Kafka 
broker.
+   * 
+   *  {@code
*  KAFKA_PUT('topic', ["message1"], { "bootstrap.servers": 
"kafka-broker-1:6667" })
+   *  }
+   * 
*/
   @Stellar(
   namespace = "KAFKA",
   name = "PUT",
-  description = "Sends messages to a Kafka topic.",
+  description = "Sends messages to a Kafka topic. ",
   params = {
   "topic - The name of the Kafka topic.",
   "messages - A list of messages to write.",
@@ -355,45 +367,82 @@ public class KafkaFunctions {
 
 @Override
 public Object apply(List args, Context context) throws 
ParseException {
-
   String topic = ConversionUtils.convert(args.get(0), String.class);
-  List messages = ConversionUtils.convert(args.get(1), List.class);
 
-  // build the properties for kafka
+  List messages;
+  if(args.get(1) instanceof String) {
+// a single message needs sent
+String msg = ConversionUtils.convert(args.get(1), String.class);
+messages = Collections.singletonList(msg);
+
+  } else {
+// a list of messages; all need sent
+messages = ConversionUtils.convert(args.get(1), List.class);
+  }
+
+  // are there any overrides?
   Map overrides = new HashMap<>();
   if(args.size() > 2) {
 overrides = ConversionUtils.convert(args.get(2), Map.class);
   }
-  Properties properties = buildKafkaProperties(overrides, context);
 
   // send the messages
-  try {
-send(topic, messages, properties);
-
-  } catch(InterruptedException | ExecutionException e) {
-throw new ParseException(e.getMessage(), e);
-  }
+  Properties properties = buildKafkaProperties(overrides, context);
+

[23/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-deployment/vagrant/quick-dev-platform/index.html
--
diff --git 
a/site/current-book/metron-deployment/vagrant/quick-dev-platform/index.html 
b/site/current-book/metron-deployment/vagrant/quick-dev-platform/index.html
deleted file mode 100644
index f72df78..000
--- a/site/current-book/metron-deployment/vagrant/quick-dev-platform/index.html
+++ /dev/null
@@ -1,406 +0,0 @@
-
-
-http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
-  
-
-
-
-
-Metron  Quick Development Platform
-
-
-
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
-
-  
-
-  
-
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Quick Development Platform
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
-  
-
-
-  
-
-  
-
-
-
-User Documentation
-   











  


-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   




   
-  
-
-  
-  
-Deployment
-
-  
-  
-
-  
-  
-Kerberos-ambari-setup
-
-  
-  
-
-  
-  
-Kerberos-manual-setup
-
-  
-  
-
-  
-  
-Amazon-ec2
-
-
-  
-
-  
-  
-Other-examples
-  
-  
-  
-
-  
-  
-Ambari
-
-  
-  
-
-  
-  
-Ansible-docker
-
-

[36/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-analytics/metron-profiler/index.html
--
diff --git a/site/current-book/metron-analytics/metron-profiler/index.html 
b/site/current-book/metron-analytics/metron-profiler/index.html
index c3bf7ad..fe498af 100644
--- a/site/current-book/metron-analytics/metron-profiler/index.html
+++ b/site/current-book/metron-analytics/metron-profiler/index.html
@@ -1,295 +1,168 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Metron Profiler
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Metron Profiler
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Metron Profiler
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   

 
-  
-
-  
-  
-Analytics
-
-  
-  
-
-  
-  
-Maas-service
-
-  
-  
-
-Profiler
-  
-  
-  
-
-  
-  
-Profiler-client
-
-
-  
-
-  
-  
-Statistics
-  
-  
-
-  
-  
-
-  
-  
-Docker
-
-

[44/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/css/apache-maven-fluido-1.3.0.min.css
--
diff --git a/site/current-book/css/apache-maven-fluido-1.3.0.min.css 
b/site/current-book/css/apache-maven-fluido-1.3.0.min.css
deleted file mode 100644
index 7db4133..000
--- a/site/current-book/css/apache-maven-fluido-1.3.0.min.css
+++ /dev/null
@@ -1,9 +0,0 @@
-/*!
- * Bootstrap v2.1.0
- *
- * Copyright 2012 Twitter, Inc
- * Licensed under the Apache License v2.0
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Designed and built with all the love in the world @twitter by @mdo and @fat.
- 
*/article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio,canvas,video{display:inline-block;*display:inline;*zoom:1}audio:not([controls]){display:none}html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}a:focus{outline:thin
 dotted #333;outline:5px auto 
-webkit-focus-ring-color;outline-offset:-2px}a:hover,a:active{outline:0}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{height:auto;max-width:100%;vertical-align:middle;border:0;-ms-interpolation-mode:bicubic}#map_canvas
 
img{max-width:none}button,input,select,textarea{margin:0;font-size:100%;vertical-align:middle}button,input{*overflow:visible;line-height:normal}button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0}button,input[type="button"],input[type="reset"],input[type="submit"]{cursor:pointer;-webkit-appearance:button}input[type="search"]{-webkit-box-sizing:content-box;-moz-box-sizing:con
 
tent-box;box-sizing:content-box;-webkit-appearance:textfield}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button{-webkit-appearance:none}textarea{overflow:auto;vertical-align:top}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;line-height:0;content:""}.clearfix:after{clear:both}.hide-text{font:0/0
 
a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}body{margin:0;font-family:"Helvetica
 
Neue",Helvetica,Arial,sans-serif;font-size:14px;line-height:20px;color:#333;background-color:#fff}a{color:#08c;text-decoration:none}a:hover{color:#005580;text-decoration:underline}.img-rounded{-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.img-polaroid{padding:4px;background-color:#fff;border:1px
 solid #ccc;border:1px solid rgba(0,0,0,0.2);-webkit-box-shadow:0
  1px 3px rgba(0,0,0,0.1);-moz-box-shadow:0 1px 3px 
rgba(0,0,0,0.1);box-shadow:0 1px 3px 
rgba(0,0,0,0.1)}.img-circle{-webkit-border-radius:500px;-moz-border-radius:500px;border-radius:500px}.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;margin-left:20px}.container,.navbar-static-top
 .container,.navbar-fixed-top .container,.navbar-fixed-bottom 
.container{width:940px}.span12{width:940px}.span11{width:860px}.span10{width:780px}.span9{width:700px}.span8{width:620px}.span7{width:540px}.span6{width:460px}.span5{width:380px}.span4{width:300px}.span3{width:220px}.span2{width:140px}.span1{width:60px}.offset12{margin-left:980px}.offset11{margin-left:900px}.offset10{margin-left:820px}.offset9{margin-left:740px}.offset8{margin-left:660px}.offset7{margin-left:580px}.offset6{margin-left:500px}.offset5{margin-left:420px}.offset4{margin-left:340px}.offset3{margin-left:260px}.offset2{margin-left:180px}.offs
 
et1{margin-left:100px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid
 
[class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.127659574468085%;*margin-left:2.074468085106383%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid
 [class*="span"]:first-child{margin-left:0}.row-fluid 
.span12{width:100%;*width:99.94680851063829%}.row-fluid 
.span11{width:91.48936170212765%;*width:91.43617021276594%}.row-fluid 
.span10{width:82.97872340425532%;*width:82.92553191489361%}.row-fluid 
.span9{width:74.46808510638297%;*width:74.41489361702126%}.row-fluid 
.span8{width:65.95744680851064%;*width:65.90425531914893%}.row-fluid 
.span7{width:57.44680851063829%;*width:57.39361702127659%}.row-fluid 
.span6{width:48.93617021276595%;*width:48.88297872340425%}.row-fluid 
.span5{width:40.42553191489362%;*width:40.37234042553192%}.row-fluid 
.span4{width:31.9
 14893617021278%;*width:31.861702127659576%}.row-fluid 
.span3{width:23.404255319148934%;*width:23.351063829787233%}.row-fluid 
.span2{width:14.893617021276595%;*width:14.840425531914894%}.row-fluid 
.span1{width:6.382978723404255%;*width:6.329787234042553%}.row-fluid

[34/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-analytics/metron-statistics/index.html
--
diff --git a/site/current-book/metron-analytics/metron-statistics/index.html 
b/site/current-book/metron-analytics/metron-statistics/index.html
index 29084e8..0c3515d 100644
--- a/site/current-book/metron-analytics/metron-statistics/index.html
+++ b/site/current-book/metron-analytics/metron-statistics/index.html
@@ -1,756 +1,599 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Statistics and Mathematical Functions
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Statistics and Mathematical Functions
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Statistics and Mathematical Functions
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   

 
-  
-
-  
-  
-Analytics
-
-  
-  
-
-  
-  
-Maas-service
-
-  
-  
-
-  
-  
-Profiler
-
-  
-  
-
-  
-  
-Profiler-client
-
-
-  
-
-Statistics
-  
-  
-  
-
-  
-  
-HLLP
-
-  
-
-  
-
-  
-  
-
-  
-  
-Docker
-
-

[17/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-platform/metron-enrichment/index.html
--
diff --git a/site/current-book/metron-platform/metron-enrichment/index.html 
b/site/current-book/metron-platform/metron-enrichment/index.html
index 584c36f..de1e4fa 100644
--- a/site/current-book/metron-platform/metron-enrichment/index.html
+++ b/site/current-book/metron-platform/metron-enrichment/index.html
@@ -1,291 +1,165 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Enrichment
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Enrichment
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Enrichment
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   



 
-  
-
-  
-  
-Deployment
-  
-  
-  
-
-  
-  
-Alerts
-
-  
-  
-
-  
-  
-Config
-
-  
-  
-
-  
-  
-Rest
-
-

[41/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/index.html
--
diff --git a/site/current-book/index.html b/site/current-book/index.html
index 729fff7..f8d6f85 100644
--- a/site/current-book/index.html
+++ b/site/current-book/index.html
@@ -1,274 +1,180 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Apache Metron
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Apache Metron
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Apache Metron
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  

-  
-
-Metron
-  
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   



 
-  
-
-  
-  
-Deployment
-  
-  
-  
-
-  
-  
-Alerts
-
-  
-  
-
-  
-  
-Config
-
-  
-  
-
-  
-  
-Rest
-
-   


 
-  
-
-

[50/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr

2018-06-11 Thread cestella

Merge branch 'master' into feature/METRON-1416-upgrade-solr


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/41708876
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/41708876
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/41708876

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 41708876336d009b3a2ff2c533bdf0294af1477b
Parents: f241f87 32d132b
Author: cstella 
Authored: Mon Jun 11 17:50:48 2018 -0400
Committer: cstella 
Committed: Mon Jun 11 17:50:48 2018 -0400

--
 KEYS|59 +
 Upgrading.md| 2 +-
 dev-utilities/release-utils/metron-rc-check |77 +-
 .../release-utils/validate-jira-for-release |75 +-
 metron-analytics/metron-maas-common/pom.xml | 2 +-
 metron-analytics/metron-maas-service/pom.xml| 2 +-
 metron-analytics/metron-profiler-client/pom.xml | 2 +-
 .../profiler/client/stellar/GetProfileTest.java | 3 +-
 .../client/stellar/ProfilerFunctionsTest.java   |15 +-
 metron-analytics/metron-profiler-common/pom.xml | 2 +-
 metron-analytics/metron-profiler/README.md  |15 +
 metron-analytics/metron-profiler/pom.xml| 2 +-
 .../src/main/flux/profiler/remote.yaml  | 3 +-
 metron-analytics/metron-statistics/pom.xml  | 2 +-
 .../StellarStatisticsFunctionsTest.java | 5 +-
 .../sampling/SamplerFunctionsTest.java  | 3 +-
 metron-analytics/pom.xml| 2 +-
 metron-contrib/metron-docker/pom.xml| 2 +-
 metron-contrib/metron-performance/pom.xml   | 2 +-
 metron-contrib/pom.xml  | 2 +-
 metron-deployment/Kerberos-manual-setup.md  | 8 +-
 metron-deployment/README.md |22 +-
 metron-deployment/amazon-ec2/conf/defaults.yml  | 2 +-
 .../ansible/playbooks/docker_probe_install.yml  | 2 +-
 .../roles/ambari_master/defaults/main.yml   | 2 +-
 .../roles/metron_pcapservice/defaults/main.yml  | 2 +-
 metron-deployment/development/centos6/README.md | 8 +-
 .../centos6/ansible/inventory/group_vars/all| 2 +-
 .../development/ubuntu14/README.md  | 6 +-
 .../ubuntu14/ansible/inventory/group_vars/all   | 2 +-
 .../ambari/elasticsearch-mpack/pom.xml  | 4 +-
 .../packaging/ambari/metron-mpack/pom.xml   | 4 +-
 .../CURRENT/package/files/bro_index.template| 2 +-
 .../CURRENT/package/files/error_index.template  | 2 +-
 .../package/files/metaalert_index.template  | 2 +-
 .../CURRENT/package/files/snort_index.template  | 2 +-
 .../CURRENT/package/files/yaf_index.template| 2 +-
 .../CURRENT/package/scripts/metron_service.py   | 5 +
 .../metron-mpack/src/main/resources/mpack.json  | 2 +-
 .../packaging/docker/deb-docker/pom.xml | 4 +-
 .../packaging/docker/rpm-docker/pom.xml | 4 +-
 metron-deployment/pom.xml   | 2 +-
 .../e2e/mock-data/alerts_ui_e2e_index.template  | 2 +-
 metron-interface/metron-alerts/package.json | 2 +-
 metron-interface/metron-alerts/pom.xml  | 2 +-
 .../alert-details/alert-details.component.ts| 2 +-
 .../alerts/alerts-list/alerts-list.component.ts | 2 +-
 .../table-view/table-view.component.html|10 +-
 .../table-view/table-view.component.ts  | 2 +-
 .../metron-alerts/src/app/model/alert-source.ts | 2 +-
 .../src/app/service/global-config.service.ts|10 +-
 metron-interface/metron-config/package.json | 2 +-
 metron-interface/metron-config/pom.xml  | 2 +-
 .../metron-config/scripts/package.json  | 2 +-
 metron-interface/metron-rest-client/pom.xml | 2 +-
 metron-interface/metron-rest/pom.xml| 2 +-
 .../rest/service/impl/SearchServiceImpl.java|10 +-
 .../src/main/resources/application.yml  | 2 +-
 .../metron-rest/src/main/scripts/metron-rest.sh | 5 +-
 .../MetaAlertControllerIntegrationTest.java | 4 +-
 .../service/impl/SearchServiceImplTest.java | 3 +-
 metron-interface/pom.xml| 2 +-
 metron-platform/Performance-tuning-guide.md | 4 +-
 metron-platform/README.md   | 2 +-
 metron-platform/elasticsearch-shaded/pom.xml| 2 +-
 metron-platform/metron-api/pom.xml  | 2 +-
 metron-platform/metron-common/README.md | 5 +
 metron-platform/metron-common/pom.xml   | 2 +-
 .../org/apache/metron/common/Constants.java | 1 +
 .../metron/common/bolt/ConfiguredBolt.java  |14 +-
 .../common/bolt/ConfiguredEnrichmentBolt.java   |11 +-
 .../common/bolt/ConfiguredIndexingBolt.java |13 +-

[38/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

  
-  
-  
-Use-cases
-  
-  
-
-
-
-
-
-  
-
-   
-
-
-
- http://maven.apache.org/; title="Built 
by Maven" class="poweredBy">
-
-  
-  
+
+  User Documentation
+Metron
+
+CONTRIBUTING
+Upgrading
+Analytics
+
+Maas-service
+Profiler
+Profiler-client
+Statistics
+
+
+Docker
+Performance
+Deployment
+Alerts
+Config
+Rest
+Platform
+Sensors
+Stellar-3rd-party-example
+Stellar-common
+Stellar-zeppelin
+Use-cases
+
+
+
+  
+  
+
+
+
+
+http://maven.apache.org/; title="Built by Maven" 
class="poweredBy">
+
   
 
-
-
-
-  
-Model Management Infrastructure
+
+
+Model Management Infrastructure
 
 
 Introduction
-One of the main features envisioned and requested is the ability to augment 
the threat intelligence and enrichment processes with insights derived from 
machine learning or statistical models. The challenges with this sort of 
infrastructure are
-
+One of the main features envisioned and requested is the ability to augment 
the threat intelligence and enrichment processes with insights derived from 
machine learning or statistical models.  The challenges with this sort of 
infrastructure are
 
-  
+
 Applying the model may be sufficiently computationally/resource intensive 
that we need to support scaling via load balancing, which will require service 
discovery and management.
-  
 Models require out of band and frequent training to react to growing 
threats and new patterns that emerge.
-  
-Models should be language/environment agnostic as much as possible. These 
should include small-data and big-data libraries and languages.
+Models should be language/environment agnostic as much as possible.  These 
should include small-data and big-data libraries and languages.
 
-To support a high throughput environment that is manageable, it is evident 
that 
-
+To support a high throughput environment that is manageable, it is evident 
that
 
-  
+
 Multiple versions of models will need to be exposed
-  
 Deployment should happen using Yarn to manage resources
-  
 Clients should have new model endpoints pushed to them
 
 
 Architecture
 
 To support these requirements, the following components have been 
created:
-
 
-  
+
 A Yarn application which will listen for model deployment requests and 
upon execution, register their endpoints in zookeeper:
-  
 
-
+
 Operation type: ADD, REMOVE, LIST
-
 Model Name
-
 Model Version
-
 Memory requirements (in megabytes)
-
 Number of instances
-  
-  
+
+
 A command line deployment client which will localize the model payload 
onto HDFS and submit a model request
-  
 A Java client which will interact with zookeeper and receive updates about 
model state changes (new deployments, removals, etc.)
-  
 A series of Stellar functions for interacting with models deployed via the 
Model as a Service infrastructure.
 
 
 maas_service.sh
-The maas_service.sh script starts the Yarn application which will 
listen for requests. Right now the queue for the requests is a distributed 
queue stored in http://curator.apache.org/curator-recipes/distributed-queue.html;>zookeeper
 for convenience.
+The maas_service.sh script starts the Yarn application which will 
listen for requests.  Right now the queue for the requests is a distributed 
queue stored in http://curator.apache.org/curator-recipes/distributed-queue.html;>zookeeper
 for convenience.
 
-
-
-./maas_service.sh
+
+
+./maas_service.sh
 usage: MaaSClient
  -c,--create  Flag to indicate whether to create
   the domain specified with -domain.
@@ -311,23 +177,21 @@ usage: MaaSClient
   given domain
  -zq,--zk_quorum argZookeeper Quorum
  -zr,--zk_root arg  Zookeeper Root
-
+
+
 
 maas_deploy.sh
-The maas_deploy.sh script allows users to deploy models and their 
collateral from their local disk to the cluster. It is assumed that the 
-
+The maas_deploy.sh script allows users to deploy models and their 
collateral from their local disk to the cluster. It is assumed that the
 
-  
+
 Collateral has exactly one .sh script capable of starting the 
endpoint
-  
 The model service executable will expose itself as a URL endpoint (e.g. as 
a REST interface, but not necessarily)
-  
 The model service executable will write out to local disk a JSON blob 
indicating the endpoint (see https://g

[05/50] [abbrv] metron git commit: METRON-1594: KafkaWriter is asynchronous and may lose data on node failure (mmiklavc via mmiklavc) closes apache/metron#1045

2018-06-11 Thread cestella

METRON-1594: KafkaWriter is asynchronous and may lose data on node failure 
(mmiklavc via mmiklavc) closes apache/metron#1045


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/523c38cf
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/523c38cf
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/523c38cf

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 523c38cf6399e2e3974a51a2cd0fe47e096b0bdf
Parents: b6808f7
Author: mmiklavc 
Authored: Wed Jun 6 15:40:55 2018 -0600
Committer: Michael Miklavcic 
Committed: Wed Jun 6 15:40:55 2018 -0600

--
 metron-analytics/metron-profiler/README.md  |  15 ++
 .../src/main/flux/profiler/remote.yaml  |   3 +-
 metron-platform/metron-common/README.md |   5 +
 .../metron/common/bolt/ConfiguredBolt.java  |  14 +-
 .../common/bolt/ConfiguredEnrichmentBolt.java   |  11 +-
 .../common/bolt/ConfiguredIndexingBolt.java |  13 +-
 .../common/bolt/ConfiguredParserBolt.java   |  13 +-
 .../common/bolt/ConfiguredProfilerBolt.java |   9 +-
 .../common/configuration/Configurations.java|   7 +-
 .../configuration/EnrichmentConfigurations.java |  26 +++
 .../configuration/IndexingConfigurations.java   |  12 +-
 .../configuration/ParserConfigurations.java |   1 +
 .../profiler/ProfilerConfigurations.java|  25 +++
 .../writer/ConfigurationStrategy.java   |  44 +
 .../writer/ConfigurationsStrategies.java| 144 +++
 .../writer/EnrichmentWriterConfiguration.java   | 110 
 .../writer/ParserWriterConfiguration.java   |   6 +-
 .../writer/ProfilerWriterConfiguration.java | 109 
 .../configuration/ParserConfigurationsTest.java | 120 +
 .../profiler/ProfilerConfigTest.java|  13 +-
 .../writer/ConfigurationsStrategiesTest.java|  79 +
 .../EnrichmentWriterConfigurationTest.java  |  54 ++
 .../writer/IndexingWriterConfigurationTest.java |  70 
 .../writer/ParserWriterConfigurationTest.java   |  72 
 .../writer/ProfilerWriterConfigurationTest.java |  54 ++
 .../writer/IndexingWriterConfigurationTest.java |  70 
 .../writer/ParserWriterConfigurationTest.java   |  38 
 metron-platform/metron-enrichment/README.md |  15 +-
 .../main/flux/enrichment/remote-splitjoin.yaml  |   9 +-
 .../main/flux/enrichment/remote-unified.yaml|   9 +-
 .../bolt/BulkMessageWriterBoltTest.java |  64 ---
 .../src/main/flux/indexing/batch/remote.yaml|   4 +-
 .../flux/indexing/random_access/remote.yaml |   4 +-
 metron-platform/metron-parsers/README.md|   6 +-
 .../apache/metron/parsers/bolt/ParserBolt.java  | 108 +++-
 .../metron/parsers/bolt/WriterHandler.java  |  63 +--
 .../metron/parsers/bolt/ParserBoltTest.java | 176 +--
 .../integration/WriterBoltIntegrationTest.java  |   6 +-
 .../metron/writer/bolt/BatchTimeoutHelper.java  |   6 +-
 .../writer/bolt/BulkMessageWriterBolt.java  |  48 ++---
 .../apache/metron/writer/kafka/KafkaWriter.java |  92 +++---
 41 files changed, 1426 insertions(+), 321 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/523c38cf/metron-analytics/metron-profiler/README.md
--
diff --git a/metron-analytics/metron-profiler/README.md 
b/metron-analytics/metron-profiler/README.md
index 79cdd44..1a17e10 100644
--- a/metron-analytics/metron-profiler/README.md
+++ b/metron-analytics/metron-profiler/README.md
@@ -538,6 +538,8 @@ The Profiler runs as an independent Storm topology.  The 
configuration for the P
 | [`profiler.hbase.batch`](#profilerhbasebatch)
 | The number of puts that are written to HBase in a single batch.
 | 
[`profiler.hbase.flush.interval.seconds`](#profilerhbaseflushintervalseconds) | 
The maximum number of seconds between batch writes to HBase.
 | [`topology.kryo.register`](#topologykryoregister)
 | Storm will use Kryo serialization for these classes.
+| [`profiler.writer.batchSize`](#profilerwriterbatchsize)  
 | The number of records to batch when writing to Kakfa.
+| [`profiler.writer.batchTimeout`](#profilerwriterbatchtimeout)
 | The timeout in ms for batching when writing to Kakfa.
 
 
 ### `profiler.input.topic`
@@ -852,6 +854,19 @@ More information on accessing profile data can be found in 
the [Profiler Client]
 
 More information on using the [`STATS_*` functions in Stellar can be found 
here](../../metron-platform/metron-common).
 
+### `profiler.writer.batchSize`
+
+*Default*: 15
+
+The number of records to batch when writing to Kakfa. This is managed in the 
global configuration and does not require a

[27/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-deployment/packaging/ambari/index.html
--
diff --git a/site/current-book/metron-deployment/packaging/ambari/index.html 
b/site/current-book/metron-deployment/packaging/ambari/index.html
index 8013381..bac5758 100644
--- a/site/current-book/metron-deployment/packaging/ambari/index.html
+++ b/site/current-book/metron-deployment/packaging/ambari/index.html
@@ -1,505 +1,331 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Ambari Management Pack Development
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Ambari Management Pack Development
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Ambari Management Pack Development
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   




   
-  
-
-  
-  
-Deployment
-
-  
-  
-
-  
-  
-Kerberos-ambari-setup
-
-  
-  
-
-  
-  
-Kerberos-manual-setup
-
-  
-  
-
-  
-

[08/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-stellar/stellar-zeppelin/index.html
--
diff --git a/site/current-book/metron-stellar/stellar-zeppelin/index.html 
b/site/current-book/metron-stellar/stellar-zeppelin/index.html
new file mode 100644
index 000..54bc800
--- /dev/null
+++ b/site/current-book/metron-stellar/stellar-zeppelin/index.html
@@ -0,0 +1,239 @@
+
+
+http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
+  
+
+
+
+
+Metron  Stellar Interpreter for Apache Zeppelin
+
+
+
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
+
+  
+
+  
+
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Stellar Interpreter for Apache Zeppelin
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
+  
+  
+
+  
+
+  User Documentation
+Metron
+
+CONTRIBUTING
+Upgrading
+Analytics
+Docker
+Performance
+Deployment
+Alerts
+Config
+Rest
+Platform
+Sensors
+Stellar-3rd-party-example
+Stellar-common
+Stellar-zeppelin
+Use-cases
+
+
+
+  
+  
+
+
+
+
+http://maven.apache.org/; title="Built by Maven" 
class="poweredBy">
+
+  
+
+
+
+Stellar Interpreter for Apache Zeppelin
+https://zeppelin.apache.org/;>Apache 
Zeppelin is a web-based notebook that enables data-driven, interactive data 
analytics and collaborative documents with SQL, Scala and more.  This project 
provides a means to run the Stellar REPL directly within a Zeppelin 
Notebook.
+
+
+Prerequisites
+Installation
+Usage
+
+
+Prerequisites
+
+
+https://zeppelin.apache.org/;>Apache 
Zeppelin 0.7.3
+This is tested with version 0.7.3.  Other versions may work, but are not 
supported.
+
+
+Installation
+Currently, you need to manually install the Stellar Interpreter in 
Zeppelin. In the future this step could be automated by the Metron Mpack.
+To install the Stellar Interpreter in your Apache Zeppelin installation, 
follow these instructions.  This is paraphrased from the https://zeppelin.apache.org/docs/latest/development/writingzeppelininterpreter.html#install-your-interpreter-binary;>Zeppelin
 docs.
+
+
+
+
+Build and install Metron. Metron and its dependencies will be retrieved 
from your local Maven repository.
+
+
+
+cd $METRON_HOME
+mvn clean install -DskipTests
+
+
+
+
+If you do not already have Zeppelin installed, https://zeppelin.apache.org/download.html;>download and unpack Apache 
Zeppelin.  Then change directories to the root of your Zeppelin 
download.
+
+
+
+cd $ZEPPELIN_HOME
+
+
+
+
+Use Zeppelins installation utility to install the Stellar 
Interpreter.
+If Zeppelin was already installed, make sure that it is stopped before 
running this command.  Update the version, 0.5.0 in the example 
below, to whatever is appropriate for your environment.
+
+
+
+bin/install-interpreter.sh --name stellar --artifact 
org.apache.metron:stellar-zeppelin:0.5.0
+
+
+
+
+Start Zeppelin.
+
+
+
+bin/zeppelin-daemon.sh start
+
+
+
+
+Navigate to Zeppelin running at http://localhost:8080/;>http://localhost:8080/.  The Stellar 
Interpreter should be ready for use with a basic set of functions.
+
+
+
+Usage
+
+
+
+
+Create a new notebook.
+
+
+
+
+Click on Notebook  Create new note.
+
+
+
+Set the default Interpreter to stellar.
+When creating the notebook, if you define stellar as the default 
interpreter, then there is no need to enter %stellar at the top of 
each code block.
+If stellar is not the default interpreter, then you must enter 
%stellar at the top of a code block containing Stellar code.
+
+
+
+
+
+In the first block, add the following Stellar, then click Run.
+
+
+
+2 in [2,3,4]
+
+
+
+
+In the next block, check which functions are available to you.
+When executing Stellars magic functions, you must explicitly define 
which interpreter should be used in the code block.  If you define 
stellar as the default interpreter when creating a notebook, 
then this is only required when using Stellars magic functions.
+
+
+
+%stellar
+
+%functions
+
+
+You will only see the functions defined within 
stellar-common since that is the only library that we added to the 
interpreter.
+
+
+
+Add additional Stellar functions to your session.
+
+
+
+
+Go back to the Stellar interpreter configuration and add another dependency 
as follows.
+
+
+
+org.apache.metron:metron-statistics:0.5.0
+
+
+
+
+Go back to your notebook and run %functions again.  You will now 
see the additional functions defined within

[16/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-platform/metron-indexing/index.html
--
diff --git a/site/current-book/metron-platform/metron-indexing/index.html 
b/site/current-book/metron-platform/metron-indexing/index.html
index 3d79323..6d4b8d6 100644
--- a/site/current-book/metron-platform/metron-indexing/index.html
+++ b/site/current-book/metron-platform/metron-indexing/index.html
@@ -1,378 +1,200 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Indexing
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Indexing
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Indexing
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   



 
-  
-
-  
-  
-Deployment
-  
-  
-  
-
-  
-  
-Alerts
-
-  
-  
-
-  
-  
-Config
-
-  
-  
-
-  
-  
-Rest
-
-

[02/50] [abbrv] metron git commit: METRON-1553 Validate JIRA Script Error (nickwallen) closes apache/metron#1013

2018-06-11 Thread cestella

METRON-1553 Validate JIRA Script Error (nickwallen) closes apache/metron#1013


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/45e3ed87
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/45e3ed87
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/45e3ed87

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 45e3ed87ca4aceda7c8387389fc3a286ee5264d9
Parents: dbcc52c
Author: nickwallen 
Authored: Tue Jun 5 11:50:58 2018 -0400
Committer: nickallen 
Committed: Tue Jun 5 11:50:58 2018 -0400

--
 .../release-utils/validate-jira-for-release | 75 
 1 file changed, 46 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/45e3ed87/dev-utilities/release-utils/validate-jira-for-release
--
diff --git a/dev-utilities/release-utils/validate-jira-for-release 
b/dev-utilities/release-utils/validate-jira-for-release
index df5776c..5fd4367 100755
--- a/dev-utilities/release-utils/validate-jira-for-release
+++ b/dev-utilities/release-utils/validate-jira-for-release
@@ -137,35 +137,52 @@ for i in "$@"; do
   esac
 done
 
-WORKDIR="~/tmp"
 
 # ensure all required values are set
 if [ -z "$VERSION" ]; then
-   echo "Missing -v/--version is is required"
-   exit 1
+  help
+  echo "Missing -v/--version is is required"
+  exit 1
 fi
 if [ -z "$START" ]; then
-   echo "Missing -s/--start which is required"
-   exit 1
+  help
+  echo "Missing -s/--start which is required"
+  exit 1
 fi
 if [ -z "$END" ]; then
-   echo "Missing -e/--end which is required"
-   exit 1
+  help
+  echo "Missing -e/--end which is required"
+  exit 1
 fi
 if [ -z "$REPO" ]; then
-   echo "Missing -r/--repo which is required"
-   exit 1
+  help
+  echo "Missing -r/--repo which is required"
+  exit 1
 fi
 if [ -z "$BRANCH" ]; then
-   echo "Missing -b/--branch which is required"
-   exit 1
+  help
+  echo "Missing -b/--branch which is required"
+  exit 1
 fi
 
-# clone the metron repo and fetch all tags
-git clone $REPO "metron-$VERSION"
+TMPDIR="$HOME/tmp"
+WORKDIR="$TMPDIR/metron-$VERSION"
+
+# warn the user if the working directory exists
+if [ -d "$WORKDIR" ]; then
+  read -p "  directory exists [$WORKDIR].  overwrite existing repo? [yN] " -n 
1 -r
+  echo
+  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+exit 1
+  fi
+fi
+
+# fetch the repo and all tags
+rm -rf "$WORKDIR"
+git clone $REPO "$WORKDIR"
+cd "$WORKDIR"
 git checkout $BRANCH
-cd "$WORKDIR/metron-$VERSION"
-git fetch --all --tags
+git fetch --tags
 
 # find all JIRAs that have been committed since the last release
 GET_JIRAS="git log $START..$END --oneline | grep -E -o 'METRON[- ]*[0-9]+'"
@@ -177,21 +194,21 @@ printf "$FORMAT_STR" "JIRA" "STATUS" "FIX VERSION" 
"ASSIGNEE" "FIX"
 # for each JIRA since the last release tag...
 eval $GET_JIRAS | while read JIRA ; do
 
-   # fetch the JIRA content
-   
URL="https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$JIRA/$JIRA.xml;
-CONTENT=`curl -s $URL`
+  # fetch the JIRA content
+  
URL="https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$JIRA/$JIRA.xml;
+  CONTENT=`curl -s $URL`
 
-   # painfully extract some fields
-   STATUS=`echo "$CONTENT" | grep "]*>" | sed 
's/^.*]*>//' | sed 's/<.status>.*$//'`
-   ASSIGNEE=`echo "$CONTENT" | grep "]*>" | sed 
's/^.*]*>//' | sed 's/<.assignee>.*$//'`
-   FIXV=`echo "$CONTENT" | grep "]*>" | sed 
's/^.*]*>//' | sed 's/<.fixVersion>.*$//'`
+  # painfully extract some fields
+  STATUS=`echo "$CONTENT" | grep "]*>" | sed 's/^.*]*>//' 
| sed 's/<.status>.*$//'`
+  ASSIGNEE=`echo "$CONTENT" | grep "]*>" | sed 
's/^.*]*>//' | sed 's/<.assignee>.*$//'`
+  FIXV=`echo "$CONTENT" | grep "]*>" | sed 
's/^.*]*>//' | sed 's/<.fixVersion>.*$//'`
 
-   # the link is only populated, if there is something to fix
-   LINK=""
-   if [ "$FIXV" != "$VERSION" ] || [ "$STATUS" != "Done" ]; then
-   LINK="https://issues.apache.org/jira/browse/$JIRA;
-   fi
+  # the link is only populated, if there is something to fix
+  LINK=""
+  if [ "$FIXV" != "$VERSION" ] || [ "$STATUS" != "Done" ]; then
+LINK="https://issues.apache.org/jira/browse/$JIRA;
+  fi
 
-   # show the JIRA
-   printf "$FORMAT_STR" "$JIRA" "$STATUS" "$FIXV" "$ASSIGNEE" "$LINK"
+  # show the JIRA
+  printf "$FORMAT_STR" "$JIRA" "$STATUS" "$FIXV" "$ASSIGNEE" "$LINK"
 done

[06/50] [abbrv] metron git commit: METRON-1568: Stellar should have a _ special variable which returns the message in map form closes apache/incubator-metron#1021

2018-06-11 Thread cestella

METRON-1568: Stellar should have a _ special variable which returns the message 
in map form closes apache/incubator-metron#1021


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1b9828e6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1b9828e6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1b9828e6

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 1b9828e6a68fd39fd308ebb80f1d79cd490ceaec
Parents: 523c38c
Author: cstella 
Authored: Thu Jun 7 10:54:05 2018 -0400
Committer: cstella 
Committed: Thu Jun 7 10:54:05 2018 -0400

--
 .../enrichment/handler/StellarConfig.java   |  11 +-
 .../StellarTransformationTest.java  |  30 ++-
 .../adapters/stellar/StellarAdapterTest.java|  26 ++
 .../integration/EnrichmentIntegrationTest.java  |   2 +
 .../main/config/zookeeper/enrichments/test.json |   1 +
 .../metron/pcap/filter/PcapFieldResolver.java   |   5 +
 metron-stellar/stellar-common/README.md |   1 +
 .../metron/stellar/common/utils/ConcatMap.java  | 256 +++
 .../common/utils/StellarProcessorUtils.java |  51 +++-
 .../metron/stellar/dsl/MapVariableResolver.java |   8 +
 .../metron/stellar/dsl/VariableResolver.java|   1 +
 .../dsl/functions/DataStructureFunctions.java   |   4 +
 .../stellar/common/utils/ConcatMapTest.java |  83 ++
 .../stellar/dsl/functions/BasicStellarTest.java |  18 ++
 14 files changed, 487 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
index 62e0263..9f96a22 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
@@ -19,6 +19,7 @@ package 
org.apache.metron.common.configuration.enrichment.handler;
 
 import org.apache.metron.stellar.common.StellarAssignment;
 import org.apache.metron.stellar.common.StellarProcessor;
+import org.apache.metron.stellar.dsl.VariableResolver;
 import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -142,8 +143,14 @@ public class StellarConfig implements Config {
   {
 
 Map messageSegment = new HashMap<>();
-for(String variable : stellarFields) {
-  messageSegment.put(variable, message.get(variable));
+if(stellarFields.contains(VariableResolver.ALL_FIELDS)) {
+  //we need to include all of the fields here.
+  messageSegment.putAll(message);
+}
+else {
+  for (String variable : stellarFields) {
+messageSegment.put(variable, message.get(variable));
+  }
 }
 return messageSegment;
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
--
diff --git 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
index fc91844..3b7c7bb 100644
--- 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
+++ 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
@@ -78,6 +78,32 @@ public class StellarTransformationTest {
   @Multiline
   public static String badConfig;
 
+  /**
+   { "fieldTransformations" : [
+{ "transformation" : "STELLAR"
+,"output" : [ "new_field"]
+,"config" : {
+  "new_field" : "MAP_GET('source.type', _)"
+}
+}
+]
+  }
+   */
+ @Multiline
+ public static String configAll;
+
+  @Test
+  public void testConfigAll() throws Exception {
+SensorParserConfig c = 
SensorParserConfig.fromBytes(Bytes.toBytes(configAll));
+JSONObject input = new JSONObject();
+input.put("source.type", "test");
+for (FieldTransformer handler : c.getFieldTransformations()) {
+  handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
+}
+Assert.assertEquals(2,

[10/50] [abbrv] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053

2018-06-11 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-sensors/pycapa/index.html
--
diff --git a/site/current-book/metron-sensors/pycapa/index.html 
b/site/current-book/metron-sensors/pycapa/index.html
index 8c46464..f81aee9 100644
--- a/site/current-book/metron-sensors/pycapa/index.html
+++ b/site/current-book/metron-sensors/pycapa/index.html
@@ -1,313 +1,199 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Metron  Pycapa
-
+
 
 
-
-  
-
-
-  
-
-$( document ).ready( function() { $( 
'.carousel' ).carousel( { interval: 3500 } ) } );
-  
-
-
-  
-
-
-
-
-  
-
-http://metron.apache.org/; 
id="bannerLeft">
-   
 
-
-  
-  
+
+
+  $( document ).ready( function() { $( '.carousel' ).carousel( { 
interval: 3500 } ) } );
+
+  
+  
+
+  
+http://metron.apache.org/; 
id="bannerLeft">
+
 
   
 
   
 
-
-
-  
-http://www.apache.org; class="externalLink" 
title="Apache">
-Apache
-
-  /
-
-http://metron.apache.org/; class="externalLink" 
title="Metron">
-Metron
-
-  /
-
-
-Documentation
-
-  /
-Pycapa
-
-
-
-  Last Published: 
2018-01-03 |
-  Version: 0.4.2
-
-
+  http://www.apache.org; class="externalLink" 
title="Apache">Apache/
+  http://metron.apache.org/; class="externalLink" 
title="Metron">Metron/
+  Documentation/
+Pycapa
+| 
Last Published: 2018-06-07
+  Version: 0.5.0
+
   
-
-
   
-
+
   
-
-
-
-User Documentation
-   











  
  
-  
-
-  
-  
-Metron
-
-  
-  
-
-  
-  
-Upgrading
-
-   
   
-  
-
-  
-  
-Analytics
-  
-  
-  
-
-  
-  
-Docker
-
-   



 
-  
-
-  
-  
-Deployment
-  
-  
-  
-
-  
-  
-Alerts
-
-  
-  
-
-  
-  
-Config
-
-  
-  
-
-  
-  
-Rest
-
-

metron git commit: METRON-1601: Rename metaalert alert nested field to metron_alert to avoid collision closes apache/incubator-metron#1049

2018-06-08 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 40796c06a -> edec7b18b


METRON-1601: Rename metaalert alert nested field to metron_alert to avoid 
collision closes apache/incubator-metron#1049


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/edec7b18
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/edec7b18
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/edec7b18

Branch: refs/heads/master
Commit: edec7b18bc40e40f35916489ba2fde2ca4916dc5
Parents: 40796c0
Author: cstella 
Authored: Fri Jun 8 09:49:54 2018 -0400
Committer: cstella 
Committed: Fri Jun 8 09:49:54 2018 -0400

--
 .../METRON/CURRENT/package/files/bro_index.template   |  2 +-
 .../METRON/CURRENT/package/files/error_index.template |  2 +-
 .../METRON/CURRENT/package/files/metaalert_index.template |  2 +-
 .../METRON/CURRENT/package/files/snort_index.template |  2 +-
 .../METRON/CURRENT/package/files/yaf_index.template   |  2 +-
 .../e2e/mock-data/alerts_ui_e2e_index.template|  2 +-
 .../app/alerts/alert-details/alert-details.component.ts   |  2 +-
 .../src/app/alerts/alerts-list/alerts-list.component.ts   |  2 +-
 .../alerts-list/table-view/table-view.component.html  | 10 +-
 .../alerts/alerts-list/table-view/table-view.component.ts |  2 +-
 .../metron-alerts/src/app/model/alert-source.ts   |  2 +-
 .../controller/MetaAlertControllerIntegrationTest.java|  4 ++--
 metron-platform/metron-elasticsearch/README.md|  8 
 .../ElasticsearchMetaAlertIntegrationTest.java| 10 +-
 .../java/org/apache/metron/indexing/dao/MetaAlertDao.java |  2 +-
 .../apache/metron/indexing/dao/InMemoryMetaAlertDao.java  |  2 +-
 16 files changed, 28 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/edec7b18/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
index 30f2591..17ad4d2 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -143,7 +143,7 @@
 "uid": {
   "type": "keyword"
 },
-"alert": {
+"metron_alert": {
   "type": "nested"
 },
 "ip_src_addr": {

http://git-wip-us.apache.org/repos/asf/metron/blob/edec7b18/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template
index 00aaf87..d119509 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template
@@ -38,7 +38,7 @@
 "error_type": {
   "type": "keyword"
 },
-"alert": {
+"metron_alert": {
   "type": "nested"
 }
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/edec7b18/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
index 24e8357..05d5e32 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
@@ -36,7 +36,7 @@
   "type": "date",
   "format":

metron git commit: METRON-1568: Stellar should have a _ special variable which returns the message in map form closes apache/incubator-metron#1021

2018-06-07 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 523c38cf6 -> 1b9828e6a


METRON-1568: Stellar should have a _ special variable which returns the message 
in map form closes apache/incubator-metron#1021


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1b9828e6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1b9828e6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1b9828e6

Branch: refs/heads/master
Commit: 1b9828e6a68fd39fd308ebb80f1d79cd490ceaec
Parents: 523c38c
Author: cstella 
Authored: Thu Jun 7 10:54:05 2018 -0400
Committer: cstella 
Committed: Thu Jun 7 10:54:05 2018 -0400

--
 .../enrichment/handler/StellarConfig.java   |  11 +-
 .../StellarTransformationTest.java  |  30 ++-
 .../adapters/stellar/StellarAdapterTest.java|  26 ++
 .../integration/EnrichmentIntegrationTest.java  |   2 +
 .../main/config/zookeeper/enrichments/test.json |   1 +
 .../metron/pcap/filter/PcapFieldResolver.java   |   5 +
 metron-stellar/stellar-common/README.md |   1 +
 .../metron/stellar/common/utils/ConcatMap.java  | 256 +++
 .../common/utils/StellarProcessorUtils.java |  51 +++-
 .../metron/stellar/dsl/MapVariableResolver.java |   8 +
 .../metron/stellar/dsl/VariableResolver.java|   1 +
 .../dsl/functions/DataStructureFunctions.java   |   4 +
 .../stellar/common/utils/ConcatMapTest.java |  83 ++
 .../stellar/dsl/functions/BasicStellarTest.java |  18 ++
 14 files changed, 487 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
index 62e0263..9f96a22 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
@@ -19,6 +19,7 @@ package 
org.apache.metron.common.configuration.enrichment.handler;
 
 import org.apache.metron.stellar.common.StellarAssignment;
 import org.apache.metron.stellar.common.StellarProcessor;
+import org.apache.metron.stellar.dsl.VariableResolver;
 import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -142,8 +143,14 @@ public class StellarConfig implements Config {
   {
 
 Map messageSegment = new HashMap<>();
-for(String variable : stellarFields) {
-  messageSegment.put(variable, message.get(variable));
+if(stellarFields.contains(VariableResolver.ALL_FIELDS)) {
+  //we need to include all of the fields here.
+  messageSegment.putAll(message);
+}
+else {
+  for (String variable : stellarFields) {
+messageSegment.put(variable, message.get(variable));
+  }
 }
 return messageSegment;
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
--
diff --git 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
index fc91844..3b7c7bb 100644
--- 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
+++ 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
@@ -78,6 +78,32 @@ public class StellarTransformationTest {
   @Multiline
   public static String badConfig;
 
+  /**
+   { "fieldTransformations" : [
+{ "transformation" : "STELLAR"
+,"output" : [ "new_field"]
+,"config" : {
+  "new_field" : "MAP_GET('source.type', _)"
+}
+}
+]
+  }
+   */
+ @Multiline
+ public static String configAll;
+
+  @Test
+  public void testConfigAll() throws Exception {
+SensorParserConfig c = 
SensorParserConfig.fromBytes(Bytes.toBytes(configAll));
+JSONObject input = new JSONObject();
+input.put("source.type", "test");
+for (FieldTransformer handler : c.getFieldTransformations()) {
+  handler.transformAndUpdate(input,

metron git commit: METRON-1579: Stellar should return the expression that failed in the exception closes apache/incubator-metron#1033

2018-05-31 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 63cd41433 -> acab9436d


METRON-1579: Stellar should return the expression that failed in the exception 
closes apache/incubator-metron#1033


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/acab9436
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/acab9436
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/acab9436

Branch: refs/heads/master
Commit: acab9436d028cf047e21948f604f5dab1507a983
Parents: 63cd414
Author: cstella 
Authored: Thu May 31 10:40:17 2018 -0700
Committer: cstella 
Committed: Thu May 31 10:40:17 2018 -0700

--
 .../profiler/client/stellar/GetProfileTest.java |  3 ++-
 .../client/stellar/ProfilerFunctionsTest.java   | 15 ++-
 .../StellarStatisticsFunctionsTest.java |  5 ++--
 .../sampling/SamplerFunctionsTest.java  |  3 ++-
 .../adapters/stellar/StellarAdapter.java| 14 +-
 .../management/IndexingConfigFunctionsTest.java |  7 ++---
 .../management/ThreatTriageFunctionsTest.java   |  7 ++---
 .../stellar/common/BaseStellarProcessor.java| 28 ++--
 .../common/StellarPredicateProcessor.java   |  6 +
 .../shell/specials/AssignmentCommandTest.java   |  7 +++--
 .../dsl/functions/OrdinalFunctionsTest.java |  9 ---
 11 files changed, 78 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/acab9436/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java
--
diff --git 
a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java
 
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java
index 6ed6e64..0eacb42 100644
--- 
a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java
+++ 
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.metron.hbase.mock.MockHBaseTableProvider;
 import org.apache.metron.profiler.client.ProfileWriter;
 import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
 import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
 import 
org.apache.metron.stellar.dsl.functions.resolver.SingletonFunctionResolver;
 import org.apache.metron.profiler.ProfileMeasurement;
@@ -266,7 +267,7 @@ public class GetProfileTest {
   /**
* Initialization should fail if the required context values are missing.
*/
-  @Test(expected = IllegalStateException.class)
+  @Test(expected = ParseException.class)
   public void testMissingContext() {
 Context empty = Context.EMPTY_CONTEXT();
 

http://git-wip-us.apache.org/repos/asf/metron/blob/acab9436/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java
--
diff --git 
a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java
 
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java
index e1c6aa8..1670e8c 100644
--- 
a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java
+++ 
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java
@@ -25,6 +25,7 @@ import org.apache.metron.profiler.StandAloneProfiler;
 import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor;
 import org.apache.metron.stellar.common.StellarStatefulExecutor;
 import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
 import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
@@ -145,12 +146,12 @@ public class ProfilerFunctionsTest {
 assertEquals(0, profiler.getRouteCount());
   }
 
-  @Test(expected = IllegalArgumentException.class)
+  @Test(expected = ParseException.class)
   public void testProfilerInitNoArgs() {
 run("PROFILER_INIT()", StandAloneProfiler.class);
   }
 
-  @Test(expected = IllegalArgumentException.class)
+  @Test(expected = ParseException.class)
   public void testProfilerInitInvalidArg() {
 run("PROFILER_INIT({ \"invalid\": 2 })", StandAloneProfiler.class);
   }
@@ -273,17 +274,17 @@

metron git commit: METRON-1515: Errors loading stellar functions currently bomb the entire topology, they should be recoverable closes apache/incubator-metron#985

2018-04-16 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 1d3e7fcd7 -> 3fcbf8b4e


METRON-1515: Errors loading stellar functions currently bomb the entire 
topology, they should be recoverable closes apache/incubator-metron#985


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3fcbf8b4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3fcbf8b4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3fcbf8b4

Branch: refs/heads/master
Commit: 3fcbf8b4e4e38f9c50842b8af857092b091c7c40
Parents: 1d3e7fc
Author: cstella 
Authored: Mon Apr 16 15:12:11 2018 -0400
Committer: cstella 
Committed: Mon Apr 16 15:12:11 2018 -0400

--
 .../resolver/ClasspathFunctionResolver.java | 45 +++-
 .../resolver/ClasspathFunctionResolverTest.java | 30 +
 2 files changed, 65 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/3fcbf8b4/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
index 85aa015..b17233a 100644
--- 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java
@@ -34,6 +34,7 @@ import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.Stellar;
 import org.apache.metron.stellar.dsl.StellarFunction;
 
+import org.atteo.classindex.ClassFilter;
 import org.atteo.classindex.ClassIndex;
 import org.reflections.util.FilterBuilder;
 
@@ -219,6 +220,17 @@ public class ClasspathFunctionResolver extends 
BaseFunctionResolver {
 }
   }
 
+  protected Iterable getStellarClasses(ClassLoader cl) {
+return ClassIndex.getAnnotated(Stellar.class, cl);
+  }
+
+  protected boolean includeClass(Class c, FilterBuilder filterBuilder)
+  {
+boolean isAssignable = StellarFunction.class.isAssignableFrom(c);
+boolean isFiltered = filterBuilder.apply(c.getCanonicalName());
+return isAssignable && isFiltered;
+  }
+
   /**
* Returns a set of classes that should undergo further interrogation for 
resolution
* (aka discovery) of Stellar functions.
@@ -254,16 +266,29 @@ public class ClasspathFunctionResolver extends 
BaseFunctionResolver {
 Set classes = new HashSet<>();
 Set ret = new HashSet<>();
 for(ClassLoader cl : cls) {
-  for(Class c : ClassIndex.getAnnotated(Stellar.class, cl)) {
-LOG.debug("{}: Found class: {}", cl.getClass().getCanonicalName(), 
c.getCanonicalName());
-boolean isAssignable = StellarFunction.class.isAssignableFrom(c);
-boolean isFiltered = filterBuilder.apply(c.getCanonicalName());
-if( isAssignable && isFiltered ) {
-  String className = c.getName();
-  if(!classes.contains(className)) {
-LOG.debug("{}: Added class: {}", cl.getClass().getCanonicalName(), 
className);
-ret.add((Class) c);
-classes.add(className);
+  for(Class c : getStellarClasses(cl)) {
+try {
+  LOG.debug("{}: Found class: {}", cl.getClass().getCanonicalName(), 
c.getCanonicalName());
+  if (includeClass(c, filterBuilder)) {
+String className = c.getName();
+if (!classes.contains(className)) {
+  LOG.debug("{}: Added class: {}", 
cl.getClass().getCanonicalName(), className);
+  ret.add((Class) c);
+  classes.add(className);
+}
+  }
+}
+catch(Error le) {
+  //we have had some error loading a stellar function.  This could 
mean that
+  //the classpath is unstable (e.g. old copies of jars are on the 
classpath).
+  try {
+LOG.error("Skipping class " + c.getName() + ": " + le.getMessage()
++ ", please check that there are not old versions of 
stellar functions on the classpath.", le);
+  }
+  catch(Error ie) {
+//it's possible that getName() will throw an exception if the 
class is VERY malformed.
+LOG.error("Skipping class: " + le.getMessage()
++ ", please check that there are not old versions of 
stellar functions on the classpath.", le);
   }
 }
   }

metron git commit: METRON-1521: JSONMapParser is no longer serializable closes apache/incubator-metron#991

2018-04-12 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master ea6992fd9 -> eb5b2d42b


METRON-1521: JSONMapParser is no longer serializable closes 
apache/incubator-metron#991


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/eb5b2d42
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/eb5b2d42
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/eb5b2d42

Branch: refs/heads/master
Commit: eb5b2d42b83901c05c15e00b9c3c5bce5dd72ab9
Parents: ea6992f
Author: cstella 
Authored: Thu Apr 12 09:33:12 2018 -0400
Committer: cstella 
Committed: Thu Apr 12 09:33:12 2018 -0400

--
 .../java/org/apache/metron/parsers/json/JSONMapParser.java | 4 ++--
 .../org/apache/metron/parsers/integration/ParserDriver.java| 6 +-
 2 files changed, 7 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/eb5b2d42/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
--
diff --git 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
index bddf35d..f5d67f9 100644
--- 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
+++ 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
@@ -89,8 +89,7 @@ public class JSONMapParser extends BasicParser {
   public static final String JSONP_QUERY = "jsonpQuery";
 
   private MapStrategy mapStrategy = MapStrategy.DROP;
-  private TypeRef>> typeRef = new 
TypeRef>>() {
-  };
+  private transient TypeRef>> typeRef = null;
   private String jsonpQuery = null;
 
 
@@ -99,6 +98,7 @@ public class JSONMapParser extends BasicParser {
 String strategyStr = (String) config.getOrDefault(MAP_STRATEGY_CONFIG, 
MapStrategy.DROP.name());
 mapStrategy = MapStrategy.valueOf(strategyStr);
 if (config.containsKey(JSONP_QUERY)) {
+  typeRef = new TypeRef>>() { };
   jsonpQuery = (String) config.get(JSONP_QUERY);
   Configuration.setDefaults(new Configuration.Defaults() {
 

http://git-wip-us.apache.org/repos/asf/metron/blob/eb5b2d42/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
--
diff --git 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
index deb0217..b03ea80 100644
--- 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
+++ 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
@@ -18,6 +18,7 @@
 package org.apache.metron.parsers.integration;
 
 import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang.SerializationUtils;
 import org.apache.metron.common.configuration.ConfigurationsUtils;
 import org.apache.metron.common.configuration.FieldValidator;
 import org.apache.metron.common.configuration.ParserConfigurations;
@@ -42,6 +43,7 @@ import org.mockito.Matchers;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -54,7 +56,7 @@ import static org.mockito.Mockito.when;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class ParserDriver {
+public class ParserDriver implements Serializable {
   private static final Logger LOG = LoggerFactory.getLogger(ParserBolt.class);
   public static class CollectingWriter implements MessageWriter{
 List output;
@@ -151,6 +153,8 @@ public class ParserDriver {
 
   public ProcessorResult> run(List in) {
 ShimParserBolt bolt = new ShimParserBolt(new ArrayList<>());
+byte[] b = SerializationUtils.serialize(bolt);
+ShimParserBolt b2 = (ShimParserBolt) SerializationUtils.deserialize(b);
 OutputCollector collector = mock(OutputCollector.class);
 bolt.prepare(null, null, collector);
 for(byte[] record : in) {

metron git commit: METRON-1504: Enriching missing values does not match the semantics between the new enrichment topology and old closes apache/incubator-metron#976

2018-04-09 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 46bc63dbc -> 3ba9ae251


METRON-1504: Enriching missing values does not match the semantics between the 
new enrichment topology and old closes apache/incubator-metron#976


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3ba9ae25
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3ba9ae25
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3ba9ae25

Branch: refs/heads/master
Commit: 3ba9ae25126bc393e3b9307f4ffe63ac66a909f4
Parents: 46bc63d
Author: cstella 
Authored: Mon Apr 9 11:47:51 2018 -0400
Committer: cstella 
Committed: Mon Apr 9 11:47:51 2018 -0400

--
 .../enrichment/parallel/ParallelEnricher.java   |  3 +
 .../integration/EnrichmentIntegrationTest.java  | 12 +++-
 .../parallel/ParallelEnricherTest.java  | 59 +++-
 3 files changed, 70 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
--
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
index 6ddb892..b10c148 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
@@ -170,6 +170,9 @@ public class ParallelEnricher {
 for(Object o : m.keySet()) {
   String field = (String) o;
   Object value = m.get(o);
+  if(value == null) {
+continue;
+  }
   CacheKey cacheKey = new CacheKey(field, value, config);
   String prefix = adapter.getOutputPrefix(cacheKey);
   Supplier supplier = () -> {

http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
--
diff --git 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index 267ca62..3c55c95 100644
--- 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -95,7 +95,15 @@ public class EnrichmentIntegrationTest extends 
BaseIntegrationTest {
 
   private static List getInputMessages(String path){
 try{
-  return TestUtils.readSampleData(path);
+  List ret = TestUtils.readSampleData(path);
+  {
+//we want one of the fields without a destination IP to ensure that 
enrichments can function
+Map sansDestinationIp = JSONUtils.INSTANCE.load(new 
String(ret.get(ret.size() -1))
+   , 
JSONUtils.MAP_SUPPLIER);
+sansDestinationIp.remove(Constants.Fields.DST_ADDR.getName());
+ret.add(JSONUtils.INSTANCE.toJSONPretty(sansDestinationIp));
+  }
+  return ret;
 }catch(IOException ioe){
   return null;
 }
@@ -262,8 +270,6 @@ public class EnrichmentIntegrationTest extends 
BaseIntegrationTest {
 
 //ensure we always have a source ip and destination ip
 Assert.assertNotNull(jsonDoc.get(SRC_IP));
-Assert.assertNotNull(jsonDoc.get(DST_IP));
-
 Assert.assertNotNull(jsonDoc.get("ALL_CAPS"));
 Assert.assertNotNull(jsonDoc.get("map.blah"));
 Assert.assertNull(jsonDoc.get("map"));

http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
--
diff --git 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
index 4a4573b..d4fcdf4 100644
--- 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
+++

[2/2] metron git commit: METRON-1483: Create a tool to monitor performance of the topologies closes apache/incubator-metron#958

2018-03-20 Thread cestella

METRON-1483: Create a tool to monitor performance of the topologies closes 
apache/incubator-metron#958


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/46ad9d93
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/46ad9d93
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/46ad9d93

Branch: refs/heads/master
Commit: 46ad9d93b4385da0f8668f2ba84212d54d00ba4b
Parents: e3eeec3
Author: cstella 
Authored: Tue Mar 20 09:36:32 2018 -0400
Committer: cstella 
Committed: Tue Mar 20 09:36:32 2018 -0400

--
 metron-contrib/metron-performance/README.md | 205 
 .../performance_measurement.png | Bin 0 -> 5790 bytes
 metron-contrib/metron-performance/pom.xml   | 134 +
 .../src/main/assembly/assembly.xml  |  42 ++
 .../metron/performance/load/LoadGenerator.java  | 175 +++
 .../metron/performance/load/LoadOptions.java| 499 +++
 .../performance/load/MessageGenerator.java  |  48 ++
 .../metron/performance/load/SendToKafka.java| 107 
 .../load/monitor/AbstractMonitor.java   |  49 ++
 .../load/monitor/EPSGeneratedMonitor.java   |  53 ++
 .../monitor/EPSThroughputWrittenMonitor.java|  77 +++
 .../performance/load/monitor/MonitorNaming.java |  23 +
 .../performance/load/monitor/MonitorTask.java   |  44 ++
 .../performance/load/monitor/Results.java   |  51 ++
 .../load/monitor/writers/CSVWriter.java |  67 +++
 .../load/monitor/writers/ConsoleWriter.java |  65 +++
 .../load/monitor/writers/Writable.java  |  40 ++
 .../load/monitor/writers/Writer.java|  86 
 .../performance/sampler/BiasedSampler.java  | 113 +
 .../metron/performance/sampler/Sampler.java |  24 +
 .../performance/sampler/UnbiasedSampler.java|  28 ++
 .../metron/performance/util/KafkaUtil.java  |  56 +++
 .../src/main/scripts/load_tool.sh   |  36 ++
 .../performance/load/LoadOptionsTest.java   |  93 
 .../performance/load/SendToKafkaTest.java   |  49 ++
 .../metron/performance/sampler/SamplerTest.java | 145 ++
 metron-contrib/pom.xml  |  15 +
 .../common-services/METRON/CURRENT/metainfo.xml |   4 +
 .../packaging/docker/deb-docker/pom.xml |   6 +
 .../docker/rpm-docker/SPECS/metron.spec |  21 +
 .../packaging/docker/rpm-docker/pom.xml |   6 +
 31 files changed, 2361 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/46ad9d93/metron-contrib/metron-performance/README.md
--
diff --git a/metron-contrib/metron-performance/README.md 
b/metron-contrib/metron-performance/README.md
new file mode 100644
index 000..8981349
--- /dev/null
+++ b/metron-contrib/metron-performance/README.md
@@ -0,0 +1,205 @@
+
+# Performance Utilities
+
+This project creates some useful performance monitoring and measurement
+utilities.
+
+## `load-tool.sh`
+
+The Load tool is intended to do the following:
+* Generate a load at a specific events per second into kafka
+  * The messages are taken from a template file, where there is a message 
template per line
+  * The load can be biased (e.g. 80% of the load can be comprised of 20% of 
the templates)
+* Monitor the kafka offsets for a topic to determine the events per second 
written
+  * This could be the topic that you are generating load on
+  * This could be another topic that represents the output of some topology 
(e.g. generate load on `enrichments` and monitor `indexing` to determine the 
throughput of the enrichment topology).
+
+```
+usage: Generator
+ -bs,--sample_bias  The discrete distribution to bias
+   the sampling. This is a CSV of 2
+   columns.  The first column is the %
+   of the templates and the 2nd column
+   is the probability (0-100) that
+   it's chosen.  For instance:
+   20,80
+   80,20
+   implies that 20% of the templates
+   will comprise 80% of the output and
+   the remaining 80% of the templates
+   will comprise 20% of the output.
+ -c,--csvA CSV file to emit monitoring data
+   to.  The format is a CSV with the
+   following schema: timestamp, (name,
+   eps, historical_mean,
+

metron git commit: METRON-1397 Support for JSON Path and complex documents in JSONMapParser closes apache/incubator-metron#914

2018-03-15 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 85d12475c -> 9c5d9d766


METRON-1397 Support for JSON Path and complex documents in JSONMapParser closes 
apache/incubator-metron#914


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9c5d9d76
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9c5d9d76
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9c5d9d76

Branch: refs/heads/master
Commit: 9c5d9d76644fc07bae36644906f52e0422f33d0e
Parents: 85d1247
Author: ottobackwards 
Authored: Thu Mar 15 14:17:31 2018 -0400
Committer: cstella 
Committed: Thu Mar 15 14:17:31 2018 -0400

--
 dependencies_with_url.csv   |   3 +
 .../docker/rpm-docker/SPECS/metron.spec |   1 +
 .../jsonMapQuery/parsed/jsonMapExampleParsed|   2 +
 .../data/jsonMapQuery/raw/jsonMapExampleOutput  |   1 +
 metron-platform/metron-parsers/README.md|  12 ++
 metron-platform/metron-parsers/pom.xml  |   5 +
 .../config/zookeeper/parsers/jsonMapQuery.json  |   5 +
 .../metron/parsers/json/JSONMapParser.java  | 145 +
 .../JSONMapQueryIntegrationTest.java|  36 
 .../validation/SampleDataValidation.java|   2 +-
 .../parsers/json/JSONMapParserQueryTest.java| 201 +++
 .../metron/test/utils/ValidationUtils.java  |  46 -
 12 files changed, 406 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index e2b947b..1e73eb1 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -22,6 +22,9 @@ com.flipkart.zjsonpatch:zjsonpatch:jar:0.3.4:compile,Apache 
v2, https://github.c
 com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD 
license,http://code.google.com/p/protobuf
 com.google.protobuf:protobuf-java:jar:2.6.1:compile,New BSD 
license,http://code.google.com/p/protobuf
 com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/
+com.jayway.jsonpath:json-path:jar:2.3.0:compile,Apache 
v2,https://github.com/json-path/JsonPath
+net.minidev:accessors-smart:jar:1.2:compile,Apache 
v2,https://github.com/netplex/json-smart-v2
+net.minidev:json-smart:jar:2.3:compile,Apache 
v2,https://github.com/netplex/json-smart-v2
 com.maxmind.db:maxmind-db:jar:1.2.1:compile,CC-BY-SA 
3.0,https://github.com/maxmind/MaxMind-DB
 com.maxmind.geoip2:geoip2:jar:2.8.0:compile,Apache 
v2,https://github.com/maxmind/GeoIP2-java
 com.sun.xml.bind:jaxb-impl:jar:2.2.3-1:compile,CDDL,http://jaxb.java.net/

http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec 
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 265d595..cc01d7c 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -147,6 +147,7 @@ This package installs the Metron Parser files
 %{metron_home}/bin/start_parser_topology.sh
 %{metron_home}/config/zookeeper/parsers/bro.json
 %{metron_home}/config/zookeeper/parsers/jsonMap.json
+%{metron_home}/config/zookeeper/parsers/jsonMapQuery.json
 %{metron_home}/config/zookeeper/parsers/snort.json
 %{metron_home}/config/zookeeper/parsers/squid.json
 %{metron_home}/config/zookeeper/parsers/websphere.json

http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed
--
diff --git 
a/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed
 
b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed
new file mode 100644
index 000..e614bda
--- /dev/null
+++ 
b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed
@@ -0,0 +1,2 @@
+{ "string" : "bar", "number" : 2, "ignored" : [ "blah" ], "original_string":"{ 
\"string\" : \"bar\", \"number\" : 2, \"ignored\" : [ \"blah\" ] 
}","timestamp":1, 
"source.type":"jsonMapQuery","guid":"this-is-random-uuid-will-be-36-chars" }
+{ "number" : 7 , "original_string" : "{ \"number\" : 7 }", 
"source.type":"jsonMapQuery","timestamp":1,"guid":"this-is-random-uuid-will-be-36-chars"}

[2/2] metron git commit: METRON-1299 In MetronError tests, don't test for HostName if getHostName wouldn't work closes apache/incubator-metron#924

2018-03-15 Thread cestella

METRON-1299 In MetronError tests, don't test for HostName if getHostName 
wouldn't work closes apache/incubator-metron#924


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/85d12475
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/85d12475
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/85d12475

Branch: refs/heads/master
Commit: 85d12475cf363f0b9d6bdd973ec9860550f73980
Parents: e69ce21
Author: ottobackwards 
Authored: Thu Mar 15 14:13:44 2018 -0400
Committer: cstella 
Committed: Thu Mar 15 14:13:44 2018 -0400

--
 .../metron/common/error/MetronErrorTest.java  | 18 +++---
 1 file changed, 15 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/85d12475/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
--
diff --git 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
index 5e505a8..e7390de 100644
--- 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
+++ 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
@@ -18,15 +18,16 @@
 package org.apache.metron.common.error;
 
 import com.google.common.collect.Sets;
-import com.google.common.primitives.Bytes;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.metron.common.Constants;
 import org.json.simple.JSONObject;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 
-import static org.apache.metron.common.Constants.ErrorFields.RAW_MESSAGE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -53,7 +54,18 @@ public class MetronErrorTest {
 assertEquals(Constants.ErrorType.PARSER_ERROR.getType(), 
errorJSON.get(Constants.ErrorFields.ERROR_TYPE.getName()));
 assertEquals("error", errorJSON.get(Constants.SENSOR_TYPE));
 assertEquals("sensorType", 
errorJSON.get(Constants.ErrorFields.FAILED_SENSOR_TYPE.getName()));
-assertTrue(((String) 
errorJSON.get(Constants.ErrorFields.HOSTNAME.getName())).length() > 0);
+
+String hostName = null;
+try {
+  hostName = InetAddress.getLocalHost().getHostName();
+} catch (UnknownHostException uhe) {
+  // unable to get the hostname on this machine, don't test it
+}
+
+if (!StringUtils.isEmpty(hostName)) {
+  assertTrue(((String) 
errorJSON.get(Constants.ErrorFields.HOSTNAME.getName())).length() > 0);
+  assertEquals(hostName, (String) 
errorJSON.get(Constants.ErrorFields.HOSTNAME.getName()));
+}
 assertTrue(((long) 
errorJSON.get(Constants.ErrorFields.TIMESTAMP.getName())) > 0);
   }

[1/2] metron git commit: METRON-1485 Upgrade vagrant for dev environments closes apache/incubator-metron#959

2018-03-15 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 26c5d3065 -> 85d12475c


METRON-1485 Upgrade vagrant for dev environments closes 
apache/incubator-metron#959


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e69ce213
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e69ce213
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e69ce213

Branch: refs/heads/master
Commit: e69ce213aaad84fd49d972c329a4bf4bf7d4b2ad
Parents: 26c5d30
Author: JonZeolla 
Authored: Thu Mar 15 14:09:28 2018 -0400
Committer: cstella 
Committed: Thu Mar 15 14:09:28 2018 -0400

--
 metron-deployment/development/centos6/README.md  | 2 +-
 metron-deployment/development/ubuntu14/README.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/e69ce213/metron-deployment/development/centos6/README.md
--
diff --git a/metron-deployment/development/centos6/README.md 
b/metron-deployment/development/centos6/README.md
index 5132c30..bd8553c 100644
--- a/metron-deployment/development/centos6/README.md
+++ b/metron-deployment/development/centos6/README.md
@@ -31,7 +31,7 @@ The computer used to deploy Apache Metron will need to have 
the following compon
 
  - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0)
  - [Docker](https://www.docker.com/community-edition)
- - [Vagrant](https://www.vagrantup.com) 1.8+
+ - [Vagrant](https://www.vagrantup.com) 2.0+
  - [Vagrant Hostmanager 
Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager)
  - [Virtualbox](https://virtualbox.org) 5.0+
  - Python 2.7

http://git-wip-us.apache.org/repos/asf/metron/blob/e69ce213/metron-deployment/development/ubuntu14/README.md
--
diff --git a/metron-deployment/development/ubuntu14/README.md 
b/metron-deployment/development/ubuntu14/README.md
index af7d3a3..5856911 100644
--- a/metron-deployment/development/ubuntu14/README.md
+++ b/metron-deployment/development/ubuntu14/README.md
@@ -31,7 +31,7 @@ The computer used to deploy Apache Metron will need to have 
the following compon
 
  - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0)
  - [Docker](https://www.docker.com/community-edition)
- - [Vagrant](https://www.vagrantup.com) 1.8+
+ - [Vagrant](https://www.vagrantup.com) 2.0+
  - [Vagrant Hostmanager 
Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager)
  - [Virtualbox](https://virtualbox.org) 5.0+
  - Python 2.7

metron git commit: METRON-1490: Better error message when user specifies an enrichment type that doesn't exist closes apache/metron#963

2018-03-15 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 03a4aa384 -> 37662d3bd


METRON-1490: Better error message when user specifies an enrichment type that 
doesn't exist closes apache/metron#963


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/37662d3b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/37662d3b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/37662d3b

Branch: refs/heads/master
Commit: 37662d3bd8a4162bef4474be7ab60a2fabbe3e2f
Parents: 03a4aa3
Author: cstella 
Authored: Thu Mar 15 12:05:39 2018 -0400
Committer: cstella 
Committed: Thu Mar 15 12:05:39 2018 -0400

--
 .../enrichment/parallel/ParallelEnricher.java   |  5 +++
 .../parallel/ParallelEnricherTest.java  | 39 +++-
 2 files changed, 43 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/37662d3b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
--
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
index 2238c92..6ddb892 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
@@ -18,6 +18,7 @@
 package org.apache.metron.enrichment.parallel;
 
 import com.github.benmanes.caffeine.cache.stats.CacheStats;
+import com.google.common.base.Joiner;
 import org.apache.metron.common.Constants;
 import 
org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler;
@@ -152,6 +153,10 @@ public class ParallelEnricher {
 for(Map.Entry task : tasks.entrySet()) {
   //task is the list of enrichment tasks for the task.getKey() adapter
   EnrichmentAdapter adapter = 
enrichmentsByType.get(task.getKey());
+  if(adapter == null) {
+throw new IllegalStateException("Unable to find an adapter for " + 
task.getKey()
++ ", possible adapters are: " + 
Joiner.on(",").join(enrichmentsByType.keySet()));
+  }
   for(JSONObject m : task.getValue()) {
 /* now for each unit of work (each of these only has one element in 
them)
  * the key is the field name and the value is value associated with 
that field.

http://git-wip-us.apache.org/repos/asf/metron/blob/37662d3b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
--
diff --git 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
index c3a3109..4a4573b 100644
--- 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
+++ 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.metron.enrichment.parallel;
 
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableMap;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.Constants;
@@ -24,6 +25,7 @@ import 
org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.enrichment.adapters.stellar.StellarAdapter;
 import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.StellarFunctions;
 import org.json.simple.JSONObject;
@@ -32,6 +34,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class ParallelEnricherTest {
@@ -61,6 +64,7 @@ public class ParallelEnricherTest {
   private static ParallelEnricher enricher;
   private static Context stellarContext;
   private static AtomicInteger numAccesses = new AtomicInteger(0);
+  private static Map enrichmentsByType;
   @BeforeClass
   public static void setup() {
 ConcurrencyContext infrastructure = new ConcurrencyContext();
@@ -75,7 +79,8 @@ public class

metron git commit: METRON-1471: Migrate shuffle connections to local or shuffle closes apache/metron#949

2018-03-08 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master abb152b83 -> 8bc7732e4


METRON-1471: Migrate shuffle connections to local or shuffle closes 
apache/metron#949


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8bc7732e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8bc7732e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8bc7732e

Branch: refs/heads/master
Commit: 8bc7732e4c904085e4f78469abded84635971632
Parents: abb152b
Author: cstella 
Authored: Thu Mar 8 17:17:57 2018 -0500
Committer: cstella 
Committed: Thu Mar 8 17:17:57 2018 -0500

--
 .../metron-profiler/src/main/flux/profiler/remote.yaml | 6 +++---
 .../metron-indexing/src/main/flux/indexing/batch/remote.yaml   | 4 ++--
 .../src/main/flux/indexing/random_access/remote.yaml   | 4 ++--
 .../apache/metron/parsers/topology/ParserTopologyBuilder.java  | 4 ++--
 4 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
--
diff --git 
a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml 
b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
index 752640a..9ec5ba4 100644
--- a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
+++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
@@ -168,7 +168,7 @@ streams:
 from: "kafkaSpout"
 to: "splitterBolt"
 grouping:
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE
 
 -   name: "splitter -> builder"
 from: "splitterBolt"
@@ -182,11 +182,11 @@ streams:
 to: "hbaseBolt"
 grouping:
 streamId: "hbase"
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE
 
 -   name: "builder -> kafka"
 from: "builderBolt"
 to: "kafkaBolt"
 grouping:
 streamId: "kafka"
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE

http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
--
diff --git 
a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml 
b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
index 85e3baa..3087dd9 100644
--- a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
+++ b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
@@ -158,7 +158,7 @@ streams:
 from: "kafkaSpout"
 to: "hdfsIndexingBolt"
 grouping:
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE
 
 
 -   name: "hdfsBolt -> errorIndexingBolt"
@@ -166,4 +166,4 @@ streams:
 to: "indexingErrorBolt"
 grouping:
 streamId: "error"
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE

http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
--
diff --git 
a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
 
b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
index cadc1ec..429ba45 100644
--- 
a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
+++ 
b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
@@ -130,11 +130,11 @@ streams:
 from: "kafkaSpout"
 to: "indexingBolt"
 grouping:
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE
 
 -   name: "indexingBolt -> errorIndexingBolt"
 from: "indexingBolt"
 to: "indexingErrorBolt"
 grouping:
 streamId: "error"
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE

http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
--
diff --git 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index 1039e56..2865dd6 100644
--- 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++ 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@

metron git commit: METRON-1467: Replace guava caches in places where the keyspace might be large closes apache/metron#947

2018-03-07 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 9fb0d06e6 -> abb152b83


METRON-1467: Replace guava caches in places where the keyspace might be large 
closes apache/metron#947


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/abb152b8
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/abb152b8
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/abb152b8

Branch: refs/heads/master
Commit: abb152b83631001ad067849dfaefd2d4e3b0cdb4
Parents: 9fb0d06
Author: cstella 
Authored: Wed Mar 7 11:20:56 2018 -0500
Committer: cstella 
Committed: Wed Mar 7 11:20:56 2018 -0500

--
 metron-interface/metron-rest/pom.xml|  5 +++
 metron-platform/metron-enrichment/pom.xml   |  2 +-
 .../enrichment/bolt/GenericEnrichmentBolt.java  | 19 ---
 .../apache/metron/enrichment/bolt/JoinBolt.java | 34 +---
 .../bolt/GenericEnrichmentBoltTest.java |  2 +-
 .../metron/enrichment/bolt/JoinBoltTest.java|  7 ++--
 metron-stellar/stellar-common/pom.xml   |  5 +++
 .../stellar/common/BaseStellarProcessor.java| 31 +++---
 .../stellar/dsl/functions/DateFunctions.java|  8 ++---
 pom.xml |  1 +
 10 files changed, 56 insertions(+), 58 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/abb152b8/metron-interface/metron-rest/pom.xml
--
diff --git a/metron-interface/metron-rest/pom.xml 
b/metron-interface/metron-rest/pom.xml
index 44bad97..dcdea2b 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -38,6 +38,11 @@
 2.6.4
 
 
+  
+  com.github.ben-manes.caffeine
+  caffeine
+  ${global_caffeine_version}
+
 
 org.springframework.kafka
 spring-kafka

http://git-wip-us.apache.org/repos/asf/metron/blob/abb152b8/metron-platform/metron-enrichment/pom.xml
--
diff --git a/metron-platform/metron-enrichment/pom.xml 
b/metron-platform/metron-enrichment/pom.xml
index bcfb41b..1dffd8b 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -70,7 +70,7 @@
 
   com.github.ben-manes.caffeine
   caffeine
-  2.6.2
+  ${global_caffeine_version}
 
 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/abb152b8/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
--
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
index 7d67d2d..0677453 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
@@ -18,13 +18,13 @@
 
 package org.apache.metron.enrichment.bolt;
 
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang3.StringUtils;
+
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt;
 import org.apache.metron.common.configuration.ConfigurationType;
@@ -146,13 +146,8 @@ public class GenericEnrichmentBolt extends 
ConfiguredEnrichmentBolt {
   throw new IllegalStateException("MAX_TIME_RETAIN_MINUTES must be 
specified");
 if (this.adapter == null)
   throw new IllegalStateException("Adapter must be specified");
-loader = new CacheLoader() {
-  @Override
-  public JSONObject load(CacheKey key) throws Exception {
-return adapter.enrich(key);
-  }
-};
-cache = CacheBuilder.newBuilder().maximumSize(maxCacheSize)
+loader = key -> adapter.enrich(key);
+cache = Caffeine.newBuilder().maximumSize(maxCacheSize)
 .expireAfterWrite(maxTimeRetain, TimeUnit.MINUTES)
 .build(loader);
 boolean success = 
adapter.initializeAdapter(getConfigurations().getGlobalConfig());
@@ -228,7 +223,7 @@ public class GenericEnrichmentBolt extends

metron git commit: METRON-1463: Adjust the groupings and shuffles in enrichment to be more efficient closes apache/metron#944

2018-03-07 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 1d95b8316 -> 9fb0d06e6


METRON-1463: Adjust the groupings and shuffles in enrichment to be more 
efficient closes apache/metron#944


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9fb0d06e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9fb0d06e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9fb0d06e

Branch: refs/heads/master
Commit: 9fb0d06e644597032da58c5df020a841b93d19c0
Parents: 1d95b83
Author: cstella 
Authored: Wed Mar 7 09:21:52 2018 -0500
Committer: cstella 
Committed: Wed Mar 7 09:21:52 2018 -0500

--
 .../src/main/flux/enrichment/remote.yaml| 42 +---
 .../enrichment/bolt/EnrichmentJoinBolt.java |  4 +-
 .../enrichment/bolt/EnrichmentSplitterBolt.java |  4 +-
 .../enrichment/bolt/GenericEnrichmentBolt.java  |  2 +-
 .../enrichment/bolt/ThreatIntelJoinBolt.java|  4 +-
 .../bolt/ThreatIntelSplitterBolt.java   |  4 +-
 6 files changed, 28 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
--
diff --git 
a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml 
b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
index 7ee1b2e..fd7ceff 100644
--- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
@@ -411,7 +411,7 @@ streams:
 from: "kafkaSpout"
 to: "enrichmentSplitBolt"
 grouping:
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE
 
 #enrichment
 -   name: "enrichmentSplit -> host"
@@ -420,14 +420,15 @@ streams:
 grouping:
 streamId: "host"
 type: FIELDS
-args: ["key"]
+args: ["message"]
+
 -   name: "enrichmentSplit -> geo"
 from: "enrichmentSplitBolt"
 to: "geoEnrichmentBolt"
 grouping:
 streamId: "geo"
 type: FIELDS
-args: ["key"]
+args: ["message"]
 
 -   name: "enrichmentSplit -> stellar"
 from: "enrichmentSplitBolt"
@@ -435,7 +436,7 @@ streams:
 grouping:
 streamId: "stellar"
 type: FIELDS
-args: ["key"]
+args: ["message"]
 
 
 -   name: "enrichmentSplit -> simpleHBaseEnrichmentBolt"
@@ -444,7 +445,7 @@ streams:
 grouping:
 streamId: "hbaseEnrichment"
 type: FIELDS
-args: ["key"]
+args: ["message"]
 
 -   name: "splitter -> join"
 from: "enrichmentSplitBolt"
@@ -453,6 +454,7 @@ streams:
 streamId: "message"
 type: FIELDS
 args: ["key"]
+
 -   name: "geo -> join"
 from: "geoEnrichmentBolt"
 to: "enrichmentJoinBolt"
@@ -460,6 +462,7 @@ streams:
 streamId: "geo"
 type: FIELDS
 args: ["key"]
+
 -   name: "stellar -> join"
 from: "stellarEnrichmentBolt"
 to: "enrichmentJoinBolt"
@@ -468,8 +471,6 @@ streams:
 type: FIELDS
 args: ["key"]
 
-
-
 -   name: "simpleHBaseEnrichmentBolt -> join"
 from: "simpleHBaseEnrichmentBolt"
 to: "enrichmentJoinBolt"
@@ -477,6 +478,7 @@ streams:
 streamId: "hbaseEnrichment"
 type: FIELDS
 args: ["key"]
+
 -   name: "host -> join"
 from: "hostEnrichmentBolt"
 to: "enrichmentJoinBolt"
@@ -491,32 +493,28 @@ streams:
 to: "enrichmentErrorOutputBolt"
 grouping:
 streamId: "error"
-type: FIELDS
-args: ["message"]
+type: LOCAL_OR_SHUFFLE
 
 -   name: "stellarEnrichmentBolt -> enrichmentErrorOutputBolt"
 from: "stellarEnrichmentBolt"
 to: "enrichmentErrorOutputBolt"
 grouping:
 streamId: "error"
-type: FIELDS
-args: ["message"]
+type: LOCAL_OR_SHUFFLE
 
 -   name: "hostEnrichmentBolt -> enrichmentErrorOutputBolt"
 from: "hostEnrichmentBolt"
 to: "enrichmentErrorOutputBolt"
 grouping:
 streamId: "error"
-type: FIELDS
-args: ["message"]
+type: LOCAL_OR_SHUFFLE
 
 -   name: "simpleHBaseEnrichmentBolt -> enrichmentErrorOutputBolt"
 from: "simpleHBaseEnrichmentBolt"
 to: "enrichmentErrorOutputBolt"
 grouping:
 streamId: "error"
-type: FIELDS
-args: ["message"]
+type: LOCAL_OR_SHUFFLE
 
 #threat intel

[2/4] metron git commit: METRON-1460: Create a complementary non-split-join enrichment topology closes apache/metron#940

2018-03-07 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/1d95b831/metron-platform/metron-enrichment/unified_enrichment_arch.svg
--
diff --git a/metron-platform/metron-enrichment/unified_enrichment_arch.svg 
b/metron-platform/metron-enrichment/unified_enrichment_arch.svg
new file mode 100644
index 000..e42d394
--- /dev/null
+++ b/metron-platform/metron-enrichment/unified_enrichment_arch.svg
@@ -0,0 +1,14 @@
+
+http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd;>
+http://www.w3.org/2000/svg; 
xmlns:xlink="http://www.w3.org/1999/xlink; width="611px" height="671px" 
version="1.1" content="mxfile userAgent=Mozilla/5.0 (Macintosh; Intel 
Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 
Safari/537.36 version=8.3.6 editor=www.draw.io 
type=devicediagram 
id=831675e5-f95b-138c-0e1a-018afd21d13b 
name=Page-17b3Z0qvIkiX8NHn5tzEKuGSeQWKGO+Z5nnn6hm9nnqqsk1XVbXXqtP1l2pa5t4QQBOER7ss9llb8BtPtwU/hUKh9kja/QUBy/AYzv0EQDoH338+B89cBBEd/HcinMvl1CPyXA2Z5pb8fBH4/upZJOv/pxKXvm6Uc/nww7rsujZc/HQunqd//fFrWN3++6xDm6d8dMOOw+fujbpksxR+P9fqX40Ja5sUfdwZfxK9PojCu86lfu9/v9xsEZz9/fn3chn9c6/cHnYsw6fd/dQhmf4Ppqe+XX6/ag06bp2v/6LZf3+P+nU//1u4p7Zb/ky/8bpYtbNb0jxb/tGs5/+iLYmmb+xX4G0z9rbXA/SYJ5yJNfn8TNmXe3a/j+77pdB/Y0mkp7w4lf/+gLZPkueJzjeG5cnvkz9D5X7/MBf3697nsM2SA//XCntfPeIKe63f9Ehe/3+une3/u/NOmZerrv1np7kAqK5uG7pt++mk/jKZEnMDP8b5bfh9qP9f89c2/OjMpp3tUlf3T8LlfnytT
 
v3fU/Vjp8e92Nvg3E94zI+3bdJnO+5Tfv0Cg/+v3/v59WoAI8uv9/i+D7I8pUPzr8QX8fjD8fVznf7v2v9j2fvG7ef/a1K+/MzXbTWVctE/r/63R/7BS2f7MlL+ZUwmjtHn3c/l770T9svTtfULzfED9bfT/0atd36V/MRqWfngGzTz8mrxZeTz2pH7uRv5xFPjjyP06CZfwN5j89Rbihi7/DaJLh9KNHZD5vCfvP5ppF6yd3684/P6LOmjSv/+lr1f3G0Rd98ugARr24xgImYIJjRqRAbfpmiWyDDXwEm1neJ8I+9e8SDErxmQODJIkeexK26ReD7WmfXSWiWJQzt/GJe84+TpdqQtDEmdWiOoUS6NdOMTzYqgR8T27fOOZ9UbTfn3kr6OzyqMy4C093cXpCSyAp/t+dfaaZALEiKR0Fh1zieNaYGZ93y6Dwx6jPhZ+/qNw/P43bp2Gvm7Dch1rle9dqNmr1HahFy5aPmKKqTpOQCYHV656gC2aPhW0g+9vHyeuvgXulIbCdTJlndK1+txmv0c2Fwkf1E7lWL/fhukmLJeFZyF7f/J5R/PTKbeFGs9wqLdLtbLs4QbdE7U6TMkl6acuHBSdshUvhfrTMON1DdcuC1qngfe4pCb7KiQays3HdIxiHUEpBwHcaqwPHHErQuR9Erk9T+vJJ3ikpiCbOBY6DRZSFToRpZ1mzGmc7pTnkulJEYd4qwYkqubD6v2tBLC1dzudxnZfyDHbzpk+2kYZI5/okq8PHoOStMHnMYAxz11Oz9sjUuD5NOkdXzrur3FVgJtL8cog0LnfuTglTI8JanjWKYtDNDZUYBw8S0m4h+EHgkyIninMfuySld59pnl6wkrrWHG2i7nMz4TM+4sndGmWVXxngC22EeDTh+iu3lemtNf1AYmnOVoBXdq6qsL9GlogQV7uOUjZHgOMoNhqQI+93PsAtvCiG1zpB8gVqCde1X3Mo6V7VFBowwYto1Xku4TWD
 
b1Y/CC54UwHOktWcn21pYL26j3PuZSBu+3daQbm9rc9KMlkhbHMaly35ACsJ2OKFka9KgK/Nlxh3gRTKyhLWIXgNkmVhLa79/fX6K3ljh1DGQTpipm/j3AApJvxjKDQ5+6f5deQJbN3iWN371Bq4M8YBrg6rKonR58kxAJvOP0ZE3yd3Gc09uv1/swbaQA8TxSAJAjHkjWFo0V4goufj1l2BW8DysY844VCaABj5ftVsAWEXpPnjLi6V0Tt1DogVHArnxU+kvCDIxTje71P7AbFn+gI3KtNyIdnMqXRBEMDjrB0ksqCdym+TSLnLLsg3Jd1Epqu4Whb3MW8UlKahHZvgckdRWymMV0J/+lPoaWHZ4qIWr6HXiNFmPDxDxOLF9J4uiVTCTg7IP/EmDJUAntMgKS/H4DbDjgfnzk20f7tyblXH6q820Uod6zbVSDaXB80weB29syhLsD4umnMVUrJynjOf0usrqrGpYG5XgIC1rf8WxiQ8BlQW3AC+P6WZ0uFa5wtpV7rI0KRadcEkHheLmfWt7xH5dszcHg6k/5pisQc0HJXVXFENRblazVXfJ5Ouu+eRR3WjQCS0LCmGSGuwJVDewr64tS36IPvtd3H+zTANnF1mj9948Qm4Joju7qR77SFpHloEdLPhOL2GoTfHitWhnhOJEvJsggYDpzWsptDpmwOCLkMsm5nt6enzM8bdYwcjkS9MAZQTyBmczAd8UXE4RVuJ3mv29d+D9K2Ofx4TZXope66td8eTleAushSpuRTY2IjoAva3X+G4jPdKeyA91+uVWEJhjxCDNhV/7RuB0J01Vwt4K9Ph4DACaXfUquPfcDeLToQg+BSldZgrkS6r0dkJsXqgu0LIqm/2lkJQjboda7dxWcKdI+tztttnFE3x1zQZGObX4VLx4lYAKiwPc6g0FSpt5OIuEzw6cgPGJSHWWf+roqYEwf0ceBRxq/3/Tj5cQbv5rlucAxcSxj2SfXsO6CJMX
 
LfgoTr1ek5Hlu7uP/pydGJku1zBweroGZ00d8yxD5+KyGM4KcveKqgWj4Mi/72muDbYRHpjSV5dCJIKjAe+CmtTJoXdRuibJ/GLrdeyeU2mDVb2vg209gG3HaOF3e9u8elshdM8Z04ChCP2xxfMQt75URLuJ3woWz/FYTEExToVCo6dsxtWwl8oZfbT0MrTaQAUpKqUO6GraFfTesBxCfh+tnqYtuBisdfePqV2eg99bks4525C+NK7jfUiiAxm1VUTPTAe1ObUNU11aCvzwDACuOuceKV9Mk7pW6dYH2j2rsLWpdhXiehbEcgFldUuTm5Hn1HzD+hpDu3smqPgwxJHctWhZsS5f1+hx9D74PhMBjbhq9D3xisTfmnOXqJrvWDIkbqczEwjnze8oV8xrCwE+meChRX0fvA1hr3pl3qGX9Ihby6fJCC+KqrQvDR28u6TgBh7+lN3DN1gF5iwIe6XTUYUTmCThtAz8y0jNa8TweY+liQOsasK1I8SOHaHsAEz+/opNTuNIImZTwBoipEBaTHdc8PRX+9ZSSczbBTnOlis4rK/JiZTvc85tdsUQUCacxSadgP/Jjn/JOg9nsrsnkZp9cF84VHf/Kn9y7+QTAc/qonxkVbikBNBFGp+vmQP5UdG5yMZdnG9evhGtGGd0lKR2ZegGz0feMpyVL3wDxx6gYyn5ruFNXeMB3q7+g7gVPDy3R8fGqzC2tDu0SuuYOwfsViv2Sh0155LDXnBXY9jYA7ChRAOq6zNjZghFy1S1noCM7A+34G9Zk1gfDMxpDhVDYmr6vVvOltq9l6eyjL5g4V8bwHSjhejIiWkAe1lcrNfJ64sI0xAwbIYLR9Zr9kuznvmMJVD/T2rPid39flyB25yvTKwa6VwDZO5c8QCBt8+rzDy4F/9/2WLuDrJa3FsU9rb/FaQTIKjiKIZtR2FxOVZ/6MuVZ4r3nXcB/QgswA3XgDRiLquKdkF+xa8DgT1cIqCRZz0Qg

[1/4] metron git commit: METRON-1460: Create a complementary non-split-join enrichment topology closes apache/metron#940

2018-03-07 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 486be49e7 -> 1d95b8316


http://git-wip-us.apache.org/repos/asf/metron/blob/1d95b831/metron-platform/metron-enrichment/unified_enrichment_arch_diagram.xml
--
diff --git 
a/metron-platform/metron-enrichment/unified_enrichment_arch_diagram.xml 
b/metron-platform/metron-enrichment/unified_enrichment_arch_diagram.xml
new file mode 100644
index 000..269a0d2
--- /dev/null
+++ b/metron-platform/metron-enrichment/unified_enrichment_arch_diagram.xml
@@ -0,0 +1,14 @@
+
+
+7b3Z0qvIkiX8NHn5tzEKuGSeQWKGO+Z5nnn6hm9nnqqsk1XVbXXqtP1l2pa5t4QQBOER7ss9llb8BtPtwU/hUKh9kja/QUBy/AYzv0EQDoH338+B89cBBEd/HcinMvl1CPyXA2Z5pb8fBH4/upZJOv/pxKXvm6Uc/nww7rsujZc/HQunqd//fFrWN3++6xDm6d8dMOOw+fujbpksxR+P9fqX40Ja5sUfdwZfxK9PojCu86lfu9/v9xsEZz9/fn3chn9c6/cHnYsw6fd/dQhmf4Ppqe+XX6/ag06bp2v/6LZf3+P+nU//1u4p7Zb/ky/8bpYtbNb0jxb/tGs5/+iLYmmb+xX4G0z9rbXA/SYJ5yJNfn8TNmXe3a/j+77pdB/Y0mkp7w4lf/+gLZPkueJzjeG5cnvkz9D5X7/MBf3697nsM2SA//XCntfPeIKe63f9Ehe/3+une3/u/NOmZerrv1np7kAqK5uG7pt++mk/jKZEnMDP8b5bfh9qP9f89c2/OjMpp3tUlf3T8LlfnytTv3fU/Vjp8e92Nvg3E94zI+3bdJnO+5Tfv0Cg/+v3/v59WoAI8uv9/i+D7I8pUPzr8QX8fjD8fVznf7v2v9j2fvG7ef/a1K+/MzXbTWVctE/r/63R/7BS2f7MlL+ZUwmjtHn3c/l770T9svTtfULzfED9bfT/0atd36V/MRqWfngGzTz8mrxZeTz2pH7uRv5xFPjjyP06CZfwN5j
 
89Rbihi7/DaJLh9KNHZD5vCfvP5ppF6yd3684/P6LOmjSv/+lr1f3G0Rd98ugARr24xgImYIJjRqRAbfpmiWyDDXwEm1neJ8I+9e8SDErxmQODJIkeexK26ReD7WmfXSWiWJQzt/GJe84+TpdqQtDEmdWiOoUS6NdOMTzYqgR8T27fOOZ9UbTfn3kr6OzyqMy4C093cXpCSyAp/t+dfaaZALEiKR0Fh1zieNaYGZ93y6Dwx6jPhZ+/qNw/P43bp2Gvm7Dch1rle9dqNmr1HahFy5aPmKKqTpOQCYHV656gC2aPhW0g+9vHyeuvgXulIbCdTJlndK1+txmv0c2Fwkf1E7lWL/fhukmLJeFZyF7f/J5R/PTKbeFGs9wqLdLtbLs4QbdE7U6TMkl6acuHBSdshUvhfrTMON1DdcuC1qngfe4pCb7KiQays3HdIxiHUEpBwHcaqwPHHErQuR9Erk9T+vJJ3ikpiCbOBY6DRZSFToRpZ1mzGmc7pTnkulJEYd4qwYkqubD6v2tBLC1dzudxnZfyDHbzpk+2kYZI5/okq8PHoOStMHnMYAxz11Oz9sjUuD5NOkdXzrur3FVgJtL8cog0LnfuTglTI8JanjWKYtDNDZUYBw8S0m4h+EHgkyIninMfuySld59pnl6wkrrWHG2i7nMz4TM+4sndGmWVXxngC22EeDTh+iu3lemtNf1AYmnOVoBXdq6qsL9GlogQV7uOUjZHgOMoNhqQI+93PsAtvCiG1zpB8gVqCde1X3Mo6V7VFBowwYto1Xku4TWDb1Y/CC54UwHOktWcn21pYL26j3PuZSBu+3daQbm9rc9KMlkhbHMaly35ACsJ2OKFka9KgK/Nlxh3gRTKyhLWIXgNkmVhLa79/fX6K3ljh1DGQTpipm/j3AApJvxjKDQ5+6f5deQJbN3iWN371Bq4M8YBrg6rKonR58kxAJvOP0ZE3yd3Gc09uv1/swbaQA8TxSAJAjHkjWFo0V4
 
goufj1l2BW8DysY844VCaABj5ftVsAWEXpPnjLi6V0Tt1DogVHArnxU+kvCDIxTje71P7AbFn+gI3KtNyIdnMqXRBEMDjrB0ksqCdym+TSLnLLsg3Jd1Epqu4Whb3MW8UlKahHZvgckdRWymMV0J/+lPoaWHZ4qIWr6HXiNFmPDxDxOLF9J4uiVTCTg7IP/EmDJUAntMgKS/H4DbDjgfnzk20f7tyblXH6q820Uod6zbVSDaXB80weB29syhLsD4umnMVUrJynjOf0usrqrGpYG5XgIC1rf8WxiQ8BlQW3AC+P6WZ0uFa5wtpV7rI0KRadcEkHheLmfWt7xH5dszcHg6k/5pisQc0HJXVXFENRblazVXfJ5Ouu+eRR3WjQCS0LCmGSGuwJVDewr64tS36IPvtd3H+zTANnF1mj9948Qm4Joju7qR77SFpHloEdLPhOL2GoTfHitWhnhOJEvJsggYDpzWsptDpmwOCLkMsm5nt6enzM8bdYwcjkS9MAZQTyBmczAd8UXE4RVuJ3mv29d+D9K2Ofx4TZXope66td8eTleAushSpuRTY2IjoAva3X+G4jPdKeyA91+uVWEJhjxCDNhV/7RuB0J01Vwt4K9Ph4DACaXfUquPfcDeLToQg+BSldZgrkS6r0dkJsXqgu0LIqm/2lkJQjboda7dxWcKdI+tztttnFE3x1zQZGObX4VLx4lYAKiwPc6g0FSpt5OIuEzw6cgPGJSHWWf+roqYEwf0ceBRxq/3/Tj5cQbv5rlucAxcSxj2SfXsO6CJMXLfgoTr1ek5Hlu7uP/pydGJku1zBweroGZ00d8yxD5+KyGM4KcveKqgWj4Mi/72muDbYRHpjSV5dCJIKjAe+CmtTJoXdRuibJ/GLrdeyeU2mDVb2vg209gG3HaOF3e9u8elshdM8Z04ChCP2xxfMQt75URLuJ3woWz/FYTEExToVCo6dsxtWwl8oZfbT0MrTaQAUpKqUO6GraFfT
 
esBxCfh+tnqYtuBisdfePqV2eg99bks4525C+NK7jfUiiAxm1VUTPTAe1ObUNU11aCvzwDACuOuceKV9Mk7pW6dYH2j2rsLWpdhXiehbEcgFldUuTm5Hn1HzD+hpDu3smqPgwxJHctWhZsS5f1+hx9D74PhMBjbhq9D3xisTfmnOXqJrvWDIkbqczEwjnze8oV8xrCwE+meChRX0fvA1hr3pl3qGX9Ihby6fJCC+KqrQvDR28u6TgBh7+lN3DN1gF5iwIe6XTUYUTmCThtAz8y0jNa8TweY+liQOsasK1I8SOHaHsAEz+/opNTuNIImZTwBoipEBaTHdc8PRX+9ZSSczbBTnOlis4rK/JiZTvc85tdsUQUCacxSadgP/Jjn/JOg9nsrsnkZp9cF84VHf/Kn9y7+QTAc/qonxkVbikBNBFGp+vmQP5UdG5yMZdnG9evhGtGGd0lKR2ZegGz0feMpyVL3wDxx6gYyn5ruFNXeMB3q7+g7gVPDy3R8fGqzC2tDu0SuuYOwfsViv2Sh0155LDXnBXY9jYA7ChRAOq6zNjZghFy1S1noCM7A+34G9Zk1gfDMxpDhVDYmr6vVvOltq9l6eyjL5g4V8bwHSjhejIiWkAe1lcrNfJ64sI0xAwbIYLR9Zr9kuznvmMJVD/T2rPid39flyB25yvTKwa6VwDZO5c8QCBt8+rzDy4F/9/2WLuDrJa3FsU9rb/FaQTIKjiKIZtR2FxOVZ/6MuVZ4r3nXcB/QgswA3XgDRiLquKdkF+xa8DgT1cIqCRZz0Qgi8UUEEOKTcgnamp3dkYh7/r/hiRymV18JYviGXjtAapjEnkaQNUjtkgMer2dGogCusdBj5svJ7aYwk6lvfP4UXSZDAX3Ugwc5C5T6KVghEm0IG9fzMvOxSdsSBhLAeMEiiNlZubchvZqcsvkoFi0GWLWJxDQccjJyeWT0yib1ENWY5PRKJq9iRhkzRBOovb8EzKc6nCbDBoQNyB

[3/4] metron git commit: METRON-1460: Create a complementary non-split-join enrichment topology closes apache/metron#940

2018-03-07 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/1d95b831/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java
--
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java
new file mode 100644
index 000..5f82b1c
--- /dev/null
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.parallel;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.function.Function;
+
+/**
+ * The strategy to use to construct the thread pool.
+ */
+public enum WorkerPoolStrategies {
+  /**
+   * Fixed thread pool
+   */
+  FIXED(numThreads -> Executors.newFixedThreadPool(numThreads)),
+  /**
+   * Work stealing thread pool.
+   */
+  WORK_STEALING(numThreads -> Executors.newWorkStealingPool(numThreads))
+  ;
+  Function creator;
+  WorkerPoolStrategies(Function creator) {
+this.creator = creator;
+  }
+
+  public ExecutorService create(int numThreads) {
+return creator.apply(numThreads);
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/1d95b831/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java
--
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java
index ab3d462..63d39c5 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java
@@ -21,6 +21,7 @@ import com.google.common.base.Function;
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.metron.common.configuration.enrichment.EnrichmentConfig;
 import 
org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
@@ -28,6 +29,7 @@ import org.apache.metron.enrichment.lookup.EnrichmentLookup;
 import org.apache.metron.enrichment.lookup.handler.KeyWithContext;
 import org.apache.metron.hbase.TableProvider;
 import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.json.simple.JSONObject;
 import sun.management.Sensor;
 
 import javax.annotation.Nullable;
@@ -118,4 +120,18 @@ public class EnrichmentUtils {
 }
   }
 
+  public static JSONObject adjustKeys(JSONObject enrichedMessage, JSONObject 
enrichedField, String field, String prefix) {
+if ( !enrichedField.isEmpty()) {
+  for (Object enrichedKey : enrichedField.keySet()) {
+if(!StringUtils.isEmpty(prefix)) {
+  enrichedMessage.put(field + "." + enrichedKey, 
enrichedField.get(enrichedKey));
+}
+else {
+  enrichedMessage.put(enrichedKey, enrichedField.get(enrichedKey));
+}
+  }
+}
+return enrichedMessage;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/1d95b831/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
--
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
index 7898ccd..870d709 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
+++

metron git commit: METRON-1457 Move ASF links to main page in the Metron website (anandsubbu) closes apache/metron#938

2018-03-02 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/asf-site 76bcea2ff -> 3b27ee523


METRON-1457 Move ASF links to main page in the Metron website (anandsubbu) 
closes apache/metron#938


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3b27ee52
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3b27ee52
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3b27ee52

Branch: refs/heads/asf-site
Commit: 3b27ee523512e9730dc803bda471d78b750eb835
Parents: 76bcea2
Author: Anand Subramanian 
Authored: Fri Mar 2 10:01:00 2018 -0500
Committer: cstella 
Committed: Fri Mar 2 10:01:00 2018 -0500

--
 about/index.html|  6 ++--
 community/index.html|  6 ++--
 develop/bylaws.html |  6 ++--
 develop/coding.html |  6 ++--
 develop/index.html  |  6 ++--
 docs/arch.html  |  6 ++--
 docs/hive-ddl.html  |  6 ++--
 docs/index.html |  6 ++--
 docs/sensor.html|  6 ++--
 docs/vizandalerts.html  |  6 ++--
 documentation/index.html|  6 ++--
 downloads/index.html|  6 ++--
 feed.xml|  4 +--
 help/index.html |  6 ++--
 index.html  | 36 +---
 news/2015/12/06/metron-incubator/index.html |  6 ++--
 news/2015/12/12/new-logo/index.html |  6 ++--
 news/2016/01/08/debo-committer/index.html   |  6 ++--
 news/index.html |  6 ++--
 news/releases/index.html|  6 ++--
 talks/index.html|  6 ++--
 21 files changed, 72 insertions(+), 82 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/3b27ee52/about/index.html
--
diff --git a/about/index.html b/about/index.html
index b114f3f..c2c7584 100644
--- a/about/index.html
+++ b/about/index.html
@@ -127,8 +127,6 @@
 
 News
 
-ASF
-
 
  
Download 
 
@@ -268,8 +266,8 @@
   
 
 
-Copyright Â© 2018, The Apache Software Foundation. Licensed under 
the http://www.apache.org/licenses/LICENSE-2.0;>Apache 
License, Version 2.0.
-Apache Metron and its logo are trademarks of the The Apache 
Software Foundation.
+Copyright Â© 2018, The Apache Software Foundation. Licensed under 
the https://www.apache.org/licenses/LICENSE-2.0;>Apache 
License, Version 2.0.
+Apache Metron and its logo are trademarks of The Apache Software 
Foundation.
 
 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/3b27ee52/community/index.html
--
diff --git a/community/index.html b/community/index.html
index d86b09c..9d4b735 100644
--- a/community/index.html
+++ b/community/index.html
@@ -127,8 +127,6 @@
 
 News
 
-ASF
-
 
  
Download 
 
@@ -273,8 +271,8 @@
   
 
 
-Copyright Â© 2018, The Apache Software Foundation. Licensed under 
the http://www.apache.org/licenses/LICENSE-2.0;>Apache 
License, Version 2.0.
-Apache Metron and its logo are trademarks of the The Apache 
Software Foundation.
+Copyright Â© 2018, The Apache Software Foundation. Licensed under 
the https://www.apache.org/licenses/LICENSE-2.0;>Apache 
License, Version 2.0.
+Apache Metron and its logo are trademarks of The Apache Software 
Foundation.
 
 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/3b27ee52/develop/bylaws.html
--
diff --git a/develop/bylaws.html b/develop/bylaws.html
index f7e6d3f..16fdbd9 100644
--- a/develop/bylaws.html
+++ b/develop/bylaws.html
@@ -127,8 +127,6 @@
 
 News
 
-ASF
-
 
  
Download 
 
@@ -467,8 +465,8 @@ possible.
   
 
 
-Copyright Â© 2018, The Apache Software Foundation. Licensed under 
the http://www.apache.org/licenses/LICENSE-2.0;>Apache 
License, Version 2.0.
-Apache Metron and its logo are trademarks of the The Apache 
Software Foundation.
+Copyright Â© 2018, The Apache Software Foundation. Licensed under 
the https://www.apache.org/licenses/LICENSE-2.0;>Apache 
License, Version 2.0.
+Apache Metron and its logo are trademarks of The Apache Software 
Foundation.
 
 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/3b27ee52/develop/coding.html
--
diff --git a/develop/coding.html

[2/2] metron git commit: Merge branch 'asf-site' of https://git-wip-us.apache.org/repos/asf/metron into asf-site

2018-02-15 Thread cestella

Merge branch 'asf-site' of https://git-wip-us.apache.org/repos/asf/metron into 
asf-site


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/76bcea2f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/76bcea2f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/76bcea2f

Branch: refs/heads/asf-site
Commit: 76bcea2ff4b31f26da8eb5d604b62e4db68dd005
Parents: 4dba341 3199a57
Author: cstella 
Authored: Thu Feb 15 15:35:54 2018 -0500
Committer: cstella 
Committed: Thu Feb 15 15:35:54 2018 -0500

--

--

[1/2] metron git commit: METRON-1386: Fix Metron Website Required Links closes apache/incubator-metron#935

2018-02-15 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/asf-site 3199a5704 -> 76bcea2ff


METRON-1386: Fix Metron Website Required Links closes 
apache/incubator-metron#935


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4dba3413
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4dba3413
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4dba3413

Branch: refs/heads/asf-site
Commit: 4dba3413585a8b089fb2feadc28fbdf643175f74
Parents: 79b38ea
Author: Anand Subramanian 
Authored: Thu Feb 15 15:35:24 2018 -0500
Committer: cstella 
Committed: Thu Feb 15 15:35:24 2018 -0500

--
 about/index.html|   5 +-
 asf/index.html  | 276 
 community/index.html|   9 +-
 develop/bylaws.html | 511 +++
 develop/coding.html | 246 +++
 develop/index.html  |  13 +-
 docs/arch.html  |   5 +-
 docs/hive-ddl.html  |  13 +-
 docs/index.html |  13 +-
 docs/sensor.html|  13 +-
 docs/vizandalerts.html  |   5 +-
 documentation/index.html|   7 +-
 downloads/index.html|   5 +-
 feed.xml|  30 +-
 help/index.html |   9 +-
 index.html  |   5 +-
 news/2015/12/06/metron-incubator/index.html |   5 +-
 news/2015/12/12/new-logo/index.html |   7 +-
 news/2016/01/08/debo-committer/index.html   |   5 +-
 news/index.html |   5 +-
 news/releases/index.html|   5 +-
 talks/index.html|   5 +-
 22 files changed, 1150 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/4dba3413/about/index.html
--
diff --git a/about/index.html b/about/index.html
index f50de1b..b114f3f 100644
--- a/about/index.html
+++ b/about/index.html
@@ -127,6 +127,8 @@
 
 News
 
+ASF
+
 
  
Download 
 
@@ -266,7 +268,8 @@
   
 
 
-The contents of this website are Â© 2016 Apache Software Foundation 
under the terms of the Apache License v2. Apache Metron and its logo are 
trademarks of the Apache Software Foundation.
+Copyright Â© 2018, The Apache Software Foundation. Licensed under 
the http://www.apache.org/licenses/LICENSE-2.0;>Apache 
License, Version 2.0.
+Apache Metron and its logo are trademarks of the The Apache 
Software Foundation.
 
 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/4dba3413/asf/index.html
--
diff --git a/asf/index.html b/asf/index.html
new file mode 100644
index 000..86eb6c1
--- /dev/null
+++ b/asf/index.html
@@ -0,0 +1,276 @@
+
+
+http://ogp.me/ns#; 
ng-app="hortonWorksApp">@charset 
"UTF-8";[ng\:cloak],[ng-cloak],[data-ng-cloak],[x-ng-cloak],.ng-cloak,.x-ng-cloak,.ng-hide:not(.ng-hide-animate){display:none
 
!important;}ng\:form{display:block;}.ng-animate-shim{visibility:hidden;}.ng-anchor{position:absolute;}
+
+
+http://gmpg.org/xfn/11;>
+Apache Software Foundation
+
+
+img.wp-smiley,
+img.emoji {
+display: inline !important;
+border: none !important;
+box-shadow: none !important;
+height: 1em !important;
+width: 1em !important;
+margin: 0 .07em !important;
+vertical-align: -0.1em !important;
+background: none !important;
+padding: 0 !important;
+}
+
+
+
+
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new 
Date();a=s.createElement(o),
+
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
+
+ga('create', 'UA-78972211-1', 'auto');
+ga('send', 'pageview');
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+window.lsfDataLayer = window.lsfDataLayer || [];
+
+
+
+  body {
+font-family: "roboto-condensed",

metron git commit: METRON-1386: Fix Metron Website Required Links closes apache/incubator-metron#935

2018-02-15 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master e59059bd9 -> fa5cff2c3


METRON-1386: Fix Metron Website Required Links closes 
apache/incubator-metron#935


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fa5cff2c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fa5cff2c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fa5cff2c

Branch: refs/heads/master
Commit: fa5cff2c3534c5d30f265b0118faffe971e5063b
Parents: e59059b
Author: Anand Subramanian 
Authored: Thu Feb 15 15:32:37 2018 -0500
Committer: cstella 
Committed: Thu Feb 15 15:32:37 2018 -0500

--
 site/_includes/footer.html|  3 +-
 site/_includes/primary-nav-items.html |  2 +
 site/asf/index.md | 90 ++
 site/community/index.md   |  4 ++
 4 files changed, 98 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/fa5cff2c/site/_includes/footer.html
--
diff --git a/site/_includes/footer.html b/site/_includes/footer.html
index 55c47a9..07615f4 100644
--- a/site/_includes/footer.html
+++ b/site/_includes/footer.html
@@ -1,6 +1,7 @@
 
 
 
-The contents of this website are Â© 2016 Apache Software Foundation 
under the terms of the Apache License v2. Apache Metron and its logo are 
trademarks of the Apache Software Foundation.
+Copyright Â© 2018, The Apache Software Foundation. Licensed under 
the http://www.apache.org/licenses/LICENSE-2.0;>Apache 
License, Version 2.0.
+Apache Metron and its logo are trademarks of the The Apache 
Software Foundation.
 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/fa5cff2c/site/_includes/primary-nav-items.html
--
diff --git a/site/_includes/primary-nav-items.html 
b/site/_includes/primary-nav-items.html
index 2799976..3b97c63 100644
--- a/site/_includes/primary-nav-items.html
+++ b/site/_includes/primary-nav-items.html
@@ -9,6 +9,8 @@
 
 News
 
+ASF
+
 
  
Download 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/fa5cff2c/site/asf/index.md
--
diff --git a/site/asf/index.md b/site/asf/index.md
new file mode 100644
index 000..ca98112
--- /dev/null
+++ b/site/asf/index.md
@@ -0,0 +1,90 @@
+---
+layout: default
+title: Apache Software Foundation
+---
+
+
+
+
+
+
+
+
+  real-time big data security 
+
+
+
+
+
+
+  
+
+
+Home
+License
+Events
+Sponsorship
+Security
+Thanks
+  
+
+  
+
+
+
+
+Home
+
+
+To know more about The Apache Software Foundation, visit:
+
+
+https://www.apache.org; 
target="_blank">APACHE
+
+
+
+
+
+  License
+
+
+https://www.apache.org/licenses/; 
target="_blank">APACHE LICENSE
+
+
+
+
+
+  Events
+
+
+To know more about latest happenings, go to:
+https://www.apache.org/events/current-event; target="_blank">EVENTS
+
+
+
+
+
+  Sponsorship
+
+
+https://www.apache.org/foundation/sponsorship.html; 
target="_blank">SPONSORSHIP
+
+
+
+
+
+  Security
+
+
+https://www.apache.org/security/; 
target="_blank">SECURITY
+
+
+
+
+
+  Thanks
+
+
+https://www.apache.org/foundation/thanks.html; target="_blank">THANKS
+
+

http://git-wip-us.apache.org/repos/asf/metron/blob/fa5cff2c/site/community/index.md
--
diff --git a/site/community/index.md b/site/community/index.md
index e848748..7c09b14 100644
--- a/site/community/index.md
+++ b/site/community/index.md
@@ -85,6 +85,10 @@ title: Apache Metron Community
   Jon Zeolla jonzeolla Committer 
 
   JJ Meyer jmeyer Committer 
+
+  Raghu Mitra iraghumitra Committer 
+
+  Anand Subramanian anandsubbu Committer

metron git commit: METRON-1455: Patch and Replace methods in the REST UpdateController return 400 this closes apache/incubator-metron#937

2018-02-15 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 15b98dea4 -> e59059bd9


METRON-1455: Patch and Replace methods in the REST UpdateController return 400 
this closes apache/incubator-metron#937


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e59059bd
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e59059bd
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e59059bd

Branch: refs/heads/master
Commit: e59059bd9707a6ca46c4137d796b8f2943f06b43
Parents: 15b98de
Author: cstella 
Authored: Thu Feb 15 13:00:36 2018 -0500
Committer: cstella 
Committed: Thu Feb 15 13:00:36 2018 -0500

--
 .../apache/metron/common/utils/JSONUtils.java   |  7 ++
 .../dao/ElasticsearchMetaAlertDao.java  | 15 -
 .../apache/metron/indexing/dao/IndexDao.java| 23 +---
 .../indexing/dao/update/PatchRequest.java   |  7 +++---
 4 files changed, 26 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/e59059bd/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
index 135546d..c02f19d 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
@@ -184,4 +184,11 @@ public enum JSONUtils {
 return toJSONPretty(JsonPatch.apply(patchNode, sourceNode));
   }
 
+  public Map applyPatch(List> patch, 
Map source) {
+JsonNode originalNode = convert(source, JsonNode.class);
+JsonNode patchNode = convert(patch, JsonNode.class);
+JsonNode patched = JsonPatch.apply(patchNode, originalNode);
+return _mapper.get().convertValue(patched, new TypeReference>() { });
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/e59059bd/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
--
diff --git 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index 9740272..2311a2b 100644
--- 
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ 
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -473,12 +473,15 @@ public class ElasticsearchMetaAlertDao implements 
MetaAlertDao {
   }
 
   protected boolean isPatchAllowed(PatchRequest request) {
-Iterator patchIterator = request.getPatch().iterator();
-while(patchIterator.hasNext()) {
-  JsonNode patch = (JsonNode) patchIterator.next();
-  String path = patch.path("path").asText();
-  if (STATUS_PATH.equals(path) || ALERT_PATH.equals(path)) {
-return false;
+if(request.getPatch() != null && !request.getPatch().isEmpty()) {
+  for(Map patch : request.getPatch()) {
+Object pathObj = patch.get("path");
+if(pathObj != null && pathObj instanceof String) {
+  String path = (String)pathObj;
+  if (STATUS_PATH.equals(path) || ALERT_PATH.equals(path)) {
+return false;
+  }
+}
   }
 }
 return true;

http://git-wip-us.apache.org/repos/asf/metron/blob/e59059bd/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
--
diff --git 
a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
 
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
index 2c057d4..fe546bd 100644
--- 
a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
+++ 
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
@@ -17,16 +17,11 @@
  */
 package org.apache.metron.indexing.dao;
 
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JsonNode;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
-import

[4/5] metron git commit: METRON-1451 On Centos full dev, Metron Indexing shows up as stopped (anandsubbu via merrimanr) closes apache/metron#932

2018-02-08 Thread cestella

METRON-1451 On Centos full dev, Metron Indexing shows up as stopped (anandsubbu 
via merrimanr) closes apache/metron#932


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fa86663e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fa86663e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fa86663e

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: fa86663ecdb166f889c9c0c4c34373aec68dbd30
Parents: 48d9d25
Author: anandsubbu 
Authored: Thu Feb 8 16:19:39 2018 -0600
Committer: merrimanr 
Committed: Thu Feb 8 16:19:39 2018 -0600

--
 .../ansible/roles/ambari_config/tasks/dependencies.yml   | 11 ---
 1 file changed, 4 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/fa86663e/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
--
diff --git 
a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml 
b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
index fdb21c7..52d6ca1 100644
--- a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
+++ b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
@@ -15,10 +15,7 @@
 #  limitations under the License.
 #
 ---
-- name: Install python-requests on CentOS
-  yum: name=python-requests
-  when: ansible_distribution == "CentOS"
-
-- name: Install python-requests on Ubuntu
-  apt: name=python-requests force=yes
-  when: ansible_distribution == "Ubuntu"
+- name: Install python-requests module
+  pip:
+name: requests
+version: 2.6.1

[3/5] metron git commit: METRON-1444: Add Ubuntu Repositories for Elasticsearch to the Mpack (mmiklavc via mmiklavc) closes apache/metron#928

2018-02-08 Thread cestella

METRON-1444: Add Ubuntu Repositories for Elasticsearch to the Mpack (mmiklavc 
via mmiklavc) closes apache/metron#928


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/48d9d256
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/48d9d256
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/48d9d256

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 48d9d25696a71ff07791b86348dc978db7a56a9d
Parents: e5f24f7
Author: mmiklavc 
Authored: Thu Feb 8 11:53:47 2018 -0700
Committer: Michael Miklavcic 
Committed: Thu Feb 8 11:53:47 2018 -0700

--
 .../roles/ambari_common/defaults/main.yml   |  6 +--
 .../ambari_common/tasks/elasticsearch-repo.yml  | 40 
 .../ansible/roles/ambari_common/tasks/main.yml  |  1 -
 .../packaging/ambari/metron-mpack/README.md | 14 ---
 .../ELASTICSEARCH/5.6.2/repos/repoinfo.xml  | 13 +++
 .../KIBANA/5.6.2/repos/repoinfo.xml | 19 ++
 6 files changed, 34 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/48d9d256/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
--
diff --git a/metron-deployment/ansible/roles/ambari_common/defaults/main.yml 
b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
index 0614e0f..e7968f5 100644
--- a/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
@@ -17,7 +17,5 @@
 ---
 hadoop_logrotate_frequency: daily
 hadoop_logrotate_retention: 30
-centos_ambari_install_url: 
http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.5.2.0/ambari.repo
-ubuntu_ambari_repo: 
http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/2.5.2.0
-ubuntu_elasticsearch_packages_repo: 
https://artifacts.elastic.co/packages/5.x/apt
-ubuntu_elasticsearch_curator_repo: https://packages.elastic.co/curator/5/debian
+centos_ambari_install_url: 
http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.6.1.0/ambari.repo
+ubuntu_ambari_repo: 
http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/2.6.1.0

http://git-wip-us.apache.org/repos/asf/metron/blob/48d9d256/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml
--
diff --git 
a/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml 
b/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml
deleted file mode 100644
index 730607d..000
--- a/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#

-
-#
-# for CentOS...
-#
-# this is not needed for CentOS.  the elasticsearch repository is automatically
-# added by the mpack.  there is currently an issue that needs resolved that
-# prevents us from doing the same in Ubuntu
-#
-
-#
-# for Ubuntu...
-#
-- name: Setup Elasticsearch packages repo on Ubuntu
-  shell: echo "deb {{ ubuntu_elasticsearch_packages_repo }} stable main" | 
sudo tee -a /etc/apt/sources.list.d/elasticsearch.list
-  when: ansible_distribution == "Ubuntu"
-
-- name: Setup Elasticsearch curator repo on Ubuntu
-  shell: echo "deb {{ ubuntu_elasticsearch_curator_repo }} stable main" | sudo 
tee -a /etc/apt/sources.list.d/elasticsearch.list
-  when: ansible_distribution == "Ubuntu"
-
-- name: Update package cache
-  apt: update_cache=yes
-  when: ansible_distribution == "Ubuntu"

http://git-wip-us.apache.org/repos/asf/metron/blob/48d9d256/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
--
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/main.yml 
b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
index b3258a9..f3fd905 100644
---

[1/5] metron git commit: METRON-1447 Heap Size Not Set Correctly by MPack for ES 5.x (nickwallen) closes apache/metron#927

2018-02-08 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr 67d6c0edd -> 9b25084c8


METRON-1447 Heap Size Not Set Correctly by MPack for ES 5.x (nickwallen) closes 
apache/metron#927


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/31f3aa8c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/31f3aa8c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/31f3aa8c

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 31f3aa8cb0f4c3821c264e375aa14eda207adcb0
Parents: 38b7d5e
Author: nickwallen 
Authored: Wed Feb 7 14:50:06 2018 -0500
Committer: nickallen 
Committed: Wed Feb 7 14:50:06 2018 -0500

--
 .../5.6.2/configuration/elastic-jvm-options.xml | 144 +++
 .../5.6.2/configuration/elastic-sysconfig.xml   |  12 +-
 .../ELASTICSEARCH/5.6.2/metainfo.xml|   1 +
 .../5.6.2/package/scripts/elastic_commands.py   |  13 ++
 .../5.6.2/package/scripts/params.py |   3 +
 5 files changed, 163 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
new file mode 100644
index 000..5c6aaca
--- /dev/null
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
@@ -0,0 +1,144 @@
+
+
+
+
+
+
+heap_size
+512m
+JVM heap size
+
+
+content
+The jinja template for the Elasticsearch JVM options 
file.
+
+## JVM configuration
+
+
+## IMPORTANT: JVM heap size
+
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See 
https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms{{heap_size}}
+-Xmx{{heap_size}}
+
+
+## Expert settings
+
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+
+
+## GC configuration
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# pre-touch memory pages used by the JVM during initialization
+-XX:+AlwaysPreTouch
+
+## basic
+
+# force the server VM (remove on 32-bit client JVMs)
+-server
+
+# explicitly set the stack size (reduce to 320k on 32-bit client JVMs)
+-Xss1m
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+-Djna.nosys=true
+
+# use old-style file permissions on JDK9
+-Djdk.io.permissionsUseCanonicalPath=true
+
+# flags to configure Netty
+-Dio.netty.noUnsafe=true
+-Dio.netty.noKeySetOptimization=true
+-Dio.netty.recycler.maxCapacityPerThread=0
+
+# log4j 2
+-Dlog4j.shutdownHookEnabled=false
+-Dlog4j2.disable.jmx=true
+-Dlog4j.skipJansi=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${heap.dump.path}
+
+## GC logging
+
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${loggc}
+
+# By default, the GC log file will not rotate.
+# By uncommenting the lines below, the GC log file
+# will be rotated every 128MB at most 32 times.
+#-XX:+UseGCLogFileRotation

[2/5] metron git commit: METRON-1273: Website documentation link should point to the current site-book (JonZeolla via mmiklavc) closes apache/metron#812

2018-02-08 Thread cestella

METRON-1273: Website documentation link should point to the current site-book 
(JonZeolla via mmiklavc) closes apache/metron#812


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e5f24f7b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e5f24f7b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e5f24f7b

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: e5f24f7bdce6a119adb8be543752d458f721d48d
Parents: 31f3aa8
Author: JonZeolla 
Authored: Wed Feb 7 13:58:11 2018 -0700
Committer: Michael Miklavcic 
Committed: Wed Feb 7 13:58:11 2018 -0700

--
 site/documentation/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/e5f24f7b/site/documentation/index.md
--
diff --git a/site/documentation/index.md b/site/documentation/index.md
index 6b38ed6..e44953c 100644
--- a/site/documentation/index.md
+++ b/site/documentation/index.md
@@ -67,7 +67,7 @@ title: Apache Metron Documentation
 
 
  Apache Metron documentation provides information on quickly 
getting started with Metron, performing a full installation, adding additional 
data sources, and using Metron to triage alerts. Metron documentation is 
currently a work in progress. Please check back again later as we continue to 
grow this documentation set.
-https://cwiki.apache.org/confluence/display/METRON/Documentation; 
target="_blank">LEARN MORE
+https://metron.apache.org/current-book/index.html; target="_blank">LEARN 
MORE

[5/5] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr

2018-02-08 Thread cestella

Merge branch 'master' into feature/METRON-1416-upgrade-solr


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9b25084c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9b25084c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9b25084c

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 9b25084c83e484b8119caf134a473fcb6306fa6a
Parents: 67d6c0e fa86663
Author: cstella 
Authored: Thu Feb 8 17:22:12 2018 -0500
Committer: cstella 
Committed: Thu Feb 8 17:22:12 2018 -0500

--
 .../roles/ambari_common/defaults/main.yml   |   6 +-
 .../ambari_common/tasks/elasticsearch-repo.yml  |  40 --
 .../ansible/roles/ambari_common/tasks/main.yml  |   1 -
 .../roles/ambari_config/tasks/dependencies.yml  |  11 +-
 .../packaging/ambari/metron-mpack/README.md |  14 --
 .../ELASTICSEARCH/5.6.2/repos/repoinfo.xml  |  13 ++
 .../KIBANA/5.6.2/repos/repoinfo.xml |  19 +++
 .../5.6.2/configuration/elastic-jvm-options.xml | 144 +++
 .../5.6.2/configuration/elastic-sysconfig.xml   |  12 +-
 .../ELASTICSEARCH/5.6.2/metainfo.xml|   1 +
 .../5.6.2/package/scripts/elastic_commands.py   |  13 ++
 .../5.6.2/package/scripts/params.py |   3 +
 site/documentation/index.md |   2 +-
 13 files changed, 202 insertions(+), 77 deletions(-)
--

[7/8] metron git commit: METRON-1446: Fix openjdk issue with Ubuntu (mmiklavc via mmiklavc) closes apache/metron#926

2018-02-07 Thread cestella

METRON-1446: Fix openjdk issue with Ubuntu (mmiklavc via mmiklavc) closes 
apache/metron#926


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/38b7d5e3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/38b7d5e3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/38b7d5e3

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 38b7d5e339320e71cd4d35d31b8d118a3e6d0be7
Parents: fcff059
Author: mmiklavc 
Authored: Tue Feb 6 09:57:15 2018 -0700
Committer: Michael Miklavcic 
Committed: Tue Feb 6 09:57:15 2018 -0700

--
 .../ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/38b7d5e3/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml
--
diff --git 
a/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml 
b/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml
index 8337b81..4eb0945 100644
--- a/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml
+++ b/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml
@@ -20,7 +20,7 @@
   register: jdk_dir
 
 - name: Install openjdk repository
-  shell: add-apt-repository ppa:openjdk-r/ppa
+  shell: add-apt-repository ppa:openjdk-r/ppa -y
   when: not jdk_dir.stat.exists
 
 - name: Update package cache

[4/8] metron git commit: METRON-1438 STELLAR: Move shell functions to common from metron-management (ottobackwards) closes apache/metron#920

2018-02-07 Thread cestella

METRON-1438 STELLAR: Move shell functions to common from metron-management 
(ottobackwards) closes apache/metron#920


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c26abbba
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c26abbba
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c26abbba

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: c26abbbaeaeea3551e218cb9aa8ba97b32655958
Parents: 3d3c43c
Author: ottobackwards 
Authored: Fri Feb 2 09:58:59 2018 -0500
Committer: otto 
Committed: Fri Feb 2 09:58:59 2018 -0500

--
 metron-platform/metron-management/README.md |  32 +-
 .../metron/management/ShellFunctions.java   | 302 ---
 .../metron/management/ShellFunctionsTest.java   | 171 ---
 metron-stellar/stellar-common/README.md |  31 ++
 metron-stellar/stellar-common/pom.xml   |   5 +
 .../stellar/common/shell/cli/PausableInput.java |  23 +-
 .../stellar/dsl/functions/ShellFunctions.java   | 301 ++
 .../dsl/functions/ShellFunctionsTest.java   | 176 +++
 8 files changed, 529 insertions(+), 512 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-platform/metron-management/README.md
--
diff --git a/metron-platform/metron-management/README.md 
b/metron-platform/metron-management/README.md
index 812583c..bf939c2 100644
--- a/metron-platform/metron-management/README.md
+++ b/metron-platform/metron-management/README.md
@@ -35,14 +35,13 @@ project.
 * [Functions](#functions)
 * [Grok Functions](#grok-functions)
 * [File Functions](#file-functions)
-* [Shell Functions](#shell-functions)
 * [Configuration Functions](#configuration-functions)
 * [Parser Functions](#parser-functions)
 * [Indexing Functions](#indexing-functions)
 * [Enrichment Functions](#enrichment-functions)
 * [Threat Triage Functions](#threat-triage-functions)
 * [Examples](#examples)
-*  [Iterate to Find a Valid Grok 
Pattern](#iterate-to-find-a-valid-grok-pattern)
+* [Iterate to Find a Valid Grok 
Pattern](#iterate-to-find-a-valid-grok-pattern)
 * [Manage Stellar Field 
Transformations](#manage-stellar-field-transformations)
 * [Manage Stellar Enrichments](#manage-stellar-enrichments)
 * [Manage Threat Triage Rules](#manage-threat-triage-rules)
@@ -132,35 +131,6 @@ The functions are split roughly into a few sections:
   * path - The path of the file
 * Returns: true if the file was written and false otherwise.
 
-### Shell Functions 
-
-* `SHELL_EDIT`
-  * Description: Open an editor (optionally initialized with text) and return 
whatever is saved from the editor.  The editor to use is pulled from `EDITOR` 
or `VISUAL` environment variable.
-  * Input:
-* string - (Optional) A string whose content is used to initialize the 
editor.
-  * Returns: The content that the editor saved after editor exit.
-* `SHELL_GET_EXPRESSION`
-  * Description: Get a stellar expression from a variable
-  * Input:
-* variable - variable name
-  * Returns: The stellar expression associated with the variable.
-* `SHELL_LIST_VARS`
-  * Description: Return the variables in a tabular form
-  * Input:
-* wrap : Length of string to wrap the columns
-  * Returns: A tabular representation of the variables.
-* `SHELL_MAP2TABLE`
-  * Description: Take a map and return a table
-  * Input:
-* map - Map
-  * Returns: The map in table form
-* `SHELL_VARS2MAP`
-  * Description: Take a set of variables and return a map
-  * Input:
-* variables* - variable names to use to create map 
-  * Returns: A map associating the variable name with the stellar expression.
-
-
 ### Configuration Functions
 
 * `CONFIG_GET`

http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
--
diff --git 
a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
 
b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
deleted file mode 100644
index afac7f0..000
--- 
a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this

[8/8] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr

2018-02-07 Thread cestella

Merge branch 'master' into feature/METRON-1416-upgrade-solr


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/67d6c0ed
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/67d6c0ed
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/67d6c0ed

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 67d6c0edd731f528af759ca11de875cbf77f6b2c
Parents: f525dcf 38b7d5e
Author: cstella 
Authored: Wed Feb 7 11:43:34 2018 -0500
Committer: cstella 
Committed: Wed Feb 7 11:43:34 2018 -0500

--
 .../roles/java_jdk/tasks/install_jdk_ubuntu.yml |   2 +-
 .../packaging/ambari/metron-mpack/README.md |  14 +
 metron-deployment/scripts/platform-info.sh  |   4 +-
 .../advanced-config-form.component.html |   2 +-
 .../advanced-config-form.component.spec.ts  |  34 +++
 .../advanced-config-form.component.ts   |  20 +-
 .../src/main/config/rest_application.yml|   3 +-
 .../apache/metron/rest/MetronRestConstants.java |   6 +-
 .../metron/rest/controller/StormController.java |  81 +++--
 .../metron/rest/service/StormAdminService.java  |   4 +-
 .../service/impl/StormAdminServiceImpl.java |   8 +-
 .../rest/service/impl/StormCLIWrapper.java  |  16 +-
 .../src/main/resources/application-test.yml |   3 +-
 .../src/main/resources/application-vagrant.yml  |   4 +-
 .../StormControllerIntegrationTest.java | 102 ---
 .../rest/mock/MockStormCLIClientWrapper.java|  93 --
 .../metron/rest/mock/MockStormRestTemplate.java |  22 +-
 .../service/impl/StormAdminServiceImplTest.java |   8 +-
 .../rest/service/impl/StormCLIWrapperTest.java  |  17 +-
 metron-platform/metron-management/README.md |  32 +-
 .../metron/management/ShellFunctions.java   | 302 ---
 .../metron/management/ShellFunctionsTest.java   | 171 ---
 metron-stellar/stellar-common/NOTICE|   6 +
 metron-stellar/stellar-common/README.md |  74 +
 metron-stellar/stellar-common/pom.xml   |   6 +
 .../src/main/assembly/assembly.xml  |   2 +-
 .../src/main/assembly/stand-alone-assembly.xml  |  57 
 .../stellar/common/shell/cli/PausableInput.java |  23 +-
 .../stellar/dsl/functions/ShellFunctions.java   | 301 ++
 .../src/main/scripts/deployed/stellar   |  36 +++
 .../src/main/scripts/stand-alone/stellar|  21 ++
 .../stellar-common/src/main/scripts/stellar |  36 ---
 .../dsl/functions/ShellFunctionsTest.java   | 176 +++
 33 files changed, 998 insertions(+), 688 deletions(-)
--

[5/8] metron git commit: METRON-1443 Missing Critical MPack Install Instruction for Ubuntu (nickwallen) closes apache/metron#925

2018-02-07 Thread cestella

METRON-1443 Missing Critical MPack Install Instruction for Ubuntu (nickwallen) 
closes apache/metron#925


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/06305052
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/06305052
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/06305052

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 06305052a186ff911d204c755804e7225d05bf28
Parents: c26abbb
Author: nickwallen 
Authored: Mon Feb 5 09:17:57 2018 -0500
Committer: nickallen 
Committed: Mon Feb 5 09:17:57 2018 -0500

--
 .../packaging/ambari/metron-mpack/README.md   | 14 ++
 1 file changed, 14 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/06305052/metron-deployment/packaging/ambari/metron-mpack/README.md
--
diff --git a/metron-deployment/packaging/ambari/metron-mpack/README.md 
b/metron-deployment/packaging/ambari/metron-mpack/README.md
index 5179b5f..4b0b073 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/README.md
+++ b/metron-deployment/packaging/ambari/metron-mpack/README.md
@@ -28,6 +28,20 @@ This allows you to easily install Metron using a simple, 
guided process.  This a
 
 * A [Node.js](https://nodejs.org/en/download/package-manager/) repository 
installed on the host running the Management and Alarm UI.
 
+* When installing on Ubuntu the Elasticsearch repository must be defined 
manually. This is NOT defined by the Mpack like it is on CentOS.  This is an 
open bug that needs addressed in the Mpack.  See the 
[Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html)
 documentation for more specific instructions. 
+  ```
+  $ cat >/etc/apt/sources.list.d/elasticsearch.list << EOL
+  deb https://packages.elastic.co/curator/5/debian stable main
+  deb https://artifacts.elastic.co/packages/5.x/apt stable main
+  EOL
+
+  $ wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo 
apt-key add -
+
+  $ apt-get update
+  ```
+
+
+
 ### Quick Start
 
 1. Build the Metron MPack. Execute the following command from the project's 
root directory.

[6/8] metron git commit: METRON-1442: Split rest end points for indexing topology into random access indexing and batch indexing this closes apache/incubator-metron#923

2018-02-07 Thread cestella

METRON-1442: Split rest end points for indexing topology into random access 
indexing and batch indexing this closes apache/incubator-metron#923


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fcff0596
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fcff0596
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fcff0596

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: fcff0596c7d2b2546d89283fb90fbc8c10b31f1f
Parents: 0630505
Author: MohanDV 
Authored: Mon Feb 5 09:48:47 2018 -0500
Committer: cstella 
Committed: Mon Feb 5 09:48:47 2018 -0500

--
 .../src/main/config/rest_application.yml|   3 +-
 .../apache/metron/rest/MetronRestConstants.java |   6 +-
 .../metron/rest/controller/StormController.java |  81 +++
 .../metron/rest/service/StormAdminService.java  |   4 +-
 .../service/impl/StormAdminServiceImpl.java |   8 +-
 .../rest/service/impl/StormCLIWrapper.java  |  16 +--
 .../src/main/resources/application-test.yml |   3 +-
 .../src/main/resources/application-vagrant.yml  |   4 +-
 .../StormControllerIntegrationTest.java | 102 ++-
 .../rest/mock/MockStormCLIClientWrapper.java|  93 -
 .../metron/rest/mock/MockStormRestTemplate.java |  22 ++--
 .../service/impl/StormAdminServiceImplTest.java |   8 +-
 .../rest/service/impl/StormCLIWrapperTest.java  |  17 ++--
 13 files changed, 233 insertions(+), 134 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/config/rest_application.yml
--
diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml 
b/metron-interface/metron-rest/src/main/config/rest_application.yml
index 6e4fb66..4cc51ff 100644
--- a/metron-interface/metron-rest/src/main/config/rest_application.yml
+++ b/metron-interface/metron-rest/src/main/config/rest_application.yml
@@ -46,7 +46,8 @@ storm:
   enrichment:
 script.path: ${METRON_HOME}/bin/start_enrichment_topology.sh
   indexing:
-script.path: ${METRON_HOME}/bin/start_elasticsearch_topology.sh
+randomaccess.script.path: 
${METRON_HOME}/bin/start_elasticsearch_topology.sh
+batch.script.path: ${METRON_HOME}/bin/start_hdfs_topology.sh
 
 kerberos:
   enabled: ${SECURITY_ENABLED}

http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
--
diff --git 
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
 
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index 4e8d7f2..f18d4cf 100644
--- 
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++ 
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -35,10 +35,12 @@ public class MetronRestConstants {
   public static final String TOPOLOGY_SUMMARY_URL = "/api/v1/topology/summary";
   public static final String TOPOLOGY_URL = "/api/v1/topology";
   public static final String ENRICHMENT_TOPOLOGY_NAME = "enrichment";
-  public static final String INDEXING_TOPOLOGY_NAME = "indexing";
+  public static final String BATCH_INDEXING_TOPOLOGY_NAME = "batch_indexing";
+  public static final String RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME = 
"random_access_indexing";
   public static final String PARSER_SCRIPT_PATH_SPRING_PROPERTY = 
"storm.parser.script.path";
   public static final String ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY = 
"storm.enrichment.script.path";
-  public static final String INDEXING_SCRIPT_PATH_SPRING_PROPERTY = 
"storm.indexing.script.path";
+  public static final String BATCH_INDEXING_SCRIPT_PATH_SPRING_PROPERTY = 
"storm.indexing.batch.script.path";
+  public static final String 
RANDOM_ACCESS_INDEXING_SCRIPT_PATH_SPRING_PROPERTY = 
"storm.indexing.randomaccess.script.path";
   public static final String PARSER_TOPOLOGY_OPTIONS_SPRING_PROPERTY = 
"storm.parser.topology.options";
   public static final String KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY = 
"kafka.security.protocol";
 

http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java
--
diff --git 
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java
 
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java
index 292c668..d1af1c5 100644
---

[1/8] metron git commit: METRON-1091 STELLAR Shell: Stand Alone installation (ottobackwards) closes apache/metron#690

2018-02-07 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr f525dcfd7 -> 67d6c0edd


METRON-1091 STELLAR Shell: Stand Alone installation (ottobackwards) closes 
apache/metron#690


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/124becd1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/124becd1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/124becd1

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 124becd1a35d00e620d4eaed5d5d84c0abbe8ba9
Parents: 644e951
Author: ottobackwards 
Authored: Tue Jan 30 16:34:16 2018 -0500
Committer: otto 
Committed: Tue Jan 30 16:34:16 2018 -0500

--
 metron-stellar/stellar-common/NOTICE|  6 +++
 metron-stellar/stellar-common/README.md | 43 +++
 metron-stellar/stellar-common/pom.xml   |  1 +
 .../src/main/assembly/assembly.xml  |  2 +-
 .../src/main/assembly/stand-alone-assembly.xml  | 57 
 .../src/main/scripts/deployed/stellar   | 36 +
 .../src/main/scripts/stand-alone/stellar| 21 
 .../stellar-common/src/main/scripts/stellar | 36 -
 8 files changed, 165 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/124becd1/metron-stellar/stellar-common/NOTICE
--
diff --git a/metron-stellar/stellar-common/NOTICE 
b/metron-stellar/stellar-common/NOTICE
new file mode 100644
index 000..1a19f3d
--- /dev/null
+++ b/metron-stellar/stellar-common/NOTICE
@@ -0,0 +1,6 @@
+   Apache Metron
+   Copyright 2015-2018 The Apache Software Foundation
+
+   This product includes software developed at
+   The Apache Software Foundation (http://www.apache.org/).
+

http://git-wip-us.apache.org/repos/asf/metron/blob/124becd1/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md 
b/metron-stellar/stellar-common/README.md
index 98aea77..a2f1242 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -1422,6 +1422,49 @@ Please note that functions are loading lazily in the 
background and will be unav
 %functions
 ABS, APPEND_IF_MISSING, BIN, BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, 
CHOMP, CHOP, COUNT_MATCHES, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, 
DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENDS_WITH, 
FILL_LEFT, FILL_RIGHT, FILTER, FORMAT, GET, GET_FIRST, GET_LAST, HLLP_ADD, 
HLLP_CARDINALITY, HLLP_INIT, HLLP_MERGE, IN_SUBNET, IS_DATE, IS_DOMAIN, 
IS_EMAIL, IS_EMPTY, IS_INTEGER, IS_IP, IS_URL, JOIN, LENGTH, LIST_ADD, 
MAAS_GET_ENDPOINT, MAAS_MODEL_APPLY, MAP, MAP_EXISTS, MAP_GET, MONTH, 
OUTLIER_MAD_ADD, OUTLIER_MAD_SCORE, OUTLIER_MAD_STATE_MERGE, 
PREPEND_IF_MISSING, PROFILE_FIXED, PROFILE_GET, PROFILE_WINDOW, 
PROTOCOL_TO_NAME, REDUCE, REGEXP_MATCH, SPLIT, STARTS_WITH, STATS_ADD, 
STATS_BIN, STATS_COUNT, STATS_GEOMETRIC_MEAN, STATS_INIT, STATS_KURTOSIS, 
STATS_MAX, STATS_MEAN, STATS_MERGE, STATS_MIN, STATS_PERCENTILE, 
STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS, 
STATS_SUM, STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, STRING_ENTROPY, 
SYS
 TEM_ENV_GET, SYSTEM_PROPERTY_GET, TO_DOUBLE, TO_EPOCH_TIMESTAMP, TO_FLOAT, 
TO_INTEGER, TO_LONG, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, 
URL_TO_PATH, URL_TO_PORT, URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR 
 ```
+## Stellar Shell Stand Alone
+
+The Stellar Shell is also packaged as a stand alone application.  It can be 
unpacked on any supported
+operating system. 
+
+> Only the base Stellar functions are available as packaged.  Other functions, 
such as those in metron-profiler and metron-management are not available.
+
+
+```bash
+metron-stellar/stellar-common/target/stellar-common-0.4.3-stand-alone.tar.gz
+```
+
+When unpacked, the following structure will be created:
+
+```bash
+.
+âââ bin
+âÂ Â  âââ stellar
+âââ lib
+âââ stellar-common-0.4.3-uber.jar
+```
+
+To run the Stellar Shell run the following from the directory you unpacked to:
+
+```bash
+bin/stellar
+```
+
+```bash
+-> % bin/stellar
+Stellar, Go!
+Please note that functions are loading lazily in the background and will be 
unavailable until loaded fully.
+[Stellar]>>> Functions loaded, you may refer to functions now...
+
+[Stellar]>>> %functions
+ABS, APPEND_IF_MISSING, BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, 
CEILING, CHOMP, CHOP, COS, COUNT_MATCHES, DAY_OF_MONTH, DAY_OF_WEEK, 
DAY_OF_YEAR, DECODE, DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, 
DOMAIN_TO_TLD, ENCODE, ENDS_WITH, EXP, FILL_LEFT, FILL_RIGHT,

[3/8] metron git commit: METRON-1435 Management UI cannot save json objects in advanced config (merrimanr) closes apache/metron#917

2018-02-07 Thread cestella

METRON-1435 Management UI cannot save json objects in advanced config 
(merrimanr) closes apache/metron#917


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3d3c43c7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3d3c43c7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3d3c43c7

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 3d3c43c716b0a6d58b57d6d5ba83109bc49661e4
Parents: 567d106
Author: merrimanr 
Authored: Fri Feb 2 08:37:29 2018 -0600
Committer: merrimanr 
Committed: Fri Feb 2 08:37:29 2018 -0600

--
 .../advanced-config-form.component.html |  2 +-
 .../advanced-config-form.component.spec.ts  | 34 
 .../advanced-config-form.component.ts   | 20 +++-
 3 files changed, 54 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/3d3c43c7/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html
--
diff --git 
a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html
 
b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html
index 1a30ee7..945be5f 100644
--- 
a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html
+++ 
b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html
@@ -19,7 +19,7 @@
 
 
 
-
+
 
 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/3d3c43c7/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.spec.ts
--
diff --git 
a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.spec.ts
 
b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.spec.ts
index a8f0ed0..ed80c54 100644
--- 
a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.spec.ts
+++ 
b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.spec.ts
@@ -136,6 +136,18 @@ describe('Component: AdvancedConfigFormComponent', () => {
   expect(component.configForm.controls['field2'].value).toEqual('value2');
   expect(component.configForm.controls['field3'].value).toEqual('value3');
 
+  component.newConfigKey = 'field1';
+  component.newConfigValue = '["newValue1"]';
+  component.saveNewConfig();
+  expect(Object.keys(component.config).length).toEqual(3);
+  expect(component.config['field1']).toEqual(['newValue1']);
+
+  component.newConfigKey = 'field1';
+  component.newConfigValue = '{"key":"newValue1"}';
+  component.saveNewConfig();
+  expect(Object.keys(component.config).length).toEqual(3);
+  expect(component.config['field1']).toEqual({key: 'newValue1'});
+
   component.removeConfig('field1');
   expect(Object.keys(component.config).length).toEqual(2);
   expect(component.config['field2']).toEqual('value2');
@@ -146,6 +158,28 @@ describe('Component: AdvancedConfigFormComponent', () => {
   
expect(component.configForm.controls['newConfigValue'].value).toEqual('enter 
value');
   expect(component.configForm.controls['field2'].value).toEqual('value2');
   expect(component.configForm.controls['field3'].value).toEqual('value3');
+
+
   }));
 
+it('verify display and save values',  async(() => {
+let component: AdvancedConfigFormComponent =  
fixture.componentInstance;
+component.config = {'field1': 'value1', 'field2': 'value2'};
+component.ngOnInit();
+
+expect(component.displayValue('field1')).toEqual('value1');
+
+component.saveValue('field1', '["value1","value2"]');
+expect(component.config['field1']).toEqual(['value1', 'value2']);
+
expect(component.displayValue('field1')).toEqual('["value1","value2"]');
+
+component.saveValue('field1', '["value1","value2"');
+expect(component.config['field1']).toEqual('["value1","value2"');
+expect(component.displayValue('field1')).toEqual('["value1","value2"');
+
+component.saveValue('field1', '{"key1":"value1"}');
+expect(component.config['field1']).toEqual({'key1': 'value1'});
+expect(component.displayValue('field1')).toEqual('{"key1":"value1"}');
+}));
+
 });

[4/7] metron git commit: METRON-1441: Create complementary Solr schemas for the main sensors this closes apache/metron#922

2018-02-07 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/f525dcfd/metron-platform/metron-solr/src/main/config/schema/metaalert/solrconfig.xml
--
diff --git 
a/metron-platform/metron-solr/src/main/config/schema/metaalert/solrconfig.xml 
b/metron-platform/metron-solr/src/main/config/schema/metaalert/solrconfig.xml
new file mode 100644
index 000..fff9d84
--- /dev/null
+++ 
b/metron-platform/metron-solr/src/main/config/schema/metaalert/solrconfig.xml
@@ -0,0 +1,1601 @@
+
+
+
+
+
+  
+
+  
+  7.2.0
+
+  
+
+  
+  
+  
+
+  
+  
+
+  
+  
+
+  
+
+  
+  
+
+  
+  
+
+  
+  ${solr.data.dir:}
+
+
+  
+  
+
+  
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+${solr.lock.type:native}
+
+
+
+  
+  
+  
+  
+  
+  
+
+
+
+ true
+  
+
+
+  
+  
+  
+  
+  
+  
+
+  
+  
+
+
+
+  ${solr.ulog.dir:}
+  ${solr.ulog.numVersionBuckets:65536}
+
+
+
+ 
+   ${solr.autoCommit.maxTime:15000}
+   false
+ 
+
+
+
+ 
+   ${solr.autoSoftCommit.maxTime:-1}
+ 
+
+
+
+  
+
+  
+  
+
+  
+  
+
+
+1024
+
+
+
+-1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+true
+
+   
+   
+
+   
+   20
+
+   
+   200
+
+   
+
+
+  
+
+  
+
+
+  
+
+  static firstSearcher warming in solrconfig.xml
+
+  
+
+
+
+false
+
+  
+
+
+  
+  
+
+
+
+
+
+
+
+
+  
+
+  
+  
+  
+
+ 
+   explicit
+   10
+   
+   
+   
+   false
+ 
+
+
+
+
+
+
+
+
+
+  
+  
+ 
+   explicit
+   json
+   true
+   text
+ 
+  
+
+  
+  
+ 
+   explicit
+
+   
+   velocity
+   browse
+   layout
+   Solritas
+
+   
+   edismax
+   
+  text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+  title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   100%
+   *:*
+   10
+   *,score
+
+   
+ text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+ title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename
+   3
+
+   
+   on
+   true
+   cat
+   manu_exact
+   content_type
+   author_s
+   ipod
+   GB
+   1
+   cat,inStock
+   after
+   price
+   0
+   600
+   50
+   popularity
+   0
+   10
+   3
+   manufacturedate_dt
+   NOW/YEAR-10YEARS
+   NOW
+   +1YEAR
+   before
+   after
+
+   
+   on
+   content features title name
+   true
+   html
+   b
+   /b
+   0
+   title
+   0
+   name
+   3
+   200
+   content
+   750
+
+   
+   on
+   false
+   5
+   2
+   5
+   true
+   true
+   5
+   3
+ 
+
+ 
+ 
+   spellcheck
+ 
+  
+
+
+  
+
+  text
+
+  
+
+  
+
+  
+  
+
+  true
+  ignored_
+
+  
+  true
+  links
+  ignored_
+
+  
+
+  
+
+   
+  
+
+text_general
+
+
+
+
+
+  default
+  text
+  solr.DirectSolrSpellChecker
+  
+  internal
+  
+  0.5
+  
+  2
+  
+  1
+  
+  5
+  
+  4
+  
+  0.01
+  
+
+
+
+
+  wordbreak
+  solr.WordBreakSolrSpellChecker
+  name
+  true
+  true
+  10
+
+
+
+
+
+
+
+
+
+
+  
+
+  
+  
+
+  
+  default
+  wordbreak
+  on
+  true
+  10
+  5
+  5
+  true
+  true
+  10
+  5
+
+
+  spellcheck
+
+  
+
+  
+  
+
+  mySuggester
+  FuzzyLookupFactory
+  DocumentDictionaryFactory
+  cat
+  price
+  string
+  false
+
+  
+
+  
+
+  true
+  10
+
+
+  suggest
+
+  
+
+
+  
+  
+
+  
+  
+
+  true
+
+
+  tvComponent
+
+  
+
+  
+  
+
+
+
+  lingo3g
+  true
+  com.carrotsearch.lingo3g.Lingo3GClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  lingo
+  org.carrot2.clustering.lingo.LingoClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  stc
+  org.carrot2.clustering.stc.STCClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  kmeans
+  org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm
+  clustering/carrot2
+
+  
+
+  
+  
+
+  true
+  true
+  
+  name
+  
+  id
+  
+  features
+  
+  true
+  
+  
+  
+  false
+
+

[3/7] metron git commit: METRON-1441: Create complementary Solr schemas for the main sensors this closes apache/metron#922

2018-02-07 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/f525dcfd/metron-platform/metron-solr/src/main/config/schema/snort/solrconfig.xml
--
diff --git 
a/metron-platform/metron-solr/src/main/config/schema/snort/solrconfig.xml 
b/metron-platform/metron-solr/src/main/config/schema/snort/solrconfig.xml
new file mode 100644
index 000..fff9d84
--- /dev/null
+++ b/metron-platform/metron-solr/src/main/config/schema/snort/solrconfig.xml
@@ -0,0 +1,1601 @@
+
+
+
+
+
+  
+
+  
+  7.2.0
+
+  
+
+  
+  
+  
+
+  
+  
+
+  
+  
+
+  
+
+  
+  
+
+  
+  
+
+  
+  ${solr.data.dir:}
+
+
+  
+  
+
+  
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+${solr.lock.type:native}
+
+
+
+  
+  
+  
+  
+  
+  
+
+
+
+ true
+  
+
+
+  
+  
+  
+  
+  
+  
+
+  
+  
+
+
+
+  ${solr.ulog.dir:}
+  ${solr.ulog.numVersionBuckets:65536}
+
+
+
+ 
+   ${solr.autoCommit.maxTime:15000}
+   false
+ 
+
+
+
+ 
+   ${solr.autoSoftCommit.maxTime:-1}
+ 
+
+
+
+  
+
+  
+  
+
+  
+  
+
+
+1024
+
+
+
+-1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+true
+
+   
+   
+
+   
+   20
+
+   
+   200
+
+   
+
+
+  
+
+  
+
+
+  
+
+  static firstSearcher warming in solrconfig.xml
+
+  
+
+
+
+false
+
+  
+
+
+  
+  
+
+
+
+
+
+
+
+
+  
+
+  
+  
+  
+
+ 
+   explicit
+   10
+   
+   
+   
+   false
+ 
+
+
+
+
+
+
+
+
+
+  
+  
+ 
+   explicit
+   json
+   true
+   text
+ 
+  
+
+  
+  
+ 
+   explicit
+
+   
+   velocity
+   browse
+   layout
+   Solritas
+
+   
+   edismax
+   
+  text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+  title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   100%
+   *:*
+   10
+   *,score
+
+   
+ text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+ title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename
+   3
+
+   
+   on
+   true
+   cat
+   manu_exact
+   content_type
+   author_s
+   ipod
+   GB
+   1
+   cat,inStock
+   after
+   price
+   0
+   600
+   50
+   popularity
+   0
+   10
+   3
+   manufacturedate_dt
+   NOW/YEAR-10YEARS
+   NOW
+   +1YEAR
+   before
+   after
+
+   
+   on
+   content features title name
+   true
+   html
+   b
+   /b
+   0
+   title
+   0
+   name
+   3
+   200
+   content
+   750
+
+   
+   on
+   false
+   5
+   2
+   5
+   true
+   true
+   5
+   3
+ 
+
+ 
+ 
+   spellcheck
+ 
+  
+
+
+  
+
+  text
+
+  
+
+  
+
+  
+  
+
+  true
+  ignored_
+
+  
+  true
+  links
+  ignored_
+
+  
+
+  
+
+   
+  
+
+text_general
+
+
+
+
+
+  default
+  text
+  solr.DirectSolrSpellChecker
+  
+  internal
+  
+  0.5
+  
+  2
+  
+  1
+  
+  5
+  
+  4
+  
+  0.01
+  
+
+
+
+
+  wordbreak
+  solr.WordBreakSolrSpellChecker
+  name
+  true
+  true
+  10
+
+
+
+
+
+
+
+
+
+
+  
+
+  
+  
+
+  
+  default
+  wordbreak
+  on
+  true
+  10
+  5
+  5
+  true
+  true
+  10
+  5
+
+
+  spellcheck
+
+  
+
+  
+  
+
+  mySuggester
+  FuzzyLookupFactory
+  DocumentDictionaryFactory
+  cat
+  price
+  string
+  false
+
+  
+
+  
+
+  true
+  10
+
+
+  suggest
+
+  
+
+
+  
+  
+
+  
+  
+
+  true
+
+
+  tvComponent
+
+  
+
+  
+  
+
+
+
+  lingo3g
+  true
+  com.carrotsearch.lingo3g.Lingo3GClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  lingo
+  org.carrot2.clustering.lingo.LingoClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  stc
+  org.carrot2.clustering.stc.STCClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  kmeans
+  org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm
+  clustering/carrot2
+
+  
+
+  
+  
+
+  true
+  true
+  
+  name
+  
+  id
+  
+  features
+  
+  true
+  
+  
+  
+  false
+
+  
+  edismax

[2/7] metron git commit: METRON-1441: Create complementary Solr schemas for the main sensors this closes apache/metron#922

2018-02-07 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/f525dcfd/metron-platform/metron-solr/src/main/config/schema/yaf/solrconfig.xml
--
diff --git 
a/metron-platform/metron-solr/src/main/config/schema/yaf/solrconfig.xml 
b/metron-platform/metron-solr/src/main/config/schema/yaf/solrconfig.xml
new file mode 100644
index 000..fff9d84
--- /dev/null
+++ b/metron-platform/metron-solr/src/main/config/schema/yaf/solrconfig.xml
@@ -0,0 +1,1601 @@
+
+
+
+
+
+  
+
+  
+  7.2.0
+
+  
+
+  
+  
+  
+
+  
+  
+
+  
+  
+
+  
+
+  
+  
+
+  
+  
+
+  
+  ${solr.data.dir:}
+
+
+  
+  
+
+  
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+${solr.lock.type:native}
+
+
+
+  
+  
+  
+  
+  
+  
+
+
+
+ true
+  
+
+
+  
+  
+  
+  
+  
+  
+
+  
+  
+
+
+
+  ${solr.ulog.dir:}
+  ${solr.ulog.numVersionBuckets:65536}
+
+
+
+ 
+   ${solr.autoCommit.maxTime:15000}
+   false
+ 
+
+
+
+ 
+   ${solr.autoSoftCommit.maxTime:-1}
+ 
+
+
+
+  
+
+  
+  
+
+  
+  
+
+
+1024
+
+
+
+-1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+true
+
+   
+   
+
+   
+   20
+
+   
+   200
+
+   
+
+
+  
+
+  
+
+
+  
+
+  static firstSearcher warming in solrconfig.xml
+
+  
+
+
+
+false
+
+  
+
+
+  
+  
+
+
+
+
+
+
+
+
+  
+
+  
+  
+  
+
+ 
+   explicit
+   10
+   
+   
+   
+   false
+ 
+
+
+
+
+
+
+
+
+
+  
+  
+ 
+   explicit
+   json
+   true
+   text
+ 
+  
+
+  
+  
+ 
+   explicit
+
+   
+   velocity
+   browse
+   layout
+   Solritas
+
+   
+   edismax
+   
+  text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+  title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   100%
+   *:*
+   10
+   *,score
+
+   
+ text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+ title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename
+   3
+
+   
+   on
+   true
+   cat
+   manu_exact
+   content_type
+   author_s
+   ipod
+   GB
+   1
+   cat,inStock
+   after
+   price
+   0
+   600
+   50
+   popularity
+   0
+   10
+   3
+   manufacturedate_dt
+   NOW/YEAR-10YEARS
+   NOW
+   +1YEAR
+   before
+   after
+
+   
+   on
+   content features title name
+   true
+   html
+   b
+   /b
+   0
+   title
+   0
+   name
+   3
+   200
+   content
+   750
+
+   
+   on
+   false
+   5
+   2
+   5
+   true
+   true
+   5
+   3
+ 
+
+ 
+ 
+   spellcheck
+ 
+  
+
+
+  
+
+  text
+
+  
+
+  
+
+  
+  
+
+  true
+  ignored_
+
+  
+  true
+  links
+  ignored_
+
+  
+
+  
+
+   
+  
+
+text_general
+
+
+
+
+
+  default
+  text
+  solr.DirectSolrSpellChecker
+  
+  internal
+  
+  0.5
+  
+  2
+  
+  1
+  
+  5
+  
+  4
+  
+  0.01
+  
+
+
+
+
+  wordbreak
+  solr.WordBreakSolrSpellChecker
+  name
+  true
+  true
+  10
+
+
+
+
+
+
+
+
+
+
+  
+
+  
+  
+
+  
+  default
+  wordbreak
+  on
+  true
+  10
+  5
+  5
+  true
+  true
+  10
+  5
+
+
+  spellcheck
+
+  
+
+  
+  
+
+  mySuggester
+  FuzzyLookupFactory
+  DocumentDictionaryFactory
+  cat
+  price
+  string
+  false
+
+  
+
+  
+
+  true
+  10
+
+
+  suggest
+
+  
+
+
+  
+  
+
+  
+  
+
+  true
+
+
+  tvComponent
+
+  
+
+  
+  
+
+
+
+  lingo3g
+  true
+  com.carrotsearch.lingo3g.Lingo3GClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  lingo
+  org.carrot2.clustering.lingo.LingoClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  stc
+  org.carrot2.clustering.stc.STCClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  kmeans
+  org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm
+  clustering/carrot2
+
+  
+
+  
+  
+
+  true
+  true
+  
+  name
+  
+  id
+  
+  features
+  
+  true
+  
+  
+  
+  false
+
+  
+  edismax
+

[1/7] metron git commit: METRON-1441: Create complementary Solr schemas for the main sensors this closes apache/metron#922

2018-02-07 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr c5b6a6f27 -> f525dcfd7


http://git-wip-us.apache.org/repos/asf/metron/blob/f525dcfd/metron-platform/metron-solr/src/test/resources/example_data/bro
--
diff --git a/metron-platform/metron-solr/src/test/resources/example_data/bro 
b/metron-platform/metron-solr/src/test/resources/example_data/bro
new file mode 100644
index 000..23d3235
--- /dev/null
+++ b/metron-platform/metron-solr/src/test/resources/example_data/bro
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{"adapter.threatinteladapter.end.ts":"1517499201357","bro_timestamp":"1517499194.7338","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1517499201202","enrichmentsplitterbolt.splitter.begin.ts":"1517499201200","adapter.hostfromjsonlistadapter.end.ts":"1517499201207","adapter.geoadapter.begin.ts":"1517499201209","uid":"CUrRne3iLIxXavQtci","trans_depth":143,"protocol":"http","original_string":"HTTP
 | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 
uri:\/api\/v1\/clusters\/metron_cluster\/services\/KAFKA\/components\/KAFKA_BROKER?fields=metrics\/kafka\/server\/BrokerTopicMetrics\/AllTopicsBytesInPerSec\/1MinuteRate[1484165330,1484168930,15],metrics\/kafka\/server\/BrokerTopicMetrics\/AllTopicsBytesOutPerSec\/1MinuteRate[1484165330,1484168930,15],metrics\/kafka\/server\/BrokerTopicMetrics\/AllTopicsMessagesInPerSec\/1MinuteRate[1484165330,1484168930,15],metrics\/kafka\/controller\/KafkaController\/ActiveControllerCount[1484165330,1484168930,15],metrics\/kafka\/c
 
ontroller\/ControllerStats\/LeaderElectionRateAndTimeMs\/1MinuteRate[1484165330,1484168930,15],metrics\/kafka\/controller\/ControllerStats\/UncleanLeaderElectionsPerSec\/1MinuteRate[1484165330,1484168930,15],metrics\/kafka\/server\/ReplicaFetcherManager\/Replica-MaxLag[1484165330,1484168930,15],metrics\/kafka\/server\/ReplicaManager\/PartitionCount[1484165330,1484168930,15],metrics\/kafka\/server\/ReplicaManager\/UnderReplicatedPartitions[1484165330,1484168930,15],metrics\/kafka\/server\/ReplicaManager\/LeaderCount[1484165330,1484168930,15]=null_padding&_=1484168930776
 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:143 
host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 
(Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) 
Chrome\/55.0.2883.95 Safari\/537.36 ts:1517499194.7338 
id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1517499201359","host":"node1","enrich
 
mentjoinbolt.joiner.ts":"1517499201212","adapter.hostfromjsonlistadapter.begin.ts":"1517499201206","threatintelsplitterbolt.splitter.begin.ts":"1517499201215","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0
 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) 
Chrome\/55.0.2883.95 
Safari\/537.36","timestamp":1517499194733,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/clusters\/metron_cluster\/services\/KAFKA\/components\/KAFKA_BROKER?fields=metrics\/kafka\/server\/BrokerTopicMetrics\/AllTopicsBytesInPerSec\/1MinuteRate[1484165330,1484168930,15],metrics\/kafka\/server\/BrokerTopicMetrics\/AllTopicsBytesOutPerSec\/1MinuteRate[1484165330,1484168930,15],metrics\/kafka\/server\/BrokerTopicMetrics\/AllTopicsMessagesInPerSec\/1MinuteRate[1484165330,1484168930,15],metrics\/kafka\/controller\/KafkaController\/ActiveControllerCount[1484165330,1484168930,15],metrics\/kafka\/controller\/ControllerStats\/LeaderElectionRateAndTimeMs\/1MinuteRate[1484165330,14

[7/7] metron git commit: METRON-1441: Create complementary Solr schemas for the main sensors this closes apache/metron#922

2018-02-07 Thread cestella

METRON-1441: Create complementary Solr schemas for the main sensors this closes 
apache/metron#922


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f525dcfd
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f525dcfd
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f525dcfd

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: f525dcfd75056554c96ad759559ebff628e2fb5f
Parents: c5b6a6f
Author: cstella 
Authored: Wed Feb 7 11:28:46 2018 -0500
Committer: cstella 
Committed: Wed Feb 7 11:28:46 2018 -0500

--
 .../docker/rpm-docker/SPECS/metron.spec |   10 +
 .../apache/metron/common/error/MetronError.java |6 +-
 .../apache/metron/common/utils/JSONUtils.java   |4 +-
 .../elasticsearch/dao/ElasticsearchDao.java |1 +
 .../ElasticsearchUpdateIntegrationTest.java |1 -
 .../indexing/dao/SearchIntegrationTest.java |1 +
 metron-platform/metron-solr/README.md   |7 +
 .../src/main/config/schema/bro/managed-schema   |  693 
 .../src/main/config/schema/bro/solrconfig.xml   | 1601 ++
 .../src/main/config/schema/error/managed-schema |   58 +
 .../src/main/config/schema/error/solrconfig.xml | 1601 ++
 .../main/config/schema/metaalert/managed-schema |   41 +
 .../main/config/schema/metaalert/solrconfig.xml | 1601 ++
 .../src/main/config/schema/snort/managed-schema |   86 +
 .../src/main/config/schema/snort/solrconfig.xml | 1601 ++
 .../src/main/config/schema/yaf/managed-schema   |   92 +
 .../src/main/config/schema/yaf/solrconfig.xml   | 1601 ++
 .../apache/metron/solr/dao/SolrSearchDao.java   |1 +
 .../apache/metron/solr/schema/FieldType.java|   98 ++
 .../metron/solr/schema/SchemaTranslator.java|  194 +++
 .../apache/metron/solr/writer/SolrWriter.java   |   15 +-
 .../schema/SchemaValidationIntegrationTest.java |  199 +++
 .../src/test/resources/example_data/bro |   21 +
 .../src/test/resources/example_data/error   |   17 +
 .../src/test/resources/example_data/snort   |   21 +
 .../src/test/resources/example_data/yaf |   21 +
 .../test/error/MetronErrorJSONMatcher.java  |2 +
 27 files changed, 9585 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/f525dcfd/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec 
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 3d25478..5f97f62 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -228,6 +228,16 @@ This package installs the Metron Solr files
 %{metron_home}/bin/install_solr.sh
 %{metron_home}/bin/start_solr_topology.sh
 %{metron_home}/config/solr.properties
+%{metron_home}/config/schema/bro/managed-schema
+%{metron_home}/config/schema/bro/solrconfig.xml
+%{metron_home}/config/schema/error/managed-schema
+%{metron_home}/config/schema/error/solrconfig.xml
+%{metron_home}/config/schema/metaalert/managed-schema
+%{metron_home}/config/schema/metaalert/solrconfig.xml
+%{metron_home}/config/schema/snort/managed-schema
+%{metron_home}/config/schema/snort/solrconfig.xml
+%{metron_home}/config/schema/yaf/managed-schema
+%{metron_home}/config/schema/yaf/solrconfig.xml
 %attr(0644,root,root) %{metron_home}/lib/metron-solr-%{full_version}-uber.jar
 
 # ~~

http://git-wip-us.apache.org/repos/asf/metron/blob/f525dcfd/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
index 9a553ca..bc02c5c 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
@@ -26,10 +26,7 @@ import org.json.simple.JSONObject;
 
 import java.net.InetAddress;
 import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
+import java.util.*;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.metron.common.Constants.ERROR_TYPE;
@@ -92,6 +89,7 @@ public class MetronError {
   @SuppressWarnings({"unchecked"})
   public JSONObject

[6/7] metron git commit: METRON-1441: Create complementary Solr schemas for the main sensors this closes apache/metron#922

2018-02-07 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/f525dcfd/metron-platform/metron-solr/src/main/config/schema/bro/solrconfig.xml
--
diff --git 
a/metron-platform/metron-solr/src/main/config/schema/bro/solrconfig.xml 
b/metron-platform/metron-solr/src/main/config/schema/bro/solrconfig.xml
new file mode 100644
index 000..fff9d84
--- /dev/null
+++ b/metron-platform/metron-solr/src/main/config/schema/bro/solrconfig.xml
@@ -0,0 +1,1601 @@
+
+
+
+
+
+  
+
+  
+  7.2.0
+
+  
+
+  
+  
+  
+
+  
+  
+
+  
+  
+
+  
+
+  
+  
+
+  
+  
+
+  
+  ${solr.data.dir:}
+
+
+  
+  
+
+  
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+${solr.lock.type:native}
+
+
+
+  
+  
+  
+  
+  
+  
+
+
+
+ true
+  
+
+
+  
+  
+  
+  
+  
+  
+
+  
+  
+
+
+
+  ${solr.ulog.dir:}
+  ${solr.ulog.numVersionBuckets:65536}
+
+
+
+ 
+   ${solr.autoCommit.maxTime:15000}
+   false
+ 
+
+
+
+ 
+   ${solr.autoSoftCommit.maxTime:-1}
+ 
+
+
+
+  
+
+  
+  
+
+  
+  
+
+
+1024
+
+
+
+-1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+true
+
+   
+   
+
+   
+   20
+
+   
+   200
+
+   
+
+
+  
+
+  
+
+
+  
+
+  static firstSearcher warming in solrconfig.xml
+
+  
+
+
+
+false
+
+  
+
+
+  
+  
+
+
+
+
+
+
+
+
+  
+
+  
+  
+  
+
+ 
+   explicit
+   10
+   
+   
+   
+   false
+ 
+
+
+
+
+
+
+
+
+
+  
+  
+ 
+   explicit
+   json
+   true
+   text
+ 
+  
+
+  
+  
+ 
+   explicit
+
+   
+   velocity
+   browse
+   layout
+   Solritas
+
+   
+   edismax
+   
+  text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+  title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   100%
+   *:*
+   10
+   *,score
+
+   
+ text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+ title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename
+   3
+
+   
+   on
+   true
+   cat
+   manu_exact
+   content_type
+   author_s
+   ipod
+   GB
+   1
+   cat,inStock
+   after
+   price
+   0
+   600
+   50
+   popularity
+   0
+   10
+   3
+   manufacturedate_dt
+   NOW/YEAR-10YEARS
+   NOW
+   +1YEAR
+   before
+   after
+
+   
+   on
+   content features title name
+   true
+   html
+   b
+   /b
+   0
+   title
+   0
+   name
+   3
+   200
+   content
+   750
+
+   
+   on
+   false
+   5
+   2
+   5
+   true
+   true
+   5
+   3
+ 
+
+ 
+ 
+   spellcheck
+ 
+  
+
+
+  
+
+  text
+
+  
+
+  
+
+  
+  
+
+  true
+  ignored_
+
+  
+  true
+  links
+  ignored_
+
+  
+
+  
+
+   
+  
+
+text_general
+
+
+
+
+
+  default
+  text
+  solr.DirectSolrSpellChecker
+  
+  internal
+  
+  0.5
+  
+  2
+  
+  1
+  
+  5
+  
+  4
+  
+  0.01
+  
+
+
+
+
+  wordbreak
+  solr.WordBreakSolrSpellChecker
+  name
+  true
+  true
+  10
+
+
+
+
+
+
+
+
+
+
+  
+
+  
+  
+
+  
+  default
+  wordbreak
+  on
+  true
+  10
+  5
+  5
+  true
+  true
+  10
+  5
+
+
+  spellcheck
+
+  
+
+  
+  
+
+  mySuggester
+  FuzzyLookupFactory
+  DocumentDictionaryFactory
+  cat
+  price
+  string
+  false
+
+  
+
+  
+
+  true
+  10
+
+
+  suggest
+
+  
+
+
+  
+  
+
+  
+  
+
+  true
+
+
+  tvComponent
+
+  
+
+  
+  
+
+
+
+  lingo3g
+  true
+  com.carrotsearch.lingo3g.Lingo3GClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  lingo
+  org.carrot2.clustering.lingo.LingoClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  stc
+  org.carrot2.clustering.stc.STCClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  kmeans
+  org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm
+  clustering/carrot2
+
+  
+
+  
+  
+
+  true
+  true
+  
+  name
+  
+  id
+  
+  features
+  
+  true
+  
+  
+  
+  false
+
+  
+  edismax
+

metron git commit: METRON-1442: Split rest end points for indexing topology into random access indexing and batch indexing this closes apache/incubator-metron#923

2018-02-05 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/master 06305052a -> fcff0596c


METRON-1442: Split rest end points for indexing topology into random access 
indexing and batch indexing this closes apache/incubator-metron#923


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fcff0596
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fcff0596
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fcff0596

Branch: refs/heads/master
Commit: fcff0596c7d2b2546d89283fb90fbc8c10b31f1f
Parents: 0630505
Author: MohanDV 
Authored: Mon Feb 5 09:48:47 2018 -0500
Committer: cstella 
Committed: Mon Feb 5 09:48:47 2018 -0500

--
 .../src/main/config/rest_application.yml|   3 +-
 .../apache/metron/rest/MetronRestConstants.java |   6 +-
 .../metron/rest/controller/StormController.java |  81 +++
 .../metron/rest/service/StormAdminService.java  |   4 +-
 .../service/impl/StormAdminServiceImpl.java |   8 +-
 .../rest/service/impl/StormCLIWrapper.java  |  16 +--
 .../src/main/resources/application-test.yml |   3 +-
 .../src/main/resources/application-vagrant.yml  |   4 +-
 .../StormControllerIntegrationTest.java | 102 ++-
 .../rest/mock/MockStormCLIClientWrapper.java|  93 -
 .../metron/rest/mock/MockStormRestTemplate.java |  22 ++--
 .../service/impl/StormAdminServiceImplTest.java |   8 +-
 .../rest/service/impl/StormCLIWrapperTest.java  |  17 ++--
 13 files changed, 233 insertions(+), 134 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/config/rest_application.yml
--
diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml 
b/metron-interface/metron-rest/src/main/config/rest_application.yml
index 6e4fb66..4cc51ff 100644
--- a/metron-interface/metron-rest/src/main/config/rest_application.yml
+++ b/metron-interface/metron-rest/src/main/config/rest_application.yml
@@ -46,7 +46,8 @@ storm:
   enrichment:
 script.path: ${METRON_HOME}/bin/start_enrichment_topology.sh
   indexing:
-script.path: ${METRON_HOME}/bin/start_elasticsearch_topology.sh
+randomaccess.script.path: 
${METRON_HOME}/bin/start_elasticsearch_topology.sh
+batch.script.path: ${METRON_HOME}/bin/start_hdfs_topology.sh
 
 kerberos:
   enabled: ${SECURITY_ENABLED}

http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
--
diff --git 
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
 
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index 4e8d7f2..f18d4cf 100644
--- 
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++ 
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -35,10 +35,12 @@ public class MetronRestConstants {
   public static final String TOPOLOGY_SUMMARY_URL = "/api/v1/topology/summary";
   public static final String TOPOLOGY_URL = "/api/v1/topology";
   public static final String ENRICHMENT_TOPOLOGY_NAME = "enrichment";
-  public static final String INDEXING_TOPOLOGY_NAME = "indexing";
+  public static final String BATCH_INDEXING_TOPOLOGY_NAME = "batch_indexing";
+  public static final String RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME = 
"random_access_indexing";
   public static final String PARSER_SCRIPT_PATH_SPRING_PROPERTY = 
"storm.parser.script.path";
   public static final String ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY = 
"storm.enrichment.script.path";
-  public static final String INDEXING_SCRIPT_PATH_SPRING_PROPERTY = 
"storm.indexing.script.path";
+  public static final String BATCH_INDEXING_SCRIPT_PATH_SPRING_PROPERTY = 
"storm.indexing.batch.script.path";
+  public static final String 
RANDOM_ACCESS_INDEXING_SCRIPT_PATH_SPRING_PROPERTY = 
"storm.indexing.randomaccess.script.path";
   public static final String PARSER_TOPOLOGY_OPTIONS_SPRING_PROPERTY = 
"storm.parser.topology.options";
   public static final String KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY = 
"kafka.security.protocol";
 

http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java
--
diff --git 
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java

[1/5] metron git commit: METRON-1419: Create a SolrDao this closes apache/incubator-metron#911

2018-02-01 Thread cestella

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr 644e951c8 -> 23113a633


http://git-wip-us.apache.org/repos/asf/metron/blob/23113a63/metron-platform/metron-solr/src/test/resources/config/test/conf/solrconfig.xml
--
diff --git 
a/metron-platform/metron-solr/src/test/resources/config/test/conf/solrconfig.xml
 
b/metron-platform/metron-solr/src/test/resources/config/test/conf/solrconfig.xml
new file mode 100644
index 000..fff9d84
--- /dev/null
+++ 
b/metron-platform/metron-solr/src/test/resources/config/test/conf/solrconfig.xml
@@ -0,0 +1,1601 @@
+
+
+
+
+
+  
+
+  
+  7.2.0
+
+  
+
+  
+  
+  
+
+  
+  
+
+  
+  
+
+  
+
+  
+  
+
+  
+  
+
+  
+  ${solr.data.dir:}
+
+
+  
+  
+
+  
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+${solr.lock.type:native}
+
+
+
+  
+  
+  
+  
+  
+  
+
+
+
+ true
+  
+
+
+  
+  
+  
+  
+  
+  
+
+  
+  
+
+
+
+  ${solr.ulog.dir:}
+  ${solr.ulog.numVersionBuckets:65536}
+
+
+
+ 
+   ${solr.autoCommit.maxTime:15000}
+   false
+ 
+
+
+
+ 
+   ${solr.autoSoftCommit.maxTime:-1}
+ 
+
+
+
+  
+
+  
+  
+
+  
+  
+
+
+1024
+
+
+
+-1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+true
+
+   
+   
+
+   
+   20
+
+   
+   200
+
+   
+
+
+  
+
+  
+
+
+  
+
+  static firstSearcher warming in solrconfig.xml
+
+  
+
+
+
+false
+
+  
+
+
+  
+  
+
+
+
+
+
+
+
+
+  
+
+  
+  
+  
+
+ 
+   explicit
+   10
+   
+   
+   
+   false
+ 
+
+
+
+
+
+
+
+
+
+  
+  
+ 
+   explicit
+   json
+   true
+   text
+ 
+  
+
+  
+  
+ 
+   explicit
+
+   
+   velocity
+   browse
+   layout
+   Solritas
+
+   
+   edismax
+   
+  text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+  title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   100%
+   *:*
+   10
+   *,score
+
+   
+ text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+ title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename
+   3
+
+   
+   on
+   true
+   cat
+   manu_exact
+   content_type
+   author_s
+   ipod
+   GB
+   1
+   cat,inStock
+   after
+   price
+   0
+   600
+   50
+   popularity
+   0
+   10
+   3
+   manufacturedate_dt
+   NOW/YEAR-10YEARS
+   NOW
+   +1YEAR
+   before
+   after
+
+   
+   on
+   content features title name
+   true
+   html
+   b
+   /b
+   0
+   title
+   0
+   name
+   3
+   200
+   content
+   750
+
+   
+   on
+   false
+   5
+   2
+   5
+   true
+   true
+   5
+   3
+ 
+
+ 
+ 
+   spellcheck
+ 
+  
+
+
+  
+
+  text
+
+  
+
+  
+
+  
+  
+
+  true
+  ignored_
+
+  
+  true
+  links
+  ignored_
+
+  
+
+  
+
+   
+  
+
+text_general
+
+
+
+
+
+  default
+  text
+  solr.DirectSolrSpellChecker
+  
+  internal
+  
+  0.5
+  
+  2
+  
+  1
+  
+  5
+  
+  4
+  
+  0.01
+  
+
+
+
+
+  wordbreak
+  solr.WordBreakSolrSpellChecker
+  name
+  true
+  true
+  10
+
+
+
+
+
+
+
+
+
+
+  
+
+  
+  
+
+  
+  default
+  wordbreak
+  on
+  true
+  10
+  5
+  5
+  true
+  true
+  10
+  5
+
+
+  spellcheck
+
+  
+
+  
+  
+
+  mySuggester
+  FuzzyLookupFactory
+  DocumentDictionaryFactory
+  cat
+  price
+  string
+  false
+
+  
+
+  
+
+  true
+  10
+
+
+  suggest
+
+  
+
+
+  
+  
+
+  
+  
+
+  true
+
+
+  tvComponent
+
+  
+
+  
+  
+
+
+
+  lingo3g
+  true
+  com.carrotsearch.lingo3g.Lingo3GClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  lingo
+  org.carrot2.clustering.lingo.LingoClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  stc
+  org.carrot2.clustering.stc.STCClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  kmeans
+  org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm
+  clustering/carrot2
+
+  
+
+  
+  
+
+  true
+  true
+

[5/5] metron git commit: METRON-1419: Create a SolrDao this closes apache/incubator-metron#911

2018-02-01 Thread cestella

METRON-1419: Create a SolrDao this closes apache/incubator-metron#911


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/23113a63
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/23113a63
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/23113a63

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 23113a6337a3fc4d0bfbb708303b30bf8122f23f
Parents: 644e951
Author: merrimanr 
Authored: Thu Feb 1 16:13:46 2018 -0500
Committer: cstella 
Committed: Thu Feb 1 16:13:46 2018 -0500

--
 dependencies_with_url.csv   |3 +
 .../elasticsearch/dao/ColumnMetadataDao.java|   67 -
 .../dao/ElasticsearchColumnMetadataDao.java |   31 +-
 .../elasticsearch/dao/ElasticsearchDao.java |  651 +--
 .../dao/ElasticsearchSearchDao.java |  565 ++
 .../dao/ElasticsearchUpdateDao.java |  130 ++
 .../elasticsearch/dao/ElasticsearchDaoTest.java |8 +-
 .../ElasticsearchSearchIntegrationTest.java |  112 ++
 .../ElasticsearchUpdateIntegrationTest.java |  219 +--
 .../metron/indexing/dao/ColumnMetadataDao.java  |   39 +
 .../metron/indexing/dao/search/SearchDao.java   |   34 +
 .../metron/indexing/dao/update/UpdateDao.java   |   30 +
 .../indexing/dao/SearchIntegrationTest.java |  215 +--
 .../indexing/dao/UpdateIntegrationTest.java |  199 +++
 metron-platform/metron-solr/pom.xml |   37 +-
 .../metron/solr/dao/SolrColumnMetadataDao.java  |  120 ++
 .../org/apache/metron/solr/dao/SolrDao.java |  117 ++
 .../apache/metron/solr/dao/SolrSearchDao.java   |  310 
 .../apache/metron/solr/dao/SolrUpdateDao.java   |  100 ++
 .../SolrIndexingIntegrationTest.java|   10 +-
 .../integration/SolrSearchIntegrationTest.java  |  153 ++
 .../integration/SolrUpdateIntegrationTest.java  |   73 +
 .../integration/components/SolrComponent.java   |   22 +-
 .../resources/config/bro/conf/managed-schema|   50 +
 .../resources/config/bro/conf/solrconfig.xml| 1601 ++
 .../resources/config/snort/conf/managed-schema  |   51 +
 .../resources/config/snort/conf/solrconfig.xml  | 1601 ++
 .../resources/config/test/conf/managed-schema   |   68 +
 .../resources/config/test/conf/solrconfig.xml   | 1601 ++
 pom.xml |2 +-
 30 files changed, 7251 insertions(+), 968 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/23113a63/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index a1f431b..2bf1c76 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -21,6 +21,7 @@ com.esotericsoftware:reflectasm:jar:1.10.1:compile,New BSD 
License,http://code.g
 com.flipkart.zjsonpatch:zjsonpatch:jar:0.3.4:compile,Apache v2, 
https://github.com/flipkart-incubator/zjsonpatch
 com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD 
license,http://code.google.com/p/protobuf
 com.google.protobuf:protobuf-java:jar:2.6.1:compile,New BSD 
license,http://code.google.com/p/protobuf
+com.google.protobuf:protobuf-java:jar:3.1.0:compile,New BSD 
license,http://code.google.com/p/protobuf
 com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/
 com.maxmind.db:maxmind-db:jar:1.2.1:compile,CC-BY-SA 
3.0,https://github.com/maxmind/MaxMind-DB
 com.maxmind.geoip2:geoip2:jar:2.8.0:compile,Apache 
v2,https://github.com/maxmind/GeoIP2-java
@@ -78,6 +79,7 @@ 
org.jvnet.jaxb2_commons:jaxb2-basics-runtime:jar:0.6.5:compile,BSD,https://githu
 org.krakenapps:kraken-api:jar:2.1.1:compile, Apache v2,
 org.krakenapps:kraken-pcap:jar:1.7.1:compile, Apache v2,
 org.ow2.asm:asm:jar:4.0:compile,BSD,http://asm.ow2.org/
+org.ow2.asm:asm:jar:5.1:compile,BSD,http://asm.ow2.org/
 org.slf4j:slf4j-api:jar:1.6.1:compile,MIT,http://www.slf4j.org
 org.slf4j:slf4j-api:jar:1.7.10:compile,MIT,http://www.slf4j.org
 org.slf4j:slf4j-api:jar:1.7.5:compile,MIT,http://www.slf4j.org
@@ -91,6 +93,7 @@ 
org.slf4j:slf4j-log4j12:jar:1.7.21:compile,MIT,http://www.slf4j.org
 org.slf4j:slf4j-log4j12:jar:1.7.5:compile,MIT,http://www.slf4j.org
 org.slf4j:slf4j-log4j12:jar:1.7.7:compile,MIT,http://www.slf4j.org
 org.slf4j:slf4j-simple:jar:1.7.7:compile,MIT,http://www.slf4j.org
+org.slf4j:jcl-over-slf4j:jar:1.7.7:compile,MIT,http://www.slf4j.org
 org.slf4j:jcl-over-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
 org.slf4j:jul-to-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
 aopalliance:aopalliance:jar:1.0:compile,Public 
Domain,http://aopalliance.sourceforge.net

[4/5] metron git commit: METRON-1419: Create a SolrDao this closes apache/incubator-metron#911

2018-02-01 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/23113a63/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
--
diff --git 
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
 
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
index 325d42e..dd29af3 100644
--- 
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
+++ 
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
@@ -19,203 +19,80 @@ package org.apache.metron.elasticsearch.integration;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.collect.Iterables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.metron.common.Constants;
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
 import 
org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
-import org.apache.metron.hbase.mock.MockHTable;
-import org.apache.metron.hbase.mock.MockHBaseTableProvider;
-import org.apache.metron.indexing.dao.*;
-import org.apache.metron.indexing.dao.update.Document;
-import org.apache.metron.indexing.dao.update.ReplaceRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.*;
+import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.UpdateIntegrationTest;
+import org.apache.metron.integration.InMemoryComponent;
 
+public class ElasticsearchUpdateIntegrationTest extends UpdateIntegrationTest {
 
-public class ElasticsearchUpdateIntegrationTest {
-  private static final int MAX_RETRIES = 10;
-  private static final int SLEEP_MS = 500;
   private static final String SENSOR_NAME= "test";
-  private static final String TABLE_NAME = "modifications";
-  private static final String CF = "p";
   private static String indexDir = "target/elasticsearch_mutation";
   private static String dateFormat = ".MM.dd.HH";
   private static String index = SENSOR_NAME + "_index_" + new 
SimpleDateFormat(dateFormat).format(new Date());
-  private static MockHTable table;
-  private static IndexDao esDao;
-  private static IndexDao hbaseDao;
-  private static MultiIndexDao dao;
   private static ElasticSearchComponent es;
 
-  @BeforeClass
-  public static void setup() throws Exception {
-Configuration config = HBaseConfiguration.create();
-MockHBaseTableProvider tableProvider = new MockHBaseTableProvider();
-tableProvider.addToCache(TABLE_NAME, CF);
-table = (MockHTable)tableProvider.getTable(config, TABLE_NAME);
-// setup the client
-es = new ElasticSearchComponent.Builder()
-.withHttpPort(9211)
-.withIndexDir(new File(indexDir))
-.build();
-es.start();
+  @Override
+  protected String getIndexName() {
+return SENSOR_NAME + "_index_" + new 
SimpleDateFormat(dateFormat).format(new Date());
+  }
 
-hbaseDao = new HBaseDao();
-AccessConfig accessConfig = new AccessConfig();
-accessConfig.setTableProvider(tableProvider);
-Map globalConfig = new HashMap() {{
+  @Override
+  protected Map createGlobalConfig() throws Exception {
+return new HashMap() {{
   put("es.clustername", "metron");
   put("es.port", "9300");
   put("es.ip", "localhost");
   put("es.date.format", dateFormat);
-  put(HBaseDao.HBASE_TABLE, TABLE_NAME);
-  put(HBaseDao.HBASE_CF, CF);
 }};
-accessConfig.setGlobalConfigSupplier(() -> globalConfig);
-
-esDao = new ElasticsearchDao();
-
-dao = new MultiIndexDao(hbaseDao, esDao);
-dao.init(accessConfig);
+  }
 
+  @Override
+  protected IndexDao createDao() throws Exception {
+return new ElasticsearchDao();
   }
 
-  @AfterClass
-  public static void teardown() {
-if(es != null) {
-  es.stop();
-}
+  @Override
+  protected InMemoryComponent startIndex() throws Exception {
+es = new ElasticSearchComponent.Builder()
+.withHttpPort(9211)
+.withIndexDir(new

[2/5] metron git commit: METRON-1419: Create a SolrDao this closes apache/incubator-metron#911

2018-02-01 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/23113a63/metron-platform/metron-solr/src/test/resources/config/snort/conf/solrconfig.xml
--
diff --git 
a/metron-platform/metron-solr/src/test/resources/config/snort/conf/solrconfig.xml
 
b/metron-platform/metron-solr/src/test/resources/config/snort/conf/solrconfig.xml
new file mode 100644
index 000..fff9d84
--- /dev/null
+++ 
b/metron-platform/metron-solr/src/test/resources/config/snort/conf/solrconfig.xml
@@ -0,0 +1,1601 @@
+
+
+
+
+
+  
+
+  
+  7.2.0
+
+  
+
+  
+  
+  
+
+  
+  
+
+  
+  
+
+  
+
+  
+  
+
+  
+  
+
+  
+  ${solr.data.dir:}
+
+
+  
+  
+
+  
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+${solr.lock.type:native}
+
+
+
+  
+  
+  
+  
+  
+  
+
+
+
+ true
+  
+
+
+  
+  
+  
+  
+  
+  
+
+  
+  
+
+
+
+  ${solr.ulog.dir:}
+  ${solr.ulog.numVersionBuckets:65536}
+
+
+
+ 
+   ${solr.autoCommit.maxTime:15000}
+   false
+ 
+
+
+
+ 
+   ${solr.autoSoftCommit.maxTime:-1}
+ 
+
+
+
+  
+
+  
+  
+
+  
+  
+
+
+1024
+
+
+
+-1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+true
+
+   
+   
+
+   
+   20
+
+   
+   200
+
+   
+
+
+  
+
+  
+
+
+  
+
+  static firstSearcher warming in solrconfig.xml
+
+  
+
+
+
+false
+
+  
+
+
+  
+  
+
+
+
+
+
+
+
+
+  
+
+  
+  
+  
+
+ 
+   explicit
+   10
+   
+   
+   
+   false
+ 
+
+
+
+
+
+
+
+
+
+  
+  
+ 
+   explicit
+   json
+   true
+   text
+ 
+  
+
+  
+  
+ 
+   explicit
+
+   
+   velocity
+   browse
+   layout
+   Solritas
+
+   
+   edismax
+   
+  text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+  title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   100%
+   *:*
+   10
+   *,score
+
+   
+ text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+ title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename
+   3
+
+   
+   on
+   true
+   cat
+   manu_exact
+   content_type
+   author_s
+   ipod
+   GB
+   1
+   cat,inStock
+   after
+   price
+   0
+   600
+   50
+   popularity
+   0
+   10
+   3
+   manufacturedate_dt
+   NOW/YEAR-10YEARS
+   NOW
+   +1YEAR
+   before
+   after
+
+   
+   on
+   content features title name
+   true
+   html
+   b
+   /b
+   0
+   title
+   0
+   name
+   3
+   200
+   content
+   750
+
+   
+   on
+   false
+   5
+   2
+   5
+   true
+   true
+   5
+   3
+ 
+
+ 
+ 
+   spellcheck
+ 
+  
+
+
+  
+
+  text
+
+  
+
+  
+
+  
+  
+
+  true
+  ignored_
+
+  
+  true
+  links
+  ignored_
+
+  
+
+  
+
+   
+  
+
+text_general
+
+
+
+
+
+  default
+  text
+  solr.DirectSolrSpellChecker
+  
+  internal
+  
+  0.5
+  
+  2
+  
+  1
+  
+  5
+  
+  4
+  
+  0.01
+  
+
+
+
+
+  wordbreak
+  solr.WordBreakSolrSpellChecker
+  name
+  true
+  true
+  10
+
+
+
+
+
+
+
+
+
+
+  
+
+  
+  
+
+  
+  default
+  wordbreak
+  on
+  true
+  10
+  5
+  5
+  true
+  true
+  10
+  5
+
+
+  spellcheck
+
+  
+
+  
+  
+
+  mySuggester
+  FuzzyLookupFactory
+  DocumentDictionaryFactory
+  cat
+  price
+  string
+  false
+
+  
+
+  
+
+  true
+  10
+
+
+  suggest
+
+  
+
+
+  
+  
+
+  
+  
+
+  true
+
+
+  tvComponent
+
+  
+
+  
+  
+
+
+
+  lingo3g
+  true
+  com.carrotsearch.lingo3g.Lingo3GClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  lingo
+  org.carrot2.clustering.lingo.LingoClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  stc
+  org.carrot2.clustering.stc.STCClusteringAlgorithm
+  clustering/carrot2
+
+
+
+  kmeans
+  org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm
+  clustering/carrot2
+
+  
+
+  
+  
+
+  true
+  true
+  
+  name
+  
+  id
+  
+  features
+  
+  true
+  
+  
+  
+

[3/5] metron git commit: METRON-1419: Create a SolrDao this closes apache/incubator-metron#911

2018-02-01 Thread cestella

http://git-wip-us.apache.org/repos/asf/metron/blob/23113a63/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema
--
diff --git 
a/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema 
b/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema
new file mode 100644
index 000..3bcb2bc
--- /dev/null
+++ 
b/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema
@@ -0,0 +1,50 @@
+
+
+
+   
+   
+
+   
+
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+
+   
+
+ guid
+
+
+
+
+
+
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/metron/blob/23113a63/metron-platform/metron-solr/src/test/resources/config/bro/conf/solrconfig.xml
--
diff --git 
a/metron-platform/metron-solr/src/test/resources/config/bro/conf/solrconfig.xml 
b/metron-platform/metron-solr/src/test/resources/config/bro/conf/solrconfig.xml
new file mode 100644
index 000..fff9d84
--- /dev/null
+++ 
b/metron-platform/metron-solr/src/test/resources/config/bro/conf/solrconfig.xml
@@ -0,0 +1,1601 @@
+
+
+
+
+
+  
+
+  
+  7.2.0
+
+  
+
+  
+  
+  
+
+  
+  
+
+  
+  
+
+  
+
+  
+  
+
+  
+  
+
+  
+  ${solr.data.dir:}
+
+
+  
+  
+
+  
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+${solr.lock.type:native}
+
+
+
+  
+  
+  
+  
+  
+  
+
+
+
+ true
+  
+
+
+  
+  
+  
+  
+  
+  
+
+  
+  
+
+
+
+  ${solr.ulog.dir:}
+  ${solr.ulog.numVersionBuckets:65536}
+
+
+
+ 
+   ${solr.autoCommit.maxTime:15000}
+   false
+ 
+
+
+
+ 
+   ${solr.autoSoftCommit.maxTime:-1}
+ 
+
+
+
+  
+
+  
+  
+
+  
+  
+
+
+1024
+
+
+
+-1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+true
+
+   
+   
+
+   
+   20
+
+   
+   200
+
+   
+
+
+  
+
+  
+
+
+  
+
+  static firstSearcher warming in solrconfig.xml
+
+  
+
+
+
+false
+
+  
+
+
+  
+  
+
+
+
+
+
+
+
+
+  
+
+  
+  
+  
+
+ 
+   explicit
+   10
+   
+   
+   
+   false
+ 
+
+
+
+
+
+
+
+
+
+  
+  
+ 
+   explicit
+   json
+   true
+   text
+ 
+  
+
+  
+  
+ 
+   explicit
+
+   
+   velocity
+   browse
+   layout
+   Solritas
+
+   
+   edismax
+   
+  text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+  title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   100%
+   *:*
+   10
+   *,score
+
+   
+ text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+ title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
+   
+   text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename
+   3
+
+   
+   on
+   true
+   cat
+   manu_exact
+   content_type
+   author_s
+   ipod
+   GB
+   1
+   cat,inStock
+   after
+   price
+   0
+   600
+   50
+   popularity
+   0
+   10
+   3
+   manufacturedate_dt
+   NOW/YEAR-10YEARS
+   NOW
+   +1YEAR
+   before
+   after
+
+   
+   on
+   content features title name
+   true
+   html
+   b
+   /b
+   0
+   title
+   0
+   name
+   3
+   200
+   content
+   750
+
+   
+   on
+   false
+   5
+   2
+   5
+   true
+   true
+   5
+   3
+ 
+
+ 
+ 
+   spellcheck
+ 
+  
+
+
+  
+
+  text
+
+  
+
+  
+
+  
+  
+
+  true
+  ignored_
+
+  
+  true
+  links
+  ignored_
+
+  
+
+  
+
+   
+  
+
+text_general
+
+
+
+
+
+  default
+  text
+  solr.DirectSolrSpellChecker
+  
+  internal
+  
+  0.5
+  
+  2
+  
+  1
+  
+  5
+  
+  4
+  
+  0.01
+  
+
+
+
+
+  wordbreak
+  solr.WordBreakSolrSpellChecker
+  name
+  true
+  true
+  10
+
+
+
+
+
+
+
+
+
+
+  
+
+  
+  
+
+  
+  default
+  wordbreak
+  on
+  true
+  10
+  5
+  5
+  true
+  true
+  10
+  5
+
+
+  spellcheck
+
+  
+
+  
+  
+
+  mySuggester
+  FuzzyLookupFactory
+  DocumentDictionaryFactory
+  cat
+  price
+  string
+  false
+
+  
+
+  
+
+  true
+  10
+
+
+  suggest
+
+  
+
+
+  
+  
+
+  
+  
+
+  true

1 2 3 >

1 - 100 of 208 matches

Mail list logo