metron git commit: METRON-1471: Migrate shuffle connections to local or shuffle closes apache/metron#949

2018-03-08 Thread cestella 
Repository: metron
Updated Branches:
  refs/heads/master abb152b83 -> 8bc7732e4


METRON-1471: Migrate shuffle connections to local or shuffle closes 
apache/metron#949


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8bc7732e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8bc7732e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8bc7732e

Branch: refs/heads/master
Commit: 8bc7732e4c904085e4f78469abded84635971632
Parents: abb152b
Author: cstella 
Authored: Thu Mar 8 17:17:57 2018 -0500
Committer: cstella 
Committed: Thu Mar 8 17:17:57 2018 -0500

--
 .../metron-profiler/src/main/flux/profiler/remote.yaml | 6 +++---
 .../metron-indexing/src/main/flux/indexing/batch/remote.yaml   | 4 ++--
 .../src/main/flux/indexing/random_access/remote.yaml   | 4 ++--
 .../apache/metron/parsers/topology/ParserTopologyBuilder.java  | 4 ++--
 4 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
--
diff --git 
a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml 
b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
index 752640a..9ec5ba4 100644
--- a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
+++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
@@ -168,7 +168,7 @@ streams:
 from: "kafkaSpout"
 to: "splitterBolt"
 grouping:
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE
 
 -   name: "splitter -> builder"
 from: "splitterBolt"
@@ -182,11 +182,11 @@ streams:
 to: "hbaseBolt"
 grouping:
 streamId: "hbase"
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE
 
 -   name: "builder -> kafka"
 from: "builderBolt"
 to: "kafkaBolt"
 grouping:
 streamId: "kafka"
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE

http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
--
diff --git 
a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml 
b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
index 85e3baa..3087dd9 100644
--- a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
+++ b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
@@ -158,7 +158,7 @@ streams:
 from: "kafkaSpout"
 to: "hdfsIndexingBolt"
 grouping:
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE
 
 
 -   name: "hdfsBolt -> errorIndexingBolt"
@@ -166,4 +166,4 @@ streams:
 to: "indexingErrorBolt"
 grouping:
 streamId: "error"
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE

http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
--
diff --git 
a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
 
b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
index cadc1ec..429ba45 100644
--- 
a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
+++ 
b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
@@ -130,11 +130,11 @@ streams:
 from: "kafkaSpout"
 to: "indexingBolt"
 grouping:
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE
 
 -   name: "indexingBolt -> errorIndexingBolt"
 from: "indexingBolt"
 to: "indexingErrorBolt"
 grouping:
 streamId: "error"
-type: SHUFFLE
+type: LOCAL_OR_SHUFFLE

http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
--
diff --git 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index 1039e56..2865dd6 100644
--- 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++ 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@

metron git commit: METRON-1482 Update REST to work with Solr (merrimanr) closes apache/metron#957

2018-03-08 Thread rmerriman 
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr a7a41a55a -> cdbe7ac1f


METRON-1482 Update REST to work with Solr (merrimanr) closes apache/metron#957


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/cdbe7ac1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/cdbe7ac1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/cdbe7ac1

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: cdbe7ac1f291c8d5be505973aaf01aa1d8a91920
Parents: a7a41a5
Author: merrimanr 
Authored: Thu Mar 8 14:08:22 2018 -0600
Committer: merrimanr 
Committed: Thu Mar 8 14:08:22 2018 -0600

--
 .../CURRENT/package/scripts/rest_commands.py|   1 +
 .../metron-rest/src/main/scripts/metron-rest.sh |  29 +++--
 .../metron/solr/dao/SolrMetaAlertDao.java   | 126 +++
 .../apache/metron/solr/dao/SolrSearchDao.java   |   5 +-
 4 files changed, 150 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/cdbe7ac1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index 4f717bb..c3d3ec1 100755
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -101,6 +101,7 @@ class RestCommands:
   "export METRON_INDEX_CP={metron_indexing_classpath};"
   "export METRON_LOG_DIR={metron_log_dir};"
   "export METRON_PID_FILE={pid_file};"
+  "export METRON_RA_INDEXING_WRITER={ra_indexing_writer};"
   "{metron_home}/bin/metron-rest.sh;"
   "unset METRON_JDBC_PASSWORD;"
 ))

http://git-wip-us.apache.org/repos/asf/metron/blob/cdbe7ac1/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
--
diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh 
b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
index f9a2b69..e0eeaa4 100644
--- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
+++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
@@ -84,19 +84,32 @@ if [ ${METRON_JDBC_CLIENT_PATH} ]; then
 METRON_REST_CLASSPATH+=":${METRON_JDBC_CLIENT_PATH}"
 fi
 
-# Use a custom indexing jar if provided, else pull the metron-elasticsearch 
uber jar
+# Use metron-elasticsearch uber jar by default
+indexing_jar_pattern="${METRON_HOME}/lib/metron-elasticsearch*uber.jar"
+# Use metron-solr uber jar if ra indexing writer set to Solr
+if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then
+indexing_jar_pattern="${METRON_HOME}/lib/metron-solr*uber.jar"
+fi
+# Use a custom indexing jar if provided
 if [ ${METRON_INDEX_CP} ]; then
-echo "Default metron indexing jar is: ${METRON_INDEX_CP}"
-METRON_REST_CLASSPATH+=":${METRON_INDEX_CP}"
-else
-indexing_jar_pattern="${METRON_HOME}/lib/metron-elasticsearch*uber.jar"
-indexing_files=( ${indexing_jar_pattern} )
-echo "Default metron indexing jar is: ${indexing_files[0]}"
-METRON_REST_CLASSPATH+=":${indexing_files[0]}"
+indexing_jar_pattern="${METRON_INDEX_CP}"
 fi
+indexing_files=( ${indexing_jar_pattern} )
+echo "Metron indexing jar is: ${indexing_files[0]}"
+METRON_REST_CLASSPATH+=":${indexing_files[0]}"
 
 echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}"
 
+#Use Solr daos if ra indexing writer set to Solr
+if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then
+METRON_INDEX_DAO=" 
--index.dao.impl=org.apache.metron.solr.dao.SolrDao,org.apache.metron.indexing.dao.HBaseDao"
+METRON_METAALERT_DAO=" 
--meta.dao.impl=org.apache.metron.solr.dao.SolrMetaAlertDao"
+echo "METRON_INDEX_DAO=${METRON_INDEX_DAO}"
+echo "METRON_METAALERT_DAO=${METRON_METAALERT_DAO}"
+METRON_SPRING_OPTIONS+=${METRON_INDEX_DAO}
+METRON_SPRING_OPTIONS+=${METRON_METAALERT_DAO}
+fi
+
 echo "Starting application"
 ${JAVA_HOME}/bin/java ${METRON_JVMFLAGS} \
 -cp ${METRON_REST_CLASSPATH} \

http://git-wip-us.apache.org/repos/asf/metron/blob/cdbe7ac1/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
--
diff --git

[2/2] metron git commit: METRON-1464 Convert schemas to be compatible with Solr 5.5.2 (merrimanr) closes apache/metron#945

2018-03-08 Thread rmerriman 
METRON-1464 Convert schemas to be compatible with Solr 5.5.2 (merrimanr) closes 
apache/metron#945


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a7a41a55
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a7a41a55
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a7a41a55

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: a7a41a55aa79b7fa3f64c51a4d968f011e8b11d1
Parents: 84cac7e
Author: merrimanr 
Authored: Thu Mar 8 11:52:42 2018 -0600
Committer: merrimanr 
Committed: Thu Mar 8 11:52:42 2018 -0600

--
 .../docker/rpm-docker/SPECS/metron.spec |  10 +-
 .../src/main/config/schema/bro/managed-schema   | 693 ---
 .../src/main/config/schema/bro/schema.xml   | 693 +++
 .../src/main/config/schema/error/managed-schema |  58 --
 .../src/main/config/schema/error/schema.xml |  58 ++
 .../main/config/schema/metaalert/managed-schema |  41 --
 .../src/main/config/schema/metaalert/schema.xml |  41 ++
 .../src/main/config/schema/snort/managed-schema |  86 ---
 .../src/main/config/schema/snort/schema.xml |  86 +++
 .../src/main/config/schema/yaf/managed-schema   |  92 ---
 .../src/main/config/schema/yaf/schema.xml   |  92 +++
 11 files changed, 975 insertions(+), 975 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/a7a41a55/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec 
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 5222efc..c7d37ed 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -232,15 +232,15 @@ This package installs the Metron Solr files
 %{metron_home}/bin/start_solr_topology.sh
 %{metron_home}/bin/stop_solr.sh
 %{metron_home}/config/solr.properties
-%{metron_home}/config/schema/bro/managed-schema
+%{metron_home}/config/schema/bro/schema.xml
 %{metron_home}/config/schema/bro/solrconfig.xml
-%{metron_home}/config/schema/error/managed-schema
+%{metron_home}/config/schema/error/schema.xml
 %{metron_home}/config/schema/error/solrconfig.xml
-%{metron_home}/config/schema/metaalert/managed-schema
+%{metron_home}/config/schema/metaalert/schema.xml
 %{metron_home}/config/schema/metaalert/solrconfig.xml
-%{metron_home}/config/schema/snort/managed-schema
+%{metron_home}/config/schema/snort/schema.xml
 %{metron_home}/config/schema/snort/solrconfig.xml
-%{metron_home}/config/schema/yaf/managed-schema
+%{metron_home}/config/schema/yaf/schema.xml
 %{metron_home}/config/schema/yaf/solrconfig.xml
 %attr(0644,root,root) %{metron_home}/lib/metron-solr-%{full_version}-uber.jar
 

http://git-wip-us.apache.org/repos/asf/metron/blob/a7a41a55/metron-platform/metron-solr/src/main/config/schema/bro/managed-schema
--
diff --git 
a/metron-platform/metron-solr/src/main/config/schema/bro/managed-schema 
b/metron-platform/metron-solr/src/main/config/schema/bro/managed-schema
deleted file mode 100644
index b473395..000
--- a/metron-platform/metron-solr/src/main/config/schema/bro/managed-schema
+++ /dev/null
@@ -1,693 +0,0 @@
-
-
-
-  
-  
-  
-
-  
-  
-  
-  
-  guid
-
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-
-  
-  
-  
-  
-  
-
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-

http://git-wip-us.apache.org/repos/asf/metron/blob/a7a41a55/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml

[1/2] metron git commit: METRON-1464 Convert schemas to be compatible with Solr 5.5.2 (merrimanr) closes apache/metron#945

2018-03-08 Thread rmerriman 
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr 84cac7e85 -> a7a41a55a


http://git-wip-us.apache.org/repos/asf/metron/blob/a7a41a55/metron-platform/metron-solr/src/main/config/schema/metaalert/managed-schema
--
diff --git 
a/metron-platform/metron-solr/src/main/config/schema/metaalert/managed-schema 
b/metron-platform/metron-solr/src/main/config/schema/metaalert/managed-schema
deleted file mode 100644
index 2138a92..000
--- 
a/metron-platform/metron-solr/src/main/config/schema/metaalert/managed-schema
+++ /dev/null
@@ -1,41 +0,0 @@
-
-
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  guid
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-

http://git-wip-us.apache.org/repos/asf/metron/blob/a7a41a55/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml
--
diff --git 
a/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml 
b/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml
new file mode 100644
index 000..e36c71e
--- /dev/null
+++ b/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml
@@ -0,0 +1,41 @@
+
+
+
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  guid
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+

http://git-wip-us.apache.org/repos/asf/metron/blob/a7a41a55/metron-platform/metron-solr/src/main/config/schema/snort/managed-schema
--
diff --git 
a/metron-platform/metron-solr/src/main/config/schema/snort/managed-schema 
b/metron-platform/metron-solr/src/main/config/schema/snort/managed-schema
deleted file mode 100644
index 2cdf3ab..000
--- a/metron-platform/metron-solr/src/main/config/schema/snort/managed-schema
+++ /dev/null
@@ -1,86 +0,0 @@
-
-
-
-  
-  
-
-  
-  
-  
-  
-  guid
-
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-
-  
-  
-  
-  
-  
-
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-

http://git-wip-us.apache.org/repos/asf/metron/blob/a7a41a55/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
--
diff --git 
a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml 
b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
new file mode 100644
index 000..ae13b8d
--- /dev/null
+++ b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
@@ -0,0 +1,86 @@
+
+
+
+  
+  
+
+  
+  
+  
+  
+  guid
+
+  
+  
+  
+  
+  
+
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+
+  
+  
+  
+  
+  
+  
+  
+  
+  
+
+  
+  
+
+  
+  
+  
+  
+  
+
+  
+  
+
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+

http://git-wip-us.apache.org/repos/asf/metron/blob/a7a41a55/metron-platform/metron-solr/src/main/config/schema/yaf/managed-schema
--
diff --git 
a/metron-platform/metron-solr/src/main/config/schema/yaf/managed-schema 
b/metron-platform/metron-solr/src/main/config/schema/yaf/managed-schema
deleted file mode 100644
index edf80da..000
--- a/metron-platform/metron-solr/src/main/config/schema/yaf/managed-schema
+++ /dev/null
@@ -1,92 +0,0 @@
-
-
-
-  
-  
-
-  
-  
-  
-  
-  guid
-
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-
-  
-  
-  
-  
-  
-
-  
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-

http://git-wip-us.apache.org/repos/asf/metron/blob/a7a41a55/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
--
diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml 
b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
new file mode 100644
index 000..edccd9d
--- /dev/null
+++ b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
@@ -0,0 +1,92 @@
+
+
+
+  
+  
+
+  
+  
+  
+  
+  guid
+
+  
+  
+  
+  
+  
+
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+
+  
+  
+  
+  
+  
+  
+  
+  
+  
+
+  
+  
+
+  
+  
+  
+  
+  
+
+  
+  
+
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+  
+