[12/13] metron git commit: METRON-1548 Remove hardcoded source:type from Alerts UI (justinleet) closes apache/metron#1010

2018-05-18 Thread leet 
METRON-1548 Remove hardcoded source:type from Alerts UI (justinleet) closes 
apache/metron#1010


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/71a015fa
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/71a015fa
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/71a015fa

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 71a015fabc47b23ea0c5400fa239fee9f5c0d194
Parents: a99cadb
Author: justinjleet 
Authored: Wed May 16 17:39:07 2018 -0400
Committer: justinjleet 
Committed: Fri May 18 13:32:29 2018 -0400

--
 .../rest/service/impl/SearchServiceImpl.java| 42 ++
 .../src/main/resources/application.yml  |  2 +-
 .../SearchControllerIntegrationTest.java|  9 ++-
 .../service/impl/SearchServiceImplTest.java | 60 +---
 4 files changed, 92 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/71a015fa/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
--
diff --git 
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
 
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
index e5bab08..21d158f 100644
--- 
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
+++ 
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
@@ -23,19 +23,26 @@ import static 
org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME;
 import static 
org.apache.metron.rest.MetronRestConstants.SEARCH_FACET_FIELDS_SPRING_PROPERTY;
 
 import com.google.common.collect.Lists;
+import java.io.IOException;
 import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.metron.common.Constants;
 import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.search.FieldType;
 import org.apache.metron.indexing.dao.search.GetRequest;
 import org.apache.metron.indexing.dao.search.GroupRequest;
 import org.apache.metron.indexing.dao.search.GroupResponse;
 import org.apache.metron.indexing.dao.search.InvalidSearchException;
 import org.apache.metron.indexing.dao.search.SearchRequest;
 import org.apache.metron.indexing.dao.search.SearchResponse;
-import org.apache.metron.indexing.dao.search.FieldType;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.model.AlertsUIUserSettings;
 import org.apache.metron.rest.service.AlertsUIService;
+import org.apache.metron.rest.service.GlobalConfigService;
 import org.apache.metron.rest.service.SearchService;
 import org.apache.metron.rest.service.SensorIndexingConfigService;
 import org.slf4j.Logger;
@@ -44,11 +51,6 @@ import 
org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.env.Environment;
 import org.springframework.stereotype.Service;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Optional;
-import java.util.List;
-
 @Service
 public class SearchServiceImpl implements SearchService {
 
@@ -57,14 +59,19 @@ public class SearchServiceImpl implements SearchService {
   private IndexDao dao;
   private Environment environment;
   private SensorIndexingConfigService sensorIndexingConfigService;
+  private GlobalConfigService globalConfigService;
   private AlertsUIService alertsUIService;
 
   @Autowired
-  public SearchServiceImpl(IndexDao dao, Environment environment,
-  SensorIndexingConfigService sensorIndexingConfigService, AlertsUIService 
alertsUIService) {
+  public SearchServiceImpl(IndexDao dao,
+  Environment environment,
+  SensorIndexingConfigService sensorIndexingConfigService,
+  GlobalConfigService globalConfigService,
+  AlertsUIService alertsUIService) {
 this.dao = dao;
 this.environment = environment;
 this.sensorIndexingConfigService = sensorIndexingConfigService;
+this.globalConfigService = globalConfigService;
 this.alertsUIService = alertsUIService;
   }
 
@@ -133,11 +140,24 @@ public class SearchServiceImpl implements SearchService {
 return indices;
   }
 
-  private List getDefaultFacetFields() throws RestException {
+  @SuppressWarnings("unchecked")
+  public List getDefaultFacetFields() throws RestException {
 Optional alertUserSettings = 
alertsUIService.getAlertsUIUserSettings();
 if (!alertUserSettings.isPresent() || 
alertUserSettings.get().getFacetFields() == null) {
-  String facetFieldsProperty =

[06/13] metron git commit: METRON-1541 Mvn clean results in git status having deleted files. (justinleet via nickwallen) closes apache/metron#1003

2018-05-18 Thread leet 
METRON-1541 Mvn clean results in git status having deleted files. (justinleet 
via nickwallen) closes apache/metron#1003


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a17c1adf
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a17c1adf
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a17c1adf

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: a17c1adfd4cca63a49ef542c21cef90b18c74be0
Parents: d7edce9
Author: justinleet 
Authored: Wed May 9 16:47:29 2018 -0400
Committer: nickallen 
Committed: Wed May 9 16:47:29 2018 -0400

--
 metron-deployment/packaging/ambari/.gitignore   |  2 +
 .../enrichment-splitjoin.properties.j2  | 63 
 .../templates/enrichment-unified.properties.j2  | 60 ---
 3 files changed, 2 insertions(+), 123 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/a17c1adf/metron-deployment/packaging/ambari/.gitignore
--
diff --git a/metron-deployment/packaging/ambari/.gitignore 
b/metron-deployment/packaging/ambari/.gitignore
index ca2e75c..2f93166 100644
--- a/metron-deployment/packaging/ambari/.gitignore
+++ b/metron-deployment/packaging/ambari/.gitignore
@@ -3,3 +3,5 @@ archive.zip
 elasticsearch.properties.j2
 hdfs.properties.j2
 enrichment.properties.j2
+enrichment-splitjoin.properties.j2
+enrichment-unified.properties.j2

http://git-wip-us.apache.org/repos/asf/metron/blob/a17c1adf/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
deleted file mode 100644
index a0b21c9..000
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
+++ /dev/null
@@ -1,63 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-# Storm #
-enrichment.workers={{enrichment_workers}}
-enrichment.acker.executors={{enrichment_acker_executors}}
-topology.worker.childopts={{enrichment_topology_worker_childopts}}
-topology.auto-credentials={{topology_auto_credentials}}
-topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
-
-# Kafka #
-kafka.zk={{zookeeper_quorum}}
-kafka.broker={{kafka_brokers}}
-kafka.security.protocol={{kafka_security_protocol}}
-
-# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
-kafka.start={{enrichment_kafka_start}}
-
-enrichment.input.topic={{enrichment_input_topic}}
-enrichment.output.topic={{enrichment_output_topic}}
-enrichment.error.topic={{enrichment_error_topic}}
-threat.intel.error.topic={{threatintel_error_topic}}
-
-# JoinBolt #
-enrichment.join.cache.size={{enrichment_join_cache_size}}
-threat.intel.join.cache.size={{threatintel_join_cache_size}}
-
-# Enrichment #
-hbase.provider.impl={{enrichment_hbase_provider_impl}}
-enrichment.simple.hbase.table={{enrichment_hbase_table}}
-enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
-enrichment.host.known_hosts={{enrichment_host_known_hosts}}
-
-# Threat Intel #
-threat.intel.tracker.table={{threatintel_hbase_table}}
-threat.intel.tracker.cf={{threatintel_hbase_cf}}
-threat.intel.simple.hbase.table={{threatintel_hbase_table}}
-threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
-
-# Parallelism #
-kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}}
-enrichment.split.parallelism={{enrichment_split_parallelism}}
-enrichment.stellar.parallelism={{enrichment_stellar_parallelism}}

[04/13] metron git commit: METRON-1184 EC2 Deployment - Updating control_path to accommodate for Linux (Ahmed Shah via ottobackwards) closes apache/metron#754

2018-05-18 Thread leet 
METRON-1184 EC2 Deployment - Updating control_path to accommodate for Linux 
(Ahmed Shah via ottobackwards) closes apache/metron#754


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ca4644ba
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ca4644ba
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ca4644ba

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: ca4644ba5e94c579fed6a7c7d1e8bea05fe9ba9c
Parents: 1b1a45b
Author: Ahmed Shah 
Authored: Wed May 9 14:22:50 2018 -0400
Committer: otto 
Committed: Wed May 9 14:22:50 2018 -0400

--
 metron-deployment/amazon-ec2/ansible.cfg | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/ca4644ba/metron-deployment/amazon-ec2/ansible.cfg
--
diff --git a/metron-deployment/amazon-ec2/ansible.cfg 
b/metron-deployment/amazon-ec2/ansible.cfg
index 1f4f0ea..0fe3343 100644
--- a/metron-deployment/amazon-ec2/ansible.cfg
+++ b/metron-deployment/amazon-ec2/ansible.cfg
@@ -24,5 +24,8 @@ forks = 20
 log_path = ./ansible.log
 
 # fix for "ssh throws 'unix domain socket too long' " problem
+#[ssh_connection]
+#control_path = ~/.ssh/ansible-ssh-%%C
+
 [ssh_connection]
-control_path = ~/.ssh/ansible-ssh-%%C
+control_path = ~/.ssh/ansbile-ssh-%%h-%%r

[08/13] metron git commit: METRON-1551 Profiler Should Not Use Java Serialization (nickwallen) closes apache/metron#1012

2018-05-18 Thread leet 
METRON-1551 Profiler Should Not Use Java Serialization (nickwallen) closes 
apache/metron#1012


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9ce4ba5a
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9ce4ba5a
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9ce4ba5a

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 9ce4ba5a9c8febc3a3ed6992f73f99396d7248a9
Parents: b9453aa
Author: nickwallen 
Authored: Mon May 14 17:00:34 2018 -0400
Committer: nickallen 
Committed: Mon May 14 17:00:34 2018 -0400

--
 .../metron/profiler/ProfileMeasurement.java |   3 +-
 .../apache/metron/profiler/ProfilePeriod.java   |   3 +-
 .../metron/profiler/ProfileMeasurementTest.java | 108 +++
 .../metron/profiler/ProfilePeriodTest.java  |  49 +
 metron-analytics/metron-profiler/README.md  |  62 +++
 .../src/main/config/profiler.properties |  13 +++
 .../src/main/flux/profiler/remote.yaml  |   3 +
 .../zookeeper/profile-with-stats/profiler.json  |  12 +++
 .../integration/ProfilerIntegrationTest.java|  83 +-
 .../statistics/sampling/UniformSampler.java |   5 +-
 .../package/templates/profiler.properties.j2|  13 +++
 .../configuration/profiler/ProfileResult.java   |   4 +-
 .../profiler/ProfileResultExpressions.java  |   4 +-
 .../profiler/ProfileTriageExpressions.java  |   3 +-
 .../configuration/profiler/ProfilerConfig.java  |  12 +--
 .../apache/metron/common/utils/SerDeUtils.java  |   7 +-
 .../profiler/ProfilerConfigTest.java|  85 ++-
 .../stellar/common/utils/BloomFilter.java   |   8 +-
 .../metron/stellar/common/utils/SerDeUtils.java |  16 +--
 .../common/utils/StellarProcessorUtils.java | 101 +++--
 20 files changed, 537 insertions(+), 57 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/9ce4ba5a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
--
diff --git 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
index f6cc286..4737c3d 100644
--- 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
+++ 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
@@ -22,6 +22,7 @@ package org.apache.metron.profiler;
 
 import org.apache.metron.common.configuration.profiler.ProfileConfig;
 
+import java.io.Serializable;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -33,7 +34,7 @@ import java.util.concurrent.TimeUnit;
  * A profile contains many individual {@link ProfileMeasurement} values 
captured over a
  * period of time.  These values in aggregate form a time series.
  */
-public class ProfileMeasurement {
+public class ProfileMeasurement implements Serializable {
 
   /**
* The name of the profile that this measurement is associated with.

http://git-wip-us.apache.org/repos/asf/metron/blob/9ce4ba5a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
--
diff --git 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
index c2d8b21..cbb8275 100644
--- 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
+++ 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
@@ -20,6 +20,7 @@
 
 package org.apache.metron.profiler;
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -33,7 +34,7 @@ import static java.lang.String.format;
  * The Profiler captures a ProfileMeasurement once every ProfilePeriod.  There 
can be
  * multiple ProfilePeriods every hour.
  */
-public class ProfilePeriod {
+public class ProfilePeriod implements Serializable {
 
   /**
* A monotonically increasing number identifying the period.  The first 
period is 0

http://git-wip-us.apache.org/repos/asf/metron/blob/9ce4ba5a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileMeasurementTest.java
--
diff --git

[13/13] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr

2018-05-18 Thread leet 
Merge branch 'master' into feature/METRON-1416-upgrade-solr


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/eb336665
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/eb336665
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/eb336665

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: eb3366656de643e7436f7cf280561b94f3921c65
Parents: bc16e68 71a015f
Author: justinjleet 
Authored: Fri May 18 14:21:24 2018 -0400
Committer: justinjleet 
Committed: Fri May 18 14:21:24 2018 -0400

--
 dependencies_with_url.csv   |  56 ++-
 .../metron/profiler/ProfileMeasurement.java |   3 +-
 .../apache/metron/profiler/ProfilePeriod.java   |   3 +-
 .../metron/profiler/ProfileMeasurementTest.java | 108 +
 .../metron/profiler/ProfilePeriodTest.java  |  49 +++
 metron-analytics/metron-profiler/README.md  |  62 ++-
 .../src/main/config/profiler.properties |  13 +
 .../src/main/flux/profiler/remote.yaml  |   3 +
 .../zookeeper/profile-with-stats/profiler.json  |  12 +
 .../integration/ProfilerIntegrationTest.java|  83 +++-
 .../metron/statistics/StatisticsProvider.java   |   6 +-
 .../statistics/sampling/UniformSampler.java |   5 +-
 metron-deployment/amazon-ec2/ansible.cfg|   5 +-
 .../roles/ambari_config/vars/single_node_vm.yml |   1 +
 metron-deployment/packaging/ambari/.gitignore   |   2 +
 .../enrichment-splitjoin.properties.j2  |  63 ---
 .../templates/enrichment-unified.properties.j2  |  60 ---
 .../package/templates/profiler.properties.j2|  13 +
 metron-interface/metron-alerts/README.md|   6 +
 metron-interface/metron-alerts/proxy.conf.json  |   9 -
 .../alert-details/alert-details.component.ts|   2 +-
 .../alert-details/alerts-details.routing.ts |   2 +-
 .../alerts/alerts-list/alerts-list.component.ts |  25 +-
 .../table-view/table-view.component.ts  |  32 +-
 .../tree-view/tree-view.component.ts|  17 +-
 .../alerts/meta-alerts/meta-alerts.component.ts |  20 +-
 .../metron-alerts/src/app/app.module.ts |   4 +-
 .../src/app/model/search-request.ts |   2 +-
 .../src/app/service/authentication.service.ts   |  10 +-
 .../service/elasticsearch-localstorage-impl.ts  |   6 +-
 .../src/app/service/global-config.service.ts|  50 +++
 .../src/app/service/update.service.ts   |  10 +-
 .../metron-alerts/src/app/utils/constants.ts|   4 +-
 .../metron-alerts/src/app/utils/utils.ts|   7 +-
 metron-interface/metron-config/proxy.conf.json  |   4 +-
 metron-interface/metron-rest/README.md  |   2 +
 metron-interface/metron-rest/pom.xml|  17 +-
 .../metron/rest/MetronRestApplication.java  |   7 +
 .../apache/metron/rest/MetronRestConstants.java |   2 +
 .../metron/rest/config/JpaConfiguration.java|   6 +-
 .../metron/rest/config/WebSecurityConfig.java   |  19 +-
 .../rest/service/impl/SearchServiceImpl.java|  42 +-
 .../src/main/resources/application-test.yml |   2 +-
 .../src/main/resources/application.yml  |   2 +-
 .../metron-rest/src/main/scripts/metron-rest.sh |   2 +-
 .../SearchControllerIntegrationTest.java|   9 +-
 .../service/impl/SearchServiceImplTest.java |  60 ++-
 metron-platform/metron-common/README.md |  15 +-
 .../org/apache/metron/common/Constants.java |  10 +-
 .../configuration/SensorParserConfig.java   | 410 +++
 .../configuration/profiler/ProfileResult.java   |   4 +-
 .../profiler/ProfileResultExpressions.java  |   4 +-
 .../profiler/ProfileTriageExpressions.java  |   3 +-
 .../configuration/profiler/ProfilerConfig.java  |  12 +-
 .../common/utils/CompressionStrategies.java | 100 +
 .../common/utils/CompressionStrategy.java   |  52 +++
 .../apache/metron/common/utils/SerDeUtils.java  |   7 +-
 .../profiler/ProfilerConfigTest.java|  85 +++-
 .../common/utils/CompressionUtilsTest.java  |  62 +++
 .../nonbulk/geo/GeoEnrichmentLoader.java|  70 +++-
 .../nonbulk/geo/GeoEnrichmentLoaderTest.java|  40 +-
 .../parsers/topology/ParserTopologyBuilder.java | 139 ---
 .../parsers/topology/ParserTopologyCLI.java | 147 +--
 .../components/ParserTopologyComponent.java |  80 ++--
 .../parsers/topology/ParserTopologyCLITest.java | 122 --
 ...pleHbaseEnrichmentWriterIntegrationTest.java |  69 ++--
 .../integration/WriterBoltIntegrationTest.java  | 382 +++--
 .../apache/metron/writer/kafka/KafkaWriter.java |   5 +
 .../stellar/common/utils/BloomFilter.java   |   8 +-
 .../metron/stellar/common/utils/SerDeUtils.java |  16 +-
 .../common/utils/StellarProcessorUtils.java | 101 -
 .../metron/stellar/dsl/functions/Ordinal.java   |  36 ++
 .../stellar/dsl/functions/OrdinalFunctions.java |  48 ++-

[09/13] metron git commit: METRON-1552: Add gzip file validation check to the geo loader (mmiklavc via mmiklavc) closes apache/metron#1011

2018-05-18 Thread leet 
METRON-1552: Add gzip file validation check to the geo loader (mmiklavc via 
mmiklavc) closes apache/metron#1011


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/20eaed23
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/20eaed23
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/20eaed23

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 20eaed239b2552d0823d34f571b63d941c352bc9
Parents: 9ce4ba5
Author: mmiklavc 
Authored: Tue May 15 11:12:07 2018 -0600
Committer: Michael Miklavcic 
Committed: Tue May 15 11:12:07 2018 -0600

--
 .../common/utils/CompressionStrategies.java | 100 +++
 .../common/utils/CompressionStrategy.java   |  52 ++
 .../common/utils/CompressionUtilsTest.java  |  62 
 .../nonbulk/geo/GeoEnrichmentLoader.java|  70 +
 .../nonbulk/geo/GeoEnrichmentLoaderTest.java|  40 ++--
 5 files changed, 298 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/20eaed23/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
new file mode 100644
index 000..f9c53c8
--- /dev/null
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/CompressionStrategies.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.utils;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipException;
+
+/*
+ * Factory to provide various compression strategies.
+ */
+public enum CompressionStrategies implements CompressionStrategy {
+
+  GZIP(new CompressionStrategy() {
+@Override
+public void compress(File inFile, File outFile) throws IOException {
+  try (FileInputStream fis = new FileInputStream(inFile);
+  FileOutputStream fos = new FileOutputStream(outFile);
+  GZIPOutputStream gzipOS = new GZIPOutputStream(fos)) {
+byte[] buffer = new byte[1024];
+int len;
+while ((len = fis.read(buffer)) != -1) {
+  gzipOS.write(buffer, 0, len);
+}
+  }
+}
+
+@Override
+public void decompress(File inFile, File outFile) throws IOException {
+  try (FileInputStream fis = new FileInputStream(inFile);
+  GZIPInputStream gis = new GZIPInputStream(fis);
+  FileOutputStream fos = new FileOutputStream(outFile)) {
+byte[] buffer = new byte[1024];
+int len;
+while ((len = gis.read(buffer)) != -1) {
+  fos.write(buffer, 0, len);
+}
+  }
+
+}
+
+@Override
+public boolean test(File gzipFile) {
+  try (FileInputStream fis = new FileInputStream(gzipFile);
+  GZIPInputStream gis = new GZIPInputStream(fis)) {
+byte[] buffer = new byte[1024];
+// this will throw an exception on malformed file
+gis.read(buffer);
+  } catch (ZipException | EOFException e) {
+return false;
+  } catch (IOException e) {
+throw new IllegalStateException("Error occurred while attempting to 
validate gzip file", e);
+  }
+  return true;
+}
+  });
+
+  private CompressionStrategy strategy;
+
+  CompressionStrategies(CompressionStrategy strategy) {
+this.strategy = strategy;
+  }
+
+  @Override
+  public void compress(File inFile, File outFile) throws IOException {
+strategy.compress(inFile, outFile);
+  }
+
+  @Override
+  public void decompress(File inFile, File outFile) throws

[03/13] metron git commit: METRON-1530 Default proxy config settings in metron-contrib need to be updated (sardell via merrimanr) closes apache/metron#998

2018-05-18 Thread leet 
METRON-1530 Default proxy config settings in metron-contrib need to be updated 
(sardell via merrimanr) closes apache/metron#998


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1b1a45b7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1b1a45b7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1b1a45b7

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 1b1a45b7e9c2b8c451f970028b4fe310ef9b7894
Parents: ac05638
Author: sardell 
Authored: Wed May 9 09:43:58 2018 -0500
Committer: merrimanr 
Committed: Wed May 9 09:43:58 2018 -0500

--
 metron-interface/metron-alerts/proxy.conf.json | 9 -
 metron-interface/metron-config/proxy.conf.json | 4 ++--
 2 files changed, 2 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/1b1a45b7/metron-interface/metron-alerts/proxy.conf.json
--
diff --git a/metron-interface/metron-alerts/proxy.conf.json 
b/metron-interface/metron-alerts/proxy.conf.json
index 27d1ee9..612bd67 100644
--- a/metron-interface/metron-alerts/proxy.conf.json
+++ b/metron-interface/metron-alerts/proxy.conf.json
@@ -6,14 +6,5 @@
   "/logout": {
 "target": "http://node1:8082;,
 "secure": false
-  },
-  "/search": {
-"target": "http://node1:9200;,
-"pathRewrite": {"^/search" : ""},
-"secure": false
-  },
-  "/_cluster": {
-"target": "http://node1:9200;,
-"secure": false
   }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/1b1a45b7/metron-interface/metron-config/proxy.conf.json
--
diff --git a/metron-interface/metron-config/proxy.conf.json 
b/metron-interface/metron-config/proxy.conf.json
index 29466cc..612bd67 100644
--- a/metron-interface/metron-config/proxy.conf.json
+++ b/metron-interface/metron-config/proxy.conf.json
@@ -1,10 +1,10 @@
 {
   "/api/v1": {
-"target": "http://localhost:8080;,
+"target": "http://node1:8082;,
 "secure": false
   },
   "/logout": {
-"target": "http://localhost:8080;,
+"target": "http://node1:8082;,
 "secure": false
   }
 }

[11/13] metron git commit: METRON-1548 Remove hardcoded source:type from Alerts UI (sardell via justinleet) closes apache/metron#1010

2018-05-18 Thread leet 
METRON-1548 Remove hardcoded source:type from Alerts UI (sardell via 
justinleet) closes apache/metron#1010


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a99cadb0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a99cadb0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a99cadb0

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: a99cadb018b96cf96955df4b6e4acade28530a41
Parents: 4458085
Author: Shane Ardell 
Authored: Mon May 7 13:19:42 2018 +0200
Committer: justinjleet 
Committed: Fri May 18 13:32:29 2018 -0400

--
 metron-interface/metron-alerts/README.md|  6 +++
 .../alert-details/alert-details.component.ts|  2 +-
 .../alert-details/alerts-details.routing.ts |  2 +-
 .../alerts/alerts-list/alerts-list.component.ts | 25 --
 .../table-view/table-view.component.ts  | 32 +++--
 .../tree-view/tree-view.component.ts| 17 +--
 .../alerts/meta-alerts/meta-alerts.component.ts | 20 ++--
 .../metron-alerts/src/app/app.module.ts |  4 +-
 .../src/app/model/search-request.ts |  2 +-
 .../src/app/service/authentication.service.ts   | 10 +++-
 .../service/elasticsearch-localstorage-impl.ts  |  6 ++-
 .../src/app/service/global-config.service.ts| 50 
 .../src/app/service/update.service.ts   | 10 +++-
 .../metron-alerts/src/app/utils/constants.ts|  4 +-
 .../metron-alerts/src/app/utils/utils.ts|  7 +--
 metron-platform/metron-common/README.md | 15 +++---
 16 files changed, 174 insertions(+), 38 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/a99cadb0/metron-interface/metron-alerts/README.md
--
diff --git a/metron-interface/metron-alerts/README.md 
b/metron-interface/metron-alerts/README.md
index 0cc2fba..2d34505 100644
--- a/metron-interface/metron-alerts/README.md
+++ b/metron-interface/metron-alerts/README.md
@@ -99,6 +99,12 @@ rest:
   port: REST applciation port
 ```
 
+## Global Configuration Properties
+
+### `source.type.field`
+
+The source type format used. Defaults to `source:type`.
+
 ## Usage
 
 After configuration is complete, the Management UI can be managed as a service:

http://git-wip-us.apache.org/repos/asf/metron/blob/a99cadb0/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --git 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index 8335ad7..e1c1685 100644
--- 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++ 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -124,7 +124,7 @@ export class AlertDetailsComponent implements OnInit {
   ngOnInit() {
 this.activatedRoute.params.subscribe(params => {
   this.alertId = params['guid'];
-  this.alertSourceType = params['sourceType'];
+  this.alertSourceType = params['source.type.field'];
   this.alertIndex = params['index'];
   this.isMetaAlert = (this.alertIndex === META_ALERTS_INDEX && 
this.alertSourceType !== META_ALERTS_SENSOR_TYPE) ? true : false;
   this.getData();

http://git-wip-us.apache.org/repos/asf/metron/blob/a99cadb0/metron-interface/metron-alerts/src/app/alerts/alert-details/alerts-details.routing.ts
--
diff --git 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alerts-details.routing.ts
 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alerts-details.routing.ts
index 0cb9c9c..c4080b8 100644
--- 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alerts-details.routing.ts
+++ 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alerts-details.routing.ts
@@ -20,5 +20,5 @@ import { RouterModule } from '@angular/router';
 import {AlertDetailsComponent} from './alert-details.component';
 
 export const routing: ModuleWithProviders = RouterModule.forChild([
-{ path: 'details/:sourceType/:guid/:index', component: 
AlertDetailsComponent, outlet: 'dialog'}
+{ path: 'details/:source.type.field/:guid/:index', component: 
AlertDetailsComponent, outlet: 'dialog'}
 ]);

http://git-wip-us.apache.org/repos/asf/metron/blob/a99cadb0/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
--
diff --git 
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts

[05/13] metron git commit: METRON-1461 MIN MAX stellar function should take a stats or list object and return min/max (MohanDV via nickwallen) closes apache/metron#942

2018-05-18 Thread leet 
METRON-1461 MIN MAX stellar function should take a stats or list object and 
return min/max (MohanDV via nickwallen) closes apache/metron#942


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d7edce97
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d7edce97
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d7edce97

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: d7edce974341a6fca0e8e87b11baed5ad0d5d0c1
Parents: ca4644b
Author: MohanDV 
Authored: Wed May 9 16:33:32 2018 -0400
Committer: nickallen 
Committed: Wed May 9 16:33:32 2018 -0400

--
 .../metron/statistics/StatisticsProvider.java   |  6 +--
 .../metron/stellar/dsl/functions/Ordinal.java   | 36 +++
 .../stellar/dsl/functions/OrdinalFunctions.java | 48 +---
 .../dsl/functions/OrdinalFunctionsTest.java | 41 -
 4 files changed, 110 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/d7edce97/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java
--
diff --git 
a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java
 
b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java
index 860aa4e..e737484 100644
--- 
a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java
+++ 
b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/StatisticsProvider.java
@@ -20,14 +20,14 @@
 package org.apache.metron.statistics;
 
 
+import org.apache.metron.stellar.dsl.functions.Ordinal;
+
 /**
  * Provides statistical functions.
  */
-public interface StatisticsProvider {
+public interface StatisticsProvider extends Ordinal{
   void addValue(double value);
   long getCount();
-  double getMin();
-  double getMax();
   double getMean();
   double getSum();
   double getVariance();

http://git-wip-us.apache.org/repos/asf/metron/blob/d7edce97/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java
new file mode 100644
index 000..d3bd9ce
--- /dev/null
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/Ordinal.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.dsl.functions;
+
+/**
+ * Interface that provides the statistical function get max and min from the 
implementing object.
+ */
+public interface Ordinal {
+
+/**
+ * get the min value
+ * @return min value
+ */
+double getMin();
+
+/**
+ * get the max value
+ * @return max value
+ */
+double getMax();
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/d7edce97/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
index 49e9369..6ac9ff5 100644
--- 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
@@ -22,12 +22,10 @@ import com.google.common.collect.Iterables;
 import org.apache.metron.stellar.common.utils.ConversionUtils;
 import

[07/13] metron git commit: METRON-1549: Add empty object test to WriterBoltIntegrationTest implementation (mmiklavc via mmiklavc) closes apache/metron#1009

2018-05-18 Thread leet 
METRON-1549: Add empty object test to WriterBoltIntegrationTest implementation 
(mmiklavc via mmiklavc) closes apache/metron#1009


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b9453aab
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b9453aab
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b9453aab

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: b9453aabd781c7c67258d9506af176fbcab85be1
Parents: a17c1ad
Author: mmiklavc 
Authored: Fri May 11 12:04:01 2018 -0600
Committer: Michael Miklavcic 
Committed: Fri May 11 12:04:01 2018 -0600

--
 .../integration/WriterBoltIntegrationTest.java  | 315 ++-
 1 file changed, 231 insertions(+), 84 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/b9453aab/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
--
diff --git 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
index cde08bc..d565147 100644
--- 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
+++ 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
@@ -17,40 +17,56 @@
  */
 package org.apache.metron.writers.integration;
 
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
 import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import javax.annotation.Nullable;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.configuration.SensorParserConfig;
-import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.common.field.validation.FieldValidation;
 import org.apache.metron.common.utils.JSONUtils;
 import 
org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
-import org.apache.metron.integration.*;
+import org.apache.metron.integration.BaseIntegrationTest;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.Processor;
+import org.apache.metron.integration.ProcessorResult;
+import org.apache.metron.integration.ReadinessState;
 import org.apache.metron.integration.components.KafkaComponent;
-import org.apache.metron.integration.processors.KafkaMessageSet;
 import org.apache.metron.integration.components.ZKServerComponent;
+import org.apache.metron.integration.processors.KafkaMessageSet;
 import org.apache.metron.integration.processors.KafkaProcessor;
-import org.apache.metron.parsers.csv.CSVParser;
 import 
org.apache.metron.parsers.integration.components.ParserTopologyComponent;
-import org.apache.metron.test.utils.UnitTestHelper;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.stellar.dsl.Context;
 import org.json.simple.JSONObject;
-import org.json.simple.parser.ParseException;
 import org.junit.Assert;
 import org.junit.Test;
 
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.util.*;
-
 public class WriterBoltIntegrationTest extends BaseIntegrationTest {
+  private ZKServerComponent zkServerComponent;
+  private KafkaComponent kafkaComponent;
+  private ConfigUploadComponent configUploadComponent;
+  private ParserTopologyComponent parserTopologyComponent;
 
-  public static class MockValidator implements FieldValidation{
+  public static class MockValidator implements FieldValidation {
 
 @Override
 public boolean isValid(Map input, Map 
validationConfig, Map globalConfig, Context context) {
-  if(input.get("action").equals("invalid")) {
+  if (input.get("action").equals("invalid")) {
 return false;
   }
   return true;
@@ -60,6 +76,7 @@ public class WriterBoltIntegrationTest extends 
BaseIntegrationTest {
 public void initialize(Map validationConfig, Map globalConfig) {
 }
   }
+
   /**
* {
*   "fieldValidations" : [
@@ -68,7 +85,7 @@ public class WriterBoltIntegrationTest extends 
BaseIntegrationTest {

[01/13] metron git commit: METRON-1543 Unable to Set Parser Output Topic in Sensor Config (nickwallen) closes apache/metron#1007

2018-05-18 Thread leet 
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr bc16e6825 -> eb3366656


METRON-1543 Unable to Set Parser Output Topic in Sensor Config (nickwallen) 
closes apache/metron#1007


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3bb926df
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3bb926df
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3bb926df

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 3bb926df5d253a907bbf8dab4b76b78dd32993ea
Parents: 2b4f0b8
Author: nickwallen 
Authored: Wed May 2 15:06:03 2018 -0400
Committer: nickallen 
Committed: Wed May 2 15:06:03 2018 -0400

--
 .../org/apache/metron/common/Constants.java |  10 +-
 .../configuration/SensorParserConfig.java   | 410 +++
 .../parsers/topology/ParserTopologyBuilder.java | 139 ---
 .../parsers/topology/ParserTopologyCLI.java | 147 +--
 .../components/ParserTopologyComponent.java |  80 ++--
 .../parsers/topology/ParserTopologyCLITest.java | 122 --
 ...pleHbaseEnrichmentWriterIntegrationTest.java |  69 ++--
 .../integration/WriterBoltIntegrationTest.java  | 109 ++---
 .../apache/metron/writer/kafka/KafkaWriter.java |   5 +
 9 files changed, 676 insertions(+), 415 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/3bb926df/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index b939a92..12b541c 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -17,9 +17,7 @@
  */
 package org.apache.metron.common;
 
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 public class Constants {
@@ -37,9 +35,17 @@ public class Constants {
   public static final String SIMPLE_HBASE_THREAT_INTEL = "hbaseThreatIntel";
   public static final String GUID = "guid";
 
+  /**
+   * The key in the global configuration that defines the global parser error 
topic.
+   *
+   * This value is used only if the error topic is left undefined in a 
sensor's parser configuration.
+   */
+  public static final String PARSER_ERROR_TOPIC_GLOBALS_KEY = 
"parser.error.topic";
+
   public interface Field {
 String getName();
   }
+
   public enum Fields implements Field {
  SRC_ADDR("ip_src_addr")
 ,SRC_PORT("ip_src_port")

http://git-wip-us.apache.org/repos/asf/metron/blob/3bb926df/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
index d347481..1dfb045 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
@@ -18,6 +18,9 @@
 package org.apache.metron.common.configuration;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.metron.common.utils.JSONUtils;
 
 import java.io.IOException;
@@ -27,35 +30,171 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+/**
+ * The configuration object that defines a parser for a given sensor.  Each
+ * sensor has its own parser configuration.
+ */
 public class SensorParserConfig implements Serializable {
 
+  /**
+   * The class name of the parser.
+   */
   private String parserClassName;
+
+  /**
+   * Allows logic to be defined to filter or ignore messages.  Messages that 
have been
+   * filtered will not be parsed.
+   *
+   * This should be a fully qualified name of a class that implements the
+   * org.apache.metron.parsers.interfaces.MessageFilter interface.
+   */
   private String filterClassName;
+
+  /**
+   * The input topic containing the sensor telemetry to parse.
+   */
   private String sensorTopic;
+
+  /**
+   * The output topic where the parsed telemetry will be written.
+   */
+  private String outputTopic;
+
+  /**
+   * The

[2/2] metron git commit: METRON-1548 Remove hardcoded source:type from Alerts UI (sardell via justinleet) closes apache/metron#1010

2018-05-18 Thread leet 
METRON-1548 Remove hardcoded source:type from Alerts UI (sardell via 
justinleet) closes apache/metron#1010


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a99cadb0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a99cadb0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a99cadb0

Branch: refs/heads/master
Commit: a99cadb018b96cf96955df4b6e4acade28530a41
Parents: 4458085
Author: Shane Ardell 
Authored: Mon May 7 13:19:42 2018 +0200
Committer: justinjleet 
Committed: Fri May 18 13:32:29 2018 -0400

--
 metron-interface/metron-alerts/README.md|  6 +++
 .../alert-details/alert-details.component.ts|  2 +-
 .../alert-details/alerts-details.routing.ts |  2 +-
 .../alerts/alerts-list/alerts-list.component.ts | 25 --
 .../table-view/table-view.component.ts  | 32 +++--
 .../tree-view/tree-view.component.ts| 17 +--
 .../alerts/meta-alerts/meta-alerts.component.ts | 20 ++--
 .../metron-alerts/src/app/app.module.ts |  4 +-
 .../src/app/model/search-request.ts |  2 +-
 .../src/app/service/authentication.service.ts   | 10 +++-
 .../service/elasticsearch-localstorage-impl.ts  |  6 ++-
 .../src/app/service/global-config.service.ts| 50 
 .../src/app/service/update.service.ts   | 10 +++-
 .../metron-alerts/src/app/utils/constants.ts|  4 +-
 .../metron-alerts/src/app/utils/utils.ts|  7 +--
 metron-platform/metron-common/README.md | 15 +++---
 16 files changed, 174 insertions(+), 38 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/a99cadb0/metron-interface/metron-alerts/README.md
--
diff --git a/metron-interface/metron-alerts/README.md 
b/metron-interface/metron-alerts/README.md
index 0cc2fba..2d34505 100644
--- a/metron-interface/metron-alerts/README.md
+++ b/metron-interface/metron-alerts/README.md
@@ -99,6 +99,12 @@ rest:
   port: REST applciation port
 ```
 
+## Global Configuration Properties
+
+### `source.type.field`
+
+The source type format used. Defaults to `source:type`.
+
 ## Usage
 
 After configuration is complete, the Management UI can be managed as a service:

http://git-wip-us.apache.org/repos/asf/metron/blob/a99cadb0/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --git 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index 8335ad7..e1c1685 100644
--- 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++ 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -124,7 +124,7 @@ export class AlertDetailsComponent implements OnInit {
   ngOnInit() {
 this.activatedRoute.params.subscribe(params => {
   this.alertId = params['guid'];
-  this.alertSourceType = params['sourceType'];
+  this.alertSourceType = params['source.type.field'];
   this.alertIndex = params['index'];
   this.isMetaAlert = (this.alertIndex === META_ALERTS_INDEX && 
this.alertSourceType !== META_ALERTS_SENSOR_TYPE) ? true : false;
   this.getData();

http://git-wip-us.apache.org/repos/asf/metron/blob/a99cadb0/metron-interface/metron-alerts/src/app/alerts/alert-details/alerts-details.routing.ts
--
diff --git 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alerts-details.routing.ts
 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alerts-details.routing.ts
index 0cb9c9c..c4080b8 100644
--- 
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alerts-details.routing.ts
+++ 
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alerts-details.routing.ts
@@ -20,5 +20,5 @@ import { RouterModule } from '@angular/router';
 import {AlertDetailsComponent} from './alert-details.component';
 
 export const routing: ModuleWithProviders = RouterModule.forChild([
-{ path: 'details/:sourceType/:guid/:index', component: 
AlertDetailsComponent, outlet: 'dialog'}
+{ path: 'details/:source.type.field/:guid/:index', component: 
AlertDetailsComponent, outlet: 'dialog'}
 ]);

http://git-wip-us.apache.org/repos/asf/metron/blob/a99cadb0/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
--
diff --git 
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts

[1/2] metron git commit: METRON-1548 Remove hardcoded source:type from Alerts UI (justinleet) closes apache/metron#1010

2018-05-18 Thread leet 
Repository: metron
Updated Branches:
  refs/heads/master 4458085f2 -> 71a015fab


METRON-1548 Remove hardcoded source:type from Alerts UI (justinleet) closes 
apache/metron#1010


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/71a015fa
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/71a015fa
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/71a015fa

Branch: refs/heads/master
Commit: 71a015fabc47b23ea0c5400fa239fee9f5c0d194
Parents: a99cadb
Author: justinjleet 
Authored: Wed May 16 17:39:07 2018 -0400
Committer: justinjleet 
Committed: Fri May 18 13:32:29 2018 -0400

--
 .../rest/service/impl/SearchServiceImpl.java| 42 ++
 .../src/main/resources/application.yml  |  2 +-
 .../SearchControllerIntegrationTest.java|  9 ++-
 .../service/impl/SearchServiceImplTest.java | 60 +---
 4 files changed, 92 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/71a015fa/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
--
diff --git 
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
 
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
index e5bab08..21d158f 100644
--- 
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
+++ 
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
@@ -23,19 +23,26 @@ import static 
org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME;
 import static 
org.apache.metron.rest.MetronRestConstants.SEARCH_FACET_FIELDS_SPRING_PROPERTY;
 
 import com.google.common.collect.Lists;
+import java.io.IOException;
 import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.metron.common.Constants;
 import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.search.FieldType;
 import org.apache.metron.indexing.dao.search.GetRequest;
 import org.apache.metron.indexing.dao.search.GroupRequest;
 import org.apache.metron.indexing.dao.search.GroupResponse;
 import org.apache.metron.indexing.dao.search.InvalidSearchException;
 import org.apache.metron.indexing.dao.search.SearchRequest;
 import org.apache.metron.indexing.dao.search.SearchResponse;
-import org.apache.metron.indexing.dao.search.FieldType;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.model.AlertsUIUserSettings;
 import org.apache.metron.rest.service.AlertsUIService;
+import org.apache.metron.rest.service.GlobalConfigService;
 import org.apache.metron.rest.service.SearchService;
 import org.apache.metron.rest.service.SensorIndexingConfigService;
 import org.slf4j.Logger;
@@ -44,11 +51,6 @@ import 
org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.env.Environment;
 import org.springframework.stereotype.Service;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Optional;
-import java.util.List;
-
 @Service
 public class SearchServiceImpl implements SearchService {
 
@@ -57,14 +59,19 @@ public class SearchServiceImpl implements SearchService {
   private IndexDao dao;
   private Environment environment;
   private SensorIndexingConfigService sensorIndexingConfigService;
+  private GlobalConfigService globalConfigService;
   private AlertsUIService alertsUIService;
 
   @Autowired
-  public SearchServiceImpl(IndexDao dao, Environment environment,
-  SensorIndexingConfigService sensorIndexingConfigService, AlertsUIService 
alertsUIService) {
+  public SearchServiceImpl(IndexDao dao,
+  Environment environment,
+  SensorIndexingConfigService sensorIndexingConfigService,
+  GlobalConfigService globalConfigService,
+  AlertsUIService alertsUIService) {
 this.dao = dao;
 this.environment = environment;
 this.sensorIndexingConfigService = sensorIndexingConfigService;
+this.globalConfigService = globalConfigService;
 this.alertsUIService = alertsUIService;
   }
 
@@ -133,11 +140,24 @@ public class SearchServiceImpl implements SearchService {
 return indices;
   }
 
-  private List getDefaultFacetFields() throws RestException {
+  @SuppressWarnings("unchecked")
+  public List getDefaultFacetFields() throws RestException {
 Optional alertUserSettings = 
alertsUIService.getAlertsUIUserSettings();
 if (!alertUserSettings.isPresent() || 
alertUserSettings.get().getFacetFields() == null) {

metron git commit: METRON-1567 Large error message can t be written in Solr (justinleet) closes apache/metron#1020

2018-05-18 Thread leet 
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr 832296f0d -> bc16e6825


METRON-1567 Large error message cant be written in Solr (justinleet) 
closes apache/metron#1020


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bc16e682
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bc16e682
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bc16e682

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: bc16e68252de98846bba58aafb2d7d445caac9c1
Parents: 832296f
Author: justinleet 
Authored: Fri May 18 11:34:30 2018 -0400
Committer: leet 
Committed: Fri May 18 11:34:30 2018 -0400

--
 .../src/main/config/schema/error/schema.xml |  4 +-
 .../integration/SolrUpdateIntegrationTest.java  | 49 ++--
 2 files changed, 48 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/bc16e682/metron-platform/metron-solr/src/main/config/schema/error/schema.xml
--
diff --git 
a/metron-platform/metron-solr/src/main/config/schema/error/schema.xml 
b/metron-platform/metron-solr/src/main/config/schema/error/schema.xml
index 5f9f9a3..4aa80ef 100644
--- a/metron-platform/metron-solr/src/main/config/schema/error/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/error/schema.xml
@@ -29,7 +29,6 @@
 
   
   
-  
   
   
   
@@ -40,6 +39,8 @@
   
   guid
 
+  
+  
   
   
 
@@ -55,4 +56,5 @@
   
   
   
+  
 

http://git-wip-us.apache.org/repos/asf/metron/blob/bc16e682/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
--
diff --git 
a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
 
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
index 232c727..dc08967 100644
--- 
a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
+++ 
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
@@ -17,11 +17,14 @@
  */
 package org.apache.metron.solr.integration;
 
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-
+import org.apache.commons.lang3.StringUtils;
 import org.apache.metron.indexing.dao.IndexDao;
 import org.apache.metron.indexing.dao.MultiIndexDao;
 import org.apache.metron.indexing.dao.UpdateIntegrationTest;
@@ -29,10 +32,13 @@ import org.apache.metron.indexing.dao.update.Document;
 import org.apache.metron.integration.InMemoryComponent;
 import org.apache.metron.solr.dao.SolrDao;
 import org.apache.metron.solr.integration.components.SolrComponent;
-import org.junit.Assert;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
+  @Rule
+  public final ExpectedException exception = ExpectedException.none();
 
   protected static SolrComponent solrComponent;
 
@@ -55,7 +61,9 @@ public class SolrUpdateIntegrationTest extends 
UpdateIntegrationTest {
 
   @Override
   protected InMemoryComponent startIndex() throws Exception {
-solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, 
"../metron-solr/src/main/config/schema/bro").build();
+solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, 
"../metron-solr/src/main/config/schema/bro")
+.addCollection("error", "../metron-solr/src/main/config/schema/error")
+.build();
 solrComponent.start();
 return solrComponent;
   }
@@ -98,6 +106,39 @@ public class SolrUpdateIntegrationTest extends 
UpdateIntegrationTest {
 Document indexedDocument = dao.getLatest("bro_1", SENSOR_NAME);
 
 // assert no extra expanded fields are included
-Assert.assertEquals(8, indexedDocument.getDocument().size());
+assertEquals(8, indexedDocument.getDocument().size());
+  }
+
+  @Test
+  public void testHugeErrorFields() throws Exception {
+dao = new MultiIndexDao(createDao());
+dao.init(getAccessConfig());
+
+String hugeString = StringUtils.repeat("test ", 1_000_000);
+String hugeStringTwo = hugeString + "-2";
+
+Map documentMap = new HashMap<>();
+documentMap.put("guid", "error_guid");
+// Needs to be over 32kb
+documentMap.put("raw_message", hugeString);
+documentMap.put("raw_message_1", hugeStringTwo);
+Document errorDoc = new Document(documentMap, "error", "error", 0L);