Repository: metron Updated Branches: refs/heads/METRON-1416-upgrade-solr [created] f5aabf171
Merge remote-tracking branch 'apache/master' into SOLR-MERGE-MASTER-20180621 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/78d5dbd8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/78d5dbd8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/78d5dbd8 Branch: refs/heads/METRON-1416-upgrade-solr Commit: 78d5dbd82b8580586f547743c6c55c3f55591f0d Parents: ed4dfb9 6a325cb Author: Nick Allen <[email protected]> Authored: Thu Jun 21 16:26:28 2018 -0400 Committer: Nick Allen <[email protected]> Committed: Thu Jun 21 16:26:28 2018 -0400 ---------------------------------------------------------------------- metron-analytics/metron-maas-common/pom.xml | 2 +- metron-analytics/metron-maas-service/pom.xml | 2 +- metron-analytics/metron-profiler-client/pom.xml | 2 +- metron-analytics/metron-profiler-common/pom.xml | 2 +- metron-analytics/metron-profiler/pom.xml | 2 +- metron-analytics/metron-statistics/pom.xml | 2 +- metron-analytics/pom.xml | 2 +- metron-contrib/metron-docker/pom.xml | 2 +- metron-contrib/metron-performance/pom.xml | 2 +- metron-contrib/pom.xml | 2 +- metron-deployment/Kerberos-manual-setup.md | 4 +-- metron-deployment/amazon-ec2/conf/defaults.yml | 2 +- .../ansible/playbooks/docker_probe_install.yml | 2 +- .../roles/ambari_master/defaults/main.yml | 2 +- .../roles/metron_pcapservice/defaults/main.yml | 2 +- .../centos6/ansible/inventory/group_vars/all | 2 +- .../ubuntu14/ansible/inventory/group_vars/all | 2 +- .../ambari/elasticsearch-mpack/pom.xml | 4 +-- .../packaging/ambari/metron-mpack/pom.xml | 4 +-- .../CURRENT/configuration/metron-rest-env.xml | 12 ++++++++ .../CURRENT/package/scripts/metron_service.py | 10 +++++++ .../package/scripts/params/params_linux.py | 2 ++ .../METRON/CURRENT/themes/metron_theme.json | 20 +++++++++++++ .../metron-mpack/src/main/resources/mpack.json | 2 +- .../packaging/docker/deb-docker/pom.xml | 4 +-- .../packaging/docker/rpm-docker/pom.xml | 4 +-- metron-deployment/pom.xml | 2 +- metron-interface/metron-alerts/README.md | 6 +++- metron-interface/metron-alerts/package.json | 2 +- metron-interface/metron-alerts/pom.xml | 2 +- .../table-view/table-view.component.ts | 2 +- .../meta-alerts/meta-alerts.component.html | 6 ++-- .../alerts/meta-alerts/meta-alerts.component.ts | 4 +-- .../metron-alerts/src/app/model/filter.ts | 4 +-- metron-interface/metron-config/package.json | 2 +- metron-interface/metron-config/pom.xml | 2 +- .../metron-config/scripts/package.json | 2 +- metron-interface/metron-rest-client/pom.xml | 2 +- metron-interface/metron-rest/pom.xml | 2 +- .../src/main/resources/application.yml | 2 +- metron-interface/pom.xml | 2 +- metron-platform/Performance-tuning-guide.md | 4 +-- metron-platform/README.md | 2 +- metron-platform/elasticsearch-shaded/pom.xml | 2 +- metron-platform/metron-api/pom.xml | 2 +- metron-platform/metron-common/README.md | 3 +- metron-platform/metron-common/pom.xml | 2 +- .../src/main/scripts/cluster_info.py | 2 +- metron-platform/metron-data-management/pom.xml | 2 +- metron-platform/metron-elasticsearch/pom.xml | 2 +- .../dao/ElasticsearchMetaAlertDao.java | 30 -------------------- .../dao/ElasticsearchMetaAlertDaoTest.java | 8 ++++++ metron-platform/metron-enrichment/pom.xml | 2 +- metron-platform/metron-hbase-client/pom.xml | 2 +- metron-platform/metron-hbase/pom.xml | 2 +- metron-platform/metron-indexing/pom.xml | 2 +- metron-platform/metron-integration-test/pom.xml | 2 +- metron-platform/metron-management/pom.xml | 2 +- metron-platform/metron-parsers/pom.xml | 2 +- metron-platform/metron-pcap-backend/pom.xml | 2 +- .../src/main/scripts/pcap_zeppelin_run.sh | 2 +- metron-platform/metron-pcap/pom.xml | 2 +- metron-platform/metron-solr/pom.xml | 2 +- .../metron-storm-kafka-override/pom.xml | 2 +- metron-platform/metron-storm-kafka/pom.xml | 2 +- metron-platform/metron-test-utilities/pom.xml | 2 +- metron-platform/metron-writer/pom.xml | 2 +- metron-platform/metron-zookeeper/pom.xml | 2 +- metron-platform/pom.xml | 2 +- metron-stellar/pom.xml | 2 +- .../stellar-3rd-party-example/pom.xml | 2 +- metron-stellar/stellar-common/README.md | 4 +-- metron-stellar/stellar-common/pom.xml | 2 +- metron-stellar/stellar-zeppelin/README.md | 6 ++-- metron-stellar/stellar-zeppelin/pom.xml | 2 +- pom.xml | 2 +- site-book/pom.xml | 2 +- 77 files changed, 141 insertions(+), 114 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/78d5dbd8/metron-deployment/packaging/ambari/metron-mpack/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/78d5dbd8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py ---------------------------------------------------------------------- diff --cc metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py index 4a55f8a,1999992..58d581f --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py @@@ -138,15 -109,19 +138,25 @@@ def build_global_config_patch(params, p "op": "add", "path": "/bootstrap.servers", "value": "{{kafka_brokers}}" + }, + { + "op": "add", + "path": "/source.type.field", + "value": "{{source_type_field}}" + }, + { + "op": "add", + "path": "/threat.triage.score.field", + "value": "{{threat_triage_score_field}}" } - ] """ + patch_template = ambari_format( + """ + [ + {indexing_patches}, + {other_patches} + ] + """) File(patch_file, content=InlineTemplate(patch_template), owner=params.metron_user, http://git-wip-us.apache.org/repos/asf/metron/blob/78d5dbd8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/78d5dbd8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json ---------------------------------------------------------------------- diff --cc metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 0bb63de,d09f270..2b2d648 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@@ -837,12 -841,18 +845,24 @@@ } }, { + "config": "metron-env/solr_zookeeper_url", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-rest-env/source_type_field", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-rest-env/threat_triage_score_field", + "widget": { + "type": "text-field" + } + }, + { "config": "metron-parsers-env/parsers", "widget": { "type": "text-field" http://git-wip-us.apache.org/repos/asf/metron/blob/78d5dbd8/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.ts ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/78d5dbd8/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java ---------------------------------------------------------------------- diff --cc metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index 649077e,71fe181..55123a5 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@@ -18,23 -18,17 +18,17 @@@ package org.apache.metron.elasticsearch.dao; - import java.io.IOException; - import java.util.List; - import java.util.Map; - import java.util.Optional; --import org.apache.lucene.search.join.ScoreMode; import org.apache.metron.common.Constants; --import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; -import org.apache.metron.indexing.dao.MetaAlertDao; import org.apache.metron.indexing.dao.MultiIndexDao; +import org.apache.metron.indexing.dao.RetrieveLatestDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; +import org.apache.metron.indexing.dao.metaalert.MetaAlertDao; import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; -import org.apache.metron.indexing.dao.metaalert.MetaScores; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; @@@ -43,41 -37,40 +37,17 @@@ import org.apache.metron.indexing.dao.s import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; --import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; --import org.apache.metron.stellar.common.utils.ConversionUtils; --import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.index.IndexNotFoundException; --import org.elasticsearch.index.query.InnerHitBuilder; --import org.elasticsearch.index.query.QueryBuilder; --import org.elasticsearch.index.query.QueryBuilders; --import org.elasticsearch.index.query.QueryStringQueryBuilder; --import org.elasticsearch.search.SearchHit; import java.io.IOException; --import java.util.ArrayList; --import java.util.Arrays; --import java.util.Collection; --import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; --import java.util.Map.Entry; import java.util.Optional; --import java.util.Set; --import java.util.UUID; -import java.util.stream.Collectors; - -import static org.apache.metron.common.Constants.GUID; -import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; -import static org.elasticsearch.index.query.QueryBuilders.existsQuery; -import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import java.util.function.Supplier; - import java.util.stream.Collectors; - - import static org.apache.metron.common.Constants.GUID; - import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; - import static org.elasticsearch.index.query.QueryBuilders.boolQuery; - import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; - import static org.elasticsearch.index.query.QueryBuilders.existsQuery; - import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; - import static org.elasticsearch.index.query.QueryBuilders.termQuery; public class ElasticsearchMetaAlertDao implements MetaAlertDao { http://git-wip-us.apache.org/repos/asf/metron/blob/78d5dbd8/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java ---------------------------------------------------------------------- diff --cc metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java index 70197ea,df782bd..b1da2a4 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java @@@ -18,12 -18,31 +18,20 @@@ package org.apache.metron.elasticsearch.dao; -import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERTS_INDEX; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; - + import java.io.IOException; + import java.util.ArrayList; + import java.util.Arrays; + import java.util.Collections; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + import java.util.Optional; -import java.util.UUID; -import org.apache.metron.common.Constants; -import org.apache.metron.common.Constants.Fields; import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.HBaseDao; import org.apache.metron.indexing.dao.IndexDao; -import org.apache.metron.indexing.dao.MetaAlertDao; +import org.apache.metron.indexing.dao.MultiIndexDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; -import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; http://git-wip-us.apache.org/repos/asf/metron/blob/78d5dbd8/metron-platform/metron-indexing/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/78d5dbd8/metron-platform/metron-solr/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/78d5dbd8/metron-platform/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/78d5dbd8/pom.xml ----------------------------------------------------------------------
