Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/41708876 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/41708876 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/41708876 Branch: refs/heads/master Commit: 41708876336d009b3a2ff2c533bdf0294af1477b Parents: f241f87 32d132b Author: cstella <ceste...@gmail.com> Authored: Mon Jun 11 17:50:48 2018 -0400 Committer: cstella <ceste...@gmail.com> Committed: Mon Jun 11 17:50:48 2018 -0400 ---------------------------------------------------------------------- KEYS | 59 + Upgrading.md | 2 +- dev-utilities/release-utils/metron-rc-check | 77 +- .../release-utils/validate-jira-for-release | 75 +- metron-analytics/metron-maas-common/pom.xml | 2 +- metron-analytics/metron-maas-service/pom.xml | 2 +- metron-analytics/metron-profiler-client/pom.xml | 2 +- .../profiler/client/stellar/GetProfileTest.java | 3 +- .../client/stellar/ProfilerFunctionsTest.java | 15 +- metron-analytics/metron-profiler-common/pom.xml | 2 +- metron-analytics/metron-profiler/README.md | 15 + metron-analytics/metron-profiler/pom.xml | 2 +- .../src/main/flux/profiler/remote.yaml | 3 +- metron-analytics/metron-statistics/pom.xml | 2 +- .../StellarStatisticsFunctionsTest.java | 5 +- .../sampling/SamplerFunctionsTest.java | 3 +- metron-analytics/pom.xml | 2 +- metron-contrib/metron-docker/pom.xml | 2 +- metron-contrib/metron-performance/pom.xml | 2 +- metron-contrib/pom.xml | 2 +- metron-deployment/Kerberos-manual-setup.md | 8 +- metron-deployment/README.md | 22 +- metron-deployment/amazon-ec2/conf/defaults.yml | 2 +- .../ansible/playbooks/docker_probe_install.yml | 2 +- .../roles/ambari_master/defaults/main.yml | 2 +- .../roles/metron_pcapservice/defaults/main.yml | 2 +- metron-deployment/development/centos6/README.md | 8 +- .../centos6/ansible/inventory/group_vars/all | 2 +- .../development/ubuntu14/README.md | 6 +- .../ubuntu14/ansible/inventory/group_vars/all | 2 +- .../ambari/elasticsearch-mpack/pom.xml | 4 +- .../packaging/ambari/metron-mpack/pom.xml | 4 +- .../CURRENT/package/files/bro_index.template | 2 +- .../CURRENT/package/files/error_index.template | 2 +- .../package/files/metaalert_index.template | 2 +- .../CURRENT/package/files/snort_index.template | 2 +- .../CURRENT/package/files/yaf_index.template | 2 +- .../CURRENT/package/scripts/metron_service.py | 5 + .../metron-mpack/src/main/resources/mpack.json | 2 +- .../packaging/docker/deb-docker/pom.xml | 4 +- .../packaging/docker/rpm-docker/pom.xml | 4 +- metron-deployment/pom.xml | 2 +- .../e2e/mock-data/alerts_ui_e2e_index.template | 2 +- metron-interface/metron-alerts/package.json | 2 +- metron-interface/metron-alerts/pom.xml | 2 +- .../alert-details/alert-details.component.ts | 2 +- .../alerts/alerts-list/alerts-list.component.ts | 2 +- .../table-view/table-view.component.html | 10 +- .../table-view/table-view.component.ts | 2 +- .../metron-alerts/src/app/model/alert-source.ts | 2 +- .../src/app/service/global-config.service.ts | 10 +- metron-interface/metron-config/package.json | 2 +- metron-interface/metron-config/pom.xml | 2 +- .../metron-config/scripts/package.json | 2 +- metron-interface/metron-rest-client/pom.xml | 2 +- metron-interface/metron-rest/pom.xml | 2 +- .../rest/service/impl/SearchServiceImpl.java | 10 +- .../src/main/resources/application.yml | 2 +- .../metron-rest/src/main/scripts/metron-rest.sh | 5 +- .../MetaAlertControllerIntegrationTest.java | 4 +- .../service/impl/SearchServiceImplTest.java | 3 +- metron-interface/pom.xml | 2 +- metron-platform/Performance-tuning-guide.md | 4 +- metron-platform/README.md | 2 +- metron-platform/elasticsearch-shaded/pom.xml | 2 +- metron-platform/metron-api/pom.xml | 2 +- metron-platform/metron-common/README.md | 5 + metron-platform/metron-common/pom.xml | 2 +- .../org/apache/metron/common/Constants.java | 1 + .../metron/common/bolt/ConfiguredBolt.java | 14 +- .../common/bolt/ConfiguredEnrichmentBolt.java | 11 +- .../common/bolt/ConfiguredIndexingBolt.java | 13 +- .../common/bolt/ConfiguredParserBolt.java | 13 +- .../common/bolt/ConfiguredProfilerBolt.java | 9 +- .../common/configuration/Configurations.java | 7 +- .../configuration/ConfigurationsUtils.java | 7 + .../configuration/EnrichmentConfigurations.java | 26 + .../configuration/IndexingConfigurations.java | 28 +- .../configuration/ParserConfigurations.java | 1 + .../enrichment/handler/StellarConfig.java | 11 +- .../profiler/ProfilerConfigurations.java | 25 + .../writer/ConfigurationStrategy.java | 44 + .../writer/ConfigurationsStrategies.java | 144 + .../writer/EnrichmentWriterConfiguration.java | 110 + .../writer/IndexingWriterConfiguration.java | 5 + .../writer/ParserWriterConfiguration.java | 12 +- .../writer/ProfilerWriterConfiguration.java | 109 + .../writer/SingleBatchConfigurationFacade.java | 6 + .../writer/WriterConfiguration.java | 69 + .../common/field/DeDotFieldNameConverter.java | 46 + .../metron/common/field/FieldNameConverter.java | 32 + .../common/field/FieldNameConverters.java | 116 + .../common/field/NoopFieldNameConverter.java | 32 + .../common/interfaces/FieldNameConverter.java | 24 - .../src/main/scripts/cluster_info.py | 2 +- .../configuration/ParserConfigurationsTest.java | 120 + .../profiler/ProfilerConfigTest.java | 13 +- .../writer/ConfigurationsStrategiesTest.java | 79 + .../EnrichmentWriterConfigurationTest.java | 54 + .../writer/IndexingWriterConfigurationTest.java | 70 + .../writer/ParserWriterConfigurationTest.java | 72 + .../writer/ProfilerWriterConfigurationTest.java | 54 + .../field/DeDotFieldNameConverterTest.java | 38 + .../common/field/FieldNameConvertersTest.java | 211 + .../StellarTransformationTest.java | 30 +- .../writer/IndexingWriterConfigurationTest.java | 70 - .../writer/ParserWriterConfigurationTest.java | 38 - metron-platform/metron-data-management/pom.xml | 2 +- metron-platform/metron-elasticsearch/README.md | 8 +- metron-platform/metron-elasticsearch/pom.xml | 8 +- .../dao/ElasticsearchMetaAlertDao.java | 33 +- .../writer/ElasticsearchFieldNameConverter.java | 32 - .../writer/ElasticsearchWriter.java | 70 +- .../ElasticsearchIndexingIntegrationTest.java | 7 +- .../ElasticsearchMetaAlertIntegrationTest.java | 16 +- .../ElasticsearchFieldNameConverterTest.java | 32 - .../metron-enrichment/Performance.md | 2 +- metron-platform/metron-enrichment/README.md | 15 +- metron-platform/metron-enrichment/pom.xml | 2 +- .../main/flux/enrichment/remote-splitjoin.yaml | 9 +- .../main/flux/enrichment/remote-unified.yaml | 9 +- .../adapters/stellar/StellarAdapter.java | 14 +- .../adapters/stellar/StellarAdapterTest.java | 26 + .../bolt/BulkMessageWriterBoltTest.java | 112 +- .../integration/EnrichmentIntegrationTest.java | 2 + metron-platform/metron-hbase-client/pom.xml | 2 +- metron-platform/metron-hbase/pom.xml | 2 +- metron-platform/metron-indexing/README.md | 20 +- metron-platform/metron-indexing/pom.xml | 2 +- .../src/main/flux/indexing/batch/remote.yaml | 4 +- .../flux/indexing/random_access/remote.yaml | 4 +- .../apache/metron/indexing/dao/HBaseDao.java | 2 +- .../dao/metaalert/MetaAlertConstants.java | 2 +- .../indexing/dao/InMemoryMetaAlertDao.java | 2 +- .../integration/HBaseDaoIntegrationTest.java | 17 + .../HDFSIndexingIntegrationTest.java | 4 +- .../integration/IndexingIntegrationTest.java | 3 +- metron-platform/metron-integration-test/pom.xml | 2 +- .../main/config/zookeeper/enrichments/test.json | 1 + metron-platform/metron-management/pom.xml | 2 +- .../metron/management/KafkaFunctions.java | 573 +- .../management/IndexingConfigFunctionsTest.java | 7 +- .../KafkaFunctionsIntegrationTest.java | 353 +- .../management/ThreatTriageFunctionsTest.java | 7 +- .../metron-parsers/3rdPartyParser.md | 439 +- metron-platform/metron-parsers/README.md | 6 +- metron-platform/metron-parsers/pom.xml | 2 +- .../apache/metron/parsers/bolt/ParserBolt.java | 108 +- .../metron/parsers/bolt/WriterHandler.java | 63 +- .../src/main/scripts/start_parser_topology.sh | 27 +- .../metron/parsers/bolt/ParserBoltTest.java | 176 +- .../SimpleHBaseEnrichmentWriterTest.java | 6 +- .../integration/WriterBoltIntegrationTest.java | 6 +- metron-platform/metron-pcap-backend/pom.xml | 2 +- .../src/main/scripts/pcap_zeppelin_run.sh | 2 +- metron-platform/metron-pcap/pom.xml | 2 +- .../metron/pcap/filter/PcapFieldResolver.java | 5 + metron-platform/metron-solr/pom.xml | 2 +- .../SolrIndexingIntegrationTest.java | 3 +- .../schema/SchemaValidationIntegrationTest.java | 5 + .../metron-storm-kafka-override/pom.xml | 2 +- metron-platform/metron-storm-kafka/pom.xml | 2 +- metron-platform/metron-test-utilities/pom.xml | 2 +- metron-platform/metron-writer/pom.xml | 2 +- .../metron/writer/BulkWriterComponent.java | 36 +- .../metron/writer/bolt/BatchTimeoutHelper.java | 6 +- .../writer/bolt/BulkMessageWriterBolt.java | 138 +- .../apache/metron/writer/kafka/KafkaWriter.java | 92 +- metron-platform/metron-zookeeper/pom.xml | 2 +- metron-platform/pom.xml | 2 +- metron-stellar/pom.xml | 2 +- .../stellar-3rd-party-example/pom.xml | 2 +- metron-stellar/stellar-common/README.md | 5 +- metron-stellar/stellar-common/pom.xml | 2 +- .../stellar/common/BaseStellarProcessor.java | 28 +- .../stellar/common/CachingStellarProcessor.java | 141 +- .../common/StellarPredicateProcessor.java | 6 + .../metron/stellar/common/utils/ConcatMap.java | 256 + .../common/utils/StellarProcessorUtils.java | 51 +- .../metron/stellar/dsl/MapVariableResolver.java | 8 + .../metron/stellar/dsl/VariableResolver.java | 1 + .../dsl/functions/DataStructureFunctions.java | 4 + .../common/CachingStellarProcessorTest.java | 195 +- .../shell/specials/AssignmentCommandTest.java | 7 +- .../stellar/common/utils/ConcatMapTest.java | 83 + .../stellar/dsl/functions/BasicStellarTest.java | 18 + .../dsl/functions/OrdinalFunctionsTest.java | 9 +- metron-stellar/stellar-zeppelin/README.md | 6 +- metron-stellar/stellar-zeppelin/pom.xml | 2 +- pom.xml | 5 +- site-book/pom.xml | 2 +- site/current-book/CONTRIBUTING.html | 157 + site/current-book/Upgrading.html | 354 +- .../css/apache-maven-fluido-1.3.0.min.css | 9 - .../css/apache-maven-fluido-1.7.min.css | 17 + site/current-book/css/maven-base.css | 168 + site/current-book/css/maven-theme.css | 161 + site/current-book/css/print.css | 11 +- .../fonts/glyphicons-halflings-regular.eot | Bin 0 -> 35283 bytes .../fonts/glyphicons-halflings-regular.svg | 229 + .../fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 55016 bytes .../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 41793 bytes .../images/apache-maven-project-2.png | Bin 33442 -> 43073 bytes site/current-book/images/collapsed.gif | Bin 0 -> 53 bytes site/current-book/images/expanded.gif | Bin 0 -> 52 bytes site/current-book/images/external.png | Bin 0 -> 230 bytes site/current-book/images/icon_error_sml.gif | Bin 633 -> 1010 bytes site/current-book/images/icon_info_sml.gif | Bin 638 -> 606 bytes site/current-book/images/icon_success_sml.gif | Bin 604 -> 990 bytes site/current-book/images/icon_warning_sml.gif | Bin 625 -> 576 bytes site/current-book/images/newwindow.png | Bin 0 -> 220 bytes site/current-book/index.html | 347 +- .../js/apache-maven-fluido-1.3.0.min.js | 21 - .../js/apache-maven-fluido-1.7.min.js | 25 + site/current-book/metron-analytics/index.html | 303 +- .../metron-maas-service/index.html | 451 +- .../metron-profiler-client/index.html | 888 +- .../metron-analytics/metron-profiler/index.html | 1338 +- .../metron-statistics/HLLP.html | 10962 ++++++----------- .../metron-statistics/index.html | 870 +- .../metron-contrib/metron-docker/index.html | 540 +- .../metron-performance/index.html | 317 + .../Kerberos-ambari-setup.html | 372 +- .../Kerberos-manual-setup.html | 1218 +- .../metron-deployment/amazon-ec2/index.html | 755 +- .../metron-deployment/ansible/index.html | 128 + .../metron-deployment/ansible/roles/index.html | 140 + .../ansible/roles/opentaxii/index.html | 301 + .../ansible/roles/pcap_replay/index.html | 172 + .../ansible/roles/sensor-stubs/index.html | 221 + .../ansible/roles/sensor-test-mode/index.html | 156 + .../development/centos6/index.html | 233 + .../development/fastcapa/index.html | 249 + .../metron-deployment/development/index.html | 137 + .../development/ubuntu14/index.html | 233 + site/current-book/metron-deployment/index.html | 691 +- .../metron-deployment/other-examples/index.html | 363 +- .../manual-install/Manual_Install_CentOS6.html | 1352 +- .../ambari/elasticsearch-mpack/index.html | 190 + .../packaging/ambari/index.html | 1150 +- .../packaging/ambari/metron-mpack/index.html | 337 + .../packaging/docker/ansible-docker/index.html | 432 +- .../packaging/docker/deb-docker/index.html | 194 + .../packaging/docker/rpm-docker/index.html | 430 +- .../packaging/packer-build/index.html | 382 +- .../metron-deployment/roles/index.html | 332 - .../metron-deployment/roles/monit/index.html | 388 - .../roles/opentaxii/index.html | 480 - .../roles/pcap_replay/index.html | 355 - .../roles/sensor-stubs/index.html | 401 - .../roles/sensor-test-mode/index.html | 345 - .../vagrant/codelab-platform/index.html | 383 - .../vagrant/fastcapa-test-platform/index.html | 416 - .../vagrant/full-dev-platform/index.html | 420 - .../metron-deployment/vagrant/index.html | 305 - .../vagrant/quick-dev-platform/index.html | 406 - .../metron-interface/metron-alerts/index.html | 434 +- .../metron-interface/metron-config/index.html | 415 +- .../metron-interface/metron-rest/index.html | 2661 ++-- .../Performance-tuning-guide.html | 1123 +- site/current-book/metron-platform/index.html | 346 +- .../metron-platform/metron-api/index.html | 387 +- .../metron-platform/metron-common/index.html | 877 +- .../metron-data-management/index.html | 1348 +- .../metron-elasticsearch/index.html | 706 +- .../metron-enrichment/Performance.html | 802 ++ .../metron-enrichment/index.html | 760 +- .../metron-platform/metron-indexing/index.html | 547 +- .../metron-management/index.html | 1178 +- .../metron-parsers/3rdPartyParser.html | 467 + .../metron-platform/metron-parsers/index.html | 870 +- .../metron-parsers/parser-testing.html | 434 +- .../metron-pcap-backend/index.html | 503 +- .../metron-platform/metron-writer/index.html | 377 +- .../metron-sensors/bro-plugin-kafka/index.html | 519 - .../metron-sensors/fastcapa/index.html | 1239 +- site/current-book/metron-sensors/index.html | 299 +- .../metron-sensors/pycapa/index.html | 605 +- .../stellar-3rd-party-example/index.html | 263 +- .../stellar-common/3rdPartyStellar.html | 330 +- .../metron-stellar/stellar-common/index.html | 3603 ++---- .../metron-stellar/stellar-zeppelin/index.html | 239 + .../use-cases/forensic_clustering/index.html | 517 +- .../geographic_login_outliers/index.html | 442 +- site/current-book/use-cases/index.html | 283 +- .../use-cases/typosquat_detection/index.html | 545 + site/documentation/index.md | 14 +- 287 files changed, 26742 insertions(+), 33226 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-deployment/packaging/ambari/metron-mpack/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java ---------------------------------------------------------------------- diff --cc metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java index 82b9c11,1c92fcb..54759e4 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java @@@ -18,7 -18,8 +18,8 @@@ package org.apache.metron.rest.service.impl; import static org.apache.metron.common.Constants.ERROR_TYPE; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE; + import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; -import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE; import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME; import static org.apache.metron.rest.MetronRestConstants.SEARCH_FACET_FIELDS_SPRING_PROPERTY; http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-rest/src/main/scripts/metron-rest.sh ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java ---------------------------------------------------------------------- diff --cc metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index ab6c40c,ee3ca89..f73a640 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@@ -18,21 -18,17 +18,23 @@@ package org.apache.metron.elasticsearch.dao; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; + import org.apache.lucene.search.join.ScoreMode; import org.apache.metron.common.Constants; + import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; -import org.apache.metron.indexing.dao.MetaAlertDao; import org.apache.metron.indexing.dao.MultiIndexDao; +import org.apache.metron.indexing.dao.RetrieveLatestDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; +import org.apache.metron.indexing.dao.metaalert.MetaAlertDao; import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; -import org.apache.metron.indexing.dao.metaalert.MetaScores; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; @@@ -247,4 -714,12 +277,5 @@@ public class ElasticsearchMetaAlertDao public void setPageSize(int pageSize) { this.pageSize = pageSize; } + - private String getFieldName(String globalConfigKey, String defaultFieldName) { - if (this.elasticsearchDao == null || this.elasticsearchDao.getAccessConfig() == null) { - return defaultFieldName; - } - Map<String, Object> globalConfig = this.elasticsearchDao.getAccessConfig().getGlobalConfigSupplier().get(); - return ConfigurationsUtils.getFieldName(globalConfig, globalConfigKey, defaultFieldName); - } } http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --cc metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index 6fa6956,adc1a27..99f0490 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@@ -18,14 -18,18 +18,13 @@@ package org.apache.metron.elasticsearch.integration; -import static org.apache.metron.common.Constants.SENSOR_TYPE; -import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; -import static org.apache.metron.indexing.dao.MetaAlertDao.ALERT_FIELD; -import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERTS_INDEX; -import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_FIELD; -import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE; -import static org.apache.metron.indexing.dao.MetaAlertDao.STATUS_FIELD; -import static org.apache.metron.indexing.dao.MetaAlertDao.THREAT_FIELD_PROPERTY; +import static org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao.METAALERTS_INDEX; - import static org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.ALERT_FIELD; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_DOC; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_FIELD; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE; import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.base.Joiner; -import com.google.common.collect.Iterables; import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; @@@ -46,12 -52,23 +45,13 @@@ import org.apache.metron.elasticsearch. import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; -import org.apache.metron.indexing.dao.MetaAlertDao; -import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; -import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; ++import org.apache.metron.indexing.dao.metaalert.MetaAlertDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertIntegrationTest; import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; import org.apache.metron.indexing.dao.search.GetRequest; -import org.apache.metron.indexing.dao.search.Group; -import org.apache.metron.indexing.dao.search.GroupRequest; -import org.apache.metron.indexing.dao.search.GroupResponse; -import org.apache.metron.indexing.dao.search.GroupResult; -import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; -import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.indexing.dao.search.SortField; -import org.apache.metron.indexing.dao.update.Document; -import org.apache.metron.indexing.dao.update.OriginalNotFoundException; -import org.apache.metron.indexing.dao.update.PatchRequest; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@@ -119,6 -196,12 +119,12 @@@ public class ElasticsearchMetaAlertInte .withIndexDir(new File(INDEX_DIR)) .build(); es.start(); + } + + @Before + public void setup() throws IOException { - es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert")); ++ es.createIndexWithMapping(METAALERTS_INDEX, METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert")); + es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index")); AccessConfig accessConfig = new AccessConfig(); Map<String, Object> globalConfig = new HashMap<String, Object>() { @@@ -135,19 -218,9 +141,11 @@@ esDao = new ElasticsearchDao(); esDao.init(accessConfig); - metaDao = new ElasticsearchMetaAlertDao(esDao); + ElasticsearchMetaAlertDao elasticsearchMetaDao = new ElasticsearchMetaAlertDao(esDao); + elasticsearchMetaDao.setPageSize(5); + metaDao = elasticsearchMetaDao; } - @Before - public void setup() throws IOException { - es.createIndexWithMapping(METAALERTS_INDEX, METAALERT_DOC, - template.replace("%MAPPING_NAME%", "metaalert")); - es.createIndexWithMapping( - INDEX_WITH_SEPARATOR, "index_doc", template.replace("%MAPPING_NAME%", "index")); - } - @AfterClass public static void teardown() { if (es != null) { http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/README.md ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java ---------------------------------------------------------------------- diff --cc metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java index a055db5,0000000..daa5424 mode 100644,000000..100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java @@@ -1,30 -1,0 +1,30 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao.metaalert; + +public class MetaAlertConstants { + public static String METAALERT_TYPE = "metaalert"; + public static String METAALERT_FIELD = "metaalerts"; + public static String METAALERT_DOC = METAALERT_TYPE + "_doc"; + public static String THREAT_FIELD_DEFAULT = "threat:triage:score"; + public static String THREAT_SORT_DEFAULT = "sum"; - public static String ALERT_FIELD = "alert"; ++ public static String ALERT_FIELD = "metron_alert"; + public static String STATUS_FIELD = "status"; + public static String GROUPS_FIELD = "groups"; +} http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-solr/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java ---------------------------------------------------------------------- diff --cc metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java index 918a4ae,256f23b..2f9b285 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java @@@ -17,16 -17,10 +17,16 @@@ */ package org.apache.metron.solr.integration; +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; + import com.google.common.base.Function; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import javax.annotation.Nullable; import org.apache.metron.common.configuration.Configurations; import org.apache.metron.common.configuration.ConfigurationsUtils; - import org.apache.metron.common.interfaces.FieldNameConverter; + import org.apache.metron.common.field.FieldNameConverter; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.enrichment.integration.utils.SampleUtil; import org.apache.metron.indexing.integration.IndexingIntegrationTest; @@@ -39,10 -33,14 +39,11 @@@ import org.apache.metron.integration.co import org.apache.metron.integration.components.ZKServerComponent; import org.apache.metron.solr.integration.components.SolrComponent; -import javax.annotation.Nullable; -import java.util.List; -import java.util.Map; -import java.util.Properties; + public class SolrIndexingIntegrationTest extends IndexingIntegrationTest { - private String collection = "metron"; + private String collection = "yaf"; + private FieldNameConverter fieldNameConverter = fieldName -> fieldName; @Override public FieldNameConverter getFieldNameConverter() { http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java ---------------------------------------------------------------------- diff --cc metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java index 2b523d8,0000000..1a8e290 mode 100644,000000..100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java @@@ -1,196 -1,0 +1,201 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.solr.integration.schema; + +import com.google.common.collect.Iterables; +import com.google.common.io.Files; +import org.apache.metron.common.configuration.writer.WriterConfiguration; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.common.writer.BulkWriterResponse; +import org.apache.metron.solr.integration.components.SolrComponent; +import org.apache.metron.solr.writer.SolrWriter; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.storm.tuple.Tuple; +import org.apache.zookeeper.KeeperException; +import org.json.simple.JSONObject; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.*; + +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; +import static org.mockito.Mockito.mock; + +public class SchemaValidationIntegrationTest { + public static Iterable<String> getData(String sensor) throws IOException { + return Iterables.filter( + Files.readLines(new File("src/test/resources/example_data/" + sensor), Charset.defaultCharset()), + s -> !s.startsWith("#") && s.length() > 0 + ); + } + + public static Map<String, Object> getGlobalConfig(String sensorType, SolrComponent component) { + Map<String, Object> globalConfig = new HashMap<>(); + globalConfig.put(SOLR_ZOOKEEPER, component.getZookeeperUrl()); + return globalConfig; + } + + public static SolrComponent createSolrComponent(String sensor) throws Exception { + return new SolrComponent.Builder().build(); + } + + @Test + public void testError() throws Exception { + test("error"); + } + + @Test + public void testBro() throws Exception { + test("bro"); + } + + @Test + public void testSnort() throws Exception { + test("snort"); + } + + @Test + public void testYaf() throws Exception { + test("yaf"); + } + + public String getGuid(Map<String, Object> m) { + if(m.containsKey("guid")) { + return (String)m.get("guid"); + } + else { + return (String) m.get("original_string"); + } + } + + public void test(String sensorType) throws Exception { + SolrComponent component = null; + try { + component = createSolrComponent(sensorType); + component.start(); + component.addCollection(String.format("%s", sensorType), String.format("src/main/config/schema/%s", sensorType)); + Map<String, Object> globalConfig = getGlobalConfig(sensorType, component); + + List<JSONObject> inputs = new ArrayList<>(); + List<Tuple> tuples = new ArrayList<>(); + Map<String, Map<String, Object>> index = new HashMap<>(); + for (String message : getData(sensorType)) { + if (message.trim().length() > 0) { + Tuple t = mock(Tuple.class); + tuples.add(t); + Map<String, Object> m = JSONUtils.INSTANCE.load(message.trim(), JSONUtils.MAP_SUPPLIER); + String guid = getGuid(m); + index.put(guid, m); + inputs.add(new JSONObject(m)); + } + } + Assert.assertTrue(inputs.size() > 0); + + SolrWriter solrWriter = new SolrWriter(); + + WriterConfiguration writerConfig = new WriterConfiguration() { + @Override + public int getBatchSize(String sensorName) { + return inputs.size(); + } + + @Override + public int getBatchTimeout(String sensorName) { + return 0; + } + + @Override + public List<Integer> getAllConfiguredTimeouts() { + return new ArrayList<>(); + } + + @Override + public String getIndex(String sensorName) { + return sensorType; + } + + @Override + public boolean isEnabled(String sensorName) { + return true; + } + + @Override + public Map<String, Object> getSensorConfig(String sensorName) { + return new HashMap<String, Object>() {{ + put("index", sensorType); + put("batchSize", inputs.size()); + put("enabled", true); + }}; + } + + @Override + public Map<String, Object> getGlobalConfig() { + return globalConfig; + } + + @Override + public boolean isDefault(String sensorName) { + return false; + } ++ ++ @Override ++ public String getFieldNameConverter(String sensorName) { ++ return null; ++ } + }; + + solrWriter.init(null, null, writerConfig); + + BulkWriterResponse response = solrWriter.write(sensorType, writerConfig, tuples, inputs); + Assert.assertTrue(response.getErrors().isEmpty()); + for (Map<String, Object> m : component.getAllIndexedDocs(sensorType)) { + Map<String, Object> expected = index.get(getGuid(m)); + for (Map.Entry<String, Object> field : expected.entrySet()) { + if (field.getValue() instanceof Collection && ((Collection) field.getValue()).size() == 0) { + continue; + } + if(m.get(field.getKey()) instanceof Number) { + Number n1 = ConversionUtils.convert(field.getValue(), Double.class); + Number n2 = (Number)m.get(field.getKey()); + boolean isSame = Math.abs(n1.doubleValue() - n2.doubleValue()) < 1e-3; + if(!isSame) { + String s1 = "" + n1.doubleValue(); + String s2 = "" + n2.doubleValue(); + isSame = s1.startsWith(s2) || s2.startsWith(s1); + } + Assert.assertTrue("Unable to validate " + field.getKey() + ": " + n1 + " != " + n2, isSame); + } + else { + Assert.assertEquals("Unable to find " + field.getKey(), "" + field.getValue(), "" + m.get(field.getKey())); + } + } + } + } + finally { + if(component != null) { + component.stop(); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/pom.xml ----------------------------------------------------------------------
