Resolve merge conflicts with master.
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f30e9cc1 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f30e9cc1 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f30e9cc1 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: f30e9cc1fb9b6985e7ad7582c764d035b5cb0e1e Parents: 9cee51e cbdaee1 Author: Michael Miklavcic <michael.miklav...@gmail.com> Authored: Tue Jul 10 19:31:56 2018 -0600 Committer: Michael Miklavcic <michael.miklav...@gmail.com> Committed: Tue Jul 10 19:31:56 2018 -0600 ---------------------------------------------------------------------- dependencies_with_url.csv | 3 + .../metron/maas/service/runner/Runner.java | 2 +- metron-analytics/metron-profiler/.gitignore | 1 + .../profiler/bolt/ProfileBuilderBolt.java | 4 +- metron-deployment/Kerberos-manual-setup.md | 441 ++--- .../ansible/playbooks/metron_full_install.yml | 4 + .../ansible/playbooks/solr_install.yml | 40 + .../roles/ambari_config/vars/single_node_vm.yml | 1 + .../roles/sensor-stubs/templates/start-bro-stub | 25 +- .../sensor-stubs/templates/start-snort-stub | 25 +- .../roles/sensor-stubs/templates/start-yaf-stub | 26 +- .../ansible/roles/solr/defaults/main.yml | 23 + .../ansible/roles/solr/meta/main.yml | 19 + .../ansible/roles/solr/tasks/main.yml | 36 + metron-deployment/packaging/ambari/.gitignore | 1 + .../packaging/ambari/metron-mpack/pom.xml | 8 +- .../METRON/CURRENT/configuration/metron-env.xml | 38 +- .../configuration/metron-indexing-env.xml | 32 +- .../common-services/METRON/CURRENT/metainfo.xml | 4 + .../package/scripts/indexing_commands.py | 76 +- .../CURRENT/package/scripts/indexing_master.py | 48 +- .../CURRENT/package/scripts/metron_service.py | 102 +- .../package/scripts/params/params_linux.py | 21 +- .../package/scripts/params/status_params.py | 3 + .../CURRENT/package/scripts/rest_commands.py | 1 + .../METRON/CURRENT/themes/metron_theme.json | 18 +- .../docker/rpm-docker/SPECS/metron.spec | 17 + metron-interface/metron-alerts/.gitignore | 1 + metron-interface/metron-alerts/README.md | 3 +- .../alert-filters/alert-filters.e2e-spec.ts | 2 +- .../e2e/alerts-list/alerts-list.po.ts | 6 +- .../meta-alerts/meta-alert.e2e-spec.ts | 2 +- .../table-view/table-view.e2e-spec.ts | 90 + .../e2e/alerts-list/table-view/table-view.po.ts | 26 + .../alerts-list/tree-view/tree-view.e2e-spec.ts | 4 +- .../e2e/mock-data/alerts_ui_e2e_index.data | 16 +- .../metron-alerts/e2e/utils/e2e_util.ts | 62 +- metron-interface/metron-alerts/karma.conf.js | 2 +- .../metron-alerts/protractor.conf.js | 1 + .../alert-details/alert-details.component.ts | 135 +- .../alert-filters.component.spec.ts | 14 +- .../alerts-list/alerts-list.component.html | 5 +- .../alerts/alerts-list/alerts-list.component.ts | 7 +- .../table-view/table-view.component.html | 12 +- .../table-view/table-view.component.spec.ts | 49 +- .../table-view/table-view.component.ts | 16 +- .../tree-view/tree-view.component.spec.ts | 32 +- .../tree-view/tree-view.component.ts | 14 +- .../configure-rows.component.spec.ts | 18 +- .../save-search/save-search.component.spec.ts | 16 + .../saved-searches.component.spec.ts | 27 +- .../metron-alerts/src/app/app.component.spec.ts | 26 +- .../src/app/login/login.component.spec.ts | 40 +- .../src/app/model/comment-add-remove-request.ts | 25 + .../src/app/service/update.service.ts | 31 +- .../shared/collapse/collapse.component.spec.ts | 12 +- .../app/shared/collapse/collapse.component.ts | 2 +- .../date-picker/date-picker.component.spec.ts | 4 + .../directives/alert-search.directive.spec.ts | 21 - .../alert-severity-hexagon.directive.spec.ts | 5 +- .../directives/nav-content.directive.spec.ts | 6 +- .../list-group/list-group.component.spec.ts | 2 +- .../metron-sorter.component.spec.ts | 14 +- .../metron-table-pagination.component.spec.ts | 2 +- .../shared/pipes/center-ellipses.pipe.spec.ts | 2 +- .../pipes/column-name-translate.pipe.spec.ts | 2 +- .../src/app/shared/pipes/map-keys.pipe.spec.ts | 2 +- .../app/shared/switch/switch.component.spec.ts | 2 +- .../time-range/time-range.component.spec.ts | 12 +- .../metron-alerts/src/app/utils/constants.ts | 1 - metron-interface/metron-rest/README.md | 1 + .../apache/metron/rest/config/IndexConfig.java | 14 +- .../rest/controller/UpdateController.java | 23 + .../metron/rest/service/UpdateService.java | 3 + .../rest/service/impl/MetaAlertServiceImpl.java | 5 +- .../rest/service/impl/SearchServiceImpl.java | 2 +- .../rest/service/impl/UpdateServiceImpl.java | 21 +- .../metron-rest/src/main/scripts/metron-rest.sh | 32 +- .../MetaAlertControllerIntegrationTest.java | 6 +- .../UpdateControllerIntegrationTest.java | 96 +- .../org/apache/metron/common/Constants.java | 2 +- .../configuration/SensorParserConfig.java | 69 +- .../apache/metron/common/error/MetronError.java | 6 +- .../transformation/FieldTransformations.java | 1 + .../RegexSelectTransformation.java | 97 ++ .../metadata/DefaultRawMessageStrategy.java | 86 + .../metadata/EnvelopedRawMessageStrategy.java | 146 ++ .../common/message/metadata/MetadataUtil.java | 124 ++ .../common/message/metadata/RawMessage.java | 85 + .../message/metadata/RawMessageStrategies.java | 81 + .../message/metadata/RawMessageStrategy.java | 77 + .../common/message/metadata/RawMessageUtil.java | 59 + .../apache/metron/common/utils/JSONUtils.java | 4 +- .../common/bolt/ConfiguredParserBoltTest.java | 31 +- .../RegexSelectTransformationTest.java | 169 ++ .../message/metadata/RawMessageUtilTest.java | 282 +++ .../src/main/config/elasticsearch.properties.j2 | 2 +- .../elasticsearch/dao/ColumnMetadataDao.java | 67 - .../dao/ElasticsearchColumnMetadataDao.java | 31 +- .../elasticsearch/dao/ElasticsearchDao.java | 670 +------- .../dao/ElasticsearchMetaAlertDao.java | 669 ++------ ...ElasticsearchMetaAlertRetrieveLatestDao.java | 44 + .../dao/ElasticsearchMetaAlertSearchDao.java | 110 ++ .../dao/ElasticsearchMetaAlertUpdateDao.java | 254 +++ .../dao/ElasticsearchRetrieveLatestDao.java | 151 ++ .../dao/ElasticsearchSearchDao.java | 467 +++++ .../dao/ElasticsearchUpdateDao.java | 200 +++ .../elasticsearch/utils/ElasticsearchUtils.java | 64 + .../elasticsearch/dao/ElasticsearchDaoTest.java | 68 +- .../dao/ElasticsearchMetaAlertDaoTest.java | 238 +-- .../ElasticsearchMetaAlertIntegrationTest.java | 1224 ++----------- .../ElasticsearchSearchIntegrationTest.java | 175 +- .../ElasticsearchUpdateIntegrationTest.java | 239 +-- .../components/ElasticSearchComponent.java | 26 +- metron-platform/metron-indexing/README.md | 16 +- metron-platform/metron-indexing/pom.xml | 8 +- .../metron/indexing/dao/AccessConfig.java | 22 + .../metron/indexing/dao/ColumnMetadataDao.java | 39 + .../apache/metron/indexing/dao/HBaseDao.java | 93 +- .../apache/metron/indexing/dao/IndexDao.java | 141 +- .../metron/indexing/dao/MetaAlertDao.java | 155 -- .../metron/indexing/dao/MultiIndexDao.java | 47 + .../metron/indexing/dao/RetrieveLatestDao.java | 67 + .../metaalert/DeferredMetaAlertIndexDao.java | 42 + .../metaalert/MetaAlertAddRemoveRequest.java | 1 - .../indexing/dao/metaalert/MetaAlertConfig.java | 83 + .../dao/metaalert/MetaAlertConstants.java | 30 + .../indexing/dao/metaalert/MetaAlertDao.java | 77 + .../metaalert/MetaAlertRetrieveLatestDao.java | 25 + .../dao/metaalert/MetaAlertSearchDao.java | 35 + .../dao/metaalert/MetaAlertUpdateDao.java | 146 ++ .../indexing/dao/metaalert/MetaScores.java | 52 +- .../AbstractLuceneMetaAlertUpdateDao.java | 337 ++++ .../indexing/dao/search/AlertComment.java | 130 ++ .../metron/indexing/dao/search/SearchDao.java | 38 + .../indexing/dao/search/SearchResponse.java | 33 +- .../dao/update/CommentAddRemoveRequest.java | 78 + .../metron/indexing/dao/update/Document.java | 14 +- .../metron/indexing/dao/update/UpdateDao.java | 108 ++ .../metron/indexing/util/IndexingCacheUtil.java | 43 + .../apache/metron/indexing/dao/InMemoryDao.java | 17 + .../indexing/dao/InMemoryMetaAlertDao.java | 86 +- .../indexing/dao/SearchIntegrationTest.java | 321 ++-- .../indexing/dao/UpdateIntegrationTest.java | 306 ++++ .../dao/metaalert/MetaAlertIntegrationTest.java | 1012 +++++++++++ .../indexing/dao/metaalert/MetaScoresTest.java | 101 ++ .../AbstractLuceneMetaAlertUpdateDaoTest.java | 885 ++++++++++ .../integration/HBaseDaoIntegrationTest.java | 79 +- .../integration/IndexingIntegrationTest.java | 4 +- .../indexing/util/IndexingCacheUtilTest.java | 66 + .../components/FluxTopologyComponent.java | 9 + .../metron-parsers/ParserChaining.md | 179 ++ metron-platform/metron-parsers/README.md | 63 +- .../apache/metron/parsers/bolt/ParserBolt.java | 60 +- .../EnvelopedParserIntegrationTest.java | 209 +++ .../parsers/integration/ParserDriver.java | 2 +- .../integration/WriterBoltIntegrationTest.java | 75 +- .../src/test/resources/patterns/cisco_patterns | 6 + metron-platform/metron-pcap-backend/.gitignore | 1 + metron-platform/metron-solr/README.md | 170 ++ metron-platform/metron-solr/pom.xml | 41 +- .../src/main/config/schema/bro/schema.xml | 700 ++++++++ .../src/main/config/schema/bro/solrconfig.xml | 1601 ++++++++++++++++++ .../src/main/config/schema/error/schema.xml | 60 + .../src/main/config/schema/error/solrconfig.xml | 1601 ++++++++++++++++++ .../src/main/config/schema/metaalert/schema.xml | 58 + .../main/config/schema/metaalert/solrconfig.xml | 1601 ++++++++++++++++++ .../src/main/config/schema/snort/schema.xml | 93 + .../src/main/config/schema/snort/solrconfig.xml | 1601 ++++++++++++++++++ .../src/main/config/schema/yaf/schema.xml | 99 ++ .../src/main/config/schema/yaf/solrconfig.xml | 1601 ++++++++++++++++++ .../src/main/config/solr.properties.j2 | 2 +- .../org/apache/metron/solr/SolrConstants.java | 2 + .../metron/solr/dao/SolrColumnMetadataDao.java | 149 ++ .../org/apache/metron/solr/dao/SolrDao.java | 207 +++ .../metron/solr/dao/SolrMetaAlertDao.java | 257 +++ .../dao/SolrMetaAlertRetrieveLatestDao.java | 77 + .../metron/solr/dao/SolrMetaAlertSearchDao.java | 215 +++ .../metron/solr/dao/SolrMetaAlertUpdateDao.java | 239 +++ .../metron/solr/dao/SolrRetrieveLatestDao.java | 103 ++ .../apache/metron/solr/dao/SolrSearchDao.java | 278 +++ .../apache/metron/solr/dao/SolrUpdateDao.java | 204 +++ .../apache/metron/solr/dao/SolrUtilities.java | 143 ++ .../apache/metron/solr/schema/FieldType.java | 98 ++ .../metron/solr/schema/SchemaTranslator.java | 194 +++ .../metron/solr/writer/MetronSolrClient.java | 34 + .../apache/metron/solr/writer/SolrWriter.java | 250 ++- .../src/main/scripts/create_collection.sh | 36 + .../src/main/scripts/delete_collection.sh | 33 + .../src/main/scripts/install_solr.sh | 62 + .../metron-solr/src/main/scripts/start_solr.sh | 27 + .../src/main/scripts/start_solr_topology.sh | 2 +- .../metron-solr/src/main/scripts/stop_solr.sh | 27 + .../metron/solr/dao/SolrColumnMetadataTest.java | 149 ++ .../org/apache/metron/solr/dao/SolrDaoTest.java | 185 ++ .../metron/solr/dao/SolrMetaAlertDaoTest.java | 155 ++ .../metron/solr/dao/SolrSearchDaoTest.java | 478 ++++++ .../metron/solr/dao/SolrUpdateDaoTest.java | 241 +++ .../metron/solr/dao/SolrUtilitiesTest.java | 48 + .../SolrIndexingIntegrationTest.java | 19 +- .../SolrMetaAlertIntegrationTest.java | 412 +++++ .../SolrRetrieveLatestIntegrationTest.java | 207 +++ .../integration/SolrSearchIntegrationTest.java | 242 +++ .../integration/SolrUpdateIntegrationTest.java | 189 +++ .../integration/components/SolrComponent.java | 105 +- .../schema/SchemaValidationIntegrationTest.java | 201 +++ .../matcher/ModifiableSolrParamsMatcher.java | 55 + .../matcher/SolrInputDocumentListMatcher.java | 60 + .../solr/matcher/SolrInputDocumentMatcher.java | 50 + .../metron/solr/matcher/SolrQueryMatcher.java | 56 + .../metron/solr/writer/SolrWriterTest.java | 228 ++- .../resources/config/test/conf/managed-schema | 77 + .../resources/config/test/conf/solrconfig.xml | 1601 ++++++++++++++++++ .../src/test/resources/example_data/bro | 29 + .../src/test/resources/example_data/error | 17 + .../src/test/resources/example_data/snort | 21 + .../src/test/resources/example_data/yaf | 21 + .../test/error/MetronErrorJSONMatcher.java | 2 + metron-platform/metron-writer/README.md | 24 + .../apache/metron/writer/kafka/KafkaWriter.java | 43 +- .../metron/writer/kafka/KafkaWriterTest.java | 63 + metron-platform/pom.xml | 4 +- metron-stellar/stellar-common/README.md | 18 +- .../metron/stellar/common/StellarCompiler.java | 82 +- .../org/apache/metron/stellar/dsl/Token.java | 1 + .../stellar/dsl/functions/RegExFunctions.java | 27 +- .../stellar/dsl/functions/BasicStellarTest.java | 25 + .../metron/stellar/dsl/functions/MatchTest.java | 35 + .../dsl/functions/RegExFunctionsTest.java | 7 + pom.xml | 4 +- use-cases/forensic_clustering/README.md | 2 +- use-cases/geographic_login_outliers/README.md | 5 +- use-cases/parser_chaining/README.md | 235 +++ .../message_routing_high_level.svg | 14 + .../message_routing_high_level.xml | 14 + use-cases/typosquat_detection/README.md | 2 +- 236 files changed, 26712 insertions(+), 3958 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/dependencies_with_url.csv ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py ---------------------------------------------------------------------- diff --cc metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index 73bafbb,0ff6fb1..2fd3f0d --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@@ -161,7 -125,7 +161,8 @@@ class RestCommands "export METRON_INDEX_CP={metron_indexing_classpath};" "export METRON_LOG_DIR={metron_log_dir};" "export METRON_PID_FILE={pid_file};" + "export HDP_VERSION={hdp_version};" + "export METRON_RA_INDEXING_WRITER={ra_indexing_writer};" "{metron_home}/bin/metron-rest.sh;" "unset METRON_JDBC_PASSWORD;" )) http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-interface/metron-rest/src/main/scripts/metron-rest.sh ---------------------------------------------------------------------- diff --cc metron-interface/metron-rest/src/main/scripts/metron-rest.sh index cf7bf58,c0c9fac..7c89ae5 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh @@@ -109,8 -112,21 +112,21 @@@ METRON_REST_CLASSPATH+=":${indexing_fil echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}" + #Use Solr daos if ra indexing writer set to Solr + if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then + METRON_INDEX_DAO=" --index.dao.impl=org.apache.metron.solr.dao.SolrDao,org.apache.metron.indexing.dao.HBaseDao" + METRON_METAALERT_DAO=" --meta.dao.impl=org.apache.metron.solr.dao.SolrMetaAlertDao" + METRON_WRITER_NAME=" --index.writer.name=solr" + echo "METRON_INDEX_DAO=${METRON_INDEX_DAO}" + echo "METRON_METAALERT_DAO=${METRON_METAALERT_DAO}" + echo "METRON_WRITER_NAME=${METRON_WRITER_NAME}" + METRON_SPRING_OPTIONS+=${METRON_INDEX_DAO} + METRON_SPRING_OPTIONS+=${METRON_METAALERT_DAO} + METRON_SPRING_OPTIONS+=${METRON_WRITER_NAME} + fi + echo "Starting application" -${JAVA_HOME}/bin/java ${METRON_JVMFLAGS} \ +${JAVA_HOME}/bin/java -Dhdp.version=${HDP_VERSION} ${METRON_JVMFLAGS} \ -cp ${METRON_REST_CLASSPATH} \ org.apache.metron.rest.MetronRestApplication \ ${METRON_SPRING_OPTIONS} >> ${METRON_LOG_DIR}/metron-rest.log 2>&1 & echo $! > ${METRON_PID_FILE}; http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-platform/pom.xml ----------------------------------------------------------------------