This is an automated email from the ASF dual-hosted git repository. nickallen pushed a commit to branch feature/METRON-2088-support-hdp-3.1 in repository https://gitbox.apache.org/repos/asf/metron.git
commit e6359655733395c4c2a45b126611a823909075f7 Merge: f77946d c402e64 Author: Nick Allen <n...@nickallen.org> AuthorDate: Wed Sep 4 17:15:12 2019 -0400 Merge remote-tracking branch 'apache/master' into MERGE-MASTER-SEPT Upgrading.md | 9 + dependencies_with_url.csv | 2 +- metron-analytics/metron-maas-common/pom.xml | 1 + .../java/org/apache/metron/maas/util/RESTUtil.java | 4 +- metron-analytics/metron-maas-service/pom.xml | 6 + .../apache/metron/maas/service/runner/Runner.java | 3 +- .../metron/maas/service/MaasIntegrationTest.java | 7 +- metron-analytics/metron-profiler-client/pom.xml | 8 + .../src/main/assembly/assembly.xml | 2 +- .../profiler/client/HBaseProfilerClient.java | 34 +- .../metron/profiler/client/stellar/GetProfile.java | 61 +- .../profiler/client/stellar/VerboseProfile.java | 57 +- .../profiler/client/HBaseProfilerClientTest.java | 30 +- .../metron/profiler/client/ProfileWriter.java | 25 +- .../profiler/client/stellar/GetProfileTest.java | 35 +- .../client/stellar/VerboseProfileTest.java | 50 +- metron-analytics/metron-profiler-common/pom.xml | 6 + .../profiler/DefaultMessageDistributorTest.java | 6 +- .../profiler/hbase/SaltyRowKeyBuilderTest.java | 33 +- metron-analytics/metron-profiler-repl/pom.xml | 8 + .../src/main/assembly/assembly.xml | 2 +- .../profiler/repl/StandAloneProfilerTest.java | 3 +- metron-analytics/metron-profiler-spark/pom.xml | 3 + .../src/main/assembly/assembly.xml | 2 +- .../metron/profiler/spark/BatchProfilerConfig.java | 11 +- .../spark/function/MessageRouterFunction.java | 9 +- .../src/main/scripts/start_batch_profiler.sh | 4 +- metron-analytics/metron-profiler-storm/pom.xml | 8 + .../metron/profiler/storm/ProfileSplitterBolt.java | 11 +- .../src/main/scripts/start_profiler_topology.sh | 3 +- .../profiler/storm/ProfileSplitterBoltTest.java | 6 +- .../storm/integration/ConfigUploadComponent.java | 3 +- .../storm/integration/ProfilerIntegrationTest.java | 4 +- metron-analytics/metron-statistics/pom.xml | 3 + metron-contrib/metron-performance/pom.xml | 8 + .../src/main/assembly/assembly.xml | 2 +- .../metron/performance/load/LoadOptions.java | 8 +- .../load/monitor/writers/CSVWriter.java | 5 +- .../src/main/scripts/load_tool.sh | 2 +- .../metron/performance/load/LoadOptionsTest.java | 5 +- metron-deployment/development/centos6/README.md | 5 +- .../5.6.14/package/scripts/elastic_commands.py | 14 +- .../5.6.14/package/scripts/elastic_master.py | 7 +- .../5.6.14/package/scripts/elastic_slave.py | 3 +- .../5.6.14/package/scripts/service_check.py | 4 +- .../KIBANA/5.6.14/package/scripts/common.py | 8 +- .../KIBANA/5.6.14/package/scripts/kibana_master.py | 2 +- .../CURRENT/configuration/metron-parsers-env.xml | 2 +- .../common-services/METRON/CURRENT/metainfo.xml | 5 + .../CURRENT/package/scripts/alerts_ui_commands.py | 4 +- .../CURRENT/package/scripts/alerts_ui_master.py | 10 +- .../package/scripts/dashboard/dashboardindex.py | 10 +- .../CURRENT/package/scripts/enrichment_commands.py | 6 +- .../CURRENT/package/scripts/enrichment_master.py | 10 +- .../CURRENT/package/scripts/indexing_commands.py | 79 +- .../CURRENT/package/scripts/indexing_master.py | 32 +- .../package/scripts/management_ui_commands.py | 6 +- .../package/scripts/management_ui_master.py | 7 +- .../CURRENT/package/scripts/metron_client.py | 11 +- .../CURRENT/package/scripts/metron_security.py | 7 +- .../CURRENT/package/scripts/metron_service.py | 12 +- .../CURRENT/package/scripts/params/params.py | 2 +- .../CURRENT/package/scripts/params/params_linux.py | 9 +- .../package/scripts/params/status_params.py | 4 +- .../CURRENT/package/scripts/parser_commands.py | 9 +- .../CURRENT/package/scripts/parser_master.py | 7 +- .../CURRENT/package/scripts/pcap_commands.py | 11 +- .../METRON/CURRENT/package/scripts/pcap_master.py | 13 +- .../CURRENT/package/scripts/profiler_commands.py | 6 +- .../CURRENT/package/scripts/profiler_master.py | 13 +- .../CURRENT/package/scripts/rest_commands.py | 6 +- .../METRON/CURRENT/package/scripts/rest_master.py | 8 +- .../CURRENT/package/scripts/service_check.py | 14 +- .../package/templates/alerts-ui-app-config.json.j2 | 4 +- .../templates/management-ui-app-config.json.j2 | 4 +- .../METRON/CURRENT/service_advisor.py | 6 +- .../packaging/docker/rpm-docker/SPECS/metron.spec | 35 +- .../packaging/docker/rpm-docker/pom.xml | 6 + metron-interface/metron-alerts/package-lock.json | 31972 ++++++++----------- metron-interface/metron-alerts/package.json | 40 +- .../alert-details/alert-details.component.spec.ts | 4 +- .../alerts/alerts-list/alerts-list.component.html | 43 +- .../alerts/alerts-list/alerts-list.component.scss | 24 + .../alerts-list/alerts-list.component.spec.ts | 148 +- .../alerts/alerts-list/alerts-list.component.ts | 139 +- .../app/alerts/alerts-list/alerts-list.module.ts | 89 +- .../app/alerts/alerts-list/query-builder.spec.ts | 165 + .../src/app/alerts/alerts-list/query-builder.ts | 42 +- .../table-view/table-view.component.spec.ts | 7 +- .../alerts-list/table-view/table-view.component.ts | 29 +- .../alerts-list/tree-view/tree-view.component.html | 2 +- .../tree-view/tree-view.component.spec.ts | 13 +- .../alerts-list/tree-view/tree-view.component.ts | 63 +- .../configure-rows/configure-rows.component.html | 7 +- .../configure-rows/configure-rows.component.scss | 2 +- .../configure-rows.component.spec.ts | 14 +- .../configure-rows/configure-rows.component.ts | 5 +- .../alerts/configure-rows/configure-rows.module.ts | 22 +- .../show-hide-alert-entries.component.spec.ts | 132 + .../show-hide/show-hide-alert-entries.component.ts | 51 + .../show-hide/show-hide.service.spec.ts | 125 + .../configure-rows/show-hide/show-hide.service.ts | 70 + .../metron-alerts/src/app/app.component.html | 69 +- .../metron-alerts/src/app/app.component.scss | 31 +- .../metron-alerts/src/app/app.component.spec.ts | 37 +- .../metron-alerts/src/app/app.component.ts | 37 +- .../metron-alerts/src/app/app.module.ts | 30 +- .../metron-alerts/src/app/model/filter.spec.ts | 42 +- .../metron-alerts/src/app/model/filter.ts | 69 +- .../src/app/service/app-config.service.ts | 14 + .../app/service/elasticsearch-localstorage-impl.ts | 2 +- .../src/app/service/search.service.ts | 5 +- .../shared/context-menu/context-menu.component.ts | 18 +- .../src/app/shared/switch/switch.component.html | 2 +- .../src/app/shared/switch/switch.component.ts | 12 +- .../src/app/shared/switch/switch.module.ts | 15 +- .../shared/time-range/time-range.component.scss | 4 +- .../metron-alerts/src/app/utils/constants.ts | 18 +- .../metron-alerts/src/app/utils/httpUtil.ts | 4 +- .../metron-alerts/src/app/utils/utils.ts | 16 +- .../metron-alerts/src/assets/app-config.json | 4 +- .../metron-alerts/src/assets/images/logo-name.png | Bin 0 -> 3760 bytes .../src/assets/images/logo-symbol.png | Bin 0 -> 19063 bytes metron-interface/metron-alerts/src/slider.scss | 3 +- metron-interface/metron-alerts/src/styles.scss | 64 + metron-interface/metron-config/angular.json | 1 - metron-interface/metron-config/package-lock.json | 5931 ++-- metron-interface/metron-config/package.json | 43 +- .../metron-config/src/app/app.component.html | 69 +- .../metron-config/src/app/app.component.scss | 23 + .../metron-config/src/app/app.component.ts | 37 +- .../metron-config/src/app/app.module.ts | 21 +- .../src/app/navbar/navbar.component.scss | 10 + .../metron-config/src/app/navbar/navbar.html | 8 +- .../sensor-rule-editor.component.spec.ts | 3 + .../src/app/service/app-config.service.ts | 14 + .../src/app/service/mock.app-config.service.ts | 9 + .../sensor-enrichment-config.service.spec.ts | 4 +- .../service/sensor-enrichment-config.service.ts | 2 +- .../service/sensor-indexing-config.service.spec.ts | 10 +- .../app/service/sensor-indexing-config.service.ts | 2 +- .../metron-config/src/app/shared/metron-alerts.ts | 2 +- .../number-spinner/number-spinner.component.html | 2 +- .../number-spinner/number-spinner.component.scss | 9 +- .../metron-config/src/app/util/httpUtil.ts | 2 +- .../metron-config/src/assets/app-config.json | 4 +- .../metron-config/src/assets/images/logo-name.png | Bin 0 -> 3760 bytes .../src/assets/images/logo-symbol.png | Bin 0 -> 19063 bytes metron-interface/metron-config/src/styles.scss | 66 + metron-interface/metron-config/tsconfig.json | 1 + metron-interface/metron-rest/pom.xml | 22 + .../metron-rest/src/main/assembly/assembly.xml | 2 +- .../org/apache/metron/rest/config/GrokConfig.java | 3 +- .../org/apache/metron/rest/config/IndexConfig.java | 2 +- .../rest/service/impl/DockerStormCLIWrapper.java | 4 +- .../metron/rest/service/impl/GrokServiceImpl.java | 5 +- .../impl/SensorEnrichmentConfigServiceImpl.java | 15 +- .../impl/SensorIndexingConfigServiceImpl.java | 4 +- .../impl/SensorParserConfigServiceImpl.java | 6 +- .../metron/rest/service/impl/StormCLIWrapper.java | 4 +- .../metron/rest/user/UserSettingsClient.java | 14 +- .../metron-rest/src/main/scripts/metron-rest.sh | 2 + .../apache/metron/rest/config/HBaseConfigTest.java | 8 - .../org/apache/metron/rest/config/TestConfig.java | 33 +- .../controller/GrokControllerIntegrationTest.java | 3 +- .../controller/HdfsControllerIntegrationTest.java | 3 +- .../controller/PcapControllerIntegrationTest.java | 5 +- ...rEnrichmentConfigControllerIntegrationTest.java | 24 +- ...sorIndexingConfigControllerIntegrationTest.java | 9 +- .../UpdateControllerIntegrationTest.java | 3 +- .../metron/rest/generator/SampleDataGenerator.java | 2 +- .../rest/mock/MockPcapToPdmlScriptWrapper.java | 3 +- .../service/impl/DockerStormCLIWrapperTest.java | 4 +- .../service/impl/GlobalConfigServiceImplTest.java | 8 +- .../rest/service/impl/GrokServiceImplTest.java | 3 +- .../rest/service/impl/PcapServiceImplTest.java | 5 +- .../SensorEnrichmentConfigServiceImplTest.java | 30 +- .../impl/SensorIndexingConfigServiceImplTest.java | 8 +- .../impl/SensorParserConfigServiceImplTest.java | 12 +- .../metron/rest/user/UserSettingsClientTest.java | 22 +- metron-platform/elasticsearch-shaded/pom.xml | 2 + .../metron-common-storm/pom.xml | 6 + .../common/message/metadata/RawMessageUtil.java | 3 +- .../message/metadata/RawMessageUtilTest.java | 84 +- metron-platform/metron-common/pom.xml | 51 +- .../metron-common/src/main/assembly/assembly.xml | 2 +- .../common/configuration/ConfigurationsUtils.java | 30 +- .../common/configuration/SensorParserConfig.java | 3 +- .../enrichment/SensorEnrichmentConfig.java | 3 +- .../enrichment/SensorEnrichmentUpdateConfig.java | 12 +- .../configuration/profiler/ProfileConfig.java | 3 +- .../configuration/profiler/ProfilerConfig.java | 3 +- .../apache/metron/common/error/MetronError.java | 2 +- .../common/field/validation/SimpleValidation.java | 3 + .../field/validation/network/IPValidation.java | 2 +- .../field/validation/primitive/DateValidation.java | 3 + .../metron/common/hadoop/SequenceFileIterable.java | 10 +- .../metadata/EnvelopedRawMessageStrategy.java | 8 +- .../org/apache/metron/common/utils/KafkaUtils.java | 3 +- .../org/apache/metron/common/utils/LazyLogger.java | 94 + .../metron/common/utils/LazyLoggerFactory.java | 71 + .../apache/metron/common/utils/LazyLoggerImpl.java | 572 + .../configurations/ConfigurationsUpdater.java | 13 +- .../metron-common/src/main/scripts/stellar | 5 +- .../src/main/scripts/zk_load_configs.sh | 3 +- .../cli/ConfigurationManagerIntegrationTest.java | 9 +- .../common/configuration/ConfigurationTest.java | 3 +- .../configuration/ParserConfigurationsTest.java | 4 +- .../configuration/SensorEnrichmentConfigTest.java | 10 +- .../configuration/SensorParserConfigTest.java | 8 +- .../writer/EnrichmentWriterConfigurationTest.java | 3 +- .../writer/ParserWriterConfigurationTest.java | 4 +- .../writer/ProfilerWriterConfigurationTest.java | 3 +- .../metron/common/error/MetronErrorTest.java | 3 +- .../common/field/FieldNameConvertersTest.java | 3 +- .../validation/network/DomainValidationTest.java | 8 + .../validation/network/EmailValidationTest.java | 6 + .../validation/network/URLValidationTest.java | 8 +- .../validation/primitive/DateValidationTest.java | 4 + .../primitive/IntegerValidationTest.java | 6 + .../typosquat/TyposquattingStrategiesTest.java | 4 +- .../apache/metron/common/utils/JSONUtilsTest.java | 7 +- .../apache/metron/common/utils/KafkaUtilsTest.java | 9 +- .../metron/common/utils/LazyLoggerImplTest.java | 759 + .../common/utils/LazzyLoggerImplPerfTest.java | 118 + .../common/utils/file/ReaderSpliteratorTest.java | 5 +- .../ZKConfigurationsCacheIntegrationTest.java | 15 +- .../components/ConfigUploadComponent.java | 18 +- metron-platform/metron-data-management/pom.xml | 89 +- .../src/main/assembly/assembly.xml | 2 +- .../metron/dataloads/hbase/mr/PrunerMapper.java | 3 +- .../nonbulk/flatfile/HBaseExtractorState.java | 8 +- .../nonbulk/flatfile/importer/LocalImporter.java | 12 +- .../nonbulk/flatfile/location/RawLocation.java | 11 +- .../dataloads/nonbulk/taxii/TaxiiHandler.java | 16 +- .../src/main/scripts/flatfile_loader.sh | 7 +- .../src/main/scripts/flatfile_summarizer.sh | 7 +- .../src/main/scripts/maxmind_enrichment_load.sh | 10 +- .../src/main/scripts/prune_hdfs_files.sh | 2 +- .../src/main/scripts/threatintel_bulk_prune.sh | 2 +- .../src/main/scripts/threatintel_taxii_load.sh | 2 +- .../extractor/stix/StixExtractorTest.java | 6 +- .../metron/dataloads/hbase/mr/HBaseUtil.java | 22 +- .../mr/LeastRecentlyUsedPrunerIntegrationTest.java | 22 +- ...pleEnrichmentFlatFileLoaderIntegrationTest.java | 63 +- .../flatfile/SimpleFlatFileSummarizerTest.java | 3 +- .../nonbulk/taxii/TaxiiIntegrationTest.java | 4 +- .../metron-elasticsearch-common/pom.xml | 26 + .../bulk/ElasticsearchImportExport.java | 10 +- .../ElasticsearchUpdateIntegrationTest.java | 5 - .../metron-elasticsearch-storm/pom.xml | 19 + .../main/scripts/start_elasticsearch_topology.sh | 5 +- .../metron-enrichment-common/pom.xml | 1 + .../enrichment/adapters/cif/CIFHbaseAdapter.java | 18 +- .../adapters/simplehbase/SimpleHBaseAdapter.java | 10 +- .../adapters/stellar/StellarAdapter.java | 2 +- .../adapters/threatintel/ThreatIntelAdapter.java | 2 + .../metron/enrichment/cli/LatencySummarizer.java | 4 +- .../enrichment/converter/EnrichmentHelper.java | 7 +- .../metron/enrichment/lookup/EnrichmentLookup.java | 18 +- .../lookup/accesstracker/AccessTrackerUtil.java | 4 +- .../accesstracker/PersistentAccessTracker.java | 8 +- .../PersistentBloomTrackerCreator.java | 9 +- .../stellar/SimpleHBaseEnrichmentFunctions.java | 20 +- .../metron/enrichment/utils/EnrichmentUtils.java | 6 +- .../threatintel/ThreatIntelAdapterTest.java | 4 +- .../metron-enrichment-storm/pom.xml | 6 + .../src/main/scripts/start_enrichment_topology.sh | 3 +- .../integration/EnrichmentIntegrationTest.java | 21 +- metron-platform/metron-hbase-client/pom.xml | 2 + metron-platform/metron-hbase-server/README.md | 27 + metron-platform/metron-hbase-server/pom.xml | 6 + .../EnrichmentCoprocessorIntegrationTest.java | 11 +- .../coprocessor/EnrichmentCoprocessorTest.java | 8 +- .../metron/hbase/HBaseProjectionCriteria.java | 7 +- .../org/apache/metron/hbase/HTableProvider.java | 46 +- .../org/apache/metron/hbase/TableProvider.java | 8 +- .../apache/metron/hbase/client/HBaseClient.java | 11 +- .../metron/hbase/client/HBaseClientTest.java | 37 +- .../metron/hbase/mock/MockHBaseTableProvider.java | 18 +- .../org/apache/metron/hbase/mock/MockHTable.java | 38 +- .../metron-indexing/metron-indexing-common/pom.xml | 11 +- .../org/apache/metron/indexing/dao/HBaseDao.java | 12 +- .../metron/indexing/dao/UpdateIntegrationTest.java | 84 +- .../integration/HBaseDaoIntegrationTest.java | 5 - .../metron-indexing/metron-indexing-storm/pom.xml | 6 + .../src/main/scripts/start_hdfs_topology.sh | 3 +- .../integration/HDFSIndexingIntegrationTest.java | 3 +- .../integration/IndexingIntegrationTest.java | 25 +- metron-platform/metron-integration-test/pom.xml | 1 + .../apache/metron/integration/ProcessorResult.java | 3 +- .../components/FluxTopologyComponent.java | 8 +- .../apache/metron/integration/utils/TestUtils.java | 6 +- .../sample/data/syslog5424/parsed/Syslog5424Parsed | 4 +- .../sample/data/syslog5424/raw/Syslog5424Output | 4 +- metron-platform/metron-management/pom.xml | 3 + .../src/main/assembly/assembly.xml | 2 +- .../metron/management/ConfigurationFunctions.java | 11 +- .../apache/metron/management/GrokFunctions.java | 3 +- .../metron/management/StellarParserRunner.java | 8 +- .../management/ConfigurationFunctionsTest.java | 31 +- .../metron/management/ParserFunctionsTest.java | 14 +- metron-platform/metron-parsing/README.md | 7 +- .../metron-parsers-common/3rdPartyParser.md | 17 + .../metron-parsing/metron-parsers-common/pom.xml | 6 + .../org/apache/metron/parsers/BasicParser.java | 39 +- .../java/org/apache/metron/parsers/GrokParser.java | 35 +- .../apache/metron/parsers/ParserRunnerImpl.java | 3 +- .../org/apache/metron/parsers/csv/CSVParser.java | 8 +- .../metron/parsers/interfaces/MessageParser.java | 21 +- .../apache/metron/parsers/json/JSONMapParser.java | 5 +- .../parsers/regex/RegularExpressionsParser.java | 25 +- .../metron/parsers/syslog/BaseSyslogParser.java | 44 +- .../metron/parsers/AbstractParserConfigTest.java | 9 +- .../org/apache/metron/parsers/BasicParserTest.java | 167 + .../org/apache/metron/parsers/GrokParserTest.java | 3 +- .../apache/metron/parsers/MessageParserTest.java | 13 +- .../metron/parsers/MultiLineGrokParserTest.java | 5 +- .../parsers/MultiLineWithErrorsGrokParserTest.java | 5 +- .../metron/parsers/ParserRunnerImplTest.java | 24 +- .../apache/metron/parsers/csv/CSVParserTest.java | 29 +- .../EnvelopedParserIntegrationTest.java | 16 +- .../parsers/integration/ParserIntegrationTest.java | 9 +- .../integration/validation/ParserDriver.java | 6 +- .../validation/SampleDataValidation.java | 5 +- .../parsers/json/JSONMapParserQueryTest.java | 17 +- .../metron/parsers/json/JSONMapParserTest.java | 49 +- .../json/JSONMapParserWrappedQueryTest.java | 15 +- .../regex/RegularExpressionsParserTest.java | 32 +- .../parsers/syslog/Syslog3164ParserTest.java | 9 +- .../parsers/syslog/Syslog5424ParserTest.java | 22 +- .../metron-parsing/metron-parsers/pom.xml | 6 + .../apache/metron/parsers/asa/BasicAsaParser.java | 26 +- .../apache/metron/parsers/bro/BasicBroParser.java | 12 +- .../org/apache/metron/parsers/cef/CEFParser.java | 5 +- .../metron/parsers/fireeye/BasicFireEyeParser.java | 8 +- .../apache/metron/parsers/ise/BasicIseParser.java | 13 +- .../apache/metron/parsers/ise/JavaCharStream.java | 10 +- .../metron/parsers/lancope/BasicLancopeParser.java | 4 +- .../org/apache/metron/parsers/leef/LEEFParser.java | 7 +- .../parsers/logstash/BasicLogstashParser.java | 3 +- .../paloalto/BasicPaloAltoFirewallParser.java | 13 +- .../metron/parsers/snort/BasicSnortParser.java | 7 +- .../parsers/sourcefire/BasicSourcefireParser.java | 4 +- .../org/apache/metron/parsers/SnortParserTest.java | 43 +- .../metron/parsers/asa/BasicAsaParserTest.java | 67 +- .../metron/parsers/bro/BasicBroParserTest.java | 98 +- .../apache/metron/parsers/cef/CEFParserTest.java | 62 +- .../parsers/fireeye/BasicFireEyeParserTest.java | 32 +- .../metron/parsers/ise/BasicIseParserTest.java | 23 +- .../parsers/lancope/BasicLancopeParserTest.java | 23 +- .../apache/metron/parsers/leef/LEEFParserTest.java | 53 +- .../paloalto/BasicPaloAltoFirewallParserTest.java | 57 +- .../sourcefire/BasicSourcefireParserTest.java | 25 +- .../parsers/websphere/GrokWebSphereParserTest.java | 85 +- .../metron-parsing/metron-parsing-storm/pom.xml | 11 + .../src/main/scripts/start_parser_topology.sh | 8 +- .../integration/validation/StormParserDriver.java | 27 +- .../integration/WriterBoltIntegrationTest.java | 5 +- metron-platform/metron-pcap-backend/pom.xml | 40 +- .../src/main/assembly/assembly.xml | 2 +- .../src/main/scripts/pcap_inspector.sh | 2 +- .../src/main/scripts/pcap_query.sh | 2 +- .../src/main/scripts/start_pcap_topology.sh | 6 +- .../org/apache/metron/pcap/query/PcapCliTest.java | 8 +- metron-platform/metron-pcap/pom.xml | 10 +- .../java/org/apache/metron/pcap/mr/PcapJob.java | 9 +- .../apache/metron/pcap/utils/FileFilterUtil.java | 12 +- .../metron-solr/metron-solr-common/pom.xml | 6 + .../org/apache/metron/solr/dao/SolrSearchDao.java | 50 +- .../org/apache/metron/solr/dao/SolrUtilities.java | 5 +- .../metron/solr/schema/SchemaTranslator.java | 3 +- .../apache/metron/solr/dao/SolrSearchDaoTest.java | 22 +- .../apache/metron/solr/dao/SolrUtilitiesTest.java | 5 +- .../SolrRetrieveLatestIntegrationTest.java | 22 +- .../integration/SolrSearchIntegrationTest.java | 1 - .../integration/SolrUpdateIntegrationTest.java | 54 +- .../src/main/scripts/start_solr_topology.sh | 3 +- .../metron-storm-kafka-override/pom.xml | 2 + metron-platform/metron-storm-kafka/pom.xml | 1 + .../metron/test/converters/BinaryConverters.java | 4 +- .../apache/metron/test/filereaders/FileReader.java | 3 +- .../test/spouts/GenericInternalTestSpout.java | 3 +- .../org/apache/metron/test/utils/KafkaLoader.java | 5 +- .../metron-writer/metron-writer-common/pom.xml | 45 +- .../writer/hbase/SimpleHbaseEnrichmentWriter.java | 31 +- .../metron-writer/metron-writer-storm/pom.xml | 6 + .../org/apache/metron/writer/hdfs/HdfsWriter.java | 13 +- .../apache/metron/writer/hdfs/SourceHandler.java | 11 +- .../writer/bolt/BulkMessageWriterBoltTest.java | 3 +- .../metron/zookeeper/SimpleEventListener.java | 4 +- metron-stellar/stellar-common/README.md | 100 +- metron-stellar/stellar-common/pom.xml | 10 +- .../stellar-common/src/main/assembly/assembly.xml | 12 - .../metron/stellar/common/utils/JSONUtils.java | 6 +- .../metron/stellar/dsl/functions/RestConfig.java | 23 +- .../stellar/dsl/functions/RestFunctions.java | 657 +- .../src/main/scripts/deployed/stellar | 4 +- .../stellar/common/shell/cli/StellarShellTest.java | 5 +- .../stellar/dsl/functions/DateFunctionsTest.java | 29 +- .../functions/RestFunctionsIntegrationTest.java | 490 + .../stellar/dsl/functions/RestFunctionsTest.java | 458 +- metron-stellar/stellar-zeppelin/pom.xml | 1 + pom.xml | 7 +- 404 files changed, 24488 insertions(+), 23197 deletions(-) diff --cc metron-contrib/metron-performance/pom.xml index 4726608,40992ab..610e804 --- a/metron-contrib/metron-performance/pom.xml +++ b/metron-contrib/metron-performance/pom.xml @@@ -43,8 -43,14 +43,14 @@@ <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.apache.kafka</groupId> - <artifactId>kafka_2.10</artifactId> + <artifactId>kafka-clients</artifactId> <version>${global_kafka_version}</version> <scope>provided</scope> </dependency> diff --cc metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 694c7fa,aec0afe..80f4b87 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@@ -18,22 -18,14 +18,22 @@@ limitations under the License """ - from resource_management.libraries.script import Script - from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions import default, format + from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions.version import format_stack_version + from resource_management.libraries.script import Script +def get_hostname(config): + if 'agentLevelParams' in config.keys(): + # Ambari 2.7.x + return config['agentLevelParams']['hostname'] + else: + # Ambari 2.6.x + return config['hostname'] + config = Script.get_config() -hostname = config['hostname'] +hostname = get_hostname(config) metron_user = config['configurations']['metron-env']['metron_user'] metron_home = config['configurations']['metron-env']['metron_home'] metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir'] diff --cc metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java index 1509ae2,d363e91..d0fc444 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java @@@ -39,14 -27,10 +27,14 @@@ import org.apache.curator.framework.Cur import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.hbase.HBaseConfiguration; - import org.apache.hadoop.hbase.client.HTableInterface; + import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.zookeeper.ConfigurationsCache; import org.apache.metron.common.zookeeper.ZKConfigurationsCache; diff --cc metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml index e915d99,15df035..8edf5ab --- a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml +++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml @@@ -62,11 -68,18 +68,23 @@@ </exclusions> </dependency> <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>${global_guava_version}</version> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> <version>${global_hbase_version}</version> diff --cc metron-platform/metron-parsing/metron-parsers/pom.xml index 5c323d8,cbf9c2f..0f4bc31 --- a/metron-platform/metron-parsing/metron-parsers/pom.xml +++ b/metron-platform/metron-parsing/metron-parsers/pom.xml @@@ -46,9 -41,19 +46,15 @@@ <artifactId>metron-common</artifactId> <version>${project.parent.version}</version> </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> <!-- 3rd party dependencies --> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>${global_guava_version}</version> - </dependency> + <dependency> <groupId>io.thekraken</groupId> <artifactId>grok</artifactId>