METRON-990 Clean up and organize flux properties (merrimanr) closes apache/metron#613
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/87f65221 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/87f65221 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/87f65221 Branch: refs/heads/master Commit: 87f652215cbe9e611029f5269f6c2b30aaa8f229 Parents: cc7bbc9 Author: merrimanr <[email protected]> Authored: Wed Jul 5 11:04:34 2017 -0500 Committer: merrimanr <[email protected]> Committed: Wed Jul 5 11:04:34 2017 -0500 ---------------------------------------------------------------------- .../METRON/CURRENT/configuration/metron-env.xml | 348 ++++++-- .../package/scripts/enrichment_master.py | 1 + .../CURRENT/package/scripts/indexing_master.py | 9 + .../CURRENT/package/scripts/metron_service.py | 11 +- .../package/scripts/params/params_linux.py | 40 + .../templates/elasticsearch.properties.j2 | 49 ++ .../package/templates/enrichment.properties.j2 | 78 +- .../CURRENT/package/templates/global.json.j2 | 6 + .../METRON/CURRENT/themes/metron_theme.json | 876 +++++++++++++++++-- .../metron-rest/src/main/scripts/metron-rest | 2 +- .../src/main/config/elasticsearch.properties | 52 +- .../ElasticsearchIndexingIntegrationTest.java | 2 +- .../src/main/config/enrichment.properties | 76 +- .../src/main/flux/enrichment/remote.yaml | 32 +- .../src/main/flux/enrichment/test.yaml | 83 +- .../integration/EnrichmentIntegrationTest.java | 30 +- .../src/main/flux/indexing/remote.yaml | 16 +- .../integration/IndexingIntegrationTest.java | 15 +- .../metron-solr/src/main/config/solr.properties | 51 +- .../SolrIndexingIntegrationTest.java | 2 +- pom.xml | 1 + 21 files changed, 1427 insertions(+), 353 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index 00c48a6..a8864e3 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -137,7 +137,7 @@ </value-attributes> <on-ambari-upgrade add="true"/> </property> - <property require-input="true"> + <property> <name>metron_jdbc_platform</name> <value></value> <description>Database platform used by Metron. One of: hsqldb, h2, oracle, mysql, postgresql</description> @@ -253,69 +253,301 @@ </value-attributes> </property> <property> - <name>global-json</name> - <display-name>global.json template</display-name> - <description>This is the jinja template for global.json file</description> - <value> -{ -"es.clustername": "{{ es_cluster_name }}", -"es.ip": "{{ es_url }}", -"es.date.format": "yyyy.MM.dd.HH", -"parser.error.topic": "indexing" -} - </value> + <name>es_date_format</name> + <description>Elasticsearch Date Format</description> + <value>yyyy.MM.dd.HH</value> + <display-name>Elasticsearch Date Format</display-name> + </property> + <property> + <name>parser_error_topic</name> + <description>Parser Error Topic</description> + <value>indexing</value> + <display-name>Parser Error Topic</display-name> + </property> + <property> + <name>enrichment_workers</name> + <description>Number of Enrichment Topology Workers</description> + <value>1</value> + <display-name>Enrichment Workers</display-name> + </property> + <property> + <name>enrichment_acker_executors</name> + <description>Number of Enrichment Topology Ackers</description> + <value>1</value> + <display-name>Enrichment Ackers</display-name> + </property> + <property> + <name>enrichment_topology_worker_childopts</name> + <description>Enrichment Topology JVM Options</description> + <value/> + <display-name>Enrichment childopts</display-name> <value-attributes> - <type>content</type> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>enrichment_topology_max_spout_pending</name> + <description>Enrichment Topology Spout Max Pending Tuples</description> + <value/> + <display-name>Enrichment Max Pending</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>enrichment_kafka_start</name> + <description>Enrichment Topology Spout Offset</description> + <value>UNCOMMITTED_EARLIEST</value> + <display-name>Enrichment Offset</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>EARLIEST</value> + </entry> + <entry> + <value>LATEST</value> + </entry> + <entry> + <value>UNCOMMITTED_EARLIEST</value> + </entry> + <entry> + <value>UNCOMMITTED_LATEST</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> </value-attributes> </property> <property> - <name>elasticsearch-properties</name> - <description>The template for the elasticsearch.properties file.</description> - <display-name>elasticsearch.properties template</display-name> - <value> -##### Storm ##### -indexing.workers=1 -indexing.executors=0 -topology.worker.childopts={{topology_worker_childopts}} -topology.auto-credentials={{topology_auto_credentials}} -##### Kafka ##### -kafka.zk={{ zookeeper_quorum }} -kafka.broker={{ kafka_brokers }} -kafka.start=UNCOMMITTED_EARLIEST -kafka.security.protocol={{kafka_security_protocol}} -storm.auto.credentials=[] -##### Indexing ##### -index.input.topic=indexing -index.error.topic=indexing -writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter -##### Metrics ##### -#reporters -org.apache.metron.metrics.reporter.graphite=true -org.apache.metron.metrics.reporter.console=false -org.apache.metron.metrics.reporter.jmx=false -#Graphite Addresses -org.apache.metron.metrics.graphite.address=localhost -org.apache.metron.metrics.graphite.port=2023 -#TelemetryParserBolt -org.apache.metron.metrics.TelemetryParserBolt.acks=true -org.apache.metron.metrics.TelemetryParserBolt.emits=true -org.apache.metron.metrics.TelemetryParserBolt.fails=true -##### HDFS ##### -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| -bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy -bolt.hdfs.rotation.policy.units=DAYS -bolt.hdfs.rotation.policy.count=1 -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url={{ default_fs }} -bolt.hdfs.wip.file.path=/paloalto/wip -bolt.hdfs.finished.file.path=/paloalto/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec -index.hdfs.output={{ metron_apps_indexed_hdfs_dir }} - </value> + <name>enrichment_input_topic</name> + <description>Enrichment Input Topic</description> + <value>enrichments</value> + <display-name>Enrichment Input Topic</display-name> + </property> + <property> + <name>enrichment_output_topic</name> + <description>Enrichment Output Topic</description> + <value>indexing</value> + <display-name>Enrichment Output Topic</display-name> + </property> + <property> + <name>enrichment_error_topic</name> + <description>Enrichment Error Topic</description> + <value>indexing</value> + <display-name>Enrichment Error Topic</display-name> + </property> + <property> + <name>threatintel_error_topic</name> + <description>Threat Intel Error Topic</description> + <value>indexing</value> + <display-name>Threat Intel Error Topic</display-name> + </property> + <property> + <name>enrichment_join_cache_size</name> + <description>Enrichment Join Bolt Cache Max Size</description> + <value>100000</value> + <display-name>Enrichment Join Size</display-name> + </property> + <property> + <name>threatintel_join_cache_size</name> + <description>Threat Intel Join Bolt Cache Max Size</description> + <value>100000</value> + <display-name>Threat Intel Join Size</display-name> + </property> + <property> + <name>enrichment_host_known_hosts</name> + <description>List of Known Hosts for Host Enrichment</description> + <value>[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]</value> + <display-name>Host Enrichment</display-name> <value-attributes> <type>content</type> </value-attributes> - <on-ambari-upgrade add="true"/> + </property> + <property> + <name>enrichment_kafka_spout_parallelism</name> + <description>Enrichment Topology Kafka Spout Parallelism</description> + <value>1</value> + <display-name>Enrichment Spout Parallelism</display-name> + </property> + <property> + <name>enrichment_split_parallelism</name> + <description>Enrichment Topology Enrichment Split Bolt Parallelism</description> + <value>1</value> + <display-name>Enrichment Split Parallelism</display-name> + </property> + <property> + <name>enrichment_stellar_parallelism</name> + <description>Enrichment Topology Enrichment Stellar Bolt Parallelism</description> + <value>1</value> + <display-name>Enrichment Stellar Parallelism</display-name> + </property> + <property> + <name>enrichment_join_parallelism</name> + <description>Enrichment Topology Enrichment Join Bolt Parallelism</description> + <value>1</value> + <display-name>Enrichment Join Parallelism</display-name> + </property> + <property> + <name>threat_intel_split_parallelism</name> + <description>Enrichment Topology Threat Intel Split Bolt Parallelism</description> + <value>1</value> + <display-name>Threat Intel Spout Parallelism</display-name> + </property> + <property> + <name>threat_intel_stellar_parallelism</name> + <description>Enrichment Topology Threat Intel Stellar Bolt Parallelism</description> + <value>1</value> + <display-name>Threat Intel Spout Parallelism</display-name> + </property> + <property> + <name>threat_intel_join_parallelism</name> + <description>Enrichment Topology Threat Intel Join Bolt Parallelism</description> + <value>1</value> + <display-name>Threat Intel Join Parallelism</display-name> + </property> + <property> + <name>kafka_writer_parallelism</name> + <description>Enrichment Topology Kafka Writer Parallelism</description> + <value>1</value> + <display-name>Enrichment Kafka Writer Parallelism</display-name> + </property> + + <property> + <name>indexing_workers</name> + <description>Number of Indexing Topology Workers</description> + <value>1</value> + <display-name>Indexing Workers</display-name> + </property> + <property> + <name>indexing_acker_executors</name> + <description>Number of Indexing Topology Ackers</description> + <value>1</value> + <display-name>Enrichment Ackers</display-name> + </property> + <property> + <name>indexing_topology_worker_childopts</name> + <description>Indexing Topology JVM Options</description> + <value/> + <display-name>Indexing childopts</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>indexing_topology_max_spout_pending</name> + <description>Indexing Topology Spout Max Pending Tuples</description> + <value/> + <display-name>Indexing Max Pending</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>indexing_kafka_start</name> + <description>Indexing Topology Spout Offset</description> + <value>UNCOMMITTED_EARLIEST</value> + <display-name>Indexing Offset</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>EARLIEST</value> + </entry> + <entry> + <value>LATEST</value> + </entry> + <entry> + <value>UNCOMMITTED_EARLIEST</value> + </entry> + <entry> + <value>UNCOMMITTED_LATEST</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + </property> + <property> + <name>indexing_input_topic</name> + <description>Indexing Input Topic</description> + <value>indexing</value> + <display-name>Indexing Input Topic</display-name> + </property> + <property> + <name>indexing_error_topic</name> + <description>Indexing Error Topic</description> + <value>indexing</value> + <display-name>Indexing Error Topic</display-name> + </property> + <property> + <name>indexing_writer_class_name</name> + <description>Indexing Writer Class Name</description> + <value>org.apache.metron.elasticsearch.writer.ElasticsearchWriter</value> + <display-name>Indexing Writer Class Name</display-name> + </property> + <property> + <name>bolt_hdfs_rotation_policy</name> + <description>Class Name of HDFS Rotation Policy</description> + <value>org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy</value> + <display-name>HDFS Rotation Policy</display-name> + </property> + <property> + <name>bolt_hdfs_rotation_policy_count</name> + <description>HDFS Rotation Policy Count</description> + <value>1</value> + <display-name>HDFS Rotation Policy Count</display-name> + </property> + <property> + <name>bolt_hdfs_rotation_policy_units</name> + <description>Frequency of HDFS File Rotation</description> + <value>DAYS</value> + <display-name>HDFS Rotation Policy Units</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>SECONDS</value> + </entry> + <entry> + <value>MINUTES</value> + </entry> + <entry> + <value>HOURS</value> + </entry> + <entry> + <value>DAYS</value> + </entry> + <entry> + <value>KB</value> + </entry> + <entry> + <value>MB</value> + </entry> + <entry> + <value>GB</value> + </entry> + <entry> + <value>TB</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + </property> + <property> + <name>indexing_kafka_spout_parallelism</name> + <description>Indexing Topology Kafka Spout Parallelism</description> + <value>1</value> + <display-name>Indexing Spout Parallelism</display-name> + </property> + <property> + <name>indexing_writer_parallelism</name> + <description>Indexing Topology Writer Bolt Parallelism</description> + <value>1</value> + <display-name>Indexing Writer Parallelism</display-name> + </property> + <property> + <name>hdfs_writer_parallelism</name> + <description>Indexing Topology HDFS Bolt Parallelism</description> + <value>1</value> + <display-name>Indexing HDFS Parallelism</display-name> </property> </configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py index 4ab81d4..de3c3f8 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py @@ -105,6 +105,7 @@ class Enrichment(Script): def restart(self, env): from params import params env.set_params(params) + self.configure(env) commands = EnrichmentCommands(params) commands.restart_enrichment_topology(env) http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index f208f3a..76b6b60 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -19,6 +19,8 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute from resource_management.core.resources.system import File +from resource_management.core.source import Template +from resource_management.libraries.functions.format import format from resource_management.core.source import StaticFile from resource_management.libraries.functions import format as ambari_format from resource_management.libraries.script import Script @@ -40,6 +42,13 @@ class Indexing(Script): from params import params env.set_params(params) + Logger.info("Running indexing configure") + File(format("{metron_config_path}/elasticsearch.properties"), + content=Template("elasticsearch.properties.j2"), + owner=params.metron_user, + group=params.metron_group + ) + commands = IndexingCommands(params) metron_service.load_global_config(params) http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py index be1654f..c0b34f1 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py @@ -20,7 +20,7 @@ import subprocess from resource_management.core.logger import Logger from resource_management.core.resources.system import Directory, File from resource_management.core.resources.system import Execute -from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template from resource_management.libraries.functions import format as ambari_format from resource_management.libraries.functions.get_user_call_output import get_user_call_output from metron_security import kinit @@ -80,15 +80,12 @@ def load_global_config(params): group=params.metron_group ) - File("{0}/global.json".format(params.metron_zookeeper_config_path), + File(ambari_format("{metron_zookeeper_config_path}/global.json"), + content=Template("global.json.j2"), owner=params.metron_user, - content=InlineTemplate(params.global_json_template) + group=params.metron_group ) - File("{0}/elasticsearch.properties".format(params.metron_zookeeper_config_path + '/..'), - owner=params.metron_user, - content=InlineTemplate(params.global_properties_template)) - init_config() http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index d473325..86ce519 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -39,6 +39,7 @@ tmp_dir = Script.get_tmp_dir() hostname = config['hostname'] metron_home = status_params.metron_home parsers = status_params.parsers +parser_error_topic = config['configurations']['metron-env']['parser_error_topic'] geoip_url = config['configurations']['metron-env']['geoip_url'] geoip_hdfs_dir = "/apps/metron/geo/default/" metron_indexing_topology = status_params.metron_indexing_topology @@ -84,6 +85,7 @@ es_binary_port = config['configurations']['metron-env']['es_binary_port'] es_url = ",".join([host + ":" + es_binary_port for host in es_host_list]) es_http_port = config['configurations']['metron-env']['es_http_port'] es_http_url = es_host_list[0] + ":" + es_http_port +es_date_format = config['configurations']['metron-env']['es_date_format'] # hadoop params stack_root = Script.get_stack_root() @@ -223,3 +225,41 @@ if security_enabled: # Management UI metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", ['localhost'])[0] + +# Enrichment +enrichment_workers = config['configurations']['metron-env']['enrichment_workers'] +enrichment_acker_executors = config['configurations']['metron-env']['enrichment_acker_executors'] +enrichment_topology_worker_childopts = config['configurations']['metron-env']['enrichment_topology_worker_childopts'] +enrichment_topology_max_spout_pending = config['configurations']['metron-env']['enrichment_topology_max_spout_pending'] +enrichment_kafka_start = config['configurations']['metron-env']['enrichment_kafka_start'] +enrichment_input_topic = config['configurations']['metron-env']['enrichment_input_topic'] +enrichment_output_topic = config['configurations']['metron-env']['enrichment_output_topic'] +enrichment_error_topic = config['configurations']['metron-env']['enrichment_error_topic'] +threatintel_error_topic = config['configurations']['metron-env']['threatintel_error_topic'] +enrichment_join_cache_size = config['configurations']['metron-env']['enrichment_join_cache_size'] +threatintel_join_cache_size = config['configurations']['metron-env']['threatintel_join_cache_size'] +enrichment_host_known_hosts = config['configurations']['metron-env']['enrichment_host_known_hosts'] +enrichment_kafka_spout_parallelism = config['configurations']['metron-env']['enrichment_kafka_spout_parallelism'] +enrichment_split_parallelism = config['configurations']['metron-env']['enrichment_split_parallelism'] +enrichment_stellar_parallelism = config['configurations']['metron-env']['enrichment_stellar_parallelism'] +enrichment_join_parallelism = config['configurations']['metron-env']['enrichment_join_parallelism'] +threat_intel_split_parallelism = config['configurations']['metron-env']['threat_intel_split_parallelism'] +threat_intel_stellar_parallelism = config['configurations']['metron-env']['threat_intel_stellar_parallelism'] +threat_intel_join_parallelism = config['configurations']['metron-env']['threat_intel_join_parallelism'] +kafka_writer_parallelism = config['configurations']['metron-env']['kafka_writer_parallelism'] + +# Threat Intel +indexing_workers = config['configurations']['metron-env']['indexing_workers'] +indexing_acker_executors = config['configurations']['metron-env']['indexing_acker_executors'] +indexing_topology_worker_childopts = config['configurations']['metron-env']['indexing_topology_worker_childopts'] +indexing_topology_max_spout_pending = config['configurations']['metron-env']['indexing_topology_max_spout_pending'] +indexing_kafka_start = config['configurations']['metron-env']['indexing_kafka_start'] +indexing_input_topic = config['configurations']['metron-env']['indexing_input_topic'] +indexing_error_topic = config['configurations']['metron-env']['indexing_error_topic'] +indexing_writer_class_name = config['configurations']['metron-env']['indexing_writer_class_name'] +bolt_hdfs_rotation_policy = config['configurations']['metron-env']['bolt_hdfs_rotation_policy'] +bolt_hdfs_rotation_policy_units = config['configurations']['metron-env']['bolt_hdfs_rotation_policy_units'] +bolt_hdfs_rotation_policy_count = config['configurations']['metron-env']['bolt_hdfs_rotation_policy_count'] +indexing_kafka_spout_parallelism = config['configurations']['metron-env']['indexing_kafka_spout_parallelism'] +indexing_writer_parallelism = config['configurations']['metron-env']['indexing_writer_parallelism'] +hdfs_writer_parallelism = config['configurations']['metron-env']['hdfs_writer_parallelism'] http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/elasticsearch.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/elasticsearch.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/elasticsearch.properties.j2 new file mode 100644 index 0000000..acb0f59 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/elasticsearch.properties.j2 @@ -0,0 +1,49 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +##### Storm ##### +indexing.workers={{indexing_workers}} +indexing.acker.executors={{indexing_acker_executors}} +topology.worker.childopts={{indexing_topology_worker_childopts}} +topology.auto-credentials={{topology_auto_credentials}} +topology.max.spout.pending={{indexing_topology_max_spout_pending}} + +##### Kafka ##### +kafka.zk={{zookeeper_quorum}} +kafka.broker={{kafka_brokers}} +kafka.security.protocol={{kafka_security_protocol}} + +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start={{indexing_kafka_start}} + +indexing.input.topic={{indexing_input_topic}} +indexing.error.topic={{indexing_error_topic}} + +##### Indexing ##### +indexing.writer.class.name={{indexing_writer_class_name}} + +##### HDFS ##### +bolt.hdfs.rotation.policy={{bolt_hdfs_rotation_policy}} +bolt.hdfs.rotation.policy.units={{bolt_hdfs_rotation_policy_units}} +bolt.hdfs.rotation.policy.count={{bolt_hdfs_rotation_policy_count}} +indexing.hdfs.output={{metron_apps_indexed_hdfs_dir}} + +##### Parallelism ##### +kafka.spout.parallelism={{indexing_kafka_spout_parallelism}} +indexing.writer.parallelism={{indexing_writer_parallelism}} +hdfs.writer.parallelism={{hdfs_writer_parallelism}} http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 index bdafb45..485b938 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 @@ -16,72 +16,48 @@ # limitations under the License. #} -##### Kafka ##### +##### Storm ##### +enrichment.workers={{enrichment_workers}} +enrichment.acker.executors={{enrichment_acker_executors}} +topology.worker.childopts={{enrichment_topology_worker_childopts}} +topology.auto-credentials={{topology_auto_credentials}} +topology.max.spout.pending={{enrichment_topology_max_spout_pending}} +##### Kafka ##### kafka.zk={{zookeeper_quorum}} kafka.broker={{kafka_brokers}} kafka.security.protocol={{kafka_security_protocol}} -topology.worker.childopts={{topology_worker_childopts}} -topology.auto-credentials={{topology_auto_credentials}} -enrichment.output.topic=indexing -enrichment.error.topic=indexing -threat.intel.error.topic=indexing - -##### Metrics ##### - -#reporters -org.apache.metron.metrics.reporter.graphite=true -org.apache.metron.metrics.reporter.console=false -org.apache.metron.metrics.reporter.jmx=false - -#Graphite Addresses -org.apache.metron.metrics.graphite.address=localhost -org.apache.metron.metrics.graphite.port=2023 +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start={{enrichment_kafka_start}} -#TelemetryParserBolt -org.apache.metron.metrics.TelemetryParserBolt.acks=true -org.apache.metron.metrics.TelemetryParserBolt.emits=true -org.apache.metron.metrics.TelemetryParserBolt.fails=true +enrichment.input.topic={{enrichment_input_topic}} +enrichment.output.topic={{enrichment_output_topic}} +enrichment.error.topic={{enrichment_error_topic}} +threat.intel.error.topic={{threatintel_error_topic}} +##### JoinBolt ##### +enrichment.join.cache.size={{enrichment_join_cache_size}} +threat.intel.join.cache.size={{threatintel_join_cache_size}} -#GenericEnrichmentBolt -org.apache.metron.metrics.GenericEnrichmentBolt.acks=true -org.apache.metron.metrics.GenericEnrichmentBolt.emits=true -org.apache.metron.metrics.GenericEnrichmentBolt.fails=true - - -#TelemetryIndexingBolt -org.apache.metron.metrics.TelemetryIndexingBolt.acks=true -org.apache.metron.metrics.TelemetryIndexingBolt.emits=true -org.apache.metron.metrics.TelemetryIndexingBolt.fails=true - -##### Host Enrichment ##### - +##### Enrichment ##### hbase.provider.impl=org.apache.metron.hbase.HTableProvider enrichment.simple.hbase.table={{enrichment_table}} enrichment.simple.hbase.cf={{enrichment_cf}} -org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ -{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ -{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] - - -##### HBase ##### -bolt.hbase.table.name=pcap -bolt.hbase.table.fields=t:value -bolt.hbase.table.key.tuple.field.name=key -bolt.hbase.table.timestamp.tuple.field.name=timestamp -bolt.hbase.enable.batching=false -bolt.hbase.write.buffer.size.in.bytes=2000000 -bolt.hbase.durability=SKIP_WAL -bolt.hbase.partitioner.region.info.refresh.interval.mins=60 +enrichment.host.known_hosts={{enrichment_host_known_hosts}} ##### Threat Intel ##### - threat.intel.tracker.table={{threatintel_table}} threat.intel.tracker.cf={{threatintel_cf}} threat.intel.simple.hbase.table={{threatintel_table}} threat.intel.simple.hbase.cf={{threatintel_cf}} -threat.intel.ip.table= -threat.intel.ip.cf= +##### Parallelism ##### +kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}} +enrichment.split.parallelism={{enrichment_split_parallelism}} +enrichment.stellar.parallelism={{enrichment_stellar_parallelism}} +enrichment.join.parallelism={{enrichment_join_parallelism}} +threat.intel.split.parallelism={{threat_intel_split_parallelism}} +threat.intel.stellar.parallelism={{threat_intel_stellar_parallelism}} +threat.intel.join.parallelism={{threat_intel_join_parallelism}} +kafka.writer.parallelism={{kafka_writer_parallelism}} http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 new file mode 100644 index 0000000..61e1416 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 @@ -0,0 +1,6 @@ +{ + "es.clustername": "{{ es_cluster_name }}", + "es.ip": "{{ es_url }}", + "es.date.format": "{{es_date_format}}", + "parser.error.topic": "{{parser_error_topic}}" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 20aca46..22ca4c0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -1,71 +1,827 @@ { "configuration": { - "layouts": [{ - "name": "default", - "tabs": [{ - "name": "metron_general", - "display-name": "Index Settings", - "layout": { - "tab-columns": "1", - "tab-rows": "1", - "sections": [{ - "name": "section-general", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1", - "section-columns": "1", - "section-rows": "1", - "subsections": [{ - "name": "subsection-general-indexing", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1" - }] - }] - } - }] - }], + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "index_settings", + "display-name": "Index Settings", + "layout": { + "tab-columns": "1", + "tab-rows": "1", + "sections": [ + { + "name": "section-index-settings", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-index-settings", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { + "name": "parsers", + "display-name": "Parsers", + "layout": { + "tab-columns": "1", + "tab-rows": "1", + "sections": [ + { + "name": "section-parsers", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-parsers", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { + "name": "enrichment", + "display-name": "Enrichment", + "layout": { + "tab-columns": "1", + "tab-rows": "4", + "sections": [ + { + "name": "section-enrichment-adapters", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-enrichment-adapters", + "display-name": "Adapters", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-enrichment-kafka", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-enrichment-kafka", + "display-name": "Kafka", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-enrichment-storm", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-enrichment-storm", + "display-name": "Storm", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { + "name": "indexing", + "display-name": "Indexing", + "layout": { + "tab-columns": "1", + "tab-rows": "3", + "sections": [ + { + "name": "section-indexing-kafka", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-indexing-kafka", + "display-name": "Kafka", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-indexing-storm", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-indexing-storm", + "display-name": "Storm", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-indexing-hdfs", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-indexing-hdfs", + "display-name": "HDFS", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { + "name": "rest", + "display-name": "REST", + "layout": { + "tab-columns": "1", + "tab-rows": "1", + "sections": [ + { + "name": "section-rest", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-rest", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { + "name": "management-ui", + "display-name": "Management UI", + "layout": { + "tab-columns": "1", + "tab-rows": "1", + "sections": [ + { + "name": "section-management-ui", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-management-ui", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + } + ] + } + ], "placement": { "configuration-layout": "default", - "configs": [{ - "config": "metron-env/metron_enrichment_db_port", - "subsection-name": "subsection-general-database" - }, { + "configs": [ + { + "config": "metron-env/es_hosts", + "subsection-name": "subsection-index-settings" + }, + { + "config": "metron-env/es_binary_port", + "subsection-name": "subsection-index-settings" + }, + { + "config": "metron-env/es_http_port", + "subsection-name": "subsection-index-settings" + }, + { + "config": "metron-env/es_cluster_name", + "subsection-name": "subsection-index-settings" + }, + { + "config": "metron-env/es_date_format", + "subsection-name": "subsection-index-settings" + }, + { + "config": "metron-env/parsers", + "subsection-name": "subsection-parsers" + }, + { + "config": "metron-env/parser_error_topic", + "subsection-name": "subsection-parsers" + }, + { + "config": "metron-env/geoip_url", + "subsection-name": "subsection-enrichment-adapters" + }, + { + "config": "metron-env/enrichment_host_known_hosts", + "subsection-name": "subsection-enrichment-adapters" + }, + { + "config": "metron-env/enrichment_kafka_start", + "subsection-name": "subsection-enrichment-kafka" + }, + { + "config": "metron-env/enrichment_input_topic", + "subsection-name": "subsection-enrichment-kafka" + }, + { + "config": "metron-env/enrichment_output_topic", + "subsection-name": "subsection-enrichment-kafka" + }, + { + "config": "metron-env/enrichment_error_topic", + "subsection-name": "subsection-enrichment-kafka" + }, + { + "config": "metron-env/threatintel_error_topic", + "subsection-name": "subsection-enrichment-kafka" + }, + { + "config": "metron-env/enrichment_workers", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_acker_executors", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_topology_worker_childopts", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_topology_max_spout_pending", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_join_cache_size", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/threatintel_join_cache_size", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_kafka_spout_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_split_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_stellar_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/enrichment_join_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/threat_intel_split_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/threat_intel_stellar_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/threat_intel_join_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + { + "config": "metron-env/kafka_writer_parallelism", + "subsection-name": "subsection-enrichment-storm" + }, + + { + "config": "metron-env/indexing_kafka_start", + "subsection-name": "subsection-indexing-kafka" + }, + { + "config": "metron-env/indexing_input_topic", + "subsection-name": "subsection-indexing-kafka" + }, + { + "config": "metron-env/indexing_error_topic", + "subsection-name": "subsection-indexing-kafka" + }, + { + "config": "metron-env/indexing_workers", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/indexing_acker_executors", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/indexing_topology_worker_childopts", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/indexing_topology_max_spout_pending", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/indexing_kafka_spout_parallelism", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/indexing_writer_parallelism", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/hdfs_writer_parallelism", + "subsection-name": "subsection-indexing-storm" + }, + { + "config": "metron-env/metron_apps_indexed_hdfs_dir", + "subsection-name": "subsection-indexing-hdfs" + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy", + "subsection-name": "subsection-indexing-hdfs" + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy_units", + "subsection-name": "subsection-indexing-hdfs" + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy_count", + "subsection-name": "subsection-indexing-hdfs" + }, + + { + "config": "metron-env/metron_rest_port", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_url", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_driver", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_username", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_password", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_platform", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jdbc_client_path", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_default_grok_path", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_temp_grok_path", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_spring_profiles_active", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_jvm_flags", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_spring_options", + "subsection-name": "subsection-rest" + }, + { + "config": "metron-env/metron_management_ui_port", + "subsection-name": "subsection-management-ui" + } + ] + }, + "widgets": [ + { "config": "metron-env/es_hosts", - "subsection-name": "subsection-general-indexing" - }, { + "widget": { + "type": "text-field" + } + }, + { "config": "metron-env/es_binary_port", - "subsection-name": "subsection-general-indexing" - }, { + "widget": { + "type": "text-field" + } + }, + { "config": "metron-env/es_http_port", - "subsection-name": "subsection-general-indexing" - }, { + "widget": { + "type": "text-field" + } + }, + { "config": "metron-env/es_cluster_name", - "subsection-name": "subsection-general-indexing" - }] - }, - "widgets": [{ - "config": "metron-env/es_hosts", - "widget": { - "type": "text-field" - } - }, { - "config": "metron-env/es_binary_port", - "widget": { - "type": "text-field" - } - }, { - "config": "metron-env/es_http_port", - "widget": { - "type": "text-field" - } - }, { - "config": "metron-env/es_cluster_name", - "widget": { - "type": "text-field" + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/es_date_format", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/parsers", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/parser_error_topic", + "widget": { + "type": "text-field" + } + }, + + + { + "config": "metron-env/enrichment_workers", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/geoip_url", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_acker_executors", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_topology_worker_childopts", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_topology_max_spout_pending", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_kafka_start", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-env/enrichment_input_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_output_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_error_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/threatintel_error_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_join_cache_size", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/threatintel_join_cache_size", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_host_known_hosts", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_kafka_spout_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_split_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_stellar_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/enrichment_join_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/threat_intel_split_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/threat_intel_stellar_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/threat_intel_join_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/kafka_writer_parallelism", + "widget": { + "type": "text-field" + } + }, + + { + "config": "metron-env/indexing_workers", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_acker_executors", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_topology_worker_childopts", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_topology_max_spout_pending", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_kafka_start", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-env/indexing_input_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_error_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy_units", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-env/bolt_hdfs_rotation_policy_count", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_error_topic", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_kafka_spout_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/indexing_writer_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/hdfs_writer_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_apps_indexed_hdfs_dir", + "widget": { + "type": "text-field" + } + }, + + + { + "config": "metron-env/metron_rest_port", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jdbc_url", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jdbc_driver", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jdbc_username", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jdbc_password", + "widget": { + "type": "password" + } + }, + { + "config": "metron-env/metron_jdbc_platform", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jdbc_client_path", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_default_grok_path", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_temp_grok_path", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_spring_profiles_active", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_jvm_flags", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_spring_options", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-env/metron_management_ui_port", + "widget": { + "type": "text-field" + } } - }] + ] } } http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-interface/metron-rest/src/main/scripts/metron-rest ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest b/metron-interface/metron-rest/src/main/scripts/metron-rest index f9227a9..3fa1df0 100644 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest @@ -145,7 +145,7 @@ case "$1" in # restart) $0 stop - $0 start + $0 start $2 ;; ############################################################################## http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties index d5efcb2..1620dfd 100644 --- a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties +++ b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties @@ -16,58 +16,32 @@ ##### Storm ##### indexing.workers=1 -indexing.executors=0 +indexing.acker.executors=0 topology.worker.childopts= topology.auto-credentials=[''] +topology.max.spout.pending= ##### Kafka ##### - kafka.zk=node1:2181 kafka.broker=node1:6667 -# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST -kafka.start=UNCOMMITTED_EARLIEST kafka.security.protocol=PLAINTEXT -##### Indexing ##### -index.input.topic=indexing -index.error.topic=indexing -writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter - -##### ElasticSearch ##### - -es.ip=10.22.0.214 -es.port=9300 -es.clustername=elasticsearch - -##### Metrics ##### - -#reporters -org.apache.metron.metrics.reporter.graphite=true -org.apache.metron.metrics.reporter.console=false -org.apache.metron.metrics.reporter.jmx=false - -#Graphite Addresses - -org.apache.metron.metrics.graphite.address=localhost -org.apache.metron.metrics.graphite.port=2023 +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start=UNCOMMITTED_EARLIEST -#TelemetryParserBolt -org.apache.metron.metrics.TelemetryParserBolt.acks=true -org.apache.metron.metrics.TelemetryParserBolt.emits=true -org.apache.metron.metrics.TelemetryParserBolt.fails=true +indexing.input.topic=indexing +indexing.error.topic=indexing +##### Indexing ##### +indexing.writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter ##### HDFS ##### - -bolt.hdfs.batch.size=5000 -bolt.hdfs.field.delimiter=| bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy bolt.hdfs.rotation.policy.units=DAYS bolt.hdfs.rotation.policy.count=1 +indexing.hdfs.output=/tmp/metron/enriched -bolt.hdfs.file.rotation.size.in.mb=5 -bolt.hdfs.file.system.url=hdfs://iot01.cloud.hortonworks.com:8020 -bolt.hdfs.wip.file.path=/paloalto/wip -bolt.hdfs.finished.file.path=/paloalto/rotated -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec -index.hdfs.output=/tmp/metron/enriched +##### Parallelism ##### +kafka.spout.parallelism=1 +indexing.writer.parallelism=1 +hdfs.writer.parallelism=1 http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java index 87c0081..54e494e 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java @@ -102,7 +102,7 @@ public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTes topologyProperties.setProperty("es.clustername", "metron"); topologyProperties.setProperty("es.port", "9300"); topologyProperties.setProperty("es.ip", "localhost"); - topologyProperties.setProperty("writer.class.name", "org.apache.metron.elasticsearch.writer.ElasticsearchWriter"); + topologyProperties.setProperty("indexing.writer.class.name", "org.apache.metron.elasticsearch.writer.ElasticsearchWriter"); } @Override http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-platform/metron-enrichment/src/main/config/enrichment.properties ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties b/metron-platform/metron-enrichment/src/main/config/enrichment.properties index e293201..9592968 100644 --- a/metron-platform/metron-enrichment/src/main/config/enrichment.properties +++ b/metron-platform/metron-enrichment/src/main/config/enrichment.properties @@ -14,71 +14,51 @@ # See the License for the specific language governing permissions and # limitations under the License. +##### Storm ##### +enrichment.workers=1 +enrichment.acker.executors=0 +topology.worker.childopts= +topology.auto-credentials= +topology.max.spout.pending= ##### Kafka ##### - kafka.zk=node1:2181 kafka.broker=node1:6667 kafka.security.protocol=PLAINTEXT -topology.worker.childopts= +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start=UNCOMMITTED_EARLIEST + +enrichment.input.topic=enrichments enrichment.output.topic=indexing enrichment.error.topic=indexing threat.intel.error.topic=indexing -##### Metrics ##### - -#reporters -org.apache.metron.metrics.reporter.graphite=true -org.apache.metron.metrics.reporter.console=false -org.apache.metron.metrics.reporter.jmx=false - -#Graphite Addresses - -org.apache.metron.metrics.graphite.address=localhost -org.apache.metron.metrics.graphite.port=2023 - -#TelemetryParserBolt -org.apache.metron.metrics.TelemetryParserBolt.acks=true -org.apache.metron.metrics.TelemetryParserBolt.emits=true -org.apache.metron.metrics.TelemetryParserBolt.fails=true - - -#GenericEnrichmentBolt -org.apache.metron.metrics.GenericEnrichmentBolt.acks=true -org.apache.metron.metrics.GenericEnrichmentBolt.emits=true -org.apache.metron.metrics.GenericEnrichmentBolt.fails=true +##### JoinBolt ##### +enrichment.join.cache.size=100000 +threat.intel.join.cache.size=100000 - -#TelemetryIndexingBolt -org.apache.metron.metrics.TelemetryIndexingBolt.acks=true -org.apache.metron.metrics.TelemetryIndexingBolt.emits=true -org.apache.metron.metrics.TelemetryIndexingBolt.fails=true - -##### Host Enrichment ##### - -org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ +##### Enrichment ##### +hbase.provider.impl=org.apache.metron.hbase.HTableProvider +enrichment.simple.hbase.table=enrichment +enrichment.simple.hbase.cf=t +enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ {"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ {"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] -##### HBase ##### -bolt.hbase.table.name=pcap -bolt.hbase.table.fields=t:value -bolt.hbase.table.key.tuple.field.name=key -bolt.hbase.table.timestamp.tuple.field.name=timestamp -bolt.hbase.enable.batching=false -bolt.hbase.write.buffer.size.in.bytes=2000000 -bolt.hbase.durability=SKIP_WAL -bolt.hbase.partitioner.region.info.refresh.interval.mins=60 - ##### Threat Intel ##### - threat.intel.tracker.table=access_tracker threat.intel.tracker.cf=t -threat.intel.ip.table= -threat.intel.ip.cf= threat.intel.simple.hbase.table=threatintel threat.intel.simple.hbase.cf=t -enrichment.simple.hbase.table=enrichment -enrichment.simple.hbase.cf=t + +##### Parallelism ##### +kafka.spout.parallelism=1 +enrichment.split.parallelism=1 +enrichment.stellar.parallelism=1 +enrichment.join.parallelism=1 +threat.intel.split.parallelism=1 +threat.intel.stellar.parallelism=1 +threat.intel.join.parallelism=1 +kafka.writer.parallelism=1 http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml index 0e50f77..7ee1b2e 100644 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml @@ -16,10 +16,11 @@ name: "enrichment" config: - topology.workers: 1 - topology.acker.executors: 0 + topology.workers: ${enrichment.workers} + topology.acker.executors: ${enrichment.acker.executors} topology.worker.childopts: ${topology.worker.childopts} topology.auto-credentials: ${topology.auto-credentials} + topology.max.spout.pending: ${topology.max.spout.pending} components: @@ -56,7 +57,7 @@ components: - id: "hostEnrichmentAdapter" className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter" constructorArgs: - - '${org.apache.metron.enrichment.host.known_hosts}' + - '${enrichment.host.known_hosts}' - id: "hostEnrichment" className: "org.apache.metron.enrichment.configuration.Enrichment" constructorArgs: @@ -231,14 +232,13 @@ components: constructorArgs: - ref: "kafkaProps" # topic name - - "enrichments" + - "${enrichment.input.topic}" - "${kafka.zk}" - ref: "fields" configMethods: - name: "setFirstPollOffsetStrategy" args: - # One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST - - "UNCOMMITTED_EARLIEST" + - "${kafka.start}" spouts: @@ -246,6 +246,7 @@ spouts: className: "org.apache.metron.storm.kafka.flux.StormKafkaSpout" constructorArgs: - ref: "kafkaConfig" + parallelism: ${kafka.spout.parallelism} bolts: # Enrichment Bolts @@ -257,6 +258,8 @@ bolts: - name: "withEnrichments" args: - ref: "enrichments" + parallelism: ${enrichment.split.parallelism} + - id: "geoEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -269,6 +272,7 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + - id: "stellarEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -281,6 +285,7 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + parallelism: ${enrichment.stellar.parallelism} - id: "hostEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" @@ -294,6 +299,7 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + - id: "simpleHBaseEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -306,15 +312,18 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + - id: "enrichmentJoinBolt" className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt" constructorArgs: - "${kafka.zk}" configMethods: - name: "withMaxCacheSize" - args: [100000] + args: [${enrichment.join.cache.size}] - name: "withMaxTimeRetain" args: [10] + parallelism: ${enrichment.join.parallelism} + - id: "enrichmentErrorOutputBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" constructorArgs: @@ -336,6 +345,8 @@ bolts: - ref: "threatIntels" - name: "withMessageFieldName" args: ["message"] + parallelism: ${threat.intel.split.parallelism} + - id: "simpleHBaseThreatIntelBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -360,15 +371,19 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + parallelism: ${threat.intel.stellar.parallelism} + - id: "threatIntelJoinBolt" className: "org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt" constructorArgs: - "${kafka.zk}" configMethods: - name: "withMaxCacheSize" - args: [100000] + args: [${threat.intel.join.cache.size}] - name: "withMaxTimeRetain" args: [10] + parallelism: ${threat.intel.join.parallelism} + - id: "threatIntelErrorOutputBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" constructorArgs: @@ -387,6 +402,7 @@ bolts: - name: "withMessageWriter" args: - ref: "kafkaWriter" + parallelism: ${kafka.writer.parallelism} streams: http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml index b7fb8d4..b4481ff 100644 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml @@ -16,9 +16,14 @@ name: "enrichment" config: - topology.workers: 1 + topology.workers: ${enrichment.workers} + topology.acker.executors: ${enrichment.acker.executors} + topology.worker.childopts: ${topology.worker.childopts} + topology.auto-credentials: ${topology.auto-credentials} + topology.max.spout.pending: ${topology.max.spout.pending} components: + # Enrichment - id: "stellarEnrichmentAdapter" className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter" @@ -43,7 +48,7 @@ components: - id: "hostEnrichmentAdapter" className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter" constructorArgs: - - '${org.apache.metron.enrichment.host.known_hosts}' + - '${enrichment.host.known_hosts}' - id: "hostEnrichment" className: "org.apache.metron.enrichment.configuration.Enrichment" constructorArgs: @@ -179,40 +184,44 @@ components: - id: "kafkaProps" className: "java.util.HashMap" configMethods: - - name: "put" - args: - - "value.deserializer" - - "org.apache.kafka.common.serialization.ByteArrayDeserializer" - - name: "put" - args: - - "key.deserializer" - - "org.apache.kafka.common.serialization.ByteArrayDeserializer" - - name: "put" - args: - - "group.id" - - "enrichments" + - name: "put" + args: + - "value.deserializer" + - "org.apache.kafka.common.serialization.ByteArrayDeserializer" + - name: "put" + args: + - "key.deserializer" + - "org.apache.kafka.common.serialization.ByteArrayDeserializer" + - name: "put" + args: + - "group.id" + - "enrichments" + - name: "put" + args: + - "security.protocol" + - "${kafka.security.protocol}" + # The fields to pull out of the kafka messages - id: "fields" className: "java.util.ArrayList" configMethods: - - name: "add" - args: - - "value" + - name: "add" + args: + - "value" - id: "kafkaConfig" className: "org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder" constructorArgs: - # zookeeper hosts - - ref: "kafkaProps" - # topic name - - "enrichments" - - "${kafka.zk}" - - ref: "fields" + - ref: "kafkaProps" + # topic name + - "${enrichment.input.topic}" + - "${kafka.zk}" + - ref: "fields" configMethods: - name: "setFirstPollOffsetStrategy" args: - - "UNCOMMITTED_EARLIEST" + - "${kafka.start}" spouts: @@ -220,6 +229,7 @@ spouts: className: "org.apache.metron.storm.kafka.flux.StormKafkaSpout" constructorArgs: - ref: "kafkaConfig" + parallelism: ${kafka.spout.parallelism} bolts: # Enrichment Bolts @@ -231,6 +241,8 @@ bolts: - name: "withEnrichments" args: - ref: "enrichments" + parallelism: ${enrichment.split.parallelism} + - id: "geoEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -243,6 +255,7 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + - id: "stellarEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -255,6 +268,8 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + parallelism: ${enrichment.stellar.parallelism} + - id: "hostEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -267,6 +282,7 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + - id: "simpleHBaseEnrichmentBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -297,9 +313,11 @@ bolts: - "${kafka.zk}" configMethods: - name: "withMaxCacheSize" - args: [10000] + args: [${enrichment.join.cache.size}] - name: "withMaxTimeRetain" args: [10] + parallelism: ${enrichment.join.parallelism} + - id: "enrichmentErrorOutputBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" constructorArgs: @@ -309,6 +327,7 @@ bolts: args: - ref: "enrichmentErrorKafkaWriter" + # Threat Intel Bolts - id: "threatIntelSplitBolt" className: "org.apache.metron.enrichment.bolt.ThreatIntelSplitterBolt" @@ -320,6 +339,8 @@ bolts: - ref: "threatIntels" - name: "withMessageFieldName" args: ["message"] + parallelism: ${threat.intel.split.parallelism} + - id: "simpleHBaseThreatIntelBolt" className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" constructorArgs: @@ -344,15 +365,19 @@ bolts: args: [10000] - name: "withMaxTimeRetain" args: [10] + parallelism: ${threat.intel.stellar.parallelism} + - id: "threatIntelJoinBolt" className: "org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt" constructorArgs: - "${kafka.zk}" configMethods: - name: "withMaxCacheSize" - args: [10000] + args: [${threat.intel.join.cache.size}] - name: "withMaxTimeRetain" args: [10] + parallelism: ${threat.intel.join.parallelism} + - id: "threatIntelErrorOutputBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" constructorArgs: @@ -371,6 +396,7 @@ bolts: - name: "withMessageWriter" args: - ref: "kafkaWriter" + parallelism: ${kafka.writer.parallelism} streams: @@ -405,6 +431,7 @@ streams: type: FIELDS args: ["key"] + - name: "enrichmentSplit -> simpleHBaseEnrichmentBolt" from: "enrichmentSplitBolt" to: "simpleHBaseEnrichmentBolt" @@ -444,6 +471,7 @@ streams: args: ["key"] + - name: "simpleHBaseEnrichmentBolt -> join" from: "simpleHBaseEnrichmentBolt" to: "enrichmentJoinBolt" @@ -524,6 +552,7 @@ streams: type: FIELDS args: ["key"] + - name: "simpleHBaseThreatIntel -> join" from: "simpleHBaseThreatIntelBolt" to: "threatIntelJoinBolt" @@ -539,6 +568,7 @@ streams: streamId: "stellar" type: FIELDS args: ["key"] + - name: "threatIntelSplit -> threatIntelJoin" from: "threatIntelSplitBolt" to: "threatIntelJoinBolt" @@ -546,7 +576,6 @@ streams: streamId: "message" type: FIELDS args: ["key"] - #output - name: "threatIntelJoin -> output" from: "threatIntelJoinBolt" http://git-wip-us.apache.org/repos/asf/metron/blob/87f65221/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java index a7b2d7d..e798b72 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java @@ -124,7 +124,21 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest { final String threatIntelTableName = "threat_intel"; final String enrichmentsTableName = "enrichments"; final Properties topologyProperties = new Properties() {{ - setProperty("org.apache.metron.enrichment.host.known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"},\n" + + setProperty("enrichment.workers", "1"); + setProperty("enrichment.acker.executors", "0"); + setProperty("topology.worker.childopts", ""); + setProperty("topology.auto-credentials", "[]"); + setProperty("topology.max.spout.pending", ""); + setProperty("kafka.start", "UNCOMMITTED_EARLIEST"); + setProperty("kafka.security.protocol", "PLAINTEXT"); + setProperty("enrichment.input.topic", Constants.ENRICHMENT_TOPIC); + setProperty("enrichment.output.topic", Constants.INDEXING_TOPIC); + setProperty("enrichment.error.topic", ERROR_TOPIC); + setProperty("threat.intel.error.topic", ERROR_TOPIC); + setProperty("enrichment.join.cache.size", "1000"); + setProperty("threat.intel.join.cache.size", "1000"); + + setProperty("enrichment.host.known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"},\n" + "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"},\n" + "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"},\n" + "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}]"); @@ -135,10 +149,16 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest { setProperty("threat.intel.simple.hbase.cf", cf); setProperty("enrichment.simple.hbase.table", enrichmentsTableName); setProperty("enrichment.simple.hbase.cf", cf); - setProperty("enrichment.output.topic", Constants.INDEXING_TOPIC); - setProperty("enrichment.error.topic", ERROR_TOPIC); - setProperty("kafka.security.protocol", "PLAINTEXT"); - setProperty("storm.auto.credentials", "[]"); + + setProperty("kafka.spout.parallelism", "1"); + setProperty("enrichment.split.parallelism", "1"); + setProperty("enrichment.stellar.parallelism", "1"); + setProperty("enrichment.join.parallelism", "1"); + setProperty("threat.intel.split.parallelism", "1"); + setProperty("threat.intel.stellar.parallelism", "1"); + setProperty("threat.intel.join.parallelism", "1"); + setProperty("kafka.writer.parallelism", "1"); + }}; final ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties); final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaComponent.Topic>() {{
