METRON-1624 Set Profiler and Enrichment batch parameters in Ambari (nickwallen) closes apache/metron#1069
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b06bb5f4 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b06bb5f4 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b06bb5f4 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: b06bb5f4860c656023b9c357dfa83c2f0244b9ef Parents: b575da1 Author: nickwallen <n...@nickallen.org> Authored: Fri Jun 22 16:19:37 2018 -0400 Committer: nickallen <nickal...@apache.org> Committed: Fri Jun 22 16:19:37 2018 -0400 ---------------------------------------------------------------------- .../configuration/metron-enrichment-env.xml | 12 ++++++ .../configuration/metron-profiler-env.xml | 12 ++++++ .../CURRENT/package/scripts/metron_service.py | 20 ++++++++++ .../package/scripts/params/params_linux.py | 4 ++ .../METRON/CURRENT/themes/metron_theme.json | 41 +++++++++++++++++++- 5 files changed, 88 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/b06bb5f4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml index 81b135c..b41c455 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml @@ -89,6 +89,18 @@ <value>indexing</value> <display-name>Threat Intel Error Topic</display-name> </property> + <property> + <name>enrichment_kafka_writer_batch_size</name> + <value>15</value> + <description>The number of records to batch when writing to Kakfa.</description> + <display-name>Kafka Writer Batch Size</display-name> + </property> + <property> + <name>enrichment_kafka_writer_batch_timeout</name> + <value>0</value> + <description>The timeout in milliseconds after which a batch will be written to Kafka, even if the batch size has not been met. If unspecified, or set to `0`, this defaults to a system-determined duration.</description> + <display-name>Kafka Writer Batch Timeout</display-name> + </property> <!-- hbase parameters http://git-wip-us.apache.org/repos/asf/metron/blob/b06bb5f4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml index 8546b56..07b8d11 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml @@ -221,4 +221,16 @@ <description>The profiler storm topology acker executors</description> <display-name>Number of Acker Executors</display-name> </property> + <property> + <name>profiler_kafka_writer_batch_size</name> + <value>15</value> + <description>The number of records to batch when writing to Kakfa.</description> + <display-name>Kafka Writer Batch Size</display-name> + </property> + <property> + <name>profiler_kafka_writer_batch_timeout</name> + <value>0</value> + <description>The timeout in milliseconds after which a batch will be written to Kafka, even if the batch size has not been met. If unspecified, or set to `0`, this defaults to a system-determined duration.</description> + <display-name>Kafka Writer Batch Timeout</display-name> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/b06bb5f4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py index 1999992..e3e48ab 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py @@ -119,6 +119,26 @@ def build_global_config_patch(params, patch_file): "op": "add", "path": "/threat.triage.score.field", "value": "{{threat_triage_score_field}}" + }, + { + "op": "add", + "path": "/enrichment.writer.batchSize", + "value": "{{enrichment_kafka_writer_batch_size}}" + }, + { + "op": "add", + "path": "/enrichment.writer.batchTimeout", + "value": "{{enrichment_kafka_writer_batch_timeout}}" + }, + { + "op": "add", + "path": "/profiler.writer.batchSize", + "value": "{{profiler_kafka_writer_batch_size}}" + }, + { + "op": "add", + "path": "/profiler.writer.batchTimeout", + "value": "{{profiler_kafka_writer_batch_timeout}}" } ] """ http://git-wip-us.apache.org/repos/asf/metron/blob/b06bb5f4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index f054c00..ec01292 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -268,6 +268,8 @@ enrichment_input_topic = status_params.enrichment_input_topic enrichment_output_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic'] enrichment_error_topic = config['configurations']['metron-enrichment-env']['enrichment_error_topic'] threatintel_error_topic = config['configurations']['metron-enrichment-env']['threatintel_error_topic'] +enrichment_kafka_writer_batch_size = config['configurations']['metron-enrichment-env']['enrichment_kafka_writer_batch_size'] +enrichment_kafka_writer_batch_timeout = config['configurations']['metron-enrichment-env']['enrichment_kafka_writer_batch_timeout'] # Enrichment - Storm common parameters enrichment_workers = config['configurations']['metron-enrichment-env']['enrichment_workers'] @@ -328,6 +330,8 @@ profiler_window_lag=config['configurations']['metron-profiler-env']['profiler_wi profiler_window_lag_units=config['configurations']['metron-profiler-env']['profiler_window_lag_units'] profiler_topology_message_timeout_secs=config['configurations']['metron-profiler-env']['profiler_topology_message_timeout_secs'] profiler_topology_max_spout_pending=config['configurations']['metron-profiler-env']['profiler_topology_max_spout_pending'] +profiler_kafka_writer_batch_size = config['configurations']['metron-profiler-env']['profiler_kafka_writer_batch_size'] +profiler_kafka_writer_batch_timeout = config['configurations']['metron-profiler-env']['profiler_kafka_writer_batch_timeout'] # Indexing ra_indexing_kafka_start = config['configurations']['metron-indexing-env']['ra_indexing_kafka_start'] http://git-wip-us.apache.org/repos/asf/metron/blob/b06bb5f4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index d09f270..d757ec6 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -484,6 +484,14 @@ "subsection-name": "subsection-enrichment-kafka" }, { + "config": "metron-enrichment-env/enrichment_kafka_writer_batch_size", + "subsection-name": "subsection-enrichment-kafka" + }, + { + "config": "metron-enrichment-env/enrichment_kafka_writer_batch_timeout", + "subsection-name": "subsection-enrichment-kafka" + }, + { "config": "metron-enrichment-env/enrichment_workers", "subsection-name": "subsection-enrichment-storm" }, @@ -668,6 +676,14 @@ "subsection-name": "subsection-profiler-kafka" }, { + "config": "metron-profiler-env/profiler_kafka_writer_batch_size", + "subsection-name": "subsection-profiler-kafka" + }, + { + "config": "metron-profiler-env/profiler_kafka_writer_batch_timeout", + "subsection-name": "subsection-profiler-kafka" + }, + { "config": "metron-profiler-env/profiler_period_duration", "subsection-name": "subsection-profiler-setup" }, @@ -864,7 +880,6 @@ "type": "text-field" } }, - { "config": "metron-enrichment-env/geoip_url", "widget": { @@ -908,6 +923,18 @@ } }, { + "config": "metron-enrichment-env/enrichment_kafka_writer_batch_size", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-enrichment-env/enrichment_kafka_writer_batch_timeout", + "widget": { + "type": "text-field" + } + }, + { "config": "metron-enrichment-env/enrichment_workers", "widget": { "type": "text-field" @@ -1186,6 +1213,18 @@ } }, { + "config": "metron-profiler-env/profiler_kafka_writer_batch_size", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_kafka_writer_batch_timeout", + "widget": { + "type": "text-field" + } + }, + { "config": "metron-profiler-env/profiler_period_duration", "widget": { "type": "text-field"
