METRON-1017 Ambari components should be separate (merrimanr) closes apache/metron#634
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a9b01d1e Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a9b01d1e Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a9b01d1e Branch: refs/heads/master Commit: a9b01d1e373091662f4dff3ac3acf7cec181e103 Parents: cf165ff Author: merrimanr <[email protected]> Authored: Wed Jul 12 08:20:03 2017 -0500 Committer: merrimanr <[email protected]> Committed: Wed Jul 12 08:20:03 2017 -0500 ---------------------------------------------------------------------- .../configuration/metron-enrichment-env.xml | 174 ++++++++ .../METRON/CURRENT/configuration/metron-env.xml | 413 +------------------ .../configuration/metron-indexing-env.xml | 165 ++++++++ .../configuration/metron-management-ui-env.xml | 27 ++ .../configuration/metron-parsers-env.xml | 33 ++ .../CURRENT/configuration/metron-rest-env.xml | 112 +++++ .../common-services/METRON/CURRENT/metainfo.xml | 16 + .../package/scripts/enrichment_commands.py | 6 +- .../package/scripts/indexing_commands.py | 24 +- .../package/scripts/params/params_linux.py | 117 +++--- .../package/scripts/params/status_params.py | 19 +- .../CURRENT/package/scripts/parser_commands.py | 4 +- .../METRON/CURRENT/quicklinks/quicklinks.json | 4 +- .../METRON/CURRENT/themes/metron_theme.json | 219 +++++----- .../roles/ambari_config/vars/single_node_vm.yml | 6 +- 15 files changed, 731 insertions(+), 608 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml new file mode 100644 index 0000000..e042235 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml @@ -0,0 +1,174 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <property> + <name>geoip_url</name> + <value>http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz</value> + <description>Location of the GeoIP data to load.</description> + <display-name>GEOIP Load Datafile URL</display-name> + </property> + <property> + <name>enrichment_host_known_hosts</name> + <description>List of Known Hosts for Host Enrichment</description> + <value>[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]</value> + <display-name>Host Enrichment</display-name> + <value-attributes> + <type>content</type> + </value-attributes> + </property> + <property> + <name>enrichment_kafka_start</name> + <description>Enrichment Topology Spout Offset</description> + <value>UNCOMMITTED_EARLIEST</value> + <display-name>Enrichment Offset</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>EARLIEST</value> + </entry> + <entry> + <value>LATEST</value> + </entry> + <entry> + <value>UNCOMMITTED_EARLIEST</value> + </entry> + <entry> + <value>UNCOMMITTED_LATEST</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + </property> + <property> + <name>enrichment_input_topic</name> + <description>Enrichment Input Topic</description> + <value>enrichments</value> + <display-name>Enrichment Input Topic</display-name> + </property> + <property> + <name>enrichment_output_topic</name> + <description>Enrichment Output Topic</description> + <value>indexing</value> + <display-name>Enrichment Output Topic</display-name> + </property> + <property> + <name>enrichment_error_topic</name> + <description>Enrichment Error Topic</description> + <value>indexing</value> + <display-name>Enrichment Error Topic</display-name> + </property> + <property> + <name>threatintel_error_topic</name> + <description>Threat Intel Error Topic</description> + <value>indexing</value> + <display-name>Threat Intel Error Topic</display-name> + </property> + <property> + <name>enrichment_workers</name> + <description>Number of Enrichment Topology Workers</description> + <value>1</value> + <display-name>Enrichment Workers</display-name> + </property> + <property> + <name>enrichment_acker_executors</name> + <description>Number of Enrichment Topology Ackers</description> + <value>1</value> + <display-name>Enrichment Ackers</display-name> + </property> + <property> + <name>enrichment_topology_worker_childopts</name> + <description>Enrichment Topology JVM Options</description> + <value/> + <display-name>Enrichment childopts</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>enrichment_topology_max_spout_pending</name> + <description>Enrichment Topology Spout Max Pending Tuples</description> + <value/> + <display-name>Enrichment Max Pending</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>enrichment_join_cache_size</name> + <description>Enrichment Join Bolt Cache Max Size</description> + <value>100000</value> + <display-name>Enrichment Join Size</display-name> + </property> + <property> + <name>threatintel_join_cache_size</name> + <description>Threat Intel Join Bolt Cache Max Size</description> + <value>100000</value> + <display-name>Threat Intel Join Size</display-name> + </property> + <property> + <name>enrichment_kafka_spout_parallelism</name> + <description>Enrichment Topology Kafka Spout Parallelism</description> + <value>1</value> + <display-name>Enrichment Spout Parallelism</display-name> + </property> + <property> + <name>enrichment_split_parallelism</name> + <description>Enrichment Topology Enrichment Split Bolt Parallelism</description> + <value>1</value> + <display-name>Enrichment Split Parallelism</display-name> + </property> + <property> + <name>enrichment_stellar_parallelism</name> + <description>Enrichment Topology Enrichment Stellar Bolt Parallelism</description> + <value>1</value> + <display-name>Enrichment Stellar Parallelism</display-name> + </property> + <property> + <name>enrichment_join_parallelism</name> + <description>Enrichment Topology Enrichment Join Bolt Parallelism</description> + <value>1</value> + <display-name>Enrichment Join Parallelism</display-name> + </property> + <property> + <name>threat_intel_split_parallelism</name> + <description>Enrichment Topology Threat Intel Split Bolt Parallelism</description> + <value>1</value> + <display-name>Threat Intel Spout Parallelism</display-name> + </property> + <property> + <name>threat_intel_stellar_parallelism</name> + <description>Enrichment Topology Threat Intel Stellar Bolt Parallelism</description> + <value>1</value> + <display-name>Threat Intel Spout Parallelism</display-name> + </property> + <property> + <name>threat_intel_join_parallelism</name> + <description>Enrichment Topology Threat Intel Join Bolt Parallelism</description> + <value>1</value> + <display-name>Threat Intel Join Parallelism</display-name> + </property> + <property> + <name>kafka_writer_parallelism</name> + <description>Enrichment Topology Kafka Writer Parallelism</description> + <value>1</value> + <display-name>Enrichment Kafka Writer Parallelism</display-name> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index a8864e3..87866e8 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -31,12 +31,6 @@ <display-name>Metron apps HDFS dir</display-name> </property> <property> - <name>metron_apps_indexed_hdfs_dir</name> - <value>{{metron_apps_hdfs_dir}}/indexing/indexed</value> - <description>Indexing bolts will write to this HDFS directory</description> - <display-name>Metron apps indexed HDFS dir</display-name> - </property> - <property> <name>metron_zookeeper_config_dir</name> <value>config/zookeeper</value> <description>Metron Zookeeper config dir. Relative path to Metron home.</description> @@ -77,132 +71,17 @@ <display-name>Metron pid dir</display-name> </property> <property> - <name>metron_rest_port</name> - <value>8082</value> - <description>Port the REST application runs on</description> - <display-name>Metron REST port</display-name> - </property> - <property> - <name>metron_management_ui_port</name> - <value>4200</value> - <description>Port the Management UI runs on</description> - <display-name>Management UI port</display-name> - </property> - <property> - <name>metron_jvm_flags</name> - <description>JVM flags passed to Metron scripts</description> - <display-name>Metron JVM flags</display-name> - <value/> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - </property> - <property> - <name>metron_spring_profiles_active</name> - <description>Active Spring profiles</description> - <display-name>Active Spring profiles</display-name> - <value/> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - </property> - <property require-input="true"> - <name>metron_jdbc_driver</name> - <value></value> - <description>Class name of the JDBC Driver used by Metron</description> - <display-name>Metron JDBC Driver</display-name> - </property> - <property require-input="true"> - <name>metron_jdbc_url</name> - <value></value> - <description>JDBC Connection URL used by Metron</description> - <display-name>Metron JDBC URL</display-name> - </property> - <property require-input="true"> - <name>metron_jdbc_username</name> - <value></value> - <description>Metron JDBC Username</description> - <display-name>Metron JDBC username</display-name> - </property> - <property require-input="true"> - <name>metron_jdbc_password</name> - <value></value> - <property-type>PASSWORD</property-type> - <description>Metron JDBC Password</description> - <display-name>Metron JDBC password</display-name> - <value-attributes> - <type>password</type> - <overridable>false</overridable> - <hidden>METRON_CLIENT</hidden> - </value-attributes> - <on-ambari-upgrade add="true"/> - </property> - <property> - <name>metron_jdbc_platform</name> - <value></value> - <description>Database platform used by Metron. One of: hsqldb, h2, oracle, mysql, postgresql</description> - <display-name>Metron JDBC platform</display-name> - </property> - <property> - <name>metron_jdbc_client_path</name> - <description>Path to JDBC jar for selected platform</description> - <display-name>Metron JDBC client path</display-name> - <value/> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - </property> - <property> - <name>metron_temp_grok_path</name> - <description>Temporary local file path where grok patterns are written during testing</description> - <value>./patterns/temp</value> - <display-name>Metron temp grok path</display-name> - </property> - <property> - <name>metron_default_grok_path</name> - <description>Default HDFS directory path used when storing Grok patterns</description> - <value>/apps/metron/patterns</value> - <display-name>Metron default grok path</display-name> - </property> - <property> - <name>metron_spring_options</name> - <description>Additional Spring options not included in the rest_application.yml file</description> - <display-name>Metron Spring options</display-name> - <value/> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - </property> - <property> <name>metron_topic_retention</name> <description>Kafka Retention in GB</description> <value>10</value> <display-name>Topic Retention</display-name> </property> <property> - <name>parsers</name> - <value>bro,snort,yaf</value> - <description>Metron parsers to deploy</description> - <display-name>Metron Parsers</display-name> - </property> - <property> - <name>metron_indexing_topology</name> - <value>indexing</value> - <description>The Storm topology name for Indexing</description> - <display-name>Indexing Topology Name</display-name> - </property> - <property> <name>es_cluster_name</name> <value>metron</value> <description>Name of Elasticsearch Cluster</description> <display-name>Elasticsearch Cluster Name</display-name> </property> - <property> - <name>geoip_url</name> - <value>http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz</value> - <description>Location of the GeoIP data to load.</description> - <display-name>GEOIP Load Datafile URL</display-name> - </property> <property require-input="true"> <name>es_hosts</name> <value></value> @@ -258,296 +137,6 @@ <value>yyyy.MM.dd.HH</value> <display-name>Elasticsearch Date Format</display-name> </property> - <property> - <name>parser_error_topic</name> - <description>Parser Error Topic</description> - <value>indexing</value> - <display-name>Parser Error Topic</display-name> - </property> - <property> - <name>enrichment_workers</name> - <description>Number of Enrichment Topology Workers</description> - <value>1</value> - <display-name>Enrichment Workers</display-name> - </property> - <property> - <name>enrichment_acker_executors</name> - <description>Number of Enrichment Topology Ackers</description> - <value>1</value> - <display-name>Enrichment Ackers</display-name> - </property> - <property> - <name>enrichment_topology_worker_childopts</name> - <description>Enrichment Topology JVM Options</description> - <value/> - <display-name>Enrichment childopts</display-name> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - </property> - <property> - <name>enrichment_topology_max_spout_pending</name> - <description>Enrichment Topology Spout Max Pending Tuples</description> - <value/> - <display-name>Enrichment Max Pending</display-name> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - </property> - <property> - <name>enrichment_kafka_start</name> - <description>Enrichment Topology Spout Offset</description> - <value>UNCOMMITTED_EARLIEST</value> - <display-name>Enrichment Offset</display-name> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>EARLIEST</value> - </entry> - <entry> - <value>LATEST</value> - </entry> - <entry> - <value>UNCOMMITTED_EARLIEST</value> - </entry> - <entry> - <value>UNCOMMITTED_LATEST</value> - </entry> - </entries> - <selection-cardinality>1</selection-cardinality> - </value-attributes> - </property> - <property> - <name>enrichment_input_topic</name> - <description>Enrichment Input Topic</description> - <value>enrichments</value> - <display-name>Enrichment Input Topic</display-name> - </property> - <property> - <name>enrichment_output_topic</name> - <description>Enrichment Output Topic</description> - <value>indexing</value> - <display-name>Enrichment Output Topic</display-name> - </property> - <property> - <name>enrichment_error_topic</name> - <description>Enrichment Error Topic</description> - <value>indexing</value> - <display-name>Enrichment Error Topic</display-name> - </property> - <property> - <name>threatintel_error_topic</name> - <description>Threat Intel Error Topic</description> - <value>indexing</value> - <display-name>Threat Intel Error Topic</display-name> - </property> - <property> - <name>enrichment_join_cache_size</name> - <description>Enrichment Join Bolt Cache Max Size</description> - <value>100000</value> - <display-name>Enrichment Join Size</display-name> - </property> - <property> - <name>threatintel_join_cache_size</name> - <description>Threat Intel Join Bolt Cache Max Size</description> - <value>100000</value> - <display-name>Threat Intel Join Size</display-name> - </property> - <property> - <name>enrichment_host_known_hosts</name> - <description>List of Known Hosts for Host Enrichment</description> - <value>[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]</value> - <display-name>Host Enrichment</display-name> - <value-attributes> - <type>content</type> - </value-attributes> - </property> - <property> - <name>enrichment_kafka_spout_parallelism</name> - <description>Enrichment Topology Kafka Spout Parallelism</description> - <value>1</value> - <display-name>Enrichment Spout Parallelism</display-name> - </property> - <property> - <name>enrichment_split_parallelism</name> - <description>Enrichment Topology Enrichment Split Bolt Parallelism</description> - <value>1</value> - <display-name>Enrichment Split Parallelism</display-name> - </property> - <property> - <name>enrichment_stellar_parallelism</name> - <description>Enrichment Topology Enrichment Stellar Bolt Parallelism</description> - <value>1</value> - <display-name>Enrichment Stellar Parallelism</display-name> - </property> - <property> - <name>enrichment_join_parallelism</name> - <description>Enrichment Topology Enrichment Join Bolt Parallelism</description> - <value>1</value> - <display-name>Enrichment Join Parallelism</display-name> - </property> - <property> - <name>threat_intel_split_parallelism</name> - <description>Enrichment Topology Threat Intel Split Bolt Parallelism</description> - <value>1</value> - <display-name>Threat Intel Spout Parallelism</display-name> - </property> - <property> - <name>threat_intel_stellar_parallelism</name> - <description>Enrichment Topology Threat Intel Stellar Bolt Parallelism</description> - <value>1</value> - <display-name>Threat Intel Spout Parallelism</display-name> - </property> - <property> - <name>threat_intel_join_parallelism</name> - <description>Enrichment Topology Threat Intel Join Bolt Parallelism</description> - <value>1</value> - <display-name>Threat Intel Join Parallelism</display-name> - </property> - <property> - <name>kafka_writer_parallelism</name> - <description>Enrichment Topology Kafka Writer Parallelism</description> - <value>1</value> - <display-name>Enrichment Kafka Writer Parallelism</display-name> - </property> - <property> - <name>indexing_workers</name> - <description>Number of Indexing Topology Workers</description> - <value>1</value> - <display-name>Indexing Workers</display-name> - </property> - <property> - <name>indexing_acker_executors</name> - <description>Number of Indexing Topology Ackers</description> - <value>1</value> - <display-name>Enrichment Ackers</display-name> - </property> - <property> - <name>indexing_topology_worker_childopts</name> - <description>Indexing Topology JVM Options</description> - <value/> - <display-name>Indexing childopts</display-name> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - </property> - <property> - <name>indexing_topology_max_spout_pending</name> - <description>Indexing Topology Spout Max Pending Tuples</description> - <value/> - <display-name>Indexing Max Pending</display-name> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - </property> - <property> - <name>indexing_kafka_start</name> - <description>Indexing Topology Spout Offset</description> - <value>UNCOMMITTED_EARLIEST</value> - <display-name>Indexing Offset</display-name> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>EARLIEST</value> - </entry> - <entry> - <value>LATEST</value> - </entry> - <entry> - <value>UNCOMMITTED_EARLIEST</value> - </entry> - <entry> - <value>UNCOMMITTED_LATEST</value> - </entry> - </entries> - <selection-cardinality>1</selection-cardinality> - </value-attributes> - </property> - <property> - <name>indexing_input_topic</name> - <description>Indexing Input Topic</description> - <value>indexing</value> - <display-name>Indexing Input Topic</display-name> - </property> - <property> - <name>indexing_error_topic</name> - <description>Indexing Error Topic</description> - <value>indexing</value> - <display-name>Indexing Error Topic</display-name> - </property> - <property> - <name>indexing_writer_class_name</name> - <description>Indexing Writer Class Name</description> - <value>org.apache.metron.elasticsearch.writer.ElasticsearchWriter</value> - <display-name>Indexing Writer Class Name</display-name> - </property> - <property> - <name>bolt_hdfs_rotation_policy</name> - <description>Class Name of HDFS Rotation Policy</description> - <value>org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy</value> - <display-name>HDFS Rotation Policy</display-name> - </property> - <property> - <name>bolt_hdfs_rotation_policy_count</name> - <description>HDFS Rotation Policy Count</description> - <value>1</value> - <display-name>HDFS Rotation Policy Count</display-name> - </property> - <property> - <name>bolt_hdfs_rotation_policy_units</name> - <description>Frequency of HDFS File Rotation</description> - <value>DAYS</value> - <display-name>HDFS Rotation Policy Units</display-name> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>SECONDS</value> - </entry> - <entry> - <value>MINUTES</value> - </entry> - <entry> - <value>HOURS</value> - </entry> - <entry> - <value>DAYS</value> - </entry> - <entry> - <value>KB</value> - </entry> - <entry> - <value>MB</value> - </entry> - <entry> - <value>GB</value> - </entry> - <entry> - <value>TB</value> - </entry> - </entries> - <selection-cardinality>1</selection-cardinality> - </value-attributes> - </property> - <property> - <name>indexing_kafka_spout_parallelism</name> - <description>Indexing Topology Kafka Spout Parallelism</description> - <value>1</value> - <display-name>Indexing Spout Parallelism</display-name> - </property> - <property> - <name>indexing_writer_parallelism</name> - <description>Indexing Topology Writer Bolt Parallelism</description> - <value>1</value> - <display-name>Indexing Writer Parallelism</display-name> - </property> - <property> - <name>hdfs_writer_parallelism</name> - <description>Indexing Topology HDFS Bolt Parallelism</description> - <value>1</value> - <display-name>Indexing HDFS Parallelism</display-name> - </property> + </configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml new file mode 100644 index 0000000..e28e328 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml @@ -0,0 +1,165 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <property> + <name>indexing_kafka_start</name> + <description>Indexing Topology Spout Offset</description> + <value>UNCOMMITTED_EARLIEST</value> + <display-name>Indexing Offset</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>EARLIEST</value> + </entry> + <entry> + <value>LATEST</value> + </entry> + <entry> + <value>UNCOMMITTED_EARLIEST</value> + </entry> + <entry> + <value>UNCOMMITTED_LATEST</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + </property> + <property> + <name>indexing_input_topic</name> + <description>Indexing Input Topic</description> + <value>indexing</value> + <display-name>Indexing Input Topic</display-name> + </property> + <property> + <name>indexing_error_topic</name> + <description>Indexing Error Topic</description> + <value>indexing</value> + <display-name>Indexing Error Topic</display-name> + </property> + <property> + <name>indexing_writer_class_name</name> + <description>Indexing Writer Class Name</description> + <value>org.apache.metron.elasticsearch.writer.ElasticsearchWriter</value> + <display-name>Indexing Writer Class Name</display-name> + </property> + <property> + <name>indexing_workers</name> + <description>Number of Indexing Topology Workers</description> + <value>1</value> + <display-name>Indexing Workers</display-name> + </property> + <property> + <name>indexing_acker_executors</name> + <description>Number of Indexing Topology Ackers</description> + <value>1</value> + <display-name>Enrichment Ackers</display-name> + </property> + <property> + <name>indexing_topology_worker_childopts</name> + <description>Indexing Topology JVM Options</description> + <value/> + <display-name>Indexing childopts</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>indexing_topology_max_spout_pending</name> + <description>Indexing Topology Spout Max Pending Tuples</description> + <value/> + <display-name>Indexing Max Pending</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>indexing_kafka_spout_parallelism</name> + <description>Indexing Topology Kafka Spout Parallelism</description> + <value>1</value> + <display-name>Indexing Spout Parallelism</display-name> + </property> + <property> + <name>indexing_writer_parallelism</name> + <description>Indexing Topology Writer Bolt Parallelism</description> + <value>1</value> + <display-name>Indexing Writer Parallelism</display-name> + </property> + <property> + <name>hdfs_writer_parallelism</name> + <description>Indexing Topology HDFS Bolt Parallelism</description> + <value>1</value> + <display-name>Indexing HDFS Parallelism</display-name> + </property> + <property> + <name>metron_apps_indexed_hdfs_dir</name> + <value>{{metron_apps_hdfs_dir}}/indexing/indexed</value> + <description>Indexing bolts will write to this HDFS directory</description> + <display-name>Metron apps indexed HDFS dir</display-name> + </property> + <property> + <name>bolt_hdfs_rotation_policy</name> + <description>Class Name of HDFS Rotation Policy</description> + <value>org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy</value> + <display-name>HDFS Rotation Policy</display-name> + </property> + <property> + <name>bolt_hdfs_rotation_policy_count</name> + <description>HDFS Rotation Policy Count</description> + <value>1</value> + <display-name>HDFS Rotation Policy Count</display-name> + </property> + <property> + <name>bolt_hdfs_rotation_policy_units</name> + <description>Frequency of HDFS File Rotation</description> + <value>DAYS</value> + <display-name>HDFS Rotation Policy Units</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>SECONDS</value> + </entry> + <entry> + <value>MINUTES</value> + </entry> + <entry> + <value>HOURS</value> + </entry> + <entry> + <value>DAYS</value> + </entry> + <entry> + <value>KB</value> + </entry> + <entry> + <value>MB</value> + </entry> + <entry> + <value>GB</value> + </entry> + <entry> + <value>TB</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-management-ui-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-management-ui-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-management-ui-env.xml new file mode 100644 index 0000000..d9adddd --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-management-ui-env.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <property> + <name>metron_management_ui_port</name> + <value>4200</value> + <description>Port the Management UI runs on</description> + <display-name>Management UI port</display-name> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml new file mode 100644 index 0000000..a9a498b --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml @@ -0,0 +1,33 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <property> + <name>parsers</name> + <value>bro,snort,yaf</value> + <description>Metron parsers to deploy</description> + <display-name>Metron Parsers</display-name> + </property> + <property> + <name>parser_error_topic</name> + <description>Parser Error Topic</description> + <value>indexing</value> + <display-name>Parser Error Topic</display-name> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml new file mode 100644 index 0000000..c68f5b2 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -0,0 +1,112 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <property> + <name>metron_rest_port</name> + <value>8082</value> + <description>Port the REST application runs on</description> + <display-name>Metron REST port</display-name> + </property> + <property> + <name>metron_jvm_flags</name> + <description>JVM flags passed to Metron scripts</description> + <display-name>Metron JVM flags</display-name> + <value/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>metron_spring_profiles_active</name> + <description>Active Spring profiles</description> + <display-name>Active Spring profiles</display-name> + <value/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property require-input="true"> + <name>metron_jdbc_driver</name> + <value></value> + <description>Class name of the JDBC Driver used by Metron</description> + <display-name>Metron JDBC Driver</display-name> + </property> + <property require-input="true"> + <name>metron_jdbc_url</name> + <value></value> + <description>JDBC Connection URL used by Metron</description> + <display-name>Metron JDBC URL</display-name> + </property> + <property require-input="true"> + <name>metron_jdbc_username</name> + <value></value> + <description>Metron JDBC Username</description> + <display-name>Metron JDBC username</display-name> + </property> + <property require-input="true"> + <name>metron_jdbc_password</name> + <value></value> + <property-type>PASSWORD</property-type> + <description>Metron JDBC Password</description> + <display-name>Metron JDBC password</display-name> + <value-attributes> + <type>password</type> + <overridable>false</overridable> + <hidden>METRON_CLIENT</hidden> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>metron_jdbc_platform</name> + <value></value> + <description>Database platform used by Metron. One of: hsqldb, h2, oracle, mysql, postgresql</description> + <display-name>Metron JDBC platform</display-name> + </property> + <property> + <name>metron_jdbc_client_path</name> + <description>Path to JDBC jar for selected platform</description> + <display-name>Metron JDBC client path</display-name> + <value/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>metron_temp_grok_path</name> + <description>Temporary local file path where grok patterns are written during testing</description> + <value>./patterns/temp</value> + <display-name>Metron temp grok path</display-name> + </property> + <property> + <name>metron_default_grok_path</name> + <description>Default HDFS directory path used when storing Grok patterns</description> + <value>/apps/metron/patterns</value> + <display-name>Metron default grok path</display-name> + </property> + <property> + <name>metron_spring_options</name> + <description>Additional Spring options not included in the rest_application.yml file</description> + <display-name>Metron Spring options</display-name> + <value/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml index cd86432..b2dda25 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml @@ -67,6 +67,9 @@ <script>scripts/parser_master.py</script> <scriptType>PYTHON</scriptType> </commandScript> + <configuration-dependencies> + <config-type>metron-parsers-env</config-type> + </configuration-dependencies> </component> <component> @@ -110,6 +113,9 @@ <scriptType>PYTHON</scriptType> <timeout>600</timeout> </commandScript> + <configuration-dependencies> + <config-type>metron-enrichment-env</config-type> + </configuration-dependencies> </component> <component> @@ -173,6 +179,9 @@ </commandScript> </customCommand> </customCommands> + <configuration-dependencies> + <config-type>metron-indexing-env</config-type> + </configuration-dependencies> </component> <component> @@ -217,6 +226,9 @@ <script>scripts/rest_master.py</script> <scriptType>PYTHON</scriptType> </commandScript> + <configuration-dependencies> + <config-type>metron-rest-env</config-type> + </configuration-dependencies> </component> <component> @@ -261,6 +273,10 @@ <script>scripts/management_ui_master.py</script> <scriptType>PYTHON</scriptType> </commandScript> + <configuration-dependencies> + <config-type>metron-rest-env</config-type> + <config-type>metron-management-ui-env</config-type> + </configuration-dependencies> </component> <component> http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py index 136eade..8ca97ad 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py @@ -39,7 +39,7 @@ class EnrichmentCommands: raise ValueError("params argument is required for initialization") self.__params = params self.__enrichment_topology = params.metron_enrichment_topology - self.__enrichment_topic = params.metron_enrichment_topic + self.__enrichment_topic = params.enrichment_input_topic self.__kafka_configured = os.path.isfile(self.__params.enrichment_kafka_configured_flag_file) self.__kafka_acl_configured = os.path.isfile(self.__params.enrichment_kafka_acl_configured_flag_file) self.__hbase_configured = os.path.isfile(self.__params.enrichment_hbase_configured_flag_file) @@ -124,14 +124,14 @@ class EnrichmentCommands: def init_kafka_topics(self): Logger.info('Creating Kafka topics for enrichment') # All errors go to indexing topics, so create it here if it's not already - metron_service.init_kafka_topics(self.__params, [self.__enrichment_topic, self.__params.metron_error_topic]) + metron_service.init_kafka_topics(self.__params, [self.__enrichment_topic, self.__params.enrichment_error_topic]) self.set_kafka_configured() def init_kafka_acls(self): Logger.info('Creating Kafka ACls for enrichment') # Enrichment topic names matches group metron_service.init_kafka_acls(self.__params, - [self.__enrichment_topic, self.__params.metron_error_topic], + [self.__enrichment_topic, self.__params.enrichment_error_topic], [self.__enrichment_topic]) self.set_kafka_acl_configured() http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index da94c9d..b671c9d 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -28,7 +28,8 @@ import metron_security # Wrap major operations and functionality in this class class IndexingCommands: __params = None - __indexing = None + __indexing_topic = None + __indexing_topology = None __configured = False __acl_configured = False __hdfs_perm_configured = False @@ -37,7 +38,8 @@ class IndexingCommands: if params is None: raise ValueError("params argument is required for initialization") self.__params = params - self.__indexing = params.metron_indexing_topology + self.__indexing_topology = params.metron_indexing_topology + self.__indexing_topic = params.indexing_input_topic self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file) self.__acl_configured = os.path.isfile(self.__params.indexing_acl_configured_flag_file) @@ -70,12 +72,12 @@ class IndexingCommands: def init_kafka_topics(self): Logger.info('Creating Kafka topics for indexing') - metron_service.init_kafka_topics(self.__params, [self.__indexing]) + metron_service.init_kafka_topics(self.__params, [self.__indexing_topic]) def init_kafka_acls(self): Logger.info('Creating Kafka ACLs') # Indexed topic names matches the group - metron_service.init_kafka_acls(self.__params, [self.__indexing], [self.__indexing]) + metron_service.init_kafka_acls(self.__params, [self.__indexing_topic], [self.__indexing_topic]) def init_hdfs_dir(self): Logger.info('Setting up HDFS indexing directory') @@ -94,24 +96,24 @@ class IndexingCommands: Logger.info('Done creating HDFS indexing directory') def start_indexing_topology(self): - Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing)) + Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing_topology)) start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \ -s {1} \ -z {2}""" - Logger.info('Starting ' + self.__indexing) + Logger.info('Starting ' + self.__indexing_topology) if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, self.__params.metron_principal_name, execute_user=self.__params.metron_user) - Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum), + Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing_topology, self.__params.zookeeper_quorum), user=self.__params.metron_user) Logger.info('Finished starting indexing topology') def stop_indexing_topology(self): - Logger.info('Stopping ' + self.__indexing) - stop_cmd = 'storm kill ' + self.__indexing + Logger.info('Stopping ' + self.__indexing_topology) + stop_cmd = 'storm kill ' + self.__indexing_topology if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, @@ -146,7 +148,7 @@ class IndexingCommands: active = True topologies = metron_service.get_running_topologies(self.__params) is_running = False - if self.__indexing in topologies: - is_running = topologies[self.__indexing] in ['ACTIVE', 'REBALANCING'] + if self.__indexing_topology in topologies: + is_running = topologies[self.__indexing_topology] in ['ACTIVE', 'REBALANCING'] active &= is_running return active http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 86ce519..d8d8add 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -39,27 +39,25 @@ tmp_dir = Script.get_tmp_dir() hostname = config['hostname'] metron_home = status_params.metron_home parsers = status_params.parsers -parser_error_topic = config['configurations']['metron-env']['parser_error_topic'] -geoip_url = config['configurations']['metron-env']['geoip_url'] +parser_error_topic = config['configurations']['metron-parsers-env']['parser_error_topic'] geoip_hdfs_dir = "/apps/metron/geo/default/" -metron_indexing_topology = status_params.metron_indexing_topology metron_user = status_params.metron_user metron_group = config['configurations']['metron-env']['metron_group'] metron_log_dir = config['configurations']['metron-env']['metron_log_dir'] metron_pid_dir = config['configurations']['metron-env']['metron_pid_dir'] -metron_rest_port = config['configurations']['metron-env']['metron_rest_port'] -metron_management_ui_port = config['configurations']['metron-env']['metron_management_ui_port'] -metron_jvm_flags = '' -metron_spring_profiles_active = config['configurations']['metron-env']['metron_spring_profiles_active'] -metron_jdbc_driver = config['configurations']['metron-env']['metron_jdbc_driver'] -metron_jdbc_url = config['configurations']['metron-env']['metron_jdbc_url'] -metron_jdbc_username = config['configurations']['metron-env']['metron_jdbc_username'] -metron_jdbc_password = config['configurations']['metron-env']['metron_jdbc_password'] -metron_jdbc_platform = config['configurations']['metron-env']['metron_jdbc_platform'] -metron_jdbc_client_path = config['configurations']['metron-env']['metron_jdbc_client_path'] -metron_temp_grok_path = config['configurations']['metron-env']['metron_temp_grok_path'] -metron_default_grok_path = config['configurations']['metron-env']['metron_default_grok_path'] -metron_spring_options = config['configurations']['metron-env']['metron_spring_options'] +metron_rest_port = config['configurations']['metron-rest-env']['metron_rest_port'] +metron_management_ui_port = config['configurations']['metron-management-ui-env']['metron_management_ui_port'] +metron_jvm_flags = config['configurations']['metron-rest-env']['metron_jvm_flags'] +metron_spring_profiles_active = config['configurations']['metron-rest-env']['metron_spring_profiles_active'] +metron_jdbc_driver = config['configurations']['metron-rest-env']['metron_jdbc_driver'] +metron_jdbc_url = config['configurations']['metron-rest-env']['metron_jdbc_url'] +metron_jdbc_username = config['configurations']['metron-rest-env']['metron_jdbc_username'] +metron_jdbc_password = config['configurations']['metron-rest-env']['metron_jdbc_password'] +metron_jdbc_platform = config['configurations']['metron-rest-env']['metron_jdbc_platform'] +metron_jdbc_client_path = config['configurations']['metron-rest-env']['metron_jdbc_client_path'] +metron_temp_grok_path = config['configurations']['metron-rest-env']['metron_temp_grok_path'] +metron_default_grok_path = config['configurations']['metron-rest-env']['metron_default_grok_path'] +metron_spring_options = config['configurations']['metron-rest-env']['metron_spring_options'] metron_config_path = metron_home + '/config' metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir metron_zookeeper_config_path = status_params.metron_zookeeper_config_path @@ -131,9 +129,6 @@ if has_kafka_host: metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir'] -# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works -# well enough until we find a better way via Ambari -metron_apps_indexed_hdfs_dir = format(format(config['configurations']['metron-env']['metron_apps_indexed_hdfs_dir'])) metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention'] local_grok_patterns_dir = format("{metron_home}/patterns") @@ -175,9 +170,6 @@ threatintel_table = status_params.threatintel_table threatintel_cf = status_params.threatintel_cf # Kafka Topics -metron_enrichment_topology = status_params.metron_enrichment_topology -metron_enrichment_topic = status_params.metron_enrichment_topic -metron_error_topic = 'indexing' ambari_kafka_service_check_topic = 'ambari_kafka_service_check' consumer_offsets_topic = '__consumer_offsets' @@ -227,39 +219,48 @@ if security_enabled: metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", ['localhost'])[0] # Enrichment -enrichment_workers = config['configurations']['metron-env']['enrichment_workers'] -enrichment_acker_executors = config['configurations']['metron-env']['enrichment_acker_executors'] -enrichment_topology_worker_childopts = config['configurations']['metron-env']['enrichment_topology_worker_childopts'] -enrichment_topology_max_spout_pending = config['configurations']['metron-env']['enrichment_topology_max_spout_pending'] -enrichment_kafka_start = config['configurations']['metron-env']['enrichment_kafka_start'] -enrichment_input_topic = config['configurations']['metron-env']['enrichment_input_topic'] -enrichment_output_topic = config['configurations']['metron-env']['enrichment_output_topic'] -enrichment_error_topic = config['configurations']['metron-env']['enrichment_error_topic'] -threatintel_error_topic = config['configurations']['metron-env']['threatintel_error_topic'] -enrichment_join_cache_size = config['configurations']['metron-env']['enrichment_join_cache_size'] -threatintel_join_cache_size = config['configurations']['metron-env']['threatintel_join_cache_size'] -enrichment_host_known_hosts = config['configurations']['metron-env']['enrichment_host_known_hosts'] -enrichment_kafka_spout_parallelism = config['configurations']['metron-env']['enrichment_kafka_spout_parallelism'] -enrichment_split_parallelism = config['configurations']['metron-env']['enrichment_split_parallelism'] -enrichment_stellar_parallelism = config['configurations']['metron-env']['enrichment_stellar_parallelism'] -enrichment_join_parallelism = config['configurations']['metron-env']['enrichment_join_parallelism'] -threat_intel_split_parallelism = config['configurations']['metron-env']['threat_intel_split_parallelism'] -threat_intel_stellar_parallelism = config['configurations']['metron-env']['threat_intel_stellar_parallelism'] -threat_intel_join_parallelism = config['configurations']['metron-env']['threat_intel_join_parallelism'] -kafka_writer_parallelism = config['configurations']['metron-env']['kafka_writer_parallelism'] - -# Threat Intel -indexing_workers = config['configurations']['metron-env']['indexing_workers'] -indexing_acker_executors = config['configurations']['metron-env']['indexing_acker_executors'] -indexing_topology_worker_childopts = config['configurations']['metron-env']['indexing_topology_worker_childopts'] -indexing_topology_max_spout_pending = config['configurations']['metron-env']['indexing_topology_max_spout_pending'] -indexing_kafka_start = config['configurations']['metron-env']['indexing_kafka_start'] -indexing_input_topic = config['configurations']['metron-env']['indexing_input_topic'] -indexing_error_topic = config['configurations']['metron-env']['indexing_error_topic'] -indexing_writer_class_name = config['configurations']['metron-env']['indexing_writer_class_name'] -bolt_hdfs_rotation_policy = config['configurations']['metron-env']['bolt_hdfs_rotation_policy'] -bolt_hdfs_rotation_policy_units = config['configurations']['metron-env']['bolt_hdfs_rotation_policy_units'] -bolt_hdfs_rotation_policy_count = config['configurations']['metron-env']['bolt_hdfs_rotation_policy_count'] -indexing_kafka_spout_parallelism = config['configurations']['metron-env']['indexing_kafka_spout_parallelism'] -indexing_writer_parallelism = config['configurations']['metron-env']['indexing_writer_parallelism'] -hdfs_writer_parallelism = config['configurations']['metron-env']['hdfs_writer_parallelism'] +geoip_url = config['configurations']['metron-enrichment-env']['geoip_url'] +enrichment_host_known_hosts = config['configurations']['metron-enrichment-env']['enrichment_host_known_hosts'] +enrichment_kafka_start = config['configurations']['metron-enrichment-env']['enrichment_kafka_start'] +enrichment_input_topic = status_params.enrichment_input_topic +enrichment_output_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic'] +enrichment_error_topic = config['configurations']['metron-enrichment-env']['enrichment_error_topic'] +threatintel_error_topic = config['configurations']['metron-enrichment-env']['threatintel_error_topic'] +metron_enrichment_topology = status_params.metron_enrichment_topology +enrichment_workers = config['configurations']['metron-enrichment-env']['enrichment_workers'] +enrichment_acker_executors = config['configurations']['metron-enrichment-env']['enrichment_acker_executors'] +enrichment_topology_worker_childopts = config['configurations']['metron-enrichment-env']['enrichment_topology_worker_childopts'] +enrichment_topology_max_spout_pending = config['configurations']['metron-enrichment-env']['enrichment_topology_max_spout_pending'] +enrichment_join_cache_size = config['configurations']['metron-enrichment-env']['enrichment_join_cache_size'] +threatintel_join_cache_size = config['configurations']['metron-enrichment-env']['threatintel_join_cache_size'] +enrichment_kafka_spout_parallelism = config['configurations']['metron-enrichment-env']['enrichment_kafka_spout_parallelism'] +enrichment_split_parallelism = config['configurations']['metron-enrichment-env']['enrichment_split_parallelism'] +enrichment_stellar_parallelism = config['configurations']['metron-enrichment-env']['enrichment_stellar_parallelism'] +enrichment_join_parallelism = config['configurations']['metron-enrichment-env']['enrichment_join_parallelism'] +threat_intel_split_parallelism = config['configurations']['metron-enrichment-env']['threat_intel_split_parallelism'] +threat_intel_stellar_parallelism = config['configurations']['metron-enrichment-env']['threat_intel_stellar_parallelism'] +threat_intel_join_parallelism = config['configurations']['metron-enrichment-env']['threat_intel_join_parallelism'] +kafka_writer_parallelism = config['configurations']['metron-enrichment-env']['kafka_writer_parallelism'] + +# Indexing +indexing_kafka_start = config['configurations']['metron-indexing-env']['indexing_kafka_start'] +indexing_input_topic = status_params.indexing_input_topic +indexing_error_topic = config['configurations']['metron-indexing-env']['indexing_error_topic'] +metron_indexing_topology = status_params.metron_indexing_topology +indexing_writer_class_name = config['configurations']['metron-indexing-env']['indexing_writer_class_name'] +indexing_workers = config['configurations']['metron-indexing-env']['indexing_workers'] +indexing_acker_executors = config['configurations']['metron-indexing-env']['indexing_acker_executors'] +indexing_topology_worker_childopts = config['configurations']['metron-indexing-env']['indexing_topology_worker_childopts'] +indexing_topology_max_spout_pending = config['configurations']['metron-indexing-env']['indexing_topology_max_spout_pending'] +indexing_kafka_spout_parallelism = config['configurations']['metron-indexing-env']['indexing_kafka_spout_parallelism'] +indexing_writer_parallelism = config['configurations']['metron-indexing-env']['indexing_writer_parallelism'] +hdfs_writer_parallelism = config['configurations']['metron-indexing-env']['hdfs_writer_parallelism'] + +# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works +# well enough until we find a better way via Ambari +metron_apps_indexed_hdfs_dir = format(format(config['configurations']['metron-indexing-env']['metron_apps_indexed_hdfs_dir'])) + +bolt_hdfs_rotation_policy = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy'] +bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_units'] +bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count'] + http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 690c301..072a371 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -28,7 +28,7 @@ config = Script.get_config() metron_user = config['configurations']['metron-env']['metron_user'] # Parsers -parsers = config['configurations']['metron-env']['parsers'] +parsers = config['configurations']['metron-parsers-env']['parsers'] metron_home = config['configurations']['metron-env']['metron_home'] metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir'] metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}') @@ -38,7 +38,12 @@ rest_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_rest_ # Enrichment metron_enrichment_topology = 'enrichment' -metron_enrichment_topic = 'enrichments' +enrichment_input_topic = config['configurations']['metron-enrichment-env']['enrichment_input_topic'] +enrichment_kafka_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_configured' +enrichment_kafka_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_acl_configured' +enrichment_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_configured' +enrichment_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_acl_configured' +enrichment_geo_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_geo_configured' enrichment_table = 'enrichment' enrichment_cf = 't' @@ -46,18 +51,12 @@ threatintel_table = 'threatintel' threatintel_cf = 't' # Indexing -metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology'] +metron_indexing_topology = 'indexing' +indexing_input_topic = config['configurations']['metron-indexing-env']['indexing_input_topic'] indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' indexing_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_acl_configured' indexing_hdfs_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_hdfs_perm_configured' -# Enrichment -enrichment_kafka_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_configured' -enrichment_kafka_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_acl_configured' -enrichment_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_configured' -enrichment_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_acl_configured' -enrichment_geo_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_geo_configured' - # Storm storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py index e4cae47..a6e1f1f 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py @@ -89,7 +89,7 @@ class ParserCommands: # All errors go to indexing topics, so create it here if it's not already # Getting topics this way is a bit awkward, but I don't want to append to actual list, so copy it topics = list(self.get_parser_list()) - topics.append(self.__params.metron_error_topic) + topics.append(self.__params.enrichment_error_topic) metron_service.init_kafka_topics(self.__params, topics) def init_kafka_acls(self): @@ -97,7 +97,7 @@ class ParserCommands: # Getting topics this way is a bit awkward, but I don't want to modify the actual list, so copy it topics = list(self.get_parser_list()) - topics.append(self.__params.metron_error_topic) + topics.append(self.__params.enrichment_error_topic) # Parser group is the parser name + '_parser' metron_service.init_kafka_acls(self.__params, topics, http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json index 31fde1f..dbe2e6a 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json @@ -35,7 +35,7 @@ "https_property": "metron_rest_port", "https_default_port": "8082", "regex": "^(\\d+)$", - "site": "metron-env" + "site": "metron-rest-env" } }, { @@ -50,7 +50,7 @@ "https_property": "metron_management_ui_port", "https_default_port": "4200", "regex": "^(\\d+)$", - "site": "metron-env" + "site": "metron-management-ui-env" } } ]
