METRON-671 Refactor existing Ansible deployment to use Ambari MPack (dlyle via justinleet) closes apache/incubator-metron#436
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/68a334a8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/68a334a8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/68a334a8 Branch: refs/heads/master Commit: 68a334a8717d2a6b82f7f9651c57bc75e256ba97 Parents: 8b6fc2e Author: dlyle <dlyle65...@gmail.com> Authored: Fri Mar 10 09:31:46 2017 -0500 Committer: leet <l...@apache.org> Committed: Fri Mar 10 09:31:46 2017 -0500 ---------------------------------------------------------------------- Upgrading.md | 8 + metron-deployment/README.md | 98 +--- metron-deployment/amazon-ec2/conf/defaults.yml | 7 - metron-deployment/amazon-ec2/playbook.yml | 10 +- metron-deployment/amazon-ec2/run.sh | 7 +- .../extra_modules/ambari_cluster_state.py | 2 +- .../extra_modules/ambari_service_state.py | 352 +++++++++++ .../inventory/full-dev-platform/group_vars/all | 6 - .../inventory/full-dev-platform/hosts | 23 +- .../inventory/metron_example/group_vars/all | 100 ---- .../inventory/metron_example/hosts | 63 -- .../inventory/quick-dev-platform/group_vars/all | 6 - .../inventory/quick-dev-platform/hosts | 23 +- .../METRON/CURRENT/role_command_order.json | 6 +- .../2.3.3/configuration/elastic-env.xml | 15 +- .../2.3.3/configuration/elastic-site.xml | 8 + .../2.3.3/configuration/elastic-sysconfig.xml | 2 +- .../ELASTICSEARCH/2.3.3/metainfo.xml | 2 +- .../2.3.3/package/scripts/elastic.py | 8 +- .../2.3.3/package/scripts/params.py | 3 +- .../2.3.3/package/scripts/slave.py | 7 +- .../templates/elasticsearch.master.yaml.j2 | 4 +- .../KIBANA/4.5.1/configuration/kibana-env.xml | 19 +- .../KIBANA/4.5.1/configuration/kibana-site.xml | 2 +- .../KIBANA/4.5.1/package/scripts/params.py | 4 +- .../METRON/CURRENT/configuration/metron-env.xml | 14 +- .../package/scripts/enrichment_commands.py | 46 +- .../package/scripts/enrichment_master.py | 12 +- .../package/scripts/indexing_commands.py | 2 +- .../package/scripts/params/params_linux.py | 6 +- .../package/scripts/params/status_params.py | 4 +- .../packaging/packer-build/.gitignore | 12 + .../packaging/packer-build/README.md | 51 ++ .../packaging/packer-build/ansible.cfg | 27 + .../packaging/packer-build/ansible/playbook.yml | 28 + .../packer-build/ansible/tasks/cleanup.yml | 42 ++ .../ansible/tasks/remove_services.yml | 46 ++ .../packaging/packer-build/base-centos-6.7.json | 123 ++++ .../packaging/packer-build/bin/bento | 475 +++++++++++++++ .../packer-build/http/centos-6.7/ks.cfg | 86 +++ .../packer-build/quick-dev-centos-6.7.json | 132 +++++ .../packer-build/scripts/centos/cleanup.sh | 25 + .../packer-build/scripts/centos/networking.sh | 20 + .../packer-build/scripts/centos/update_all.sh | 20 + .../packer-build/scripts/common/metadata.sh | 6 + .../packer-build/scripts/common/minimize.sh | 28 + .../packer-build/scripts/common/reboot.sh | 20 + .../packer-build/scripts/common/sshd.sh | 20 + .../packer-build/scripts/common/sudoers.sh | 4 + .../packer-build/scripts/common/vagrant.sh | 19 + .../packer-build/scripts/common/vmtools.sh | 61 ++ metron-deployment/packer-build/.gitignore | 12 - metron-deployment/packer-build/README.md | 52 -- metron-deployment/packer-build/ansible.cfg | 27 - .../packer-build/ansible/playbook.yml | 36 -- .../packer-build/ansible/tasks/cleanup.yml | 49 -- .../packer-build/ansible/tasks/clear_topics.yml | 29 - .../ansible/tasks/remove_services.yml | 46 -- .../packer-build/ansible/tasks/stop_storm.yml | 28 - metron-deployment/packer-build/bin/bento | 469 --------------- .../packer-build/hdp-centos-6.7.json | 132 ----- .../packer-build/http/centos-6.7/ks.cfg | 86 --- .../packer-build/metron-centos-6.7.json | 131 ----- .../packer-build/scripts/centos/cleanup.sh | 25 - .../packer-build/scripts/centos/networking.sh | 20 - .../packer-build/scripts/centos/update_all.sh | 20 - .../packer-build/scripts/common/metadata.sh | 6 - .../packer-build/scripts/common/minimize.sh | 28 - .../packer-build/scripts/common/reboot.sh | 20 - .../packer-build/scripts/common/sshd.sh | 20 - .../packer-build/scripts/common/sudoers.sh | 4 - .../packer-build/scripts/common/vagrant.sh | 19 - .../packer-build/scripts/common/vmtools.sh | 61 -- metron-deployment/playbooks/ambari_install.yml | 14 + metron-deployment/playbooks/metron_build.yml | 21 + .../playbooks/metron_full_install.yml | 4 + metron-deployment/playbooks/metron_install.yml | 80 +-- metron-deployment/pom.xml | 2 +- .../roles/ambari_common/defaults/main.yml | 3 +- .../roles/ambari_config/meta/main.yml | 1 + .../roles/ambari_config/tasks/main.yml | 1 - .../roles/ambari_config/tasks/start_hdp.yml | 43 -- .../vars/multi_vagrant_cluster.yml | 99 ---- .../roles/ambari_config/vars/single_node_vm.yml | 32 +- .../roles/ambari_config/vars/small_cluster.yml | 54 +- .../roles/ambari_gather_facts/tasks/main.yml | 32 +- .../roles/ambari_master/defaults/main.yml | 2 +- .../roles/ambari_master/tasks/main.yml | 12 +- .../roles/ambari_slave/meta/main.yml | 19 + .../roles/ambari_slave/tasks/main.yml | 5 +- .../roles/bro/tasks/bro-plugin-kafka.yml | 3 + .../roles/deployment-report/tasks/main.yml | 1 - .../roles/elasticsearch/defaults/main.yml | 22 - .../elasticsearch/files/elasticsearch.repo | 24 - .../roles/elasticsearch/meta/main.yml | 24 - .../roles/elasticsearch/tasks/elasticsearch.yml | 73 --- .../roles/elasticsearch/tasks/main.yml | 24 - .../metron-elasticsearch-logrotate.yml | 27 - .../roles/java_jdk/defaults/main.yml | 2 +- metron-deployment/roles/kibana/README.md | 35 -- .../roles/kibana/defaults/main.yml | 22 - .../roles/kibana/tasks/dashboard.yml | 22 - .../roles/kibana/tasks/dependencies.yml | 27 - .../roles/kibana/tasks/elasticdump.yml | 35 -- metron-deployment/roles/kibana/tasks/kibana.yml | 47 -- metron-deployment/roles/kibana/tasks/main.yml | 21 - .../roles/kibana/templates/kibana-index.json | 52 -- .../roles/kibana/templates/kibana.repo.template | 22 - .../roles/load_web_templates/meta/main.yml | 19 + .../roles/load_web_templates/tasks/main.yml | 32 + .../roles/metron-builder/defaults/main.yml | 18 + .../roles/metron-builder/tasks/main.yml | 21 + .../roles/metron-rpms/defaults/main.yml | 18 + .../roles/metron-rpms/tasks/main.yml | 32 + .../roles/metron_common/defaults/main.yml | 19 - .../roles/metron_common/meta/main.yml | 20 - .../roles/metron_common/tasks/main.yml | 36 -- .../files/es_templates/bro_index.template | 218 ------- .../files/es_templates/snort_index.template | 183 ------ .../files/es_templates/yaf_index.template | 205 ------- .../tasks/load_templates.yml | 53 -- .../tasks/main.yml | 18 - .../roles/metron_hbase_tables/defaults/main.yml | 22 - .../roles/metron_hbase_tables/meta/main.yml | 19 - .../roles/metron_hbase_tables/tasks/main.yml | 24 - .../roles/metron_kafka_topics/defaults/main.yml | 30 - .../roles/metron_kafka_topics/meta/main.yml | 19 - .../roles/metron_kafka_topics/tasks/main.yml | 30 - metron-deployment/roles/monit/defaults/main.yml | 3 - metron-deployment/roles/monit/tasks/main.yml | 3 +- .../roles/monit/tasks/monit-definitions.yml | 20 - metron-deployment/roles/monit/tasks/scripts.yml | 55 -- .../monit/templates/monit/elasticsearch.monit | 23 - .../monit/templates/monit/enrichment.monit | 26 - .../monit/indexing-elasticsearch.monit | 26 - .../monit/templates/monit/indexing-solr.monit | 26 - .../roles/monit/templates/monit/kibana.monit | 22 - .../roles/monit/templates/monit/parsers.monit | 49 -- .../scripts/start_enrichment_topology.sh | 25 - .../monit/templates/scripts/start_topology.sh | 25 - .../monit/templates/scripts/status_topology.sh | 33 -- .../monit/templates/scripts/stop_topology.sh | 23 - metron-deployment/roles/quick_dev/meta/main.yml | 19 + .../roles/quick_dev/tasks/main.yml | 70 +++ metron-deployment/roles/solr/defaults/main.yml | 29 - metron-deployment/roles/solr/files/schema.xml | 191 ------ metron-deployment/roles/solr/meta/main.yml | 21 - metron-deployment/roles/solr/tasks/main.yml | 74 --- metron-deployment/roles/solr/templates/solr.xml | 52 -- .../roles/solr/templates/solrconfig.xml | 583 ------------------- .../vagrant/full-dev-platform/Vagrantfile | 4 +- .../vagrant/quick-dev-platform/Vagrantfile | 6 +- 152 files changed, 2214 insertions(+), 4571 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/Upgrading.md ---------------------------------------------------------------------- diff --git a/Upgrading.md b/Upgrading.md index 92e958b..b05acbc 100644 --- a/Upgrading.md +++ b/Upgrading.md @@ -152,3 +152,11 @@ When using Ansible to deploy the latest Metron bits to an existing installation, #### Workaround Set the `topology.classpath` property for storm in Ambari to `/etc/hbase/conf:/etc/hadoop/conf` +## 0.3.1 to PLACEHOLDER + +### [METRON-671: Refactor existing Ansible deployment to use Ambari MPack](https://issues.apache.org/jira/browse/METRON-671) + +#### Description +Since the Ansible Deployment uses the MPack, RPMs must be built prior to deployment. As a result, +[Docker](https://www.docker.com/) is required to perform a Quick-Dev, Full-Dev or Ansible deployment. +This effectively limits the build environment to Docker supported [platforms](https://docs.docker.com/engine/installation/#platform-support-matrix). http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/README.md b/metron-deployment/README.md index 3c9b339..ee44a2f 100644 --- a/metron-deployment/README.md +++ b/metron-deployment/README.md @@ -1,97 +1,51 @@ # Overview -This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster, Metron services, or both using ansible -playbooks. These playbooks currently only target RHEL/CentOS 6.x operating +This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster containing Metron services using Ansible. These playbooks target RHEL/CentOS 6.x operating systems. -In addition, an Ambari Management Pack can be built which can be deployed in conjuction with RPMs detailed in this README. +Installation consists of - +- Building Metron tarballs, RPMs and the Ambari MPack +- Deploying Ambari +- Leveraging Ambari to install: + * The required Hadoop Components + * Core Metron (Parsing, Enrichment, Indexing) + * Elasticsearch + * Kibana +- Starting All Services ## Prerequisites The following tools are required to run these scripts: - [Maven](https://maven.apache.org/) - [Git](https://git-scm.com/) -- [Ansible](http://www.ansible.com/) (version 2.0 or greater) - -Currently Metron must be built from source. Before running these scripts perform the following steps: - -1. Clone the Metron git repository with `git clone g...@github.com:apache/incubator-metron.git` -2. Navigate to `incubator-metron` and run `mvn clean package` +- [Ansible](http://www.ansible.com/) (version 2.0.0.2) +- [Docker](https://www.docker.com/) (Docker for Mac on OSX) These scripts depend on two files for configuration: - hosts - declares which Ansible roles will be run on which hosts - group_vars/all - various configuration settings needed to install Metron -Examples can be found in the -`incubator-metron/metron-deployment/inventory/metron_example` directory and are a good starting point. Copy this directory -into `incubator-metron/metron-deployment/inventory/` and rename it to your `project_name`. More information about Ansible files and directory -structure can be found at http://docs.ansible.com/ansible/playbooks_best_practices.html. - +For production use, it is recommended that Metron be installed on an existing cluster managed by Ambari as described in the Installing Management Pack section below. ## Ambari -The Ambari playbook will install a Hadoop cluster with all the services and configuration required by Metron. This -section can be skipped if installing Metron on a pre-existing cluster. - -Currently, this playbook supports building a local development cluster running on one node but options for other types - of clusters will be added in the future. - -### Setting up your inventory -Make sure to update the hosts file in `incubator-metron/metron-deployment/inventory/project_name/hosts` or provide an -alternate inventory file when you launch the playbooks, including the -ssh user(s) and ssh keyfile location(s). These playbooks expect two -host groups: - -- ambari_master -- ambari_slaves - -### Running the playbook -This playbook will install the Ambari server on the ambari_master, install the ambari agents on -the ambari_slaves, and create a cluster in Ambari with a blueprint for the required -Metron components. - -Navigate to `incubator-metron/metron-deployment/playbooks` and run: -`ansible-playbook -i ../inventory/project_name ambari_install.yml` - -## Metron -The Metron playbook will gather the necessary cluster settings from Ambari and install the Metron services. +The Ambari playbook will install a Hadoop cluster including the Metron Services (Parsing, Enrichment, Indexing). Ambari will also install Elasticsearch and Kibana. -### Setting up your inventory -Edit the hosts file at `incubator-metron/metron-deployment/inventory/project_name/hosts`. Declare where which hosts the -Metron services will be installed on by updating these groups: - -- enrichment - submits the topology code to Storm and requires a storm client -- search - host where Elasticsearch will be run -- web - host where the Metron UI and underlying services will run -- sensors - host where network data will be collected and published to Kafka - -The Metron topologies depend on Kafka topics and HBase tables being created beforehand. Declare a host that has Kafka and HBase clients installed by updating these groups: - -- metron_kafka_topics -- metron_hbase_tables - -If only installing Metron, these groups can be ignored: - -- ambari_master -- ambari_slaves - -### Configuring group variables -The Metron Ansible scripts depend on a set of variables. These variables can be found in the file at -`incubator-metron/metron-deployment/inventory/project_name/group_vars/all`. Edit the ambari* variables to match your Ambari -instance and update the java_home variable to match the java path on your hosts. - -### Running the playbook -Navigate to `incubator-metron/metron-deployment/playbooks` and run: -`ansible-playbook -i ../inventory/project_name metron_install.yml` +Currently, the playbooks supports building a local development cluster running on one node or deploying to a 10 node cluster on AWS EC2. ## Vagrant -A VagrantFile is included and will install a working version of the entire Metron stack. The following is required to -run this: +There are current two Vagrant modes, full-dev and quick-dev. Full-dev installs the entire Ambari/Metron stack. This is useful in testing out changes to the installation procedure. +Quick-dev re-installs the core Metron Services (Parsing, Enrichment, and Indexing)on a pre-built instance. +Use quick-dev for testing out changes to core Metron services. -- [Vagrant](https://www.vagrantup.com/) -- Hostmanager plugin for vagrant - Run `vagrant plugin install vagrant-hostmanager` on the machine where Vagrant is +### Prerequsities +- Install [Vagrant](https://www.vagrantup.com/) (5.0.16+) +- Install the Hostmanager plugin for vagrant - Run `vagrant plugin install vagrant-hostmanager` on the machine where Vagrant is installed -Navigate to `incubator-metron/metron-deployment/vagrant/full-dev-platform` and run `vagrant up`. This also provides a good -example of how to run a full end-to-end Metron install. +### Full-Dev +Navigate to `incubator-metron/metron-deployment/vagrant/full-dev-platform` and run `vagrant up`. + +### Quick-Dev +Navigate to `incubator-metron/metron-deployment/vagrant/quick-dev-platform` and run `vagrant up`. ## Ambari Management Pack An Ambari Management Pack can be built in order to make the Metron service available on top of an existing stack, rather than needing a direct stack update. http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/amazon-ec2/conf/defaults.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml index 84dbcce..c9b435b 100644 --- a/metron-deployment/amazon-ec2/conf/defaults.yml +++ b/metron-deployment/amazon-ec2/conf/defaults.yml @@ -17,19 +17,12 @@ --- # which services should be started? services_to_start: - - elasticsearch - pcap-service - - kibana - yaf - snort - snort-logs - bro - pcap-replay - - yaf-parser - - bro-parser - - snort-parser - - enrichment - - indexing # the ami for cent6 by region amis_by_region: http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/amazon-ec2/playbook.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/amazon-ec2/playbook.yml b/metron-deployment/amazon-ec2/playbook.yml index a0b15dc..fb19d89 100644 --- a/metron-deployment/amazon-ec2/playbook.yml +++ b/metron-deployment/amazon-ec2/playbook.yml @@ -26,12 +26,12 @@ - include: tasks/create-vpc.yml - include: tasks/create-open-inbound-security-group.yml - include: tasks/create-open-outbound-security-group.yml - - include: tasks/create-hosts.yml host_count=1 host_type=sensors,ambari_master,metron,ec2 + - include: tasks/create-hosts.yml host_count=1 host_type=sensors,ambari_master,ec2,monit - include: tasks/create-hosts.yml host_count=4 host_type=ambari_slave,ec2 - - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,metron_kafka_topics,metron_hbase_tables,metron,ec2,pcap_server - - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,enrichment,metron,ec2 - - include: tasks/create-hosts.yml host_count=2 host_type=search,metron,ec2 - - include: tasks/create-hosts.yml host_count=1 host_type=web,metron,ec2 + - include: tasks/create-hosts.yml host_count=1 host_type=pcap_server,monit,ec2 + - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,enrichment,metron,ec2,zeppelin + - include: tasks/create-hosts.yml host_count=2 host_type=ambari_slave,search,ec2 + - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,web,ec2 tags: - ec2 http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/amazon-ec2/run.sh ---------------------------------------------------------------------- diff --git a/metron-deployment/amazon-ec2/run.sh b/metron-deployment/amazon-ec2/run.sh index cad4fb7..bdaabf0 100755 --- a/metron-deployment/amazon-ec2/run.sh +++ b/metron-deployment/amazon-ec2/run.sh @@ -58,15 +58,10 @@ echo "=============================================================" >> $LOGFILE echo "Launching Metron[$ENV] @ $NOW"... >> $LOGFILE $DEPLOYDIR/../scripts/platform-info.sh >> $LOGFILE -# build metron -cd ../.. -mvn package -DskipTests -PHDP-2.5.0.0 -RC=$?; if [[ $RC != 0 ]]; then exit $RC; fi - # deploy metron cd $DEPLOYDIR export EC2_INI_PATH=conf/ec2.ini ansible-playbook -i ec2.py playbook.yml \ - --skip-tags="solr, sensor-stubs" \ + --skip-tags="quick_dev,sensor-stubs" \ --extra-vars="env=$ENV" \ $EXTRA_ARGS http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/extra_modules/ambari_cluster_state.py ---------------------------------------------------------------------- diff --git a/metron-deployment/extra_modules/ambari_cluster_state.py b/metron-deployment/extra_modules/ambari_cluster_state.py index a61d0f1..691d48e 100644 --- a/metron-deployment/extra_modules/ambari_cluster_state.py +++ b/metron-deployment/extra_modules/ambari_cluster_state.py @@ -385,9 +385,9 @@ def blueprint_var_to_ambari_converter(blueprint_var): this_host_map['hosts'] = this_host_list host_map.append(this_host_map) blueprint = dict() + blueprint['configurations'] = blueprint_var['required_configurations'] blueprint['host_groups'] = new_groups blueprint['Blueprints'] = {'stack_name': blueprint_var['stack_name'], 'stack_version': blueprint_var['stack_version']} - return blueprint, host_map from ansible.module_utils.basic import * http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/extra_modules/ambari_service_state.py ---------------------------------------------------------------------- diff --git a/metron-deployment/extra_modules/ambari_service_state.py b/metron-deployment/extra_modules/ambari_service_state.py new file mode 100644 index 0000000..6dbefa8 --- /dev/null +++ b/metron-deployment/extra_modules/ambari_service_state.py @@ -0,0 +1,352 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +DOCUMENTATION = ''' +--- +module: ambari_service_state +version_added: "2.1" +author: Apache Metron (Incubating : https://github.com/apache/incubator-metron ) +short_description: Start/Stop/Change Service or Component State +description: + - Start/Stop/Change Service or Component State +options: + host: + description: + The hostname for the ambari web server + port: + description: + The port for the ambari web server + username: + description: + The username for the ambari web server + password: + description: + The name of the cluster in web server + required: yes + cluster_name: + description: + The name of the cluster in ambari + required: yes + service_name: + description: + The name of the service to alter + required: no + component_name: + description: + The name of the component to alter + required: no + component_host: + description: + The host running the targeted component. Required when component_name is used. + required: no + state: + description: + The desired service/component state. + wait_for_complete: + description: + Whether to wait for the request to complete before returning. Default is False. + required: no + requirements: [ 'requests'] +''' + +EXAMPLES = ''' +# must use full relative path to any files in stored in roles/role_name/files/ +- name: Create a new ambari cluster + ambari_cluster_state: + host: localhost + port: 8080 + username: admin + password: admin + cluster_name: my_cluster + cluster_state: present + blueprint_var: roles/my_role/files/blueprint.yml + blueprint_name: hadoop + wait_for_complete: True +- name: Start the ambari cluster + ambari_cluster_state: + host: localhost + port: 8080 + username: admin + password: admin + cluster_name: my_cluster + cluster_state: started + wait_for_complete: True +- name: Stop the ambari cluster + ambari_cluster_state: + host: localhost + port: 8080 + username: admin + password: admin + cluster_name: my_cluster + cluster_state: stopped + wait_for_complete: True +- name: Delete the ambari cluster + ambari_cluster_state: + host: localhost + port: 8080 + username: admin + password: admin + cluster_name: my_cluster + cluster_state: absent +''' + +RETURN = ''' +results: + description: The content of the requests object returned from the RESTful call + returned: success + type: string +''' + +__author__ = 'apachemetron' + +import json + +try: + import requests +except ImportError: + REQUESTS_FOUND = False +else: + REQUESTS_FOUND = True + + +def main(): + + argument_spec = dict( + host=dict(type='str', default=None, required=True), + port=dict(type='int', default=None, required=True), + username=dict(type='str', default=None, required=True), + password=dict(type='str', default=None, required=True), + cluster_name=dict(type='str', default=None, required=True), + state=dict(type='str', default=None, required=True, + choices=['started', 'stopped', 'deleted']), + service_name=dict(type='str', required=False), + component_name=dict(type='str', default=None, required=False), + component_host=dict(type='str', default=None, required=False), + wait_for_complete=dict(default=False, required=False, choices=BOOLEANS), + ) + + required_together = ['component_name', 'component_host'] + + module = AnsibleModule( + argument_spec=argument_spec, + required_together=required_together + ) + + if not REQUESTS_FOUND: + module.fail_json( + msg='requests library is required for this module') + + p = module.params + + host = p.get('host') + port = p.get('port') + username = p.get('username') + password = p.get('password') + cluster_name = p.get('cluster_name') + state = p.get('state') + service_name = p.get('service_name') + component_name = p.get('component_name') + component_host = p.get('component_host') + wait_for_complete = p.get('wait_for_complete') + component_mode = False + ambari_url = 'http://{0}:{1}'.format(host, port) + + if component_name: + component_mode = True + + try: + if not cluster_exists(ambari_url, username, password, cluster_name): + module.fail_json(msg="Cluster name {0} does not exist".format(cluster_name)) + + if state in ['started', 'stopped', 'installed']: + desired_state = '' + + if state == 'started': + desired_state = 'STARTED' + elif state in ['stopped','installed']: + desired_state = 'INSTALLED' + + if component_mode: + if desired_state == 'INSTALLED': + if(can_add_component(ambari_url, username, password, cluster_name, component_name, component_host)): + add_component_to_host(ambari_url, username, password, cluster_name, component_name, component_host) + request = set_component_state(ambari_url, username, password, cluster_name, component_name, component_host, desired_state) + else: + request = set_service_state(ambari_url,username,password,cluster_name,service_name, desired_state) + if wait_for_complete: + try: + request_id = json.loads(request.content)['Requests']['id'] + except ValueError: + module.exit_json(changed=True, results=request.content) + status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2) + if status != 'COMPLETED': + module.fail_json(msg="Request failed with status {0}".format(status)) + module.exit_json(changed=True, results=request.content) + + elif state == 'deleted': + if component_mode: + request = delete_component(ambari_url, username, password, cluster_name, component_name, component_host) + else: + request = delete_service(ambari_url,username,password,cluster_name,service_name) + module.exit_json(changed=True, results=request.content) + + except requests.ConnectionError, e: + module.fail_json(msg="Could not connect to Ambari client: " + str(e.message)) + except Exception, e: + module.fail_json(msg="Ambari client exception occurred: " + str(e.message)) + + +def get_clusters(ambari_url, user, password): + r = get(ambari_url, user, password, '/api/v1/clusters') + if r.status_code != 200: + msg = 'Could not get cluster list: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + clusters = json.loads(r.content) + return clusters['items'] + + +def cluster_exists(ambari_url, user, password, cluster_name): + clusters = get_clusters(ambari_url, user, password) + return cluster_name in [item['Clusters']['cluster_name'] for item in clusters] + + +def get_request_status(ambari_url, user, password, cluster_name, request_id): + path = '/api/v1/clusters/{0}/requests/{1}'.format(cluster_name, request_id) + r = get(ambari_url, user, password, path) + if r.status_code != 200: + msg = 'Could not get cluster request status: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + service = json.loads(r.content) + return service['Requests']['request_status'] + + +def wait_for_request_complete(ambari_url, user, password, cluster_name, request_id, sleep_time): + while True: + status = get_request_status(ambari_url, user, password, cluster_name, request_id) + if status == 'COMPLETED': + return status + elif status in ['FAILED', 'TIMEDOUT', 'ABORTED', 'SKIPPED_FAILED']: + return status + else: + time.sleep(sleep_time) + + +def set_service_state(ambari_url, user, password, cluster_name, service_name, desired_state): + path = '/api/v1/clusters/{0}/services/{1}'.format(cluster_name,service_name) + request = {"RequestInfo": {"context": "Setting {0} to {1} via REST".format(service_name,desired_state)}, + "Body": {"ServiceInfo": {"state": "{0}".format(desired_state)}}} + payload = json.dumps(request) + r = put(ambari_url, user, password, path, payload) + if r.status_code not in [202, 200]: + msg = 'Could not set service state: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + return r + + +def set_component_state(ambari_url, user, password, cluster_name, component_name, component_host, desired_state): + path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name) + request = {"RequestInfo": {"context": "Setting {0} to {1} via REST".format(component_name,desired_state)}, + "Body": {"HostRoles": {"state": "{0}".format(desired_state)}}} + payload = json.dumps(request) + r = put(ambari_url, user, password, path, payload) + if r.status_code not in [202, 200]: + msg = 'Could not set component state: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + return r + + +def delete_component(ambari_url, user, password, cluster_name, component_name, component_host): + enable_maint_mode(ambari_url, user, password, cluster_name, component_name, component_host) + path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name) + r = delete(ambari_url,user,password,path) + if r.status_code not in [202, 200]: + msg = 'Could not set service state: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + return r + + +def enable_maint_mode(ambari_url, user, password, cluster_name, component_name, component_host): + path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name) + request = {"RequestInfo":{"context":"Turn On Maintenance Mode for {0}".format(component_name)}, + "Body":{"HostRoles":{"maintenance_state":"ON"}}} + payload = json.dumps(request) + r = put(ambari_url, user, password, path, payload) + if r.status_code not in [202, 200]: + msg = 'Could not set maintenance mode: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + return r + + +def delete_service(ambari_url, user, password, cluster_name, service_name): + path = '/api/v1/clusters/{0}/services/{1}'.format(cluster_name,service_name) + r = delete(ambari_url,user,password,path) + if r.status_code not in [202, 200]: + msg = 'Could not delete service: request code {0}, \ + request message {1}'.format(r.status_code, r.content) + raise Exception(msg) + return r + + +def add_component_to_host(ambari_url, user, password, cluster_name, component_name, component_host): + path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name) + r = post(ambari_url, user, password, path,'') + if r.status_code not in [202,201,200]: + msg = 'Could not add {0} to host {1}: request code {2}, \ + request message {3}'.format(component_name,component_host,r.status_code, r.content) + raise Exception(msg) + return r + + +def can_add_component(ambari_url, user, password, cluster_name, component_name, component_host): + path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name) + r = get(ambari_url, user, password, path) + return r.status_code == 404 + + +def get(ambari_url, user, password, path): + r = requests.get(ambari_url + path, auth=(user, password)) + return r + + +def put(ambari_url, user, password, path, data): + headers = {'X-Requested-By': 'ambari'} + r = requests.put(ambari_url + path, data=data, auth=(user, password), headers=headers) + return r + + +def post(ambari_url, user, password, path, data): + headers = {'X-Requested-By': 'ambari'} + r = requests.post(ambari_url + path, data=data, auth=(user, password), headers=headers) + return r + + +def delete(ambari_url, user, password, path): + headers = {'X-Requested-By': 'ambari'} + r = requests.delete(ambari_url + path, auth=(user, password), headers=headers) + return r + + +from ansible.module_utils.basic import * +if __name__ == '__main__': + main() http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/full-dev-platform/group_vars/all ---------------------------------------------------------------------- diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all index 2b4617f..56ad75d 100644 --- a/metron-deployment/inventory/full-dev-platform/group_vars/all +++ b/metron-deployment/inventory/full-dev-platform/group_vars/all @@ -17,17 +17,11 @@ # which services should be started? services_to_start: - - elasticsearch - pcap-service - - kibana - snort - snort-logs - bro - pcap-replay - - bro-parser - - snort-parser - - enrichment - - indexing # ambari ambari_host: "{{ groups.ambari_master[0] }}" http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/full-dev-platform/hosts ---------------------------------------------------------------------- diff --git a/metron-deployment/inventory/full-dev-platform/hosts b/metron-deployment/inventory/full-dev-platform/hosts index 98e8681..9bd9ea1 100644 --- a/metron-deployment/inventory/full-dev-platform/hosts +++ b/metron-deployment/inventory/full-dev-platform/hosts @@ -21,32 +21,27 @@ node1 [ambari_slave] node1 -[metron_hbase_tables] +[metron] node1 -[metron_kafka_topics] +[search] node1 -[enrichment] +[sensors] node1 -[search] +[pcap_server] node1 [web] node1 -[sensors] +[zeppelin] node1 -[pcap_server] -node1 - -[metron:children] -enrichment -search -web +[monit:children] sensors -metron_kafka_topics -metron_hbase_tables pcap_server + +[local] +127.0.0.1 http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/metron_example/group_vars/all ---------------------------------------------------------------------- diff --git a/metron-deployment/inventory/metron_example/group_vars/all b/metron-deployment/inventory/metron_example/group_vars/all deleted file mode 100644 index aeefa44..0000000 --- a/metron-deployment/inventory/metron_example/group_vars/all +++ /dev/null @@ -1,100 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -# which services should be started? -services_to_start: - - elasticsearch - - pcap-service - - kibana - - yaf - - snort - - snort-logs - - bro - - pcap-replay - - yaf-parser - - bro-parser - - snort-parser - - enrichment - - indexing - -#Ansible Variables -ansible_ssh_private_key_file: /Path/to/private/key/file #Change This -ansible_ssh_user: root - -#Ambari -ambari_host: "{{ groups.ambari_master[0] }}" -ambari_port: 8080 -ambari_user: admin -ambari_password: admin -cluster_type: small_cluster - -# hbase -pcap_hbase_table: pcap -tracker_hbase_table: access_tracker -threatintel_hbase_table: threatintel -enrichment_hbase_table: enrichment - -# kafka -num_partitions: 3 -retention_in_gb: 25 - -# metron variables -metron_version: 0.3.1 -metron_directory: /usr/metron/{{ metron_version }} -pcapservice_port: 8081 - -# sensors -sniff_interface: eth1 -bro_version: "2.4.1" -fixbuf_version: "1.7.1" -yaf_version: "2.8.0" -daq_version: "2.0.6-1" -iface: "eth0" -pycapa_repo: "https://github.com/OpenSOC/pycapa.git"; -pycapa_home: "/opt/pycapa" -snort_version: "2.9.8.0-1" -snort_alert_csv_path: "/var/log/snort/alert.csv" - -# pcap-replay -install_pcap_replay: True -install_tap: True -tap_ip: 10.0.0.1 -sensor_test_mode: True -pcap_replay_interface: tap0 -pcap_replay_home: /opt/pcap-replay - -# data directories -zookeeper_data_dir: "/data1/hadoop/zookeeper" -namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary" -namenode_name_dir: "/data1/hadoop/hdfs/namenode" -datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data" -journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode" -nodemanager_local_dirs: "/data1/hadoop/yarn/local" -timeline_ldb_store_path: "/data1/hadoop/yarn/timeline" -timeline_ldb_state_path: "/data1/hadoop/yarn/timeline" -nodemanager_log_dirs: "/data1/hadoop/yarn/log" -jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs" -storm_local_dir: "/data1/hadoop/storm" -kafka_log_dirs: "/data2/kafka-log" -elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch" - -# search -install_elasticsearch: True -install_solr: False -elasticsearch_transport_port: 9300 -elasticsearch_network_interface: eth0 -elasticsearch_web_port: 9200 http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/metron_example/hosts ---------------------------------------------------------------------- diff --git a/metron-deployment/inventory/metron_example/hosts b/metron-deployment/inventory/metron_example/hosts deleted file mode 100644 index c53466d..0000000 --- a/metron-deployment/inventory/metron_example/hosts +++ /dev/null @@ -1,63 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -[ambari_master] -node1 - -#minimum of 3 - 6 from 12 node cluser -[ambari_slave] -node2 -node3 -node4 -node5 -node6 -node7 -node8 - -[metron_hbase_tables] -node9 - -[metron_kafka_topics] -node9 - -[pcap_server] -node9 - -#3rd ambari_slave -[enrichment] -node1 - -#1 or more -[search] -node10 -node11 -node12 - -#1 only -[sensors] -node1 - -[web] -node12 - -[metron:children] -enrichment -search -web -sensors -metron_kafka_topics -metron_hbase_tables http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/quick-dev-platform/group_vars/all ---------------------------------------------------------------------- diff --git a/metron-deployment/inventory/quick-dev-platform/group_vars/all b/metron-deployment/inventory/quick-dev-platform/group_vars/all index d62e74b..e835d79 100644 --- a/metron-deployment/inventory/quick-dev-platform/group_vars/all +++ b/metron-deployment/inventory/quick-dev-platform/group_vars/all @@ -17,17 +17,11 @@ # which services should be started? services_to_start: - - elasticsearch - pcap-service - - kibana - snort - snort-logs - bro - pcap-replay - - bro-parser - - snort-parser - - enrichment - - indexing # ambari ambari_host: "{{ groups.ambari_master[0] }}" http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/quick-dev-platform/hosts ---------------------------------------------------------------------- diff --git a/metron-deployment/inventory/quick-dev-platform/hosts b/metron-deployment/inventory/quick-dev-platform/hosts index 98e8681..9bd9ea1 100644 --- a/metron-deployment/inventory/quick-dev-platform/hosts +++ b/metron-deployment/inventory/quick-dev-platform/hosts @@ -21,32 +21,27 @@ node1 [ambari_slave] node1 -[metron_hbase_tables] +[metron] node1 -[metron_kafka_topics] +[search] node1 -[enrichment] +[sensors] node1 -[search] +[pcap_server] node1 [web] node1 -[sensors] +[zeppelin] node1 -[pcap_server] -node1 - -[metron:children] -enrichment -search -web +[monit:children] sensors -metron_kafka_topics -metron_hbase_tables pcap_server + +[local] +127.0.0.1 http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json index 0b04f12..9c71887 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json @@ -4,9 +4,9 @@ "general_deps" : { "_comment" : "dependencies for all cases", "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"], - "METRON_ENRICHMENT-INSTALL" : ["METRON_INDEXING-INSTALL"], - "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"], - "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START", "METRON_INDEXING-START"], + "METRON_ENRICHMENT-INSTALL": ["METRON_INDEXING-INSTALL"], + "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_ENRICHMENT_MASTER-START"], + "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"], "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"], "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"] http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml index 2939c28..a4de039 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml @@ -23,12 +23,21 @@ <name>elastic_user</name> <value>elasticsearch</value> <property-type>USER</property-type> - <description>The user for Elasticsearch</description> + <description>Service user for Elasticsearch</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> </property> <property> - <name>user_group</name> + <name>elastic_group</name> <value>elasticsearch</value> - <description>The group for Elasticsearch</description> + <property-type>GROUP</property-type> + <description>Service group for Elasticsearch</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> </property> <property> <name>elastic_log_dir</name> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml index 3e7b6a5..cbcebdd 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml @@ -27,6 +27,14 @@ <description>Cluster name identifies your cluster</description> </property> <property> + <name>masters_also_are_datanodes</name> + <value>"false"</value> + <description>ES Masters and Slaves cannot be installed on the same nodes. Set this to "true" if you want the ES master nodes to serve as combined master/datanodes. Note: surround value in quotes.</description> + <value-attributes> + <type>string</type> + </value-attributes> + </property> + <property> <name>zen_discovery_ping_unicast_hosts</name> <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.--> <value></value> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml index 58e4916..6ef09ac 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml @@ -90,7 +90,7 @@ CONF_DIR={{conf_dir}} ES_USER={{elastic_user}} # Additional Java OPTS -ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled +ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:ErrorFile={{log_dir}}elasticsearch_err.log -XX:ParallelGCThreads=8" </value> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml index 25f98e5..53909a2 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml @@ -39,7 +39,7 @@ <name>ES_SLAVE</name> <displayName>Elasticsearch Data Node</displayName> <category>SLAVE</category> - <cardinality>3+</cardinality> + <cardinality>0+</cardinality> <commandScript> <script>scripts/elastic_slave.py</script> <scriptType>PYTHON</scriptType> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py index a472655..2e638de 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py @@ -22,12 +22,12 @@ from resource_management.core.resources.system import Directory from resource_management.core.resources.system import File from resource_management.core.source import InlineTemplate from resource_management.core.source import Template - +from resource_management.core.resources import User def elastic(): - print "INSIDE THE %s" % __file__ import params + User(params.elastic_user, action = "create", groups = params.elastic_group) params.path_data = params.path_data.replace('"', '') data_path = params.path_data.replace(' ', '').split(',') data_path[:] = [x.replace('"', '') for x in data_path] @@ -37,15 +37,15 @@ def elastic(): Directory(directories, create_parents=True, - # recursive=True, mode=0755, owner=params.elastic_user, - group=params.elastic_user + group=params.elastic_group ) print "Master env: ""{0}/elastic-env.sh".format(params.conf_dir) File("{0}/elastic-env.sh".format(params.conf_dir), owner=params.elastic_user, + group=params.elastic_group, content=InlineTemplate(params.elastic_env_sh_template) ) http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py index 8b890d4..aea81fb 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py @@ -23,6 +23,7 @@ from resource_management.libraries.script import Script # server configurations config = Script.get_config() +masters_also_are_datanodes = config['configurations']['elastic-site']['masters_also_are_datanodes'] elastic_home = config['configurations']['elastic-sysconfig']['elastic_home'] data_dir = config['configurations']['elastic-sysconfig']['data_dir'] work_dir = config['configurations']['elastic-sysconfig']['work_dir'] @@ -32,7 +33,7 @@ max_open_files = config['configurations']['elastic-sysconfig']['max_open_files'] max_map_count = config['configurations']['elastic-sysconfig']['max_map_count'] elastic_user = config['configurations']['elastic-env']['elastic_user'] -user_group = config['configurations']['elastic-env']['user_group'] +elastic_group = config['configurations']['elastic-env']['elastic_group'] log_dir = config['configurations']['elastic-env']['elastic_log_dir'] pid_dir = '/var/run/elasticsearch' pid_file = '/var/run/elasticsearch/elasticsearch.pid' http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py index 90d4d3d..15a5cf5 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py @@ -22,11 +22,12 @@ from resource_management.core.resources.system import Directory from resource_management.core.resources.system import File from resource_management.core.source import InlineTemplate from resource_management.core.source import Template - +from resource_management.core.resources import User def slave(): import params + User(params.elastic_user, action = "create", groups = params.elastic_group) params.path_data = params.path_data.replace('"', '') data_path = params.path_data.replace(' ', '').split(',') data_path[:] = [x.replace('"', '') for x in data_path] @@ -38,7 +39,7 @@ def slave(): create_parents=True, mode=0755, owner=params.elastic_user, - group=params.elastic_user, + group=params.elastic_group, cd_access="a" ) @@ -54,7 +55,7 @@ def slave(): "elasticsearch.slave.yaml.j2", configurations=configurations), owner=params.elastic_user, - group=params.elastic_user + group=params.elastic_group ) print "Master sysconfig: /etc/sysconfig/elasticsearch" http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 index 8d92f1b..ca3630a 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 @@ -28,12 +28,12 @@ discovery: zen: ping: multicast: - enabled: {{discovery_zen_ping_multicast_enabled}} + enabled: {{ discovery_zen_ping_multicast_enabled }} unicast: hosts: "{{zen_discovery_ping_unicast_hosts}}" node: - data: false + data: {{ masters_also_are_datanodes }} master: true name: {{hostname}} path: http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml index 972945d..0adf32a 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml @@ -23,13 +23,21 @@ <name>kibana_user</name> <value>kibana</value> <property-type>USER</property-type> - <description></description> + <description>Service User for Kibana</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> </property> <property> - <name>user_group</name> + <name>kabana_group</name> <value>kibana</value> <property-type>GROUP</property-type> - <description></description> + <description>Service Group for Kibana</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> </property> <property require-input="true"> <name>kibana_log_dir</name> @@ -51,4 +59,9 @@ <value>5000</value> <description>Kibana back end server port to use.</description> </property> + <property require-input="true"> + <name>kibana_default_application</name> + <value>default</value> + <description>The default application to load.</description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml index c546e2c..4373d14 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml @@ -50,7 +50,7 @@ elasticsearch.url: {{ es_url }} # kibana.index: ".kibana" # The default application to load. -# kibana.defaultAppId: "discover" +kibana.defaultAppId: "{{ kibana_default_application }}" # If your Elasticsearch is protected with basic auth, these are the user credentials # used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py index ef1e597..a5e0ec9 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py @@ -33,7 +33,7 @@ kibana_bin = '/usr/share/kibana/bin/' conf_dir = "/opt/kibana/config/" kibana_user = config['configurations']['kibana-env']['kibana_user'] -user_group = config['configurations']['kibana-env']['user_group'] +kibana_group = config['configurations']['kibana-env']['kibana_group'] log_dir = config['configurations']['kibana-env']['kibana_log_dir'] pid_dir = config['configurations']['kibana-env']['kibana_pid_dir'] pid_file = format("{pid_dir}/kibanasearch.pid") @@ -42,6 +42,8 @@ parsed = urlparse(es_url) es_host = parsed.netloc.split(':')[0] es_port = parsed.netloc.split(':')[1] kibana_port = config['configurations']['kibana-env']['kibana_server_port'] +kibana_default_application = config['configurations']['kibana-env']['kibana_default_application'] hostname = config['hostname'] java64_home = config['hostLevelParams']['java_home'] kibana_yml_template = config['configurations']['kibana-site']['content'] + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index cbff4a9..277b636 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -46,15 +46,23 @@ <name>metron_user</name> <value>metron</value> <property-type>USER</property-type> - <description>The user for Metron</description> - <display-name>Metron User</display-name> + <description>Service User for Metron</description> + <display-name>Metron User Name</display-name> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> </property> <property> <name>metron_group</name> <value>metron</value> <property-type>GROUP</property-type> - <description>The group for Metron</description> + <description>Service Group for Metron</description> <display-name>Metron Group Name</display-name> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> </property> <property> <name>metron_topic_retention</name> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py index 817f266..5878dc6 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py @@ -28,7 +28,11 @@ class EnrichmentCommands: __params = None __enrichment_topology = None __enrichment_topic = None - __configured = False + __enrichment_error_topic = None + __threat_intel_error_topic = None + __kafka_configured = False + __hbase_configured = False + __geo_configured = False def __init__(self, params): if params is None: @@ -36,13 +40,36 @@ class EnrichmentCommands: self.__params = params self.__enrichment_topology = params.metron_enrichment_topology self.__enrichment_topic = params.metron_enrichment_topic - self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file) + self.__kafka_configured = os.path.isfile(self.__params.enrichment_kafka_configured_flag_file) + self.__hbase_configured = os.path.isfile(self.__params.enrichment_hbase_configured_flag_file) + self.__geo_configured = os.path.isfile(self.__params.enrichment_geo_configured_flag_file) - def is_configured(self): - return self.__configured + def is_kafka_configured(self): + return self.__kafka_configured - def set_configured(self): - File(self.__params.enrichment_configured_flag_file, + def set_kafka_configured(self): + Logger.info("Setting Kafka Configured to True") + File(self.__params.enrichment_kafka_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) + + def is_hbase_configured(self): + return self.__hbase_configured + + def set_hbase_configured(self): + Logger.info("Setting HBase Configured to True") + File(self.__params.enrichment_hbase_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0775) + + def is_geo_configured(self): + return self.__geo_configured + + def set_geo_configured(self): + Logger.info("Setting GEO Configured to True") + File(self.__params.enrichment_geo_configured_flag_file, content="", owner=self.__params.metron_user, mode=0775) @@ -84,7 +111,7 @@ class EnrichmentCommands: type="directory", action="create_on_execute", owner=self.__params.metron_user, - group=self.__params.user_group, + group=self.__params.metron_group, mode=0775, ) @@ -101,6 +128,7 @@ class EnrichmentCommands: Logger.info("Executing command " + command) Execute(command, user=self.__params.metron_user, tries=1, logoutput=True) Logger.info("Done intializing GeoIP data") + self.set_geo_configured() def init_kafka_topics(self): Logger.info('Creating Kafka topics') @@ -128,6 +156,7 @@ class EnrichmentCommands: retention_bytes)) Logger.info("Done creating Kafka topics") + self.set_kafka_configured() def start_enrichment_topology(self): Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology)) @@ -176,6 +205,7 @@ class EnrichmentCommands: return active def create_hbase_tables(self): + Logger.info("Creating HBase Tables") add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf) Execute(add_enrichment_cmd, tries=3, @@ -191,3 +221,5 @@ class EnrichmentCommands: logoutput=False, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin' ) + Logger.info("Done creating HBase Tables") + self.set_hbase_configured() http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py index b02ba5c..362b2e2 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py @@ -48,14 +48,14 @@ class Enrichment(Script): from params import params env.set_params(params) commands = EnrichmentCommands(params) - metron_service.load_global_config(params) - if not commands.is_configured(): + if not commands.is_kafka_configured(): commands.init_kafka_topics() + if not commands.is_hbase_configured(): commands.create_hbase_tables() + if not commands.is_geo_configured(): commands.init_geo() - commands.set_configured() commands.start_enrichment_topology() @@ -79,12 +79,6 @@ class Enrichment(Script): commands = EnrichmentCommands(params) commands.restart_enrichment_topology(env) - def kafkabuild(self, env, upgrade_type=None): - from params import params - env.set_params(params) - commands = EnrichmentCommands(params) - commands.init_kafka_topics() - if __name__ == "__main__": Enrichment().execute() http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index 8ed3f9a..50af3b2 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -101,7 +101,7 @@ class IndexingCommands: type="directory", action="create_on_execute", owner=self.__params.metron_user, - group=self.__params.user_group, + group=self.__params.metron_group, mode=0775, ) Logger.info('Done creating HDFS indexing directory') http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 2b8276b..87a1f39 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -39,7 +39,7 @@ config = Script.get_config() tmp_dir = Script.get_tmp_dir() hostname = config['hostname'] -user_group = config['configurations']['cluster-env']['user_group'] +metron_group = config['configurations']['cluster-env']['metron_group'] metron_home = status_params.metron_home parsers = status_params.parsers geoip_url = config['configurations']['metron-env']['geoip_url'] @@ -51,7 +51,9 @@ metron_config_path = metron_home + '/config' metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir metron_zookeeper_config_path = status_params.metron_zookeeper_config_path parsers_configured_flag_file = status_params.parsers_configured_flag_file -enrichment_configured_flag_file = status_params.enrichment_configured_flag_file +enrichment_kafka_configured_flag_file = status_params.enrichment_kafka_configured_flag_file +enrichment_hbase_configured_flag_file = status_params.enrichment_hbase_configured_flag_file +enrichment_geo_configured_flag_file = status_params.enrichment_geo_configured_flag_file indexing_configured_flag_file = status_params.indexing_configured_flag_file global_json_template = config['configurations']['metron-env']['global-json'] global_properties_template = config['configurations']['metron-env']['elasticsearch-properties'] http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 961102f..83b4fa4 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -45,7 +45,9 @@ metron_indexing_topology = config['configurations']['metron-env']['metron_indexi indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured' # Enrichment -enrichment_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_configured' +enrichment_kafka_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_configured' +enrichment_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_configured' +enrichment_geo_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_geo_configured' # Storm storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr'] http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/.gitignore ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/packer-build/.gitignore b/metron-deployment/packaging/packer-build/.gitignore new file mode 100644 index 0000000..68bf87b --- /dev/null +++ b/metron-deployment/packaging/packer-build/.gitignore @@ -0,0 +1,12 @@ +.bundle/ +iso +*.box +virtualfloppy.vfd +packer_cache +packer.log +.DS_Store +/packer-*/ +*.variables.json +/builds/ +.kitchen +.kitchen.*.yml http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/packer-build/README.md b/metron-deployment/packaging/packer-build/README.md new file mode 100644 index 0000000..d0c5484 --- /dev/null +++ b/metron-deployment/packaging/packer-build/README.md @@ -0,0 +1,51 @@ +Build Metron Images +========================= + +Based on the fantastic [Bento](https://github.com/chef/bento) project developed by Chef. + +Images Provided +--------------------- +- base-centos-6.7: Centos 6.7 + HDP. Used in the full-dev-platform Vagrant image +- quick-dev-centos-6.7: Centos 6.7 + HDP + Metron. Used for the quick-dev-platform Vagrant image. + +Prerequisites +--------------------- +- [Packer](https://www.packer.io/) 0.12.2 +- [Virtualbox](https://www.virtualbox.org/) 5.0.16+ (Tested with 5.0.20) + +Build Both Images +---------------------- + Navigate to \<your-project-directory\>/metron-deployment/packer-build + Execute bin/bento build + + Packer will build both images and export .box files to the ./builds directory. + +Build Single Images +---------------------- + Navigate to *your-project-directory*/metron-deployment/packer-build + * Base Centos (full-dev) + ``` +bin/bento build base-centos-6.7.json +``` + * Quick Dev + ``` +bin/bento build quick-dev-centos-6.7.json +``` + +Using Your New Box File +---------------------- +Modify the relevant Vagrantfile (full-dev-platform or quick-dev-platform) replacing the lines: +``` +<pre><code>config.vm.box = "<i>box_name</i>" +config.ssh.insert_key = true</code></pre> +``` +with +``` +<pre></code>config.vm.box = "<i>test_box_name</i>" +config.vm.box = "<i>PathToBoxfile/Boxfilename</i>" +config.ssh.insert_key = true</code></pre> +``` +Launch the image as usual. + +Node: Vagrant will cache boxes, you can force Vagrant to reload your box by running <code>vagrant box remove <i>test_box_name</i></code> before launching your new image. + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/ansible.cfg ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/packer-build/ansible.cfg b/metron-deployment/packaging/packer-build/ansible.cfg new file mode 100644 index 0000000..eeb5084 --- /dev/null +++ b/metron-deployment/packaging/packer-build/ansible.cfg @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[defaults] +host_key_checking = false +library = ../../extra_modules +roles_path = ../../roles +pipelining = True +remote_user = vagrant +log_path = ./ansible.log + +# fix for "ssh throws 'unix domain socket too long' " problem +[ssh_connection] +control_path = ~/.ssh/ansible-ssh-%%C http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/ansible/playbook.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/packer-build/ansible/playbook.yml b/metron-deployment/packaging/packer-build/ansible/playbook.yml new file mode 100644 index 0000000..0aba491 --- /dev/null +++ b/metron-deployment/packaging/packer-build/ansible/playbook.yml @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- include: ../../../playbooks/metron_full_install.yml + +- hosts: packer + become: true + roles: + - role: ambari_gather_facts + tasks: + - include: tasks/cleanup.yml + tags: + - packer-cleanup + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/ansible/tasks/cleanup.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/packer-build/ansible/tasks/cleanup.yml b/metron-deployment/packaging/packer-build/ansible/tasks/cleanup.yml new file mode 100644 index 0000000..cdeab42 --- /dev/null +++ b/metron-deployment/packaging/packer-build/ansible/tasks/cleanup.yml @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Stop Monit + service: + name: monit + state: stopped + register: command_result + failed_when: "command_result|failed and 'no service or tool found for: monit' not in command_result.msg" + +- name: Delete ES Indices + uri: + method: DELETE + url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/b*,s*,y*" + +- name: Stop the ambari cluster + ambari_cluster_state: + host: "{{ groups.ambari_master[0] }}" + port: "{{ ambari_port }}" + username: "{{ ambari_user }}" + password: "{{ ambari_password }}" + cluster_name: "{{ cluster_name }}" + cluster_state: stopped + wait_for_complete: true + +- include: remove_services.yml + + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/ansible/tasks/remove_services.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/packer-build/ansible/tasks/remove_services.yml b/metron-deployment/packaging/packer-build/ansible/tasks/remove_services.yml new file mode 100644 index 0000000..9d2477c --- /dev/null +++ b/metron-deployment/packaging/packer-build/ansible/tasks/remove_services.yml @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Remove Secondary Namenode + uri: + url: "http://{{ambari_host}}:{{ambari_port}}/api/v1/clusters/metron_cluster/hosts/node1/host_components/SECONDARY_NAMENODE"; + method: DELETE + HEADER_X-Requested-By: "{{ ambari_user }}" + user: "{{ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + status_code: 200,201,404 + +- name: Remove MapReduce2 Service + uri: + url: "http://{{ambari_host}}:{{ambari_port}}/api/v1/clusters/metron_cluster/services/MAPREDUCE2"; + method: DELETE + HEADER_X-Requested-By: "{{ ambari_user }}" + user: "{{ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + status_code: 200,201,404 + +- name: Remove YARN Service + uri: + url: "http://{{ambari_host}}:{{ambari_port}}/api/v1/clusters/metron_cluster/services/YARN"; + method: DELETE + HEADER_X-Requested-By: "{{ ambari_user }}" + user: "{{ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + status_code: 200,201,404
