http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml new file mode 100644 index 0000000..8c1bc33 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Setup Ambari repo on CentOS + get_url: url="{{ centos_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo + +- name: Update package cache on CentOS + yum: name=* update_cache=yes
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml new file mode 100644 index 0000000..a0ad679 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + +- name: Setup Ambari repo on Ubuntu + shell: echo "deb {{ ubuntu_ambari_repo }} Ambari main" | sudo tee /etc/apt/sources.list.d/ambari.list + +- name: Update package cache on Ubuntu + apt: update_cache=yes http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml b/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml new file mode 100644 index 0000000..730607d --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + +# +# for CentOS... +# +# this is not needed for CentOS. the elasticsearch repository is automatically +# added by the mpack. there is currently an issue that needs resolved that +# prevents us from doing the same in Ubuntu +# + +# +# for Ubuntu... +# +- name: Setup Elasticsearch packages repo on Ubuntu + shell: echo "deb {{ ubuntu_elasticsearch_packages_repo }} stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch.list + when: ansible_distribution == "Ubuntu" + +- name: Setup Elasticsearch curator repo on Ubuntu + shell: echo "deb {{ ubuntu_elasticsearch_curator_repo }} stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch.list + when: ansible_distribution == "Ubuntu" + +- name: Update package cache + apt: update_cache=yes + when: ansible_distribution == "Ubuntu" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/hostname.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/hostname.yml b/metron-deployment/ansible/roles/ambari_common/tasks/hostname.yml new file mode 100644 index 0000000..4688332 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_common/tasks/hostname.yml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +# +# ambari uses socket.getfqdn() to find the hostname. with 'localhost.localdomain' +# in '/etc/hosts' this function will report the hostname as 'localhost.localdomain' +# rather than 'node1' as would be expected. other functions like socket.gethostname() +# will always return 'node1' as expected. ambari needs to see 'node1' to be able to +# communicate between the master and agents. + +- name: Remove ipv4 'localhost.localdomain' from /etc/hosts + lineinfile: dest=/etc/hosts state=absent regexp="^127.0.0.1(.*)localdomain(.*)$" + +- name: Remove ipv6 'localhost.localdomain' from /etc/hosts + lineinfile: dest=/etc/hosts state=absent regexp="^::1(.*)localdomain(.*)$" + +- name: Add localhost to /etc/hosts + lineinfile: dest=/etc/hosts line="127.0.0.1 localhost" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml new file mode 100644 index 0000000..27e67d5 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Stop iptables on CentOS + ignore_errors: yes + service: name=iptables state=stopped enabled=no http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml b/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml new file mode 100644 index 0000000..9fdda7e --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Disable firewall on Ubuntu + shell: ufw disable http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/logrotate.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/logrotate.yml b/metron-deployment/ansible/roles/ambari_common/tasks/logrotate.yml new file mode 100644 index 0000000..2c4a2e0 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_common/tasks/logrotate.yml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Create Logrotate Script for Hadoop Services + template: + src: "metron-hadoop-logrotate.yml" + dest: "/etc/logrotate.d/metron-ambari" + mode: 0644 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/main.yml b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml new file mode 100644 index 0000000..b3258a9 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + +- include: iptables-centos.yml + static: no + when: ansible_distribution == "CentOS" + +- include: iptables-ubuntu.yml + static: no + when: ansible_distribution == "Ubuntu" + +- include: hostname.yml + +- include: ambari-repo-centos.yml + static: no + when: ansible_distribution == "CentOS" + +- include: ambari-repo-ubuntu.yml + static: no + when: ansible_distribution == "Ubuntu" + +- include: elasticsearch-repo.yml +- include: nodejs.yml +- include: logrotate.yml http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml new file mode 100644 index 0000000..0dbf9f7 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +# +# for CentOS... +# +- name: Install Nodejs repo on CentOS + shell: curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - + args: + warn: false + when: ansible_distribution == "CentOS" + +# +# for Ubuntu... +# +- name: Install Nodejs repo on Ubuntu + shell: curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash - + args: + warn: false + when: ansible_distribution == "Ubuntu" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/templates/metron-hadoop-logrotate.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_common/templates/metron-hadoop-logrotate.yml b/metron-deployment/ansible/roles/ambari_common/templates/metron-hadoop-logrotate.yml new file mode 100644 index 0000000..d95c10b --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_common/templates/metron-hadoop-logrotate.yml @@ -0,0 +1,149 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#Hadoop HDFS Logs +/var/log/hadoop/hdfs/*.log* { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +/var/log/hadoop/hdfs/*.out { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +/var/log/hadoop/hdfs/*.audit { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +#Hadoop Yarn Logs +/var/log/hadoop/yarn/*.log { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +#Hadoop Mapreduce Logs +/var/log/hadoop/mapreduce/*.log { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +#Storm Logs +/var/log/storm/*.log { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +/var/log/storm/*.out { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +#Kafka Logs +/var/log/kafka/*.log { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +/var/log/kafka/*.err { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +#HBase Logs +/var/log/hbase/*.log* { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +/var/log/hbase/*.out { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +/var/log/hbase/*.audit { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +#Zookeeper Logs +/var/log/zookeeper/*.log { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + +/var/log/zookeeper/*.out { + {{ hadoop_logrotate_frequency }} + rotate {{ hadoop_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_config/defaults/main.yml b/metron-deployment/ansible/roles/ambari_config/defaults/main.yml new file mode 100644 index 0000000..e0de145 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_config/defaults/main.yml @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +zookeeper_data_dir: /hadoop/zookeeper +namenode_checkpoint_dir: /hadoop/hdfs/namesecondary +namenode_name_dir: /hadoop/hdfs/namenode +datanode_data_dir: /hadoop/hdfs/data +journalnode_edits_dir: /hadoop/hdfs/journalnode +jhs_recovery_store_ldb_path: /hadoop/mapreduce/jhs +nodemanager_local_dirs: /hadoop/yarn/local +timeline_ldb_store_path: /hadoop/yarn/timeline +timeline_ldb_state_path: /hadoop/yarn/timeline +nodemanager_log_dirs: /hadoop/yarn/log +storm_local_dir: /hadoop/storm +kafka_log_dirs: /kafka-log +cluster_type: small_cluster +nodemanager_mem_mb : 4096 +mapred_map_java_opts : -Xmx1024m +mapred_reduce_java_opts : -Xmx1024m +mapred_map_mem_mb : 1229 +mapred_reduce_mem_mb : 1229 +topology_classpath: '/etc/hbase/conf:/etc/hadoop/conf' +hdp_stack: "2.5" +elasticsearch_network_interface: _site_ http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_config/meta/main.yml b/metron-deployment/ansible/roles/ambari_config/meta/main.yml new file mode 100644 index 0000000..8f65a28 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_config/meta/main.yml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +dependencies: + - epel + - python-pip + - httplib2 + - java_jdk http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml new file mode 100644 index 0000000..fdb21c7 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install python-requests on CentOS + yum: name=python-requests + when: ansible_distribution == "CentOS" + +- name: Install python-requests on Ubuntu + apt: name=python-requests force=yes + when: ansible_distribution == "Ubuntu" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/main.yml b/metron-deployment/ansible/roles/ambari_config/tasks/main.yml new file mode 100644 index 0000000..d5f38b1 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_config/tasks/main.yml @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- include_vars: "{{ cluster_type }}.yml" + +- include: dependencies.yml + +- name : Wait for Ambari to start; http://{{ ambari_host }}:{{ ambari_port }} + wait_for : + host: "{{ ambari_host }}" + port: "{{ ambari_port }}" + timeout: 600 + +- name: Deploy cluster with Ambari; http://{{ ambari_host }}:{{ ambari_port }} + ambari_cluster_state: + host: "{{ ambari_host }}" + port: "{{ ambari_port }}" + username: "{{ ambari_user }}" + password: "{{ ambari_password }}" + cluster_name: "{{ cluster_name }}" + cluster_state: present + blueprint_name: "{{ blueprint_name }}" + configurations: "{{ configurations }}" + wait_for_complete: True + blueprint_var: "{{ blueprint }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml new file mode 100644 index 0000000..6a60902 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml @@ -0,0 +1,135 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +# vars file for single_node_vm blueprint + +hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER] +hadoop_slave: [APP_TIMELINE_SERVER, DATANODE, HDFS_CLIENT, NODEMANAGER, YARN_CLIENT, MAPREDUCE2_CLIENT] +spark_master: [SPARK_JOBHISTORYSERVER] +spark_slave: [SPARK_CLIENT] +storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER] +storm_slave: [SUPERVISOR] +kafka_broker: [KAFKA_BROKER] +zookeeper_master: [ZOOKEEPER_SERVER] +zookeeper_slave: [ZOOKEEPER_CLIENT] +hbase_master: [HBASE_MASTER, HBASE_CLIENT] +hbase_slave: [HBASE_REGIONSERVER] +es_master: [ES_MASTER] +kibana_master: [KIBANA_MASTER] +metron_indexing: [METRON_INDEXING] +metron_profiler: [METRON_PROFILER] +metron_enrichment_master : [METRON_ENRICHMENT_MASTER] +metron_parsers : [METRON_PARSERS] +metron_rest: [METRON_REST] +metron_management_ui: [METRON_MANAGEMENT_UI] +metron_alerts_ui: [METRON_ALERTS_UI] + +metron_components: > + {{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | + union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | union(metron_profiler) | + union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(es_master) }} + +cluster_name: "metron_cluster" +blueprint_name: "metron_blueprint" + +configurations: + - zoo.cfg: + dataDir: '{{ zookeeper_data_dir }}' + - hadoop-env: + hadoop_heapsize: 1024 + namenode_heapsize: 2048 + dtnode_heapsize: 512 + namenode_opt_permsize: 128m + - hbase-env: + hbase_regionserver_heapsize: 512 + hbase_master_heapsize: 512 + hbase_regionserver_xmn_max: 512 + - hdfs-site: + dfs.replication: 1 + dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir }}' + dfs.namenode.name.dir: '{{ namenode_name_dir }}' + dfs.datanode.data.dir: '{{ datanode_data_dir }}' + dfs.journalnode.edits.dir: '{{ journalnode_edits_dir }}' + - yarn-env: + nodemanager_heapsize: 512 + yarn_heapsize: 512 + apptimelineserver_heapsize : 512 + resourcemanager_heapsize: 1024 + - mapred-env: + jobhistory_heapsize: 256 + - mapred-site: + mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path }}' + mapreduce.map.java.opts : '{{ mapred_map_java_opts }}' + mapreduce.reduce.java.opts : '{{ mapred_reduce_java_opts }}' + mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}' + mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}' + - yarn-site: + yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs }}' + yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path }}' + yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path }}' + yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs }}' + yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}' + - storm-site: + supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]" + storm.local.dir: '{{ storm_local_dir }}' + topology.classpath: '{{ topology_classpath }}' + - kafka-env: + content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}" + - kafka-broker: + log.dirs: '{{ kafka_log_dirs }}' + delete.topic.enable: "true" + - metron-rest-env: + metron_spring_profiles_active: "dev" + - metron-parsers-env: + parsers: "bro,snort" + - elastic-site: + index_number_of_shards: 1 + index_number_of_replicas: 0 + zen_discovery_ping_unicast_hosts: "[ {{ groups.search | join(', ') }} ]" + gateway_recover_after_data_nodes: 1 + network_host: "[ _local_, {{ elasticsearch_network_interface }} ]" + masters_also_are_datanodes: "1" + +required_configurations: + - metron-env: + storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744" + es_hosts: "{{ groups.search | join(',') }}" + zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995" + - metron-rest-env: + metron_jdbc_driver: "org.h2.Driver" + metron_jdbc_url: "jdbc:h2:file:~/metrondb" + metron_jdbc_username: "root" + metron_jdbc_password: "root" + metron_jdbc_platform: "h2" + - kibana-env: + kibana_pid_dir: /var/run/kibana + kibana_es_url: http://{{ groups.search[0] }}:9200 + kibana_log_dir: /var/log/kibana + kibana_server_port: 5000 + kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9" + kibana_server_host: 0.0.0.0 + +blueprint: + stack_name: HDP + stack_version: "{{ hdp_stack }}" + required_configurations: "{{ required_configurations }}" + groups: + - name : host_group_1 + cardinality: 1 + configurations: [] + components: "{{ metron_components }}" + hosts: "{{ hdp_host_group }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml new file mode 100644 index 0000000..4ec8458 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml @@ -0,0 +1,149 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + +hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER] +app_timeline_server: [APP_TIMELINE_SERVER] +hadoop_slave: [DATANODE, NODEMANAGER] +spark_master: [SPARK_JOBHISTORYSERVER] +storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER] +storm_slave: [SUPERVISOR] +kafka_broker: [KAFKA_BROKER] +zookeeper_master: [ZOOKEEPER_SERVER] +hbase_master: [HBASE_MASTER] +hbase_slave: [HBASE_REGIONSERVER] +hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT] +es_master: [ES_MASTER] +es_slave: [ES_SLAVE] +kibana_master: [KIBANA_MASTER] +metron_indexing: [METRON_INDEXING] +metron_profiler: [METRON_PROFILER] +metron_enrichment_master : [METRON_ENRICHMENT_MASTER] +metron_parsers : [METRON_PARSERS] +metron_rest: [METRON_REST] +metron_management_ui: [METRON_MANAGEMENT_UI] +metron_alerts_ui: [METRON_ALERTS_UI] + +master_1_components: "{{ hadoop_master | union(hadoop_clients) | union(es_slave) }}" +master_1_host: + - "{{groups.ambari_slave[0]}}" +master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) | union(app_timeline_server) | union(es_slave) }}" +master_2_host: + - "{{groups.ambari_slave[1]}}" +metron_components: > + {{ metron_indexing | union(metron_profiler) | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) | union(storm_slave) | + union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }} +metron_host: + - "{{ groups.metron[0] }}" +web_components: "{{ kibana_master | union(es_master) }}" +web_host: + - "{{ groups.web[0] }}" +slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}" + +cluster_name: "metron" +blueprint_name: "metron_blueprint" + +configurations: + - zoo.cfg: + dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}' + - hadoop-env: + namenode_heapsize: 2048 + dtnode_heapsize: 1024 + - hbase-env: + hbase_regionserver_heapsize: 1024 + hbase_master_heapsize: 1024 + - hdfs-site: + dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}' + dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}' + dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}' + dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}' + - mapred-site: + mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}' + mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}' + mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}' + - yarn-site: + yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}' + yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}' + yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}' + yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}' + yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}' + - storm-site: + supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]" + storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}' + topology.classpath: '{{ topology_classpath }}' + - kafka-broker: + log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}' + - metron-rest-env: + metron_spring_profiles_active: "dev" + - metron-env: + parsers: "bro,snort,yaf" + - elastic-site: + index_number_of_shards: 2 + index_number_of_replicas: 1 + zen_discovery_ping_unicast_hosts: "[ {{ groups.web[0] }}, {{ groups.search | join(', ') }} ]" + gateway_recover_after_data_nodes: 1 + network_host: "[ _local_, {{ elasticsearch_network_interface }} ]" + +required_configurations: + - metron-env: + storm_rest_addr: "http://{{ groups.ambari_slave[1] }}:8744" + es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}" + zeppelin_server_url: "{{ groups.zeppelin[0] }}" + - metron-rest-env: + metron_jdbc_driver: "org.h2.Driver" + metron_jdbc_url: "jdbc:h2:file:~/metrondb" + metron_jdbc_username: "root" + metron_jdbc_password: "root" + metron_jdbc_platform: "h2" + - kibana-env: + kibana_pid_dir: /var/run/kibana + kibana_es_url: http://{{ groups.web[0] }}:9200 + kibana_log_dir: /var/log/kibana + kibana_server_port: 5000 + kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9" + kibana_server_host: 0.0.0.0 + +blueprint: + stack_name: HDP + stack_version: "{{ hdp_stack }}" + required_configurations: "{{ required_configurations }}" + groups: + - name : master_1 + cardinality: 1 + configuration: [] # configuration not yet implemented + components: "{{ master_1_components }}" + hosts: "{{ master_1_host }}" + - name : master_2 + cardinality: 1 + configuration: [] # configuration not yet implemented + components: "{{ master_2_components }}" + hosts: "{{ master_2_host }}" + - name : metron + cardinality: 1 + configuration: [] # configuration not yet implemented + components: "{{ metron_components }}" + hosts: "{{ metron_host }}" + - name : web + cardinality: 1 + configuration: [] # configuration not yet implemented + components: "{{ web_components }}" + hosts: "{{ web_host }}" + - name: slaves + cardinality: 1+ + configuration: [] # configuration not yet implemented + components: "{{ slave_components }}" + hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) | difference(groups.metron[0]) | difference(groups.web[0]) }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_gather_facts/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_gather_facts/meta/main.yml b/metron-deployment/ansible/roles/ambari_gather_facts/meta/main.yml new file mode 100644 index 0000000..61197e3 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_gather_facts/meta/main.yml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +dependencies: + - epel + - python-pip + - httplib2 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml b/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml new file mode 100644 index 0000000..2b37eec --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml @@ -0,0 +1,234 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +# +# cluster_name +# +- name: "Ask Ambari: cluster_name" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: cluster_name_response + when: cluster_name is undefined + +- set_fact: + cluster_name: "{{ (cluster_name_response.content | from_json)['items'][0].Clusters.cluster_name }}" + when: cluster_name is undefined + +# +# namenode_host +# +- name: "Ask Ambari: namenode_host" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/HDFS/components/NAMENODE" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: namenode_hosts_response + when: namenode_host is undefined + +- set_fact: + namenode_host: "{{ (namenode_hosts_response.content | from_json).host_components[0].HostRoles.host_name }}" + when: namenode_host is undefined + +# +# core_site_tag +# +- name: "Ask Ambari: core_site_tag" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ namenode_host }}/host_components/NAMENODE" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: core_site_tag_response + when: core_site_tag is undefined + +- set_fact: + core_site_tag: "{{ (core_site_tag_response.content | from_json).HostRoles.actual_configs['core-site'].default }}" + when: core_site_tag is undefined + +# +# hdfs_url +# +- name: "Ask Ambari: hdfs_url" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=core-site&tag={{ core_site_tag }}" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: core_site_response + when: hdfs_url is undefined + +- set_fact: + hdfs_url: "{{ (core_site_response.content | from_json)['items'][0].properties['fs.defaultFS'] }}" + when: hdfs_url is undefined + +# +# kafka_broker_hosts +# +- name: "Ask Ambari: kafka_broker_hosts" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KAFKA/components/KAFKA_BROKER" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: kafka_broker_hosts_response + when: kafka_broker_hosts is undefined + +- set_fact: + kafka_broker_hosts: "{{ (kafka_broker_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}" + when: kafka_broker_hosts is undefined + +# +# kafka_broker_tag +# +- name: "Ask Ambari: kafka_broker_tag" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ kafka_broker_hosts[0] }}/host_components/KAFKA_BROKER" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: kafka_broker_tag_response + when: kafka_broker_tag is undefined + +- set_fact: + kafka_broker_tag: "{{ (kafka_broker_tag_response.content | from_json).HostRoles.actual_configs['kafka-broker'].default }}" + when: kafka_broker_tag is undefined + +# +# kafka_broker_port +# +- name: "Ask Ambari: kafka_broker_port" + shell: > + curl -s -u {{ ambari_user }}:{{ ambari_password }} -X GET -H "X-Requested-By: ambari" "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=kafka-broker&tag={{ kafka_broker_tag }}" | python -c 'import sys, json; print json.load(sys.stdin)["items"][0]["properties"]["listeners"]' + args: + warn: false + register: kafka_broker_port_response + when: (kafka_broker_url is undefined) or (kafka_broker_port is undefined) + +- set_fact: + kafka_broker_port: "{{ kafka_broker_port_response.stdout_lines[0] | replace('PLAINTEXT://localhost:', '')}}" + when: kafka_broker_port is undefined + +- set_fact: + kafka_broker_url: "{% for host in kafka_broker_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ kafka_broker_port }}{% endfor %}" + when: kafka_broker_url is undefined + +# +# zookeeper_hosts +# +- name: "Ask Ambari: zookeeper_hosts" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/ZOOKEEPER/components/ZOOKEEPER_SERVER" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: zookeeper_hosts_response + when: zookeeper_hosts is undefined + +- set_fact: + zookeeper_hosts: "{{ (zookeeper_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}" + when: zookeeper_hosts is undefined + +# +# zookeeper_tag +# +- name: "Ask Ambari: zookeeper_tag" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ zookeeper_hosts[0] }}/host_components/ZOOKEEPER_SERVER" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: zookeeper_tag_response + when: zookeeper_tag is undefined + +- set_fact: + zookeeper_tag: "{{ (zookeeper_tag_response.content | from_json).HostRoles.actual_configs['zoo.cfg'].default }}" + when: zookeeper_tag is undefined + +# +# zookeeper_url, zookeeper_port +# +- name: "Ask Ambari: zookeeper_url, zookeeper_port" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=zoo.cfg&tag={{ zookeeper_tag }}" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: zookeeper_properties_response + when: zookeeper_url is undefined or zookeeper_port is undefined + +- set_fact: + zookeeper_port: "{{ (zookeeper_properties_response.content | from_json)['items'][0].properties['clientPort'] }}" + when: zookeeper_port is undefined + +- set_fact: + zookeeper_url: "{% for host in zookeeper_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ zookeeper_port }}{% endfor %}" + when: zookeeper_url is undefined + +- name: "Ask Ambari: metron_hosts" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/METRON/components/METRON_INDEXING" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: metron_hosts_response + when: metron_hosts is undefined + +- set_fact: + metron_hosts: "{{ (metron_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}" + when: metron_hosts is undefined + +- name: "Ask Ambari: kibana_hosts" + uri: + url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KIBANA/components/KIBANA_MASTER" + user: "{{ ambari_user }}" + password: "{{ ambari_password }}" + force_basic_auth: yes + return_content: yes + register: kibana_hosts_response + when: kibana_hosts is undefined + +- set_fact: + kibana_hosts: "{{ (kibana_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}" + when: kibana_hosts is undefined + + +# +# debug output +# +- name: debug + debug: + msg: "zookeeper_port = {{ zookeeper_port }}, + zookeeper_hosts = {{ zookeeper_hosts }}, + zookeeper_url = {{ zookeeper_url }}, + kafka_broker_port = {{ kafka_broker_port }}, + kafka_broker_hosts = {{ kafka_broker_hosts }}, + kafka_broker_url = {{ kafka_broker_url }}, + metron_hosts = {{ metron_hosts }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_master/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml new file mode 100644 index 0000000..0719b7f --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +ambari_server_mem: 2048 +ambari_mpack_version: 0.4.3.0 +metron_mpack_name: metron_mpack-{{ ambari_mpack_version }}.tar.gz +metron_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/metron-mpack/target/{{ metron_mpack_name }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_master/tasks/ambari.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/ambari.yml b/metron-deployment/ansible/roles/ambari_master/tasks/ambari.yml new file mode 100644 index 0000000..ba55f30 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_master/tasks/ambari.yml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install Ambari Server on CentOS + yum: name=ambari-server + when: ansible_distribution == "CentOS" + +- name: Install Ambari Server on Ubuntu + apt: name=ambari-server force=yes + when: ansible_distribution == "Ubuntu" + +- name: Set Ambari Server Max Memory + replace: + dest: /var/lib/ambari-server/ambari-env.sh + regexp: "\ -Xmx2048m\ " + replace: " -Xmx{{ ambari_server_mem }}m " + backup: no + +- name: Setup Ambari Server + shell: ambari-server setup -s && touch /etc/ambari-server/configured creates=/etc/ambari-server/configured + register: ambari_server_setup + failed_when: "(ambari_server_setup.stderr is defined and ambari_server_setup.stderr != '') or 'FATAL' in ambari_server_setup.stdout" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_master/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/main.yml b/metron-deployment/ansible/roles/ambari_master/tasks/main.yml new file mode 100644 index 0000000..77a2e74 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_master/tasks/main.yml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- include: ambari.yml +- include: mpack.yml + +- name: start ambari server + service: + name: ambari-server + state: restarted http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml b/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml new file mode 100644 index 0000000..16289bb --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Copy MPack to Ambari Host + copy: + src: "{{ metron_mpack_path }}" + dest: /tmp + +- name: Install MPack on Ambari Host + shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-{{ ambari_mpack_version }}.tar.gz + args: + creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-{{ ambari_mpack_version }}/addon-services http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_slave/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_slave/defaults/main.yml b/metron-deployment/ansible/roles/ambari_slave/defaults/main.yml new file mode 100644 index 0000000..f3cb4c5 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_slave/defaults/main.yml @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +ambari_installation_user: "root" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_slave/files/hostname.sh ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_slave/files/hostname.sh b/metron-deployment/ansible/roles/ambari_slave/files/hostname.sh new file mode 100644 index 0000000..cc8c1cd --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_slave/files/hostname.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +echo {{ inventory_hostname }} http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_slave/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_slave/meta/main.yml b/metron-deployment/ansible/roles/ambari_slave/meta/main.yml new file mode 100644 index 0000000..ddf6aa9 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_slave/meta/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +dependencies: + - java_jdk http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml new file mode 100644 index 0000000..62d0027 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install ambari-agent on CentOS + yum: name=ambari-agent + when: ansible_distribution == "CentOS" + +- name: Install ambari-agent on Ubuntu + apt: name=ambari-agent force=yes + when: ansible_distribution == "Ubuntu" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml new file mode 100644 index 0000000..b78a2a7 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- include: install-agent.yml + +- name: Create ambari-agent hostname script + template: + src: "../roles/ambari_slave/files/hostname.sh" + dest: "/var/lib/ambari-agent/hostname.sh" + mode: 0744 + owner: "{{ ambari_installation_user }}" + group: "{{ ambari_installation_user }}" + +- name: Configure ambari-server hostname in ambari-agent configuration + lineinfile: + dest: /etc/ambari-agent/conf/ambari-agent.ini + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + insertafter: "{{ item.insertafter }}" + backup: yes + with_items: + - { regexp: "^.*hostname=.*$", line: "hostname={{ groups.ambari_master[0] }}", insertafter: '\[server\]' } + - { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'} + +- name: Ensure ambari-agent is running + service: + name: ambari-agent + state: restarted + enabled: yes + +- name : Wait for agent to register + command : sleep 10 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/bro/meta/main.yml b/metron-deployment/ansible/roles/bro/meta/main.yml new file mode 100644 index 0000000..df226e7 --- /dev/null +++ b/metron-deployment/ansible/roles/bro/meta/main.yml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +dependencies: + - libselinux-python + - build-tools + - kafka-client + - librdkafka http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/bro.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/bro/tasks/bro.yml b/metron-deployment/ansible/roles/bro/tasks/bro.yml new file mode 100644 index 0000000..222ef0e --- /dev/null +++ b/metron-deployment/ansible/roles/bro/tasks/bro.yml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Download bro + get_url: + url: "https://www.bro.org/downloads/bro-{{ bro_version }}.tar.gz" + dest: "/tmp/bro-{{ bro_version }}.tar.gz" + +- name: Extract bro tarball + unarchive: + src: "/tmp/bro-{{ bro_version }}.tar.gz" + dest: /tmp + copy: no + creates: "/tmp/bro-{{ bro_version }}" + +- name: Compile and Install bro + shell: "{{ item }}" + environment: + CXX: /opt/rh/devtoolset-4/root/usr/bin/g++ + CC: /opt/rh/devtoolset-4/root/usr/bin/gcc + args: + chdir: "/tmp/bro-{{ bro_version }}" + creates: "{{ bro_home }}/bin/bro" + with_items: + - "./configure --prefix={{ bro_home }}" + - make + - make install + +- name: Configure bro + lineinfile: + dest: "{{ bro_home }}/etc/node.cfg" + regexp: '^interface=.*$' + line: 'interface={{ sniff_interface }}' + +- name: Install bro configuration + shell: "{{ bro_home }}/bin/broctl install" + +- name: Bro Cronjob + cron: + name: Bro Cron + minute: "{{ bro_crontab_minutes }}" + job: "{{ item }}" + with_items: + - "{{ bro_crontab_job }}" + - "{{ bro_clean_job }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/dependencies.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml new file mode 100644 index 0000000..3cd3bae --- /dev/null +++ b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install yum repositories + yum: name={{ item }} update_cache=yes + with_items: + - centos-release-scl + +- name: Install prerequisites + yum: name={{ item }} + with_items: + - cmake + - make + - gcc + - gcc-c++ + - flex + - bison + - libpcap + - libpcap-devel + - openssl-devel + - python-devel + - swig + - zlib-devel + - perl + - crontabs + - net-tools + - devtoolset-4-gcc + - devtoolset-4-gcc-c++ + - python27 + - rh-git29 + register: result + until: result.rc == 0 + retries: 5 + delay: 10 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/librdkafka.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/bro/tasks/librdkafka.yml b/metron-deployment/ansible/roles/bro/tasks/librdkafka.yml new file mode 100644 index 0000000..652d319 --- /dev/null +++ b/metron-deployment/ansible/roles/bro/tasks/librdkafka.yml @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Download librdkafka + get_url: + url: "{{ librdkafka_url }}" + dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz" + +- name: Extract librdkafka tarball + unarchive: + src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz" + dest: /tmp + copy: no + creates: "/tmp/librdkafka-{{ librdkafka_version }}" + +- name: Compile and install librdkafka + shell: "{{ item }}" + args: + chdir: "/tmp/librdkafka-{{ librdkafka_version }}" + creates: "{{ librdkafka_home }}/lib/librdkafka.so" + with_items: + - rm -rf build/ + - "./configure --prefix={{ librdkafka_home }}" + - make + - make install http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/bro/tasks/main.yml b/metron-deployment/ansible/roles/bro/tasks/main.yml new file mode 100644 index 0000000..440e4b5 --- /dev/null +++ b/metron-deployment/ansible/roles/bro/tasks/main.yml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- include: dependencies.yml +- include: librdkafka.yml +- include: bro.yml +- include: metron-bro-plugin-kafka.yml +- include: nic.yml http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml new file mode 100644 index 0000000..7043387 --- /dev/null +++ b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install bro-pkg + environment: + LD_LIBRARY_PATH: "{{ python27_lib }}" + command: "{{ python27_bin }}/pip install bro-pkg" + +- name: Configure bro-pkg + environment: + PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}" + LD_LIBRARY_PATH: "{{ python27_lib }}" + command: "{{ python27_bin }}/bro-pkg autoconfig" + +- name: Install the metron-bro-plugin-kafka package + environment: + PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}" + LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}" + CXX: /opt/rh/devtoolset-4/root/usr/bin/g++ + CC: /opt/rh/devtoolset-4/root/usr/bin/gcc + command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka --force --version {{ metron_bro_plugin_kafka_version }}" + +- name: Configure bro-kafka plugin + lineinfile: + dest: "{{ bro_home }}/share/bro/site/local.bro" + line: "{{ item }}" + with_items: + - "@load packages" + - "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);" + - "redef Kafka::topic_name = \"{{ bro_topic }}\";" + - "redef Kafka::tag_json = T;" + - "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] = \"{{ kafka_broker_url }}\");" + +- name: Deploy bro configuration changes + shell: "{{ bro_bin }}/broctl deploy" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/nic.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/bro/tasks/nic.yml b/metron-deployment/ansible/roles/bro/tasks/nic.yml new file mode 100644 index 0000000..6053618 --- /dev/null +++ b/metron-deployment/ansible/roles/bro/tasks/nic.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Turn on promiscuous mode for {{ sniff_interface }} + shell: "ip link set {{ sniff_interface }} promisc on" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/vars/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/bro/vars/main.yml b/metron-deployment/ansible/roles/bro/vars/main.yml new file mode 100644 index 0000000..11d6c31 --- /dev/null +++ b/metron-deployment/ansible/roles/bro/vars/main.yml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +bro_home: /usr/local/bro +bro_bin: "{{ bro_home }}/bin" +bro_version: 2.5.2 +bro_daemon_log: /var/log/bro.log +bro_topic: bro + +# Bro cronjob +bro_crontab_minutes: 0-59/5 +bro_crontab_job: "{{ bro_home }}/bin/broctl cron" +bro_clean_job: "rm -rf {{ bro_home }}/spool/tmp/*" + +# Bro kafka plugin +metron_bro_plugin_kafka_version: "0.1" +python27_bin: /opt/rh/python27/root/usr/bin +python27_lib: /opt/rh/python27/root/usr/lib64 +git29_bin: /opt/rh/rh-git29/root/usr/bin +httpd24_lib: /opt/rh/httpd24/root/usr/lib64 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/build-tools/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/build-tools/meta/main.yml b/metron-deployment/ansible/roles/build-tools/meta/main.yml new file mode 100644 index 0000000..ddf6aa9 --- /dev/null +++ b/metron-deployment/ansible/roles/build-tools/meta/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +dependencies: + - java_jdk http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/build-tools/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/build-tools/tasks/main.yml b/metron-deployment/ansible/roles/build-tools/tasks/main.yml new file mode 100644 index 0000000..c47ef43 --- /dev/null +++ b/metron-deployment/ansible/roles/build-tools/tasks/main.yml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install Build Tools + yum: name={{ item }} + with_items: + - "@Development tools" + - libdnet-devel + - rpm-build + - libpcap + - libpcap-devel + - pcre + - pcre-devel + - zlib + - zlib-devel + - glib2-devel + register: result + until: result.rc == 0 + retries: 5 + delay: 10 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/deployment-report/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/deployment-report/defaults/main.yml b/metron-deployment/ansible/roles/deployment-report/defaults/main.yml new file mode 100644 index 0000000..7677f48 --- /dev/null +++ b/metron-deployment/ansible/roles/deployment-report/defaults/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +timeout_secs: 120 +ambari_host: "{{ groups.ambari_master[0] }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/deployment-report/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/deployment-report/meta/main.yml b/metron-deployment/ansible/roles/deployment-report/meta/main.yml new file mode 100644 index 0000000..c3d807b --- /dev/null +++ b/metron-deployment/ansible/roles/deployment-report/meta/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +dependencies: + - ambari_gather_facts http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/deployment-report/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/deployment-report/tasks/main.yml b/metron-deployment/ansible/roles/deployment-report/tasks/main.yml new file mode 100644 index 0000000..38a5289 --- /dev/null +++ b/metron-deployment/ansible/roles/deployment-report/tasks/main.yml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the 'License'); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Sanity check Ambari web + local_action: wait_for host="{{ ambari_host }}" port="{{ ambari_port }}" timeout="{{ timeout_secs }}" + +- name: Known hosts groups + debug: var=groups + +- set_fact: + success: + - "Apache Metron deployed successfully" + - " Ambari @ http://{{ ambari_host }}:{{ ambari_port }}" + - " Zookeeper @ {{ zookeeper_url }}" + - " Kafka @ {{ kafka_broker_url }}" + - For additional information, see https://metron.apache.org/' + +- debug: var=success http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/enable-swap/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/enable-swap/defaults/main.yml b/metron-deployment/ansible/roles/enable-swap/defaults/main.yml new file mode 100644 index 0000000..7190606 --- /dev/null +++ b/metron-deployment/ansible/roles/enable-swap/defaults/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +swapfile: /swapfile +swapspace: 4G
