http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst new file mode 100644 index 0000000..42a2ca7 --- /dev/null +++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# this script is executed AFTER installation of the 'metron-config' package +# comments are stripped from this file before packaging +# environment variable substitution is performed on this file before packaging +# + +# install the service script +cp -f /usr/metron/${FULL_VERSION}/bin/metron-management-ui /etc/init.d/
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm new file mode 100644 index 0000000..f9f4bcd --- /dev/null +++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# this script is executed AFTER removal of the 'metron-management' package +# comments are stripped from this file before packaging +# environment variable substitution is performed on this file before packaging +# + +rm -f /etc/init.d/metron-management-ui http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst new file mode 100644 index 0000000..3ddedc9 --- /dev/null +++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# this script is executed BEFORE installation of the 'metron-config' package +# comments are stripped from this file before packaging +# environment variable substitution is performed on this file before packaging +# +if [ -f "/etc/init.d/metron-management-ui"]; then + # if service already exists, stop it before upgrading + /etc/init.d/metron-management-ui stop +fi + +exit 0 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm new file mode 100644 index 0000000..5b2ad74 --- /dev/null +++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# this script is executed BEFORE removal of the 'metron-management' package. +# comments are stripped from this file before packaging +# environment variable substitution is performed on this file before packaging +# + +# if service exists, stop it +if [ -f "/etc/init.d/metron-management-ui"]; then + /etc/init.d/metron-management-ui stop +fi + +exit 0 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog b/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog new file mode 100755 index 0000000..cfbf6af --- /dev/null +++ b/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# this file is added to ALL metron packages +# comments are stripped from this file before packaging +# environment variable substitution is performed on this file before packaging +# + +$PACKAGE ($FULL_VERSION) $DISTRIBUTION; urgency=medium + + * Initial release. (Closes: METRON-1351) + + -- Apache Metron <d...@metron.apache.org> Wed, 13 Dec 2017 21:19:45 +0000 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron/control ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron/control b/metron-deployment/packaging/docker/deb-docker/debian/metron/control new file mode 100755 index 0000000..d7611f9 --- /dev/null +++ b/metron-deployment/packaging/docker/deb-docker/debian/metron/control @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# this file is added to ALL metron packages +# comments are stripped from this file before packaging +# environment variable substitution is performed on this file before packaging +# + +Source: metron +Section: misc +Priority: extra +Maintainer: Apache Metron <d...@metron.apache.org> +Homepage: https://metron.apache.org/ +Package: $PACKAGE +Architecture: all +Version: $FULL_VERSION +Depends: +Description: Apache Metron + Apache Metron provides a scalable advanced security analytics framework. http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright b/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright new file mode 100755 index 0000000..d7f20c1 --- /dev/null +++ b/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# this file is added to ALL metron packages +# comments are stripped from this file before packaging +# environment variable substitution is performed on this file before packaging +# + +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: $PACKAGE +Upstream-Contact: Apache Metron <d...@metron.apache.org> + +Files: * +License: ASL-2 +Copyright: +Apache Metron +Copyright 2015-2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software developed by Cisco Systems (http://www.cisco.com) +Copyright (c) 2014 Cisco Systems. + +This product includes software developed by Chef Software (https://www.chef.io) +Copyright (c) 2012-2015, Chef Software, Inc. http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/pom.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/deb-docker/pom.xml b/metron-deployment/packaging/docker/deb-docker/pom.xml index 5770e40..7fbe47b 100644 --- a/metron-deployment/packaging/docker/deb-docker/pom.xml +++ b/metron-deployment/packaging/docker/deb-docker/pom.xml @@ -49,6 +49,12 @@ <include>**/*</include> </includes> </fileset> + <fileset> + <directory>.npm</directory> + <includes> + <include>**/*</include> + </includes> + </fileset> </filesets> </configuration> </plugin> http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/rpm-docker/pom.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml index 0b7218d..4ed2edd 100644 --- a/metron-deployment/packaging/docker/rpm-docker/pom.xml +++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml @@ -85,6 +85,12 @@ <directory>SRPMS</directory> <followSymlinks>false</followSymlinks> </fileset> + <fileset> + <directory>.npm</directory> + <includes> + <include>**/*</include> + </includes> + </fileset> </filesets> </configuration> </plugin> http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/ambari_install.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/playbooks/ambari_install.yml b/metron-deployment/playbooks/ambari_install.yml deleted file mode 100644 index c0cfef3..0000000 --- a/metron-deployment/playbooks/ambari_install.yml +++ /dev/null @@ -1,61 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- hosts: ec2 - become: true - tasks: - - include_vars: ../amazon-ec2/conf/defaults.yml - tags: - - ec2 - -- hosts: packer - become: true - tasks: - - include_vars: ../inventory/full-dev-platform/group_vars/all - tags: - - packer - -- hosts: ambari_* - become: true - roles: - - role: ambari_common - tags: - - ambari-prereqs - - hdp-install - -- hosts: ambari_master - become: true - roles: - - role: ambari_master - tags: - - ambari-server - - hdp-install - -- hosts: ambari_slave - become: true - roles: - - role: ambari_slave - tags: - - ambari-agent - - hdp-install - -- hosts: metron - become: true - roles: - - role: metron-rpms - tags: - - metron-deploy http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/docker_probe_install.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/playbooks/docker_probe_install.yml b/metron-deployment/playbooks/docker_probe_install.yml deleted file mode 100644 index 7be779a..0000000 --- a/metron-deployment/playbooks/docker_probe_install.yml +++ /dev/null @@ -1,62 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -# -# sensors -# -- hosts: localhost - tasks: - - name: add container to inventory - add_host: - name: amb-server - ansible_connection: docker - groups: sensors - changed_when: false - tags: add-host - -- hosts: sensors - vars: - metron_version: 0.4.3 - metron_directory: /usr/metron/{{ metron_version }} - bro_version: "2.5.2" - fixbuf_version: "1.7.1" - yaf_version: "2.8.0" - daq_version: "2.0.6-1" - pycapa_repo: "https://github.com/OpenSOC/pycapa.git"; - pycapa_home: "/opt/pycapa" - snort_version: "2.9.8.0-1" - snort_alert_csv_path: "/var/log/snort/alert.csv" - threat_intel_bulk_load: False - sensor_test_mode: True - install_pycapa: False - install_bro: True - install_snort: True - install_yaf: True - install_pcap_replay: True - sniff_interface: eth0 - pcap_replay_interface: "{{ sniff_interface }}" - pcapservice_port: 8081 - kafka_broker_url: amb4.service.consul:6667 - connection: docker - roles: - - { role: bro, tags: ['bro'] } - - { role: snort, tags: ['snort'] } - - { role: yaf, tags: ['yaf'] } - - { role: pcap_replay, tags: ['pcap-replay'] } - - { role: sensor-test-mode, tags: ['sensor-test-mode'] } - tags: - - sensors http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/metron_build.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/playbooks/metron_build.yml b/metron-deployment/playbooks/metron_build.yml deleted file mode 100644 index 83eb352..0000000 --- a/metron-deployment/playbooks/metron_build.yml +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- hosts: all - become: false - roles: - - role: metron-builder - tags: - - build http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/metron_full_install.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/playbooks/metron_full_install.yml b/metron-deployment/playbooks/metron_full_install.yml deleted file mode 100644 index 670f5c8..0000000 --- a/metron-deployment/playbooks/metron_full_install.yml +++ /dev/null @@ -1,34 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- hosts: all - pre_tasks: - - name: Verify Ansible Version - fail: msg="Metron Requires Ansible 2.0.0.2 or 2.2.2.0, current version is {{ ansible_version }}" - when: "ansible_version.full | version_compare('2.2.2.0', '!=') and ansible_version.full | version_compare('2.0.0.2', '!=')" - -- include: metron_build.yml - tags: - - build - -- include: ambari_install.yml - tags: - - ambari - -- include: metron_install.yml - tags: - - metron http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/metron_install.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml deleted file mode 100644 index bed615e..0000000 --- a/metron-deployment/playbooks/metron_install.yml +++ /dev/null @@ -1,102 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- hosts: ec2 - become: true - tasks: - - include_vars: ../amazon-ec2/conf/defaults.yml - tags: - - ec2 - -- hosts: packer - become: true - tasks: - - include_vars: ../inventory/full-dev-platform/group_vars/all - tags: - - packer - -# -# start installation of components in Ambari -# -- hosts: ambari_master - become: true - roles: - - role: ambari_config - tags: - - hdp-install - - hdp-deploy - -- hosts: ambari_master - become: true - roles: - - role: load_web_templates - tags: - - load_templates - -- hosts: pcap_server - become: true - roles: - - role: metron_pcapservice - tags: - - pcap-service - -# -# sensors -# -- hosts: sensors - become: true - roles: - - { role: ambari_gather_facts, tags: ['always'] } - - { role: tap_interface, tags: ['tap'] } - - { role: pycapa, tags: ['pycapa'] } - - { role: bro, tags: ['bro'] } - - { role: snort, tags: ['snort'] } - - { role: yaf, tags: ['yaf'] } - - { role: pcap_replay, tags: ['pcap-replay'] } - - { role: sensor-test-mode, tags: ['sensor-test-mode'] } - tags: - - sensors - -# -# sensor stubs - rather than deploying the sensors, deploy ligher weight -# stubs that mimic the behavior of the real sensors -# -- hosts: sensors - become: true - roles: - - { role: ambari_gather_facts } - - { role: sensor-stubs } - tags: - - sensor-stubs - -# -# monitor and start metron services with monit -# -- hosts: monit - become: true - roles: - - { role: ambari_gather_facts, tags: ['always'] } - - { role: monit, tags: ['monit'] } - - { role: monit-start, tags: ['start'] } - -# -# deployment report -# -- hosts: monit - become: false - roles: - - { role: deployment-report, tags: ['report'] } http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/README.md b/metron-deployment/roles/README.md deleted file mode 100644 index 1cb5878..0000000 --- a/metron-deployment/roles/README.md +++ /dev/null @@ -1,24 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> -# Ansible Roles - -- Monit -- OpenTaxii -- Pcap Replay -- Sensor Stubs -- Sensor Test Mode http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_common/defaults/main.yml b/metron-deployment/roles/ambari_common/defaults/main.yml deleted file mode 100644 index 2c9389c..0000000 --- a/metron-deployment/roles/ambari_common/defaults/main.yml +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -hadoop_logrotate_frequency: daily -hadoop_logrotate_retention: 30 -rhel_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.2.0/ambari.repo http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_common/meta/main.yml b/metron-deployment/roles/ambari_common/meta/main.yml deleted file mode 100644 index d7e46d6..0000000 --- a/metron-deployment/roles/ambari_common/meta/main.yml +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -dependencies: - - libselinux-python - - epel - - ntp http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_common/tasks/main.yml b/metron-deployment/roles/ambari_common/tasks/main.yml deleted file mode 100644 index 011c05f..0000000 --- a/metron-deployment/roles/ambari_common/tasks/main.yml +++ /dev/null @@ -1,54 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Check OS Version - fail: msg="Ambari HDP deployment supports CentOS 6 only." - when: (ansible_distribution != "CentOS" or ansible_distribution_major_version != "6") - -- name: Ensure iptables is stopped and is not running at boot time. - ignore_errors: yes - service: name=iptables state=stopped enabled=no - -# -# ambari uses socket.getfqdn() to find the hostname. with 'localhost.localdomain' -# in '/etc/hosts' this function will report the hostname as 'localhost.localdomain' -# rather than 'node1' as would be expected. other functions like socket.gethostname() -# will always return 'node1' as expected. ambari needs to see 'node1' to be able to -# communicate between the master and agents. - -- name: Remove ipv4 'localhost.localdomain' from /etc/hosts - lineinfile: dest=/etc/hosts state=absent regexp="^127.0.0.1(.*)localdomain(.*)$" - -- name: Remove ipv6 'localhost.localdomain' from /etc/hosts - lineinfile: dest=/etc/hosts state=absent regexp="^::1(.*)localdomain(.*)$" - -- name: Add localhost to /etc/hosts - lineinfile: dest=/etc/hosts line="127.0.0.1 localhost" - -- name: Download Ambari repo - get_url: url="{{ rhel_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo - -- name: Create Logrotate Script for Hadoop Services - template: - src: "metron-hadoop-logrotate.yml" - dest: "/etc/logrotate.d/metron-ambari" - mode: 0644 - -- name: Install Nodejs repository - shell: curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - - args: - warn: false http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml b/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml deleted file mode 100644 index d95c10b..0000000 --- a/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml +++ /dev/null @@ -1,149 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#Hadoop HDFS Logs -/var/log/hadoop/hdfs/*.log* { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -/var/log/hadoop/hdfs/*.out { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -/var/log/hadoop/hdfs/*.audit { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -#Hadoop Yarn Logs -/var/log/hadoop/yarn/*.log { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -#Hadoop Mapreduce Logs -/var/log/hadoop/mapreduce/*.log { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -#Storm Logs -/var/log/storm/*.log { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -/var/log/storm/*.out { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -#Kafka Logs -/var/log/kafka/*.log { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -/var/log/kafka/*.err { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -#HBase Logs -/var/log/hbase/*.log* { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -/var/log/hbase/*.out { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -/var/log/hbase/*.audit { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -#Zookeeper Logs -/var/log/zookeeper/*.log { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} - -/var/log/zookeeper/*.out { - {{ hadoop_logrotate_frequency }} - rotate {{ hadoop_logrotate_retention }} - missingok - notifempty - copytruncate - compress -} http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/defaults/main.yml b/metron-deployment/roles/ambari_config/defaults/main.yml deleted file mode 100644 index e0de145..0000000 --- a/metron-deployment/roles/ambari_config/defaults/main.yml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -zookeeper_data_dir: /hadoop/zookeeper -namenode_checkpoint_dir: /hadoop/hdfs/namesecondary -namenode_name_dir: /hadoop/hdfs/namenode -datanode_data_dir: /hadoop/hdfs/data -journalnode_edits_dir: /hadoop/hdfs/journalnode -jhs_recovery_store_ldb_path: /hadoop/mapreduce/jhs -nodemanager_local_dirs: /hadoop/yarn/local -timeline_ldb_store_path: /hadoop/yarn/timeline -timeline_ldb_state_path: /hadoop/yarn/timeline -nodemanager_log_dirs: /hadoop/yarn/log -storm_local_dir: /hadoop/storm -kafka_log_dirs: /kafka-log -cluster_type: small_cluster -nodemanager_mem_mb : 4096 -mapred_map_java_opts : -Xmx1024m -mapred_reduce_java_opts : -Xmx1024m -mapred_map_mem_mb : 1229 -mapred_reduce_mem_mb : 1229 -topology_classpath: '/etc/hbase/conf:/etc/hadoop/conf' -hdp_stack: "2.5" -elasticsearch_network_interface: _site_ http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/meta/main.yml b/metron-deployment/roles/ambari_config/meta/main.yml deleted file mode 100644 index 8f65a28..0000000 --- a/metron-deployment/roles/ambari_config/meta/main.yml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -dependencies: - - epel - - python-pip - - httplib2 - - java_jdk http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/tasks/main.yml b/metron-deployment/roles/ambari_config/tasks/main.yml deleted file mode 100644 index 6588d4a..0000000 --- a/metron-deployment/roles/ambari_config/tasks/main.yml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- include_vars: "{{ cluster_type }}.yml" - -- name: Install python-requests - yum: - name: python-requests - state: installed - register: result - until: result.rc == 0 - retries: 5 - delay: 10 - -- name : Wait for Ambari to start; http://{{ ambari_host }}:{{ ambari_port }} - wait_for : - host: "{{ ambari_host }}" - port: "{{ ambari_port }}" - timeout: 600 - -- name: Deploy cluster with Ambari; http://{{ ambari_host }}:{{ ambari_port }} - ambari_cluster_state: - host: "{{ ambari_host }}" - port: "{{ ambari_port }}" - username: "{{ ambari_user }}" - password: "{{ ambari_password }}" - cluster_name: "{{ cluster_name }}" - cluster_state: present - blueprint_name: "{{ blueprint_name }}" - configurations: "{{ configurations }}" - wait_for_complete: True - blueprint_var: "{{ blueprint }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/vars/single_node_vm.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml deleted file mode 100644 index 6a60902..0000000 --- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml +++ /dev/null @@ -1,135 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -# vars file for single_node_vm blueprint - -hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER] -hadoop_slave: [APP_TIMELINE_SERVER, DATANODE, HDFS_CLIENT, NODEMANAGER, YARN_CLIENT, MAPREDUCE2_CLIENT] -spark_master: [SPARK_JOBHISTORYSERVER] -spark_slave: [SPARK_CLIENT] -storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER] -storm_slave: [SUPERVISOR] -kafka_broker: [KAFKA_BROKER] -zookeeper_master: [ZOOKEEPER_SERVER] -zookeeper_slave: [ZOOKEEPER_CLIENT] -hbase_master: [HBASE_MASTER, HBASE_CLIENT] -hbase_slave: [HBASE_REGIONSERVER] -es_master: [ES_MASTER] -kibana_master: [KIBANA_MASTER] -metron_indexing: [METRON_INDEXING] -metron_profiler: [METRON_PROFILER] -metron_enrichment_master : [METRON_ENRICHMENT_MASTER] -metron_parsers : [METRON_PARSERS] -metron_rest: [METRON_REST] -metron_management_ui: [METRON_MANAGEMENT_UI] -metron_alerts_ui: [METRON_ALERTS_UI] - -metron_components: > - {{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | - union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | union(metron_profiler) | - union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(es_master) }} - -cluster_name: "metron_cluster" -blueprint_name: "metron_blueprint" - -configurations: - - zoo.cfg: - dataDir: '{{ zookeeper_data_dir }}' - - hadoop-env: - hadoop_heapsize: 1024 - namenode_heapsize: 2048 - dtnode_heapsize: 512 - namenode_opt_permsize: 128m - - hbase-env: - hbase_regionserver_heapsize: 512 - hbase_master_heapsize: 512 - hbase_regionserver_xmn_max: 512 - - hdfs-site: - dfs.replication: 1 - dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir }}' - dfs.namenode.name.dir: '{{ namenode_name_dir }}' - dfs.datanode.data.dir: '{{ datanode_data_dir }}' - dfs.journalnode.edits.dir: '{{ journalnode_edits_dir }}' - - yarn-env: - nodemanager_heapsize: 512 - yarn_heapsize: 512 - apptimelineserver_heapsize : 512 - resourcemanager_heapsize: 1024 - - mapred-env: - jobhistory_heapsize: 256 - - mapred-site: - mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path }}' - mapreduce.map.java.opts : '{{ mapred_map_java_opts }}' - mapreduce.reduce.java.opts : '{{ mapred_reduce_java_opts }}' - mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}' - mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}' - - yarn-site: - yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs }}' - yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path }}' - yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path }}' - yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs }}' - yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}' - - storm-site: - supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]" - storm.local.dir: '{{ storm_local_dir }}' - topology.classpath: '{{ topology_classpath }}' - - kafka-env: - content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}" - - kafka-broker: - log.dirs: '{{ kafka_log_dirs }}' - delete.topic.enable: "true" - - metron-rest-env: - metron_spring_profiles_active: "dev" - - metron-parsers-env: - parsers: "bro,snort" - - elastic-site: - index_number_of_shards: 1 - index_number_of_replicas: 0 - zen_discovery_ping_unicast_hosts: "[ {{ groups.search | join(', ') }} ]" - gateway_recover_after_data_nodes: 1 - network_host: "[ _local_, {{ elasticsearch_network_interface }} ]" - masters_also_are_datanodes: "1" - -required_configurations: - - metron-env: - storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744" - es_hosts: "{{ groups.search | join(',') }}" - zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995" - - metron-rest-env: - metron_jdbc_driver: "org.h2.Driver" - metron_jdbc_url: "jdbc:h2:file:~/metrondb" - metron_jdbc_username: "root" - metron_jdbc_password: "root" - metron_jdbc_platform: "h2" - - kibana-env: - kibana_pid_dir: /var/run/kibana - kibana_es_url: http://{{ groups.search[0] }}:9200 - kibana_log_dir: /var/log/kibana - kibana_server_port: 5000 - kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9" - kibana_server_host: 0.0.0.0 - -blueprint: - stack_name: HDP - stack_version: "{{ hdp_stack }}" - required_configurations: "{{ required_configurations }}" - groups: - - name : host_group_1 - cardinality: 1 - configurations: [] - components: "{{ metron_components }}" - hosts: "{{ hdp_host_group }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/vars/small_cluster.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml deleted file mode 100644 index 4ec8458..0000000 --- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml +++ /dev/null @@ -1,149 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- - -hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER] -app_timeline_server: [APP_TIMELINE_SERVER] -hadoop_slave: [DATANODE, NODEMANAGER] -spark_master: [SPARK_JOBHISTORYSERVER] -storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER] -storm_slave: [SUPERVISOR] -kafka_broker: [KAFKA_BROKER] -zookeeper_master: [ZOOKEEPER_SERVER] -hbase_master: [HBASE_MASTER] -hbase_slave: [HBASE_REGIONSERVER] -hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT] -es_master: [ES_MASTER] -es_slave: [ES_SLAVE] -kibana_master: [KIBANA_MASTER] -metron_indexing: [METRON_INDEXING] -metron_profiler: [METRON_PROFILER] -metron_enrichment_master : [METRON_ENRICHMENT_MASTER] -metron_parsers : [METRON_PARSERS] -metron_rest: [METRON_REST] -metron_management_ui: [METRON_MANAGEMENT_UI] -metron_alerts_ui: [METRON_ALERTS_UI] - -master_1_components: "{{ hadoop_master | union(hadoop_clients) | union(es_slave) }}" -master_1_host: - - "{{groups.ambari_slave[0]}}" -master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) | union(app_timeline_server) | union(es_slave) }}" -master_2_host: - - "{{groups.ambari_slave[1]}}" -metron_components: > - {{ metron_indexing | union(metron_profiler) | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) | union(storm_slave) | - union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }} -metron_host: - - "{{ groups.metron[0] }}" -web_components: "{{ kibana_master | union(es_master) }}" -web_host: - - "{{ groups.web[0] }}" -slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}" - -cluster_name: "metron" -blueprint_name: "metron_blueprint" - -configurations: - - zoo.cfg: - dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}' - - hadoop-env: - namenode_heapsize: 2048 - dtnode_heapsize: 1024 - - hbase-env: - hbase_regionserver_heapsize: 1024 - hbase_master_heapsize: 1024 - - hdfs-site: - dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}' - dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}' - dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}' - dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}' - - mapred-site: - mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}' - mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}' - mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}' - - yarn-site: - yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}' - yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}' - yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}' - yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}' - yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}' - - storm-site: - supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]" - storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}' - topology.classpath: '{{ topology_classpath }}' - - kafka-broker: - log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}' - - metron-rest-env: - metron_spring_profiles_active: "dev" - - metron-env: - parsers: "bro,snort,yaf" - - elastic-site: - index_number_of_shards: 2 - index_number_of_replicas: 1 - zen_discovery_ping_unicast_hosts: "[ {{ groups.web[0] }}, {{ groups.search | join(', ') }} ]" - gateway_recover_after_data_nodes: 1 - network_host: "[ _local_, {{ elasticsearch_network_interface }} ]" - -required_configurations: - - metron-env: - storm_rest_addr: "http://{{ groups.ambari_slave[1] }}:8744" - es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}" - zeppelin_server_url: "{{ groups.zeppelin[0] }}" - - metron-rest-env: - metron_jdbc_driver: "org.h2.Driver" - metron_jdbc_url: "jdbc:h2:file:~/metrondb" - metron_jdbc_username: "root" - metron_jdbc_password: "root" - metron_jdbc_platform: "h2" - - kibana-env: - kibana_pid_dir: /var/run/kibana - kibana_es_url: http://{{ groups.web[0] }}:9200 - kibana_log_dir: /var/log/kibana - kibana_server_port: 5000 - kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9" - kibana_server_host: 0.0.0.0 - -blueprint: - stack_name: HDP - stack_version: "{{ hdp_stack }}" - required_configurations: "{{ required_configurations }}" - groups: - - name : master_1 - cardinality: 1 - configuration: [] # configuration not yet implemented - components: "{{ master_1_components }}" - hosts: "{{ master_1_host }}" - - name : master_2 - cardinality: 1 - configuration: [] # configuration not yet implemented - components: "{{ master_2_components }}" - hosts: "{{ master_2_host }}" - - name : metron - cardinality: 1 - configuration: [] # configuration not yet implemented - components: "{{ metron_components }}" - hosts: "{{ metron_host }}" - - name : web - cardinality: 1 - configuration: [] # configuration not yet implemented - components: "{{ web_components }}" - hosts: "{{ web_host }}" - - name: slaves - cardinality: 1+ - configuration: [] # configuration not yet implemented - components: "{{ slave_components }}" - hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) | difference(groups.metron[0]) | difference(groups.web[0]) }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_gather_facts/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_gather_facts/meta/main.yml b/metron-deployment/roles/ambari_gather_facts/meta/main.yml deleted file mode 100644 index 61197e3..0000000 --- a/metron-deployment/roles/ambari_gather_facts/meta/main.yml +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -dependencies: - - epel - - python-pip - - httplib2 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_gather_facts/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_gather_facts/tasks/main.yml b/metron-deployment/roles/ambari_gather_facts/tasks/main.yml deleted file mode 100644 index 2b37eec..0000000 --- a/metron-deployment/roles/ambari_gather_facts/tasks/main.yml +++ /dev/null @@ -1,234 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -# -# cluster_name -# -- name: "Ask Ambari: cluster_name" - uri: - url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters" - user: "{{ ambari_user }}" - password: "{{ ambari_password }}" - force_basic_auth: yes - return_content: yes - register: cluster_name_response - when: cluster_name is undefined - -- set_fact: - cluster_name: "{{ (cluster_name_response.content | from_json)['items'][0].Clusters.cluster_name }}" - when: cluster_name is undefined - -# -# namenode_host -# -- name: "Ask Ambari: namenode_host" - uri: - url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/HDFS/components/NAMENODE" - user: "{{ ambari_user }}" - password: "{{ ambari_password }}" - force_basic_auth: yes - return_content: yes - register: namenode_hosts_response - when: namenode_host is undefined - -- set_fact: - namenode_host: "{{ (namenode_hosts_response.content | from_json).host_components[0].HostRoles.host_name }}" - when: namenode_host is undefined - -# -# core_site_tag -# -- name: "Ask Ambari: core_site_tag" - uri: - url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ namenode_host }}/host_components/NAMENODE" - user: "{{ ambari_user }}" - password: "{{ ambari_password }}" - force_basic_auth: yes - return_content: yes - register: core_site_tag_response - when: core_site_tag is undefined - -- set_fact: - core_site_tag: "{{ (core_site_tag_response.content | from_json).HostRoles.actual_configs['core-site'].default }}" - when: core_site_tag is undefined - -# -# hdfs_url -# -- name: "Ask Ambari: hdfs_url" - uri: - url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=core-site&tag={{ core_site_tag }}" - user: "{{ ambari_user }}" - password: "{{ ambari_password }}" - force_basic_auth: yes - return_content: yes - register: core_site_response - when: hdfs_url is undefined - -- set_fact: - hdfs_url: "{{ (core_site_response.content | from_json)['items'][0].properties['fs.defaultFS'] }}" - when: hdfs_url is undefined - -# -# kafka_broker_hosts -# -- name: "Ask Ambari: kafka_broker_hosts" - uri: - url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KAFKA/components/KAFKA_BROKER" - user: "{{ ambari_user }}" - password: "{{ ambari_password }}" - force_basic_auth: yes - return_content: yes - register: kafka_broker_hosts_response - when: kafka_broker_hosts is undefined - -- set_fact: - kafka_broker_hosts: "{{ (kafka_broker_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}" - when: kafka_broker_hosts is undefined - -# -# kafka_broker_tag -# -- name: "Ask Ambari: kafka_broker_tag" - uri: - url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ kafka_broker_hosts[0] }}/host_components/KAFKA_BROKER" - user: "{{ ambari_user }}" - password: "{{ ambari_password }}" - force_basic_auth: yes - return_content: yes - register: kafka_broker_tag_response - when: kafka_broker_tag is undefined - -- set_fact: - kafka_broker_tag: "{{ (kafka_broker_tag_response.content | from_json).HostRoles.actual_configs['kafka-broker'].default }}" - when: kafka_broker_tag is undefined - -# -# kafka_broker_port -# -- name: "Ask Ambari: kafka_broker_port" - shell: > - curl -s -u {{ ambari_user }}:{{ ambari_password }} -X GET -H "X-Requested-By: ambari" "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=kafka-broker&tag={{ kafka_broker_tag }}" | python -c 'import sys, json; print json.load(sys.stdin)["items"][0]["properties"]["listeners"]' - args: - warn: false - register: kafka_broker_port_response - when: (kafka_broker_url is undefined) or (kafka_broker_port is undefined) - -- set_fact: - kafka_broker_port: "{{ kafka_broker_port_response.stdout_lines[0] | replace('PLAINTEXT://localhost:', '')}}" - when: kafka_broker_port is undefined - -- set_fact: - kafka_broker_url: "{% for host in kafka_broker_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ kafka_broker_port }}{% endfor %}" - when: kafka_broker_url is undefined - -# -# zookeeper_hosts -# -- name: "Ask Ambari: zookeeper_hosts" - uri: - url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/ZOOKEEPER/components/ZOOKEEPER_SERVER" - user: "{{ ambari_user }}" - password: "{{ ambari_password }}" - force_basic_auth: yes - return_content: yes - register: zookeeper_hosts_response - when: zookeeper_hosts is undefined - -- set_fact: - zookeeper_hosts: "{{ (zookeeper_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}" - when: zookeeper_hosts is undefined - -# -# zookeeper_tag -# -- name: "Ask Ambari: zookeeper_tag" - uri: - url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ zookeeper_hosts[0] }}/host_components/ZOOKEEPER_SERVER" - user: "{{ ambari_user }}" - password: "{{ ambari_password }}" - force_basic_auth: yes - return_content: yes - register: zookeeper_tag_response - when: zookeeper_tag is undefined - -- set_fact: - zookeeper_tag: "{{ (zookeeper_tag_response.content | from_json).HostRoles.actual_configs['zoo.cfg'].default }}" - when: zookeeper_tag is undefined - -# -# zookeeper_url, zookeeper_port -# -- name: "Ask Ambari: zookeeper_url, zookeeper_port" - uri: - url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=zoo.cfg&tag={{ zookeeper_tag }}" - user: "{{ ambari_user }}" - password: "{{ ambari_password }}" - force_basic_auth: yes - return_content: yes - register: zookeeper_properties_response - when: zookeeper_url is undefined or zookeeper_port is undefined - -- set_fact: - zookeeper_port: "{{ (zookeeper_properties_response.content | from_json)['items'][0].properties['clientPort'] }}" - when: zookeeper_port is undefined - -- set_fact: - zookeeper_url: "{% for host in zookeeper_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ zookeeper_port }}{% endfor %}" - when: zookeeper_url is undefined - -- name: "Ask Ambari: metron_hosts" - uri: - url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/METRON/components/METRON_INDEXING" - user: "{{ ambari_user }}" - password: "{{ ambari_password }}" - force_basic_auth: yes - return_content: yes - register: metron_hosts_response - when: metron_hosts is undefined - -- set_fact: - metron_hosts: "{{ (metron_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}" - when: metron_hosts is undefined - -- name: "Ask Ambari: kibana_hosts" - uri: - url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KIBANA/components/KIBANA_MASTER" - user: "{{ ambari_user }}" - password: "{{ ambari_password }}" - force_basic_auth: yes - return_content: yes - register: kibana_hosts_response - when: kibana_hosts is undefined - -- set_fact: - kibana_hosts: "{{ (kibana_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}" - when: kibana_hosts is undefined - - -# -# debug output -# -- name: debug - debug: - msg: "zookeeper_port = {{ zookeeper_port }}, - zookeeper_hosts = {{ zookeeper_hosts }}, - zookeeper_url = {{ zookeeper_url }}, - kafka_broker_port = {{ kafka_broker_port }}, - kafka_broker_hosts = {{ kafka_broker_hosts }}, - kafka_broker_url = {{ kafka_broker_url }}, - metron_hosts = {{ metron_hosts }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_master/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_master/defaults/main.yml b/metron-deployment/roles/ambari_master/defaults/main.yml deleted file mode 100644 index bc6c08c..0000000 --- a/metron-deployment/roles/ambari_master/defaults/main.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -ambari_server_mem: 2048 -ambari_mpack_version: 0.4.3.0 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_master/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_master/tasks/main.yml b/metron-deployment/roles/ambari_master/tasks/main.yml deleted file mode 100644 index bc8e837..0000000 --- a/metron-deployment/roles/ambari_master/tasks/main.yml +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -# tasks file for ambari_master -- name: Install ambari server - yum: - name: ambari-server - register: result - until: result.rc == 0 - retries: 5 - delay: 10 - -- name: Set Ambari Server Max Memory - replace: - dest: /var/lib/ambari-server/ambari-env.sh - regexp: "\ -Xmx2048m\ " - replace: " -Xmx{{ ambari_server_mem }}m " - backup: no - -- name: Setup ambari server - shell: ambari-server setup -s && touch /etc/ambari-server/configured creates=/etc/ambari-server/configured - register: ambari_server_setup - failed_when: "(ambari_server_setup.stderr is defined and ambari_server_setup.stderr != '') or 'FATAL' in ambari_server_setup.stdout" - -- name: Copy MPack to Ambari Host - copy: - src: "{{ playbook_dir }}/../packaging/ambari/metron-mpack/target/metron_mpack-{{ ambari_mpack_version }}.tar.gz" - dest: /tmp - -- name: Install MPack on Ambari Host - shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-0.4.3.0.tar.gz - args: - creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-{{ ambari_mpack_version }}/addon-services - -- name: start ambari server - service: - name: ambari-server - state: restarted http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_slave/defaults/main.yml b/metron-deployment/roles/ambari_slave/defaults/main.yml deleted file mode 100644 index f3cb4c5..0000000 --- a/metron-deployment/roles/ambari_slave/defaults/main.yml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -ambari_installation_user: "root" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/files/hostname.sh ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_slave/files/hostname.sh b/metron-deployment/roles/ambari_slave/files/hostname.sh deleted file mode 100644 index cc8c1cd..0000000 --- a/metron-deployment/roles/ambari_slave/files/hostname.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -echo {{ inventory_hostname }} http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_slave/meta/main.yml b/metron-deployment/roles/ambari_slave/meta/main.yml deleted file mode 100644 index ddf6aa9..0000000 --- a/metron-deployment/roles/ambari_slave/meta/main.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -dependencies: - - java_jdk http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_slave/tasks/main.yml b/metron-deployment/roles/ambari_slave/tasks/main.yml deleted file mode 100644 index ca81a7a..0000000 --- a/metron-deployment/roles/ambari_slave/tasks/main.yml +++ /dev/null @@ -1,56 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- include: ../roles/metron-rpms/tasks/create_directory.yml -- include: ../roles/metron-rpms/tasks/create_repo.yml - -- name: Install ambari-agent - yum: - name: ambari-agent - state: installed - register: result - until: result.rc == 0 - retries: 5 - delay: 10 - -- name: Create ambari-agent hostname script - template: - src: "../roles/ambari_slave/files/hostname.sh" - dest: "/var/lib/ambari-agent/hostname.sh" - mode: 0744 - owner: "{{ ambari_installation_user }}" - group: "{{ ambari_installation_user }}" - -- name: Configure ambari-server hostname in ambari-agent configuration - lineinfile: - dest: /etc/ambari-agent/conf/ambari-agent.ini - regexp: "{{ item.regexp }}" - line: "{{ item.line }}" - insertafter: "{{ item.insertafter }}" - backup: yes - with_items: - - { regexp: "^.*hostname=.*$", line: "hostname={{ groups.ambari_master[0] }}", insertafter: '\[server\]' } - - { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'} - -- name: Ensure ambari-agent is running - service: - name: ambari-agent - state: restarted - enabled: yes - -- name : Wait for agent to register - command : sleep 10 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/bro/meta/main.yml b/metron-deployment/roles/bro/meta/main.yml deleted file mode 100644 index df226e7..0000000 --- a/metron-deployment/roles/bro/meta/main.yml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -dependencies: - - libselinux-python - - build-tools - - kafka-client - - librdkafka http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/bro.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/bro/tasks/bro.yml b/metron-deployment/roles/bro/tasks/bro.yml deleted file mode 100644 index 222ef0e..0000000 --- a/metron-deployment/roles/bro/tasks/bro.yml +++ /dev/null @@ -1,59 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Download bro - get_url: - url: "https://www.bro.org/downloads/bro-{{ bro_version }}.tar.gz" - dest: "/tmp/bro-{{ bro_version }}.tar.gz" - -- name: Extract bro tarball - unarchive: - src: "/tmp/bro-{{ bro_version }}.tar.gz" - dest: /tmp - copy: no - creates: "/tmp/bro-{{ bro_version }}" - -- name: Compile and Install bro - shell: "{{ item }}" - environment: - CXX: /opt/rh/devtoolset-4/root/usr/bin/g++ - CC: /opt/rh/devtoolset-4/root/usr/bin/gcc - args: - chdir: "/tmp/bro-{{ bro_version }}" - creates: "{{ bro_home }}/bin/bro" - with_items: - - "./configure --prefix={{ bro_home }}" - - make - - make install - -- name: Configure bro - lineinfile: - dest: "{{ bro_home }}/etc/node.cfg" - regexp: '^interface=.*$' - line: 'interface={{ sniff_interface }}' - -- name: Install bro configuration - shell: "{{ bro_home }}/bin/broctl install" - -- name: Bro Cronjob - cron: - name: Bro Cron - minute: "{{ bro_crontab_minutes }}" - job: "{{ item }}" - with_items: - - "{{ bro_crontab_job }}" - - "{{ bro_clean_job }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/dependencies.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/bro/tasks/dependencies.yml b/metron-deployment/roles/bro/tasks/dependencies.yml deleted file mode 100644 index 3cd3bae..0000000 --- a/metron-deployment/roles/bro/tasks/dependencies.yml +++ /dev/null @@ -1,48 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Install yum repositories - yum: name={{ item }} update_cache=yes - with_items: - - centos-release-scl - -- name: Install prerequisites - yum: name={{ item }} - with_items: - - cmake - - make - - gcc - - gcc-c++ - - flex - - bison - - libpcap - - libpcap-devel - - openssl-devel - - python-devel - - swig - - zlib-devel - - perl - - crontabs - - net-tools - - devtoolset-4-gcc - - devtoolset-4-gcc-c++ - - python27 - - rh-git29 - register: result - until: result.rc == 0 - retries: 5 - delay: 10 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/librdkafka.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/bro/tasks/librdkafka.yml b/metron-deployment/roles/bro/tasks/librdkafka.yml deleted file mode 100644 index 652d319..0000000 --- a/metron-deployment/roles/bro/tasks/librdkafka.yml +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Download librdkafka - get_url: - url: "{{ librdkafka_url }}" - dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz" - -- name: Extract librdkafka tarball - unarchive: - src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz" - dest: /tmp - copy: no - creates: "/tmp/librdkafka-{{ librdkafka_version }}" - -- name: Compile and install librdkafka - shell: "{{ item }}" - args: - chdir: "/tmp/librdkafka-{{ librdkafka_version }}" - creates: "{{ librdkafka_home }}/lib/librdkafka.so" - with_items: - - rm -rf build/ - - "./configure --prefix={{ librdkafka_home }}" - - make - - make install http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/bro/tasks/main.yml b/metron-deployment/roles/bro/tasks/main.yml deleted file mode 100644 index 440e4b5..0000000 --- a/metron-deployment/roles/bro/tasks/main.yml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- include: dependencies.yml -- include: librdkafka.yml -- include: bro.yml -- include: metron-bro-plugin-kafka.yml -- include: nic.yml http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml b/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml deleted file mode 100644 index 7043387..0000000 --- a/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml +++ /dev/null @@ -1,49 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Install bro-pkg - environment: - LD_LIBRARY_PATH: "{{ python27_lib }}" - command: "{{ python27_bin }}/pip install bro-pkg" - -- name: Configure bro-pkg - environment: - PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}" - LD_LIBRARY_PATH: "{{ python27_lib }}" - command: "{{ python27_bin }}/bro-pkg autoconfig" - -- name: Install the metron-bro-plugin-kafka package - environment: - PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}" - LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}" - CXX: /opt/rh/devtoolset-4/root/usr/bin/g++ - CC: /opt/rh/devtoolset-4/root/usr/bin/gcc - command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka --force --version {{ metron_bro_plugin_kafka_version }}" - -- name: Configure bro-kafka plugin - lineinfile: - dest: "{{ bro_home }}/share/bro/site/local.bro" - line: "{{ item }}" - with_items: - - "@load packages" - - "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);" - - "redef Kafka::topic_name = \"{{ bro_topic }}\";" - - "redef Kafka::tag_json = T;" - - "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] = \"{{ kafka_broker_url }}\");" - -- name: Deploy bro configuration changes - shell: "{{ bro_bin }}/broctl deploy" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/nic.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/bro/tasks/nic.yml b/metron-deployment/roles/bro/tasks/nic.yml deleted file mode 100644 index 6053618..0000000 --- a/metron-deployment/roles/bro/tasks/nic.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Turn on promiscuous mode for {{ sniff_interface }} - shell: "ip link set {{ sniff_interface }} promisc on"
