http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/yaf/templates/yaf ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/yaf/templates/yaf b/metron-deployment/ansible/roles/yaf/templates/yaf new file mode 100644 index 0000000..18bc4ac --- /dev/null +++ b/metron-deployment/ansible/roles/yaf/templates/yaf @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# yaf daemon +# chkconfig: 345 20 80 +# description: Runs yaf - yet another flowmeter +# processname: yaf +# +NAME=yaf +DESC="Executes yaf - yet another flowmeter" +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +DAEMON_PATH="{{ yaf_home }}" +DAEMON="{{ yaf_start }}" +DAEMONOPTS="${@:2}" + +case "$1" in + start) + printf "%-50s" "Starting $NAME..." + + # kick-off the daemon + cd $DAEMON_PATH + PID=`$DAEMON $DAEMONOPTS > /dev/null 2>&1 & echo $!` + if [ -z $PID ]; then + printf "%s\n" "Fail" + else + echo $PID > $PIDFILE + printf "%s\n" "Ok" + fi + ;; + + status) + printf "%-50s" "Checking $NAME..." + if [ -f $PIDFILE ]; then + PID=`cat $PIDFILE` + if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then + printf "%s\n" "Process dead but pidfile exists" + else + echo "Running" + fi + else + printf "%s\n" "Service not running" + fi + ;; + + stop) + printf "%-50s" "Stopping $NAME" + PID=`cat $PIDFILE` + cd $DAEMON_PATH + if [ -f $PIDFILE ]; then + kill -HUP $PID + killall $NAME + printf "%s\n" "Ok" + rm -f $PIDFILE + else + printf "%s\n" "pidfile not found" + fi + ;; + + restart) + $0 stop + $0 start + ;; + + *) + echo "Usage: $0 {status|start|stop|restart}" + exit 1 +esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/development/README.md b/metron-deployment/development/README.md new file mode 100644 index 0000000..bc99809 --- /dev/null +++ b/metron-deployment/development/README.md @@ -0,0 +1,24 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +# Metron Development Environments + +This directory contains environments useful for Metron developers. These environments are not intended for proof-of-concept, testing, or production use. These are extremely resource constrained and cannot support anything beyond the most basic work loads. + +* Metron running on CentOS 6 +* Metron running on Ubuntu 14 +* Fastcapa http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/.gitignore ---------------------------------------------------------------------- diff --git a/metron-deployment/development/centos6/.gitignore b/metron-deployment/development/centos6/.gitignore new file mode 100644 index 0000000..8000dd9 --- /dev/null +++ b/metron-deployment/development/centos6/.gitignore @@ -0,0 +1 @@ +.vagrant http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md new file mode 100644 index 0000000..5132c30 --- /dev/null +++ b/metron-deployment/development/centos6/README.md @@ -0,0 +1,105 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +Metron on CentOS 6 +================== + +This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running CentOS 6. + +Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. + +Getting Started +--------------- + +### Prerequisites + +The computer used to deploy Apache Metron will need to have the following components installed. + + - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0) + - [Docker](https://www.docker.com/community-edition) + - [Vagrant](https://www.vagrantup.com) 1.8+ + - [Vagrant Hostmanager Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager) + - [Virtualbox](https://virtualbox.org) 5.0+ + - Python 2.7 + - Maven 3.3.9 + - C++11 compliant compiler, like [GCC](https://gcc.gnu.org/projects/cxx-status.html#cxx11) + +Running the following script can help validate whether you have all the prerequisites installed and running correctly. + + ``` + metron-deployment/scripts/platform-info.sh + ``` + +#### How do I install these on MacOS? + +Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent [Homebrew](http://brew.sh/) project. + +1. Install Homebrew by following the instructions at [Homebrew](http://brew.sh/). + +1. Run the following command in a terminal to install all of the required tools. + + ``` + brew cask install vagrant virtualbox docker + brew cask install caskroom/versions/java8 + brew install [email protected] git + pip install ansible==2.2.2.0 + vagrant plugin install vagrant-hostmanager + open /Applications/Docker.app + ``` + +### Deploy Metron + +1. Ensure that the Docker service is running. + +1. Deploy Metron + + ``` + cd metron-deployment/development/centos6 + vagrant up + ``` + + Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host. + + ``` + vagrant provision + ``` + +### Explore Metron + +Navigate to the following resources to explore your newly minted Apache Metron environment. + +* [Metron Alerts](http://node1:4201) +* [Ambari](http://node1:8080) + +Connecting to the host through SSH is as simple as running the following command. +``` +vagrant ssh +``` + +### Working with Metron + +In addition to re-running the entire provisioning play book, you may now re-run an individual Ansible tag or a collection of tags in the following ways. The following commands will re-run the `sensor-stubs` role on the Vagrant image. This will install and start the sensor stub components. + +``` +vagrant --ansible-tags="sensor-stubs" provision +``` + +Tags are listed in the playbooks, some frequently used tags: ++ `hdp-install` - Install HDP ++ `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services) ++ `sensors` - Deploy and start the sensors. ++ `sensor-stubs` - Deploy and start the sensor stubs. http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/Vagrantfile ---------------------------------------------------------------------- diff --git a/metron-deployment/development/centos6/Vagrantfile b/metron-deployment/development/centos6/Vagrantfile new file mode 100644 index 0000000..ed74b76 --- /dev/null +++ b/metron-deployment/development/centos6/Vagrantfile @@ -0,0 +1,92 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require 'getoptlong' + +ansibleTags='' +ansibleSkipTags='sensors' + +begin + opts = GetoptLong.new( + [ '--ansible-tags', GetoptLong::OPTIONAL_ARGUMENT ], + [ '--ansible-skip-tags', GetoptLong::OPTIONAL_ARGUMENT ] + ) + + opts.quiet = TRUE + + opts.each do |opt, arg| + case opt + when '--ansible-tags' + ansibleTags=arg + when '--ansible-skip-tags' + ansibleSkipTags=arg + end + end +rescue Exception => ignored +#Ignore to allow other opts to be passed to Vagrant +end + +puts " Running with ansible-tags: " + ansibleTags.split(",").to_s if ansibleTags != '' +puts " Running with ansible-skip-tags: " + ansibleSkipTags.split(",").to_s if ansibleSkipTags != '' + +hosts = [{ + hostname: "node1", + ip: "192.168.66.121", + memory: "8192", + cpus: 4, + promisc: 2 # enables promisc on the 'Nth' network interface +}] + +Vagrant.configure(2) do |config| + + # all hosts built on centos 6 + config.vm.box = "metron/centos_base" + config.ssh.insert_key = true + + # enable the hostmanager plugin + config.hostmanager.enabled = true + config.hostmanager.manage_host = true + + # host definition + hosts.each_with_index do |host, index| + config.vm.define host[:hostname] do |node| + + # host settings + node.vm.hostname = host[:hostname] + node.vm.network "private_network", ip: host[:ip] + + # vm settings + node.vm.provider "virtualbox" do |vb| + vb.memory = host[:memory] + vb.cpus = host[:cpus] + + # enable promisc mode on the network interface + if host.has_key?(:promisc) + vb.customize ["modifyvm", :id, "--nicpromisc#{host[:promisc]}", "allow-all"] + end + end + end + end + + # provision the host with ansible + config.vm.provision :ansible do |ansible| + ansible.playbook = "../../ansible/playbooks/metron_full_install.yml" + ansible.sudo = true + ansible.tags = ansibleTags.split(",") if ansibleTags != '' + ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != '' + ansible.inventory_path = "ansible/inventory" + end +end http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/ansible.cfg ---------------------------------------------------------------------- diff --git a/metron-deployment/development/centos6/ansible.cfg b/metron-deployment/development/centos6/ansible.cfg new file mode 100644 index 0000000..ade50ae --- /dev/null +++ b/metron-deployment/development/centos6/ansible.cfg @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[defaults] +host_key_checking = false +library = ../../ansible/extra_modules +roles_path = ../../ansible/roles +pipelining = True +log_path = ./ansible.log + + +# fix for "ssh throws 'unix domain socket too long' " problem +[ssh_connection] +control_path = %(directory)s/%%h-%%p-%%r http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/ansible/inventory/group_vars/all ---------------------------------------------------------------------- diff --git a/metron-deployment/development/centos6/ansible/inventory/group_vars/all b/metron-deployment/development/centos6/ansible/inventory/group_vars/all new file mode 100644 index 0000000..347d82a --- /dev/null +++ b/metron-deployment/development/centos6/ansible/inventory/group_vars/all @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# only need to build the RPMs for CentOS +metron_build_packages_cmd: "shell cd {{ metron_build_dir }}/metron-deployment && mvn clean package -DskipTests -Pbuild-rpms" + +# ambari +ambari_host: "{{ groups.ambari_master[0] }}" +hdp_host_group: "{{ groups.ambari_slave }}" +ambari_port: 8080 +ambari_user: admin +ambari_password: admin +cluster_type: single_node_vm +ambari_server_mem: 512 +java_home: /usr/jdk64/jdk1.8.0_77 + +# hbase +pcap_hbase_table: pcap +tracker_hbase_table: access_tracker +threatintel_hbase_table: threatintel +enrichment_hbase_table: enrichment + +# metron +metron_version: 0.4.3 +metron_directory: /usr/metron/{{ metron_version }} +bro_version: "2.5.2" +fixbuf_version: "1.7.1" +yaf_version: "2.8.0" +daq_version: "2.0.6-1" +pycapa_repo: "https://github.com/OpenSOC/pycapa.git"; +pycapa_home: "/opt/pycapa" +snort_version: "2.9.8.0-1" +snort_alert_csv_path: "/var/log/snort/alert.csv" +threat_intel_bulk_load: False + +# data directories - only required to override defaults +zookeeper_data_dir: "/data1/hadoop/zookeeper" +namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary" +namenode_name_dir: "/data1/hadoop/hdfs/namenode" +datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data" +journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode" +nodemanager_local_dirs: "/data1/hadoop/yarn/local" +timeline_ldb_store_path: "/data1/hadoop/yarn/timeline" +timeline_ldb_state_path: "/data1/hadoop/yarn/timeline" +nodemanager_log_dirs: "/data1/hadoop/yarn/log" +jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs" +storm_local_dir: "/data1/hadoop/storm" +kafka_log_dirs: "/data1/kafka-log" +elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch" + +# sensors +sensor_test_mode: True +install_pycapa: False +install_bro: True +install_snort: True +install_yaf: False +install_pcap_replay: True +sniff_interface: eth1 +pcap_replay_interface: "{{ sniff_interface }}" +pcapservice_port: 8081 + +# search +install_elasticsearch: True +install_solr: False +solr_collection_name: Metron +solr_number_shards: 1 +solr_replication_factor: 1 +elasticsearch_transport_port: 9300 +## The elasticsearch_network_interface must be in the form specified for Elasticsearch, with leading and trailing underscores. +elasticsearch_network_interface: _eth1_ +elasticsearch_web_port: 9200 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/ansible/inventory/hosts ---------------------------------------------------------------------- diff --git a/metron-deployment/development/centos6/ansible/inventory/hosts b/metron-deployment/development/centos6/ansible/inventory/hosts new file mode 100644 index 0000000..9bd9ea1 --- /dev/null +++ b/metron-deployment/development/centos6/ansible/inventory/hosts @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[ambari_master] +node1 + +[ambari_slave] +node1 + +[metron] +node1 + +[search] +node1 + +[sensors] +node1 + +[pcap_server] +node1 + +[web] +node1 + +[zeppelin] +node1 + +[monit:children] +sensors +pcap_server + +[local] +127.0.0.1 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/.gitignore ---------------------------------------------------------------------- diff --git a/metron-deployment/development/fastcapa/.gitignore b/metron-deployment/development/fastcapa/.gitignore new file mode 100644 index 0000000..a8b42eb --- /dev/null +++ b/metron-deployment/development/fastcapa/.gitignore @@ -0,0 +1 @@ +*.retry http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/development/fastcapa/README.md b/metron-deployment/development/fastcapa/README.md new file mode 100644 index 0000000..213d115 --- /dev/null +++ b/metron-deployment/development/fastcapa/README.md @@ -0,0 +1,138 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +Fastcapa Test Environment +========================= + +Provides a test environment for the development and testing of Fastcapa. The environment is automatically validated after it is created to ensure that Fastcapa is behaving correctly. + +Two virtualized nodes are launched with Vagrant that can communicate with one another over a private network. +- The `source` node uses Metron's `pcap_replay` functionality to transmit raw network packet data over a private network. +- The `sink` node is running `fastcapa` and is capturing these network packets. +- Fastcapa then transforms and bundles the packets into a message. +- The message is sent to a Kafka broker running on the `source` node. + +Getting Started +--------------- + +The Fastcapa test environment can be executed on different operating systems. There is a sub-directory for each operating system that Fastcapa can be tested on. + +To run, simply execute `vagrant up` within the appropriate directory. For example, to run the tests on CentOS 7.1 then execute the following commands. +``` +cd centos-7.1 +vagrant up +``` + +Automated tests are executed after provisioning completes to ensure that Fastcapa and the rest of the environment is functioning properly. If you see something like the following, then the tests have passed. +``` +$ vagrant up +==> source: Running provisioner: ansible... + source: Running ansible-playbook... +... +TASK [debug] ******************************************************************* +ok: [source] => { + "msg": "Successfully received packets sent from pcap-replay!" +} +... +TASK [debug] ******************************************************************* +ok: [source] => { + "msg": "Successfully received a Kafka message from fastcapa!" +} +``` + +If the deployment process fails mid-course, running `vagrant provision` will continue the process from where it left off. This can sometimes occur when the VM reboots as part of the deployment process. The error might look like the following. +``` +TASK [fastcapa : Restart for modified kernel params] *************************** +fatal: [sink]: UNREACHABLE! => {"changed": false, "msg": "Failed to connect to the host via ssh: Shared connection to 127.0.0.1 closed.\r\n", "unreachable": true} + to retry, use: --limit @/Users/nallen/Development/metron/metron-deployment/vagrant/fastcapa-test-platform/playbook.retry + +PLAY RECAP ********************************************************************* +sink : ok=11 changed=9 unreachable=1 failed=0 +source : ok=29 changed=25 unreachable=0 failed=0 + +Ansible failed to complete successfully. Any error output should be +visible above. Please fix these errors and try again. +``` + +Going Deeper +------------ + +This section will outline in more detail the environment and how to interact with it. + +### `source` + +To validate that the `source` node is functioning properly, run the following commands. + +First, ensure that the `pcap-replay` service is running. + +``` +vagrant ssh source +sudo service pcap-replay status +``` + +Use `tcpdump` to ensure that the raw packet data is being sent over the private network. Enter 'CTRL-C' to kill the `tcpdump` process once you are able to see that packets are being sent. + +``` +sudo yum -y install tcpdump +sudo tcpdump -i enp0s8 +``` + +### `sink` + +Next validate that the `sink` is functioning properly. Run the following commands starting from the host operating system. + +First, ensure that the `fastcapa` service is running. + +``` +vagrant ssh sink +service fastcapa status +``` + +Ensure that the raw network packet data is being received by Kafka. + +``` +/usr/hdp/current/kafka-broker/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic pcap +``` + +Enter 'CTRL-C' to kill the `kafka-console-consumer` process once you are able to see that packets are being sent. These packets will appear to be gibberish in the console. This is the raw binary network packet data after all. + +FAQ +--- + +### Error Message: Timed out while waiting for the machine to boot + +``` +Timed out while waiting for the machine to boot. This means that +Vagrant was unable to communicate with the guest machine within +the configured ("config.vm.boot_timeout" value) time period. +If you look above, you should be able to see the error(s) that +Vagrant had when attempting to connect to the machine. These errors +are usually good hints as to what may be wrong. +If you're using a custom box, make sure that networking is properly +working and you're able to connect to the machine. It is a common +problem that networking isn't setup properly in these boxes. +Verify that authentication configurations are also setup properly, +as well. +If the box appears to be booting properly, you may want to increase +the timeout ("config.vm.boot_timeout") value. +â centos-7.4 git:(master) â vagrant status +Current machine states: +source running (virtualbox) +sink not created (virtualbox) +``` + +If you are unable to launch any of the Fastcapa test environments, which results in a message like the one above, then you may need to upgrade your version of Virtualbox. Success has been reported with versions of VirtualBox 5.1.22+. http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile ---------------------------------------------------------------------- diff --git a/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile b/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile new file mode 100644 index 0000000..179ca34 --- /dev/null +++ b/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Vagrant.configure("2") do |config| + + # enable hostmanager + config.hostmanager.enabled = true + config.hostmanager.manage_host = true + + # + # source + # + config.vm.define "source" do |node| + + # host settings + node.vm.hostname = "source" + node.vm.box = "bento/centos-7.1" + node.ssh.insert_key = "true" + node.vm.network :private_network, ip: "192.168.33.10", netmask: "255.255.255.0" + + # provider + node.vm.provider "virtualbox" do |vb| + vb.memory = 1024 + vb.cpus = 1 + end + end + + # + # sink + # + config.vm.define "sink" do |node| + + # host settings + node.vm.hostname = "sink" + node.vm.box = "bento/centos-7.1" + node.ssh.insert_key = "true" + node.vm.network :private_network, ip: "192.168.33.11", netmask: "255.255.255.0" + + # provider + node.vm.provider "virtualbox" do |vb| + vb.memory = 4096 + vb.cpus = 3 + + # network adapter settings; [Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio] + vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] + vb.customize ["modifyvm", :id, "--nictype2","82545EM"] + end + + # provision host + node.vm.provision :ansible do |ansible| + ansible.limit = "all" + ansible.playbook = "../playbook.yml" + ansible.extra_vars = "vars/main.yml" + end + end + + +end http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg ---------------------------------------------------------------------- diff --git a/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg b/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg new file mode 100644 index 0000000..dd78cfd --- /dev/null +++ b/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[defaults] +host_key_checking = false +library = ../../../ansible/extra_modules +roles_path = ../../../ansible/roles +pipelining = True http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.1/vars/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/development/fastcapa/centos-7.1/vars/main.yml b/metron-deployment/development/fastcapa/centos-7.1/vars/main.yml new file mode 100644 index 0000000..563fc2a --- /dev/null +++ b/metron-deployment/development/fastcapa/centos-7.1/vars/main.yml @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + +# 0.9.4+ required for fastcapa +librdkafka_version: 0.9.4 +librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz + +dpdk_device: ["00:08.0"] +dpdk_target: "x86_64-native-linuxapp-gcc" +num_huge_pages: 512 +kafka_broker_url: source:9092 +zookeeper_url: source:2181 +pcap_replay_interface: enp0s3 +kafka_broker_home: /usr/hdp/current/kafka-broker/ + +# fastcapa settings +fastcapa_portmask: 0x01 +fastcapa_kafka_config: /etc/fastcapa.conf +fastcapa_topic: pcap +fastcapa_burst_size: 32 +fastcapa_nb_rx_desc: 1024 +fastcapa_nb_rx_queue: 1 +fastcapa_tx_ring_size: 2048 + +# dummy variables for pycapa's dependence on ambari_gather_facts +cluster_name: dummy +namenode_host: dummy +core_site_tag: dummy +hdfs_url: dummy +kafka_broker_hosts: dummy +kafka_broker_tag: dummy +kafka_broker_port: dummy +zookeeper_hosts: dummy +zookeeper_tag: dummy +zookeeper_port: dummy +metron_hosts: dummy +kibana_hosts: dummy http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile ---------------------------------------------------------------------- diff --git a/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile b/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile new file mode 100644 index 0000000..c62a0f1 --- /dev/null +++ b/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Vagrant.configure("2") do |config| + + # enable hostmanager + config.hostmanager.enabled = true + config.hostmanager.manage_host = true + + # + # source + # + config.vm.define "source" do |node| + + # host settings + node.vm.hostname = "source" + node.vm.box = "bento/centos-7.4" + node.ssh.insert_key = "true" + node.vm.network :private_network, ip: "192.168.33.10", netmask: "255.255.255.0" + + # provider + node.vm.provider "virtualbox" do |vb| + vb.memory = 1024 + vb.cpus = 1 + end + end + + # + # sink + # + config.vm.define "sink" do |node| + + # host settings + node.vm.hostname = "sink" + node.vm.box = "bento/centos-7.4" + node.ssh.insert_key = "true" + node.vm.network :private_network, ip: "192.168.33.11", netmask: "255.255.255.0" + + # provider + node.vm.provider "virtualbox" do |vb| + vb.memory = 4096 + vb.cpus = 3 + + # network adapter settings; [Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio] + vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] + vb.customize ["modifyvm", :id, "--nictype2","82545EM"] + end + + # provision host + node.vm.provision :ansible do |ansible| + ansible.limit = "all" + ansible.playbook = "../playbook.yml" + ansible.extra_vars = "vars/main.yml" + end + end + + +end http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg ---------------------------------------------------------------------- diff --git a/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg b/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg new file mode 100644 index 0000000..dd78cfd --- /dev/null +++ b/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[defaults] +host_key_checking = false +library = ../../../ansible/extra_modules +roles_path = ../../../ansible/roles +pipelining = True http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.4/vars/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/development/fastcapa/centos-7.4/vars/main.yml b/metron-deployment/development/fastcapa/centos-7.4/vars/main.yml new file mode 100644 index 0000000..563fc2a --- /dev/null +++ b/metron-deployment/development/fastcapa/centos-7.4/vars/main.yml @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + +# 0.9.4+ required for fastcapa +librdkafka_version: 0.9.4 +librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz + +dpdk_device: ["00:08.0"] +dpdk_target: "x86_64-native-linuxapp-gcc" +num_huge_pages: 512 +kafka_broker_url: source:9092 +zookeeper_url: source:2181 +pcap_replay_interface: enp0s3 +kafka_broker_home: /usr/hdp/current/kafka-broker/ + +# fastcapa settings +fastcapa_portmask: 0x01 +fastcapa_kafka_config: /etc/fastcapa.conf +fastcapa_topic: pcap +fastcapa_burst_size: 32 +fastcapa_nb_rx_desc: 1024 +fastcapa_nb_rx_queue: 1 +fastcapa_tx_ring_size: 2048 + +# dummy variables for pycapa's dependence on ambari_gather_facts +cluster_name: dummy +namenode_host: dummy +core_site_tag: dummy +hdfs_url: dummy +kafka_broker_hosts: dummy +kafka_broker_tag: dummy +kafka_broker_port: dummy +zookeeper_hosts: dummy +zookeeper_tag: dummy +zookeeper_port: dummy +metron_hosts: dummy +kibana_hosts: dummy http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/playbook.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/development/fastcapa/playbook.yml b/metron-deployment/development/fastcapa/playbook.yml new file mode 100644 index 0000000..e038691 --- /dev/null +++ b/metron-deployment/development/fastcapa/playbook.yml @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +# +# the 'source' produces network traffic +# +- hosts: source + become: yes + roles: + - role: kafka-broker + - role: pcap_replay + - { role: pycapa, install_pycapa_service: False } + - { role: sensor-test-mode, pcap_replay: True, install_yaf: False, install_snort: False, install_pcap_replay: True } + tasks: + - service: name=pcap-replay state=started + +# +# the 'sink' consumes network traffic +# +- hosts: sink + become: yes + roles: + - role: librdkafka + - role: fastcapa + tasks: + - service: name=fastcapa state=started + +# +# validate the environment - needs to run on `source` node +# +- hosts: source + become: yes + tasks: + - include: tasks/validate-packets-sent.yml + - include: tasks/validate-messages-received.yml http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/tasks/validate-messages-received.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/development/fastcapa/tasks/validate-messages-received.yml b/metron-deployment/development/fastcapa/tasks/validate-messages-received.yml new file mode 100644 index 0000000..5109a0e --- /dev/null +++ b/metron-deployment/development/fastcapa/tasks/validate-messages-received.yml @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- set_fact: + kafka_consumer_cmd: "{{ kafka_broker_home}}/bin/kafka-console-consumer.sh --zookeeper {{ zookeeper_url }} --topic {{ fastcapa_topic }} --max-messages 10 --from-beginning" + +- name: Attempting to receive message(s) sent by fastcapa + shell: "{{ kafka_consumer_cmd }}" + async: 30 + register: kafka_consumer + +- name: Waiting to receive message(s) sent by fastcapa + async_status: jid={{ kafka_consumer.ansible_job_id }} + register: job + until: job.finished + retries: 30 + +- fail: + msg: "No messages received from fastcapa within timeout: {{ kafka_consumer_cmd }}" + when: job.finished != 1 + +- debug: var=job + +- debug: msg="Successfully received a Kafka message from fastcapa!" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/tasks/validate-packets-sent.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/development/fastcapa/tasks/validate-packets-sent.yml b/metron-deployment/development/fastcapa/tasks/validate-packets-sent.yml new file mode 100644 index 0000000..1b062d4 --- /dev/null +++ b/metron-deployment/development/fastcapa/tasks/validate-packets-sent.yml @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install tcpdump + yum: name=tcpdump + +- set_fact: + sniff_cmd: "tcpdump -i {{ pcap_replay_interface }} -c 10" + +- name: "Attempting to sniff packet(s)" + shell: "{{ sniff_cmd }}" + async: 30 + register: sniffer + +- name: Waiting to receive packet(s) + async_status: jid={{ sniffer.ansible_job_id }} + register: job + until: job.finished + retries: 60 + +- fail: + msg: "No packets received from pcap-replay within timeout: {{ sniff_cmd }}" + when: job.finished != 1 + +- debug: var=job + +- debug: msg="Successfully received packets sent from pcap-replay!" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/development/ubuntu14/README.md b/metron-deployment/development/ubuntu14/README.md new file mode 100644 index 0000000..af7d3a3 --- /dev/null +++ b/metron-deployment/development/ubuntu14/README.md @@ -0,0 +1,106 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +Metron on Ubuntu 14 +=================== + +This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running Ubuntu 14. + +Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. + +Getting Started +--------------- + +### Prerequisites + +The computer used to deploy Apache Metron will need to have the following components installed. + + - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0) + - [Docker](https://www.docker.com/community-edition) + - [Vagrant](https://www.vagrantup.com) 1.8+ + - [Vagrant Hostmanager Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager) + - [Virtualbox](https://virtualbox.org) 5.0+ + - Python 2.7 + - Maven 3.3.9 + - C++11 compliant compiler, like [GCC](https://gcc.gnu.org/projects/cxx-status.html#cxx11) + +Running the following script can help validate whether you have all the prerequisites installed and running correctly. + + ``` + metron-deployment/scripts/platform-info.sh + ``` + +#### How do I install these on MacOS? + +Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent [Homebrew](http://brew.sh/) project. + +1. Install Homebrew by following the instructions at [Homebrew](http://brew.sh/). + +1. Run the following command in a terminal to install all of the required tools. + + ``` + brew cask install vagrant virtualbox docker + brew cask install caskroom/versions/java8 + brew install [email protected] git + pip install ansible==2.2.2.0 + vagrant plugin install vagrant-hostmanager + open /Applications/Docker.app + ``` + +### Deploy Metron + +1. Ensure that the Docker service is running. + +1. Deploy Metron + + ``` + cd metron-deployment/development/ubuntu14 + vagrant up + ``` + + Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host. + + ``` + vagrant provision + ``` + +### Explore Metron + +Navigate to the following resources to explore your newly minted Apache Metron environment. + +* [Metron Alerts](http://node1:4201) +* [Ambari](http://node1:8080) + +Connecting to the host through SSH is as simple as running the following command. + +``` +vagrant ssh +``` + +### Working with Metron + +In addition to re-running the entire provisioning play book, you may now re-run an individual Ansible tag or a collection of tags in the following ways. The following commands will re-run the `sensor-stubs` role on the Vagrant image. This will install and start the sensor stub components. + +``` +vagrant --ansible-tags="sensor-stubs" provision +``` + +Tags are listed in the playbooks, some frequently used tags: ++ `hdp-install` - Install HDP ++ `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services) ++ `sensors` - Deploy and start the sensors. ++ `sensor-stubs` - Deploy and start the sensor stubs. http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/Vagrantfile ---------------------------------------------------------------------- diff --git a/metron-deployment/development/ubuntu14/Vagrantfile b/metron-deployment/development/ubuntu14/Vagrantfile new file mode 100644 index 0000000..55f0aea --- /dev/null +++ b/metron-deployment/development/ubuntu14/Vagrantfile @@ -0,0 +1,87 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require 'getoptlong' + +ansibleTags='' +ansibleSkipTags='sensors' + +begin + opts = GetoptLong.new( + [ '--ansible-tags', GetoptLong::OPTIONAL_ARGUMENT ], + [ '--ansible-skip-tags', GetoptLong::OPTIONAL_ARGUMENT ] + ) + + opts.quiet = TRUE + + opts.each do |opt, arg| + case opt + when '--ansible-tags' + ansibleTags=arg + when '--ansible-skip-tags' + ansibleSkipTags=arg + end + end +rescue Exception => ignored +#Ignore to allow other opts to be passed to Vagrant +end + +puts " Running with ansible-tags: " + ansibleTags.split(",").to_s if ansibleTags != '' +puts " Running with ansible-skip-tags: " + ansibleSkipTags.split(",").to_s if ansibleSkipTags != '' + +hosts = [{ + hostname: "node1", + ip: "192.168.66.121", + memory: "8192", + cpus: 4, + promisc: 2 # enables promisc on the 'Nth' network interface +}] + +Vagrant.configure(2) do |config| + + # host runs ubuntu + config.vm.box = "ubuntu/trusty64" + config.ssh.insert_key = true + + # enable the hostmanager plugin + config.hostmanager.enabled = true + config.hostmanager.manage_host = true + + # host definition + hosts.each_with_index do |host, index| + config.vm.define host[:hostname] do |node| + + # host settings + node.vm.hostname = host[:hostname] + node.vm.network "private_network", ip: host[:ip] + + # vm settings + node.vm.provider "virtualbox" do |vb| + vb.memory = host[:memory] + vb.cpus = host[:cpus] + end + end + end + + # provision the host with ansible + config.vm.provision :ansible do |ansible| + ansible.playbook = "ansible/playbook.yml" + ansible.sudo = true + ansible.tags = ansibleTags.split(",") if ansibleTags != '' + ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != '' + ansible.inventory_path = "ansible/inventory" + end +end http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/ansible.cfg ---------------------------------------------------------------------- diff --git a/metron-deployment/development/ubuntu14/ansible.cfg b/metron-deployment/development/ubuntu14/ansible.cfg new file mode 100644 index 0000000..ade50ae --- /dev/null +++ b/metron-deployment/development/ubuntu14/ansible.cfg @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[defaults] +host_key_checking = false +library = ../../ansible/extra_modules +roles_path = ../../ansible/roles +pipelining = True +log_path = ./ansible.log + + +# fix for "ssh throws 'unix domain socket too long' " problem +[ssh_connection] +control_path = %(directory)s/%%h-%%p-%%r http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all ---------------------------------------------------------------------- diff --git a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all new file mode 100644 index 0000000..429d61a --- /dev/null +++ b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# only need to build the "DEBS" for Ubuntu +metron_build_packages_cmd: "shell cd {{ metron_build_dir }}/metron-deployment && mvn clean package -DskipTests -Pbuild-debs" + +# ambari +ambari_host: "{{ groups.ambari_master[0] }}" +hdp_host_group: "{{ groups.ambari_slave }}" +ambari_port: 8080 +ambari_user: admin +ambari_password: admin +cluster_type: single_node_vm +ambari_server_mem: 512 +java_home: /usr/jdk64/jdk1.8.0_77 + +# hbase +pcap_hbase_table: pcap +tracker_hbase_table: access_tracker +threatintel_hbase_table: threatintel +enrichment_hbase_table: enrichment + +# metron +metron_version: 0.4.3 +metron_directory: /usr/metron/{{ metron_version }} +bro_version: "2.5.2" +fixbuf_version: "1.7.1" +yaf_version: "2.8.0" +daq_version: "2.0.6-1" +pycapa_repo: "https://github.com/OpenSOC/pycapa.git"; +pycapa_home: "/opt/pycapa" +snort_version: "2.9.8.0-1" +snort_alert_csv_path: "/var/log/snort/alert.csv" +threat_intel_bulk_load: False + +# data directories - only required to override defaults +zookeeper_data_dir: "/data1/hadoop/zookeeper" +namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary" +namenode_name_dir: "/data1/hadoop/hdfs/namenode" +datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data" +journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode" +nodemanager_local_dirs: "/data1/hadoop/yarn/local" +timeline_ldb_store_path: "/data1/hadoop/yarn/timeline" +timeline_ldb_state_path: "/data1/hadoop/yarn/timeline" +nodemanager_log_dirs: "/data1/hadoop/yarn/log" +jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs" +storm_local_dir: "/data1/hadoop/storm" +kafka_log_dirs: "/data1/kafka-log" +elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch" + +# sensors +sensor_test_mode: True +install_pycapa: False +install_bro: True +install_snort: True +install_yaf: False +install_pcap_replay: True +sniff_interface: eth1 +pcap_replay_interface: "{{ sniff_interface }}" +pcapservice_port: 8081 + +# search +install_elasticsearch: True +install_solr: False +solr_collection_name: Metron +solr_number_shards: 1 +solr_replication_factor: 1 +elasticsearch_transport_port: 9300 +## The elasticsearch_network_interface must be in the form specified for Elasticsearch, with leading and trailing underscores. +elasticsearch_network_interface: _eth1_ +elasticsearch_web_port: 9200 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/ansible/inventory/hosts ---------------------------------------------------------------------- diff --git a/metron-deployment/development/ubuntu14/ansible/inventory/hosts b/metron-deployment/development/ubuntu14/ansible/inventory/hosts new file mode 100644 index 0000000..9bd9ea1 --- /dev/null +++ b/metron-deployment/development/ubuntu14/ansible/inventory/hosts @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[ambari_master] +node1 + +[ambari_slave] +node1 + +[metron] +node1 + +[search] +node1 + +[sensors] +node1 + +[pcap_server] +node1 + +[web] +node1 + +[zeppelin] +node1 + +[monit:children] +sensors +pcap_server + +[local] +127.0.0.1 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/ansible/playbook.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/development/ubuntu14/ansible/playbook.yml b/metron-deployment/development/ubuntu14/ansible/playbook.yml new file mode 100644 index 0000000..fcbe99c --- /dev/null +++ b/metron-deployment/development/ubuntu14/ansible/playbook.yml @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- hosts: all + pre_tasks: + - name: Update package cache on Ubuntu + apt: update_cache=yes + roles: + - role: enable-swap + +- include: ../../../ansible/playbooks/metron_full_install.yml http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/extra_modules/ambari_cluster_state.py ---------------------------------------------------------------------- diff --git a/metron-deployment/extra_modules/ambari_cluster_state.py b/metron-deployment/extra_modules/ambari_cluster_state.py deleted file mode 100644 index 0542526..0000000 --- a/metron-deployment/extra_modules/ambari_cluster_state.py +++ /dev/null @@ -1,395 +0,0 @@ -#!/usr/bin/python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -DOCUMENTATION = ''' ---- -module: ambari_cluster_state -version_added: "2.1" -author: Mark Bittmann (https://github.com/mbittmann) -short_description: Create, delete, start or stop an ambari cluster -description: - - Create, delete, start or stop an ambari cluster -options: - host: - description: - The hostname for the ambari web server - port: - description: - The port for the ambari web server - username: - description: - The username for the ambari web server - password: - description: - The name of the cluster in web server - required: yes - cluster_name: - description: - The name of the cluster in ambari - required: yes - cluster_state: - description: - The desired state for the ambari cluster ['present', 'absent', 'started', 'stopped']. Setting the cluster - state to absent will first stop the cluster. - required: yes - blueprint_var: - description: - The path to the file defining the cluster blueprint and host mapping. Required when state == 'present' - required: no - blueprint_name: - description: - The name of the blueprint. Required when state == 'present' - required: no - wait_for_complete: - description: - Whether to wait for the request to complete before returning. Default is False. - required: no - requirements: [ 'requests'] -''' - -EXAMPLES = ''' -# must use full relative path to any files in stored in roles/role_name/files/ -- name: Create a new ambari cluster - ambari_cluster_state: - host: localhost - port: 8080 - username: admin - password: admin - cluster_name: my_cluster - cluster_state: present - blueprint_var: roles/my_role/files/blueprint.yml - blueprint_name: hadoop - wait_for_complete: True -- name: Start the ambari cluster - ambari_cluster_state: - host: localhost - port: 8080 - username: admin - password: admin - cluster_name: my_cluster - cluster_state: started - wait_for_complete: True -- name: Stop the ambari cluster - ambari_cluster_state: - host: localhost - port: 8080 - username: admin - password: admin - cluster_name: my_cluster - cluster_state: stopped - wait_for_complete: True -- name: Delete the ambari cluster - ambari_cluster_state: - host: localhost - port: 8080 - username: admin - password: admin - cluster_name: my_cluster - cluster_state: absent -''' - -RETURN = ''' -results: - description: The content of the requests object returned from the RESTful call - returned: success - type: string -created_blueprint: - description: Whether a blueprint was created - returned: success - type: boolean -status: - description: The status of the blueprint creation process - returned: success - type: string -''' - -__author__ = 'mbittmann' - -import json -try: - import requests -except ImportError: - REQUESTS_FOUND = False -else: - REQUESTS_FOUND = True - - -def main(): - - argument_spec = dict( - host=dict(type='str', default=None, required=True), - port=dict(type='int', default=None, required=True), - username=dict(type='str', default=None, required=True), - password=dict(type='str', default=None, required=True), - cluster_name=dict(type='str', default=None, required=True), - cluster_state=dict(type='str', default=None, required=True, - choices=['present', 'absent', 'started', 'stopped']), - blueprint_var=dict(type='dict', required=False), - blueprint_name=dict(type='str', default=None, required=False), - configurations=dict(type='list', default=None, required=False), - wait_for_complete=dict(default=False, required=False, type='bool'), - ) - - required_together = ['blueprint_var', 'blueprint_name'] - - module = AnsibleModule( - argument_spec=argument_spec, - required_together=required_together - ) - - if not REQUESTS_FOUND: - module.fail_json( - msg='requests library is required for this module') - - p = module.params - - host = p.get('host') - port = p.get('port') - username = p.get('password') - password = p.get('password') - cluster_name = p.get('cluster_name') - cluster_state = p.get('cluster_state') - blueprint_name = p.get('blueprint_name') - wait_for_complete = p.get('wait_for_complete') - - ambari_url = 'http://{0}:{1}'.format(host, port) - - try: - if cluster_state in ['started', 'stopped']: - if not cluster_exists(ambari_url, username, password, cluster_name): - module.fail_json(msg="Cluster name {0} does not exist".format(cluster_name)) - state = '' - if cluster_state == 'started': - state = 'STARTED' - elif cluster_state == 'stopped': - state = 'INSTALLED' - - request = set_cluster_state(ambari_url, username, password, cluster_name, state) - if wait_for_complete: - try: - request_id = json.loads(request.content)['Requests']['id'] - except ValueError: - module.exit_json(changed=True, results=request.content) - status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2) - if status != 'COMPLETED': - module.fail_json(msg="Request failed with status {0}".format(status)) - module.exit_json(changed=True, results=request.content) - elif cluster_state == 'absent': - if not cluster_exists(ambari_url, username, password, cluster_name): - module.exit_json(changed=False, msg='Skipping. Cluster does not exist') - if not can_delete_cluster(ambari_url, username, password, cluster_name): - request = set_cluster_state(ambari_url, username, password, cluster_name, 'INSTALLED') - request_id = json.loads(request.content)['Requests']['id'] - status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2) - if status != 'COMPLETED': - module.fail_json(msg="Request failed with status {0}".format(status)) - request = delete_cluster(ambari_url, username, password, cluster_name) - module.exit_json(changed=True, results=request.content) - elif cluster_state == 'present': - if not p.get('blueprint_var') or not blueprint_name: # have neither name nor file - module.fail_json(msg="Must provide blueprint_var and blueprint_name when cluster_state=='present'") - - blueprint_var = p.get('blueprint_var') - blueprint, host_map = blueprint_var_to_ambari_converter(blueprint_var) - created_blueprint = False - - if not blueprint_exists(ambari_url, username, password, blueprint_name): - create_blueprint(ambari_url, username, password, blueprint_name, blueprint) - created_blueprint = True - - if cluster_exists(ambari_url, username, password, cluster_name): - module.exit_json(changed=False, msg='Cluster {0} already exists'.format(cluster_name), - created_blueprint=created_blueprint) - - configurations = p.get('configurations') - request = create_cluster(ambari_url, username, password, cluster_name, blueprint_name, configurations, host_map) - request_id = json.loads(request.content)['Requests']['id'] - if wait_for_complete: - status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2) - if status != 'COMPLETED': - module.fail_json(msg="Request failed with status {0}".format(status)) - request_status = get_request_status(ambari_url, username, password, cluster_name, request_id) - module.exit_json(changed=True, results=request.content, - created_blueprint=created_blueprint, status=request_status) - - except requests.ConnectionError, e: - module.fail_json(msg="Could not connect to Ambari client: " + str(e.message)) - except Exception, e: - module.fail_json(msg="Ambari client exception occurred: " + str(e.message)) - - -def get_clusters(ambari_url, user, password): - r = get(ambari_url, user, password, '/api/v1/clusters') - if r.status_code != 200: - msg = 'Could not get cluster list: request code {0}, \ - request message {1}'.format(r.status_code, r.content) - raise Exception(msg) - clusters = json.loads(r.content) - return clusters['items'] - - -def cluster_exists(ambari_url, user, password, cluster_name): - clusters = get_clusters(ambari_url, user, password) - return cluster_name in [item['Clusters']['cluster_name'] for item in clusters] - - -def set_cluster_state(ambari_url, user, password, cluster_name, cluster_state): - path = '/api/v1/clusters/{0}/services'.format(cluster_name) - request = {"RequestInfo": {"context": "Setting cluster state"}, - "Body": {"ServiceInfo": {"state": "{0}".format(cluster_state)}}} - payload = json.dumps(request) - r = put(ambari_url, user, password, path, payload) - if r.status_code not in [202, 200]: - msg = 'Could not set cluster state: request code {0}, \ - request message {1}'.format(r.status_code, r.content) - raise Exception(msg) - return r - - -def create_cluster(ambari_url, user, password, cluster_name, blueprint_name, configurations, hosts_json): - path = '/api/v1/clusters/{0}'.format(cluster_name) - data = json.dumps({'blueprint': blueprint_name, 'configurations': configurations, 'host_groups': hosts_json}) - f = open('cluster.log', 'w') - f.write(data) - f.close() - r = post(ambari_url, user, password, path, data) - if r.status_code != 202: - msg = 'Could not create cluster: request code {0}, \ - request message {1}'.format(r.status_code, r.content) - raise Exception(msg) - return r - - -def get_request_status(ambari_url, user, password, cluster_name, request_id): - path = '/api/v1/clusters/{0}/requests/{1}'.format(cluster_name, request_id) - r = get(ambari_url, user, password, path) - if r.status_code != 200: - msg = 'Could not get cluster request status: request code {0}, \ - request message {1}'.format(r.status_code, r.content) - raise Exception(msg) - service = json.loads(r.content) - return service['Requests']['request_status'] - - -def wait_for_request_complete(ambari_url, user, password, cluster_name, request_id, sleep_time): - while True: - status = get_request_status(ambari_url, user, password, cluster_name, request_id) - if status == 'COMPLETED': - return status - elif status in ['FAILED', 'TIMEDOUT', 'ABORTED', 'SKIPPED_FAILED']: - return status - else: - time.sleep(sleep_time) - - -def can_delete_cluster(ambari_url, user, password, cluster_name): - path = '/api/v1/clusters/{0}/services?ServiceInfo/state=STARTED'.format(cluster_name) - r = get(ambari_url, user, password, path) - items = json.loads(r.content)['items'] - return len(items) > 0 - - -def get_blueprints(ambari_url, user, password): - path = '/api/v1/blueprints' - r = get(ambari_url, user, password, path) - if r.status_code != 200: - msg = 'Could not get blueprint list: request code {0}, \ - request message {1}'.format(r.status_code, r.content) - raise Exception(msg) - - services = json.loads(r.content) - return services['items'] - - -def create_blueprint(ambari_url, user, password, blueprint_name, blueprint_data): - data = json.dumps(blueprint_data) - f = open('blueprint.log', 'w') - f.write(data) - f.close() - path = "/api/v1/blueprints/" + blueprint_name - r = post(ambari_url, user, password, path, data) - if r.status_code != 201: - msg = 'Could not create blueprint: request code {0}, \ - request message {1}'.format(r.status_code, r.content) - raise Exception(msg) - return r - - -def blueprint_exists(ambari_url, user, password, blueprint_name): - blueprints = get_blueprints(ambari_url, user, password) - return blueprint_name in [item['Blueprints']['blueprint_name'] for item in blueprints] - - -def delete_cluster(ambari_url, user, password, cluster_name): - path = '/api/v1/clusters/{0}'.format(cluster_name) - r = delete(ambari_url, user, password, path) - if r.status_code != 200: - msg = 'Could not delete cluster: request code {0}, \ - request message {1}'.format(r.status_code, r.content) - raise Exception(msg) - return r - - -def get(ambari_url, user, password, path): - r = requests.get(ambari_url + path, auth=(user, password)) - return r - - -def put(ambari_url, user, password, path, data): - headers = {'X-Requested-By': 'ambari'} - r = requests.put(ambari_url + path, data=data, auth=(user, password), headers=headers) - return r - - -def post(ambari_url, user, password, path, data): - headers = {'X-Requested-By': 'ambari'} - r = requests.post(ambari_url + path, data=data, auth=(user, password), headers=headers) - return r - - -def delete(ambari_url, user, password, path): - headers = {'X-Requested-By': 'ambari'} - r = requests.delete(ambari_url + path, auth=(user, password), headers=headers) - return r - - -def blueprint_var_to_ambari_converter(blueprint_var): - groups = blueprint_var['groups'] - new_groups = [] - host_map = [] - for group in groups: - components = [] - for component in group['components']: - components.append({'name': component}) - group['components'] = components - hosts = group.pop('hosts') - new_groups.append(group) - this_host_map = dict() - this_host_map['name'] = group['name'] - this_host_list = [{'fqdn': host} for host in hosts] - this_host_map['hosts'] = this_host_list - host_map.append(this_host_map) - blueprint = dict() - blueprint['configurations'] = blueprint_var['required_configurations'] - blueprint['host_groups'] = new_groups - blueprint['Blueprints'] = {'stack_name': blueprint_var['stack_name'], 'stack_version': blueprint_var['stack_version']} - return blueprint, host_map - -from ansible.module_utils.basic import * -if __name__ == '__main__': - main()
