http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml b/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml new file mode 100644 index 0000000..fc6c094 --- /dev/null +++ b/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: "Allocate {{ swapspace }} for swap space" + shell: fallocate -l {{ swapspace }} {{ swapfile }} + +- name: "Permissioning {{ swapfile }}" + file: + path: "{{ swapfile }}" + mode: "600" + +- name: "Setup swap space" + shell: mkswap {{ swapfile }} + +- name: "Enable swap space" + shell: swapon {{ swapfile }} + +- name: "Enable swap on boot" + lineinfile: + dest: "/etc/fstab" + line: "{{ swapfile }} none swap sw 0 0"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/enable-swap/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml b/metron-deployment/ansible/roles/enable-swap/tasks/main.yml new file mode 100644 index 0000000..f7d108c --- /dev/null +++ b/metron-deployment/ansible/roles/enable-swap/tasks/main.yml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: "Is swap space enabled?" + shell: swapon -s | grep {{ swapfile }} + register: swapcheck + failed_when: swapcheck.rc != 0 and swapcheck.rc != 1 + +- debug: msg="Swap space is already enabled" + when: swapcheck.rc == 0 + +- debug: msg="Swap space is NOT yet enabled" + when: swapcheck.rc == 1 + +- include: enable-swap.yml + static: no + when: swapcheck.rc != 0 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/epel/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/epel/tasks/main.yml b/metron-deployment/ansible/roles/epel/tasks/main.yml new file mode 100644 index 0000000..5fecc6c --- /dev/null +++ b/metron-deployment/ansible/roles/epel/tasks/main.yml @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install EPEL repository + yum: name=epel-release + when: ansible_distribution == "CentOS" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/fastcapa/defaults/main.yml b/metron-deployment/ansible/roles/fastcapa/defaults/main.yml new file mode 100644 index 0000000..d586046 --- /dev/null +++ b/metron-deployment/ansible/roles/fastcapa/defaults/main.yml @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +# dpdk +dpdk_home: "/usr/local/dpdk" +dpdk_version: "17.08" +dpdk_sdk: "/root/dpdk-{{ dpdk_version }}" +dpdk_src_url: "http://fast.dpdk.org/rel/dpdk-{{ dpdk_version }}.tar.xz" +dpdk_target: "x86_64-native-linuxapp-gcc" +num_huge_pages: 512 +extra_cflags: -g + +# fastcapa +fastcapa_work_dir: /root/fastcapa +fastcapa_build_dir: "{{ fastcapa_work_dir }}/build/app/" +fastcapa_prefix: /usr/local/bin +fastcapa_ld_library_path: /usr/local/lib +fastcapa_bin: fastcapa + +# fastcapa settings +fastcapa_portmask: 0x01 +fastcapa_kafka_config: /etc/fastcapa.conf +fastcapa_topic: pcap +fastcapa_rx_burst_size: 32 +fastcapa_tx_burst_size: 256 +fastcapa_nb_rx_desc: 1024 +fastcapa_nb_rx_queue: 1 +fastcapa_tx_ring_size: 2048 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/fastcapa/meta/main.yml b/metron-deployment/ansible/roles/fastcapa/meta/main.yml new file mode 100644 index 0000000..d253e88 --- /dev/null +++ b/metron-deployment/ansible/roles/fastcapa/meta/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +dependencies: + - librdkafka http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/debug.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/debug.yml b/metron-deployment/ansible/roles/fastcapa/tasks/debug.yml new file mode 100644 index 0000000..06f1526 --- /dev/null +++ b/metron-deployment/ansible/roles/fastcapa/tasks/debug.yml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + - name: Install debug utilities + yum: name=yum-utils + tags: + - debug + + - name: Install debug symbols + shell: debuginfo-install -y glibc glib2 zlib + tags: + - debug http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/dependencies.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/dependencies.yml b/metron-deployment/ansible/roles/fastcapa/tasks/dependencies.yml new file mode 100644 index 0000000..5955702 --- /dev/null +++ b/metron-deployment/ansible/roles/fastcapa/tasks/dependencies.yml @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + - name: Install dependencies + yum: name={{ item }} + with_items: + - "@Development tools" + - pciutils + - net-tools + - glib2 + - glib2-devel + - git + - numactl-devel + + # + # install prerequisite packages and the latest kernel headers. need to + # ensure that the kernel headers match the current running kernel version. + # if this is not the case, the DPDK build process will fail + # + - name: Install latest kernel headers and source + yum: name={{ item }} state=latest + with_items: + - kernel + - kernel-devel + - kernel-headers http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/dpdk.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/dpdk.yml b/metron-deployment/ansible/roles/fastcapa/tasks/dpdk.yml new file mode 100644 index 0000000..a936570 --- /dev/null +++ b/metron-deployment/ansible/roles/fastcapa/tasks/dpdk.yml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + - name: "Download DPDK version {{ dpdk_version }}" + unarchive: + src: "{{ dpdk_src_url }}" + dest: "/root" + creates: "{{ dpdk_sdk }}" + copy: no + + - name: "Configure DPDK for the target environment: {{ dpdk_target }}" + shell: "make config T={{ dpdk_target }} DESTDIR={{ dpdk_home }}" + args: + chdir: "{{ dpdk_sdk }}" + creates: "{{ dpdk_home }}" + + - name: "Turn on debug flags" + lineinfile: + dest: "{{ dpdk_sdk }}/config/common_linuxapp" + regexp: 'DEBUG=n' + line: 'DEBUG=y' + tags: + - debug + + - name: "Build DPDK for the target environment: {{ dpdk_target }}" + shell: "make install T={{ dpdk_target }} DESTDIR={{ dpdk_home }} EXTRA_CFLAGS={{ extra_cflags }}" + args: + chdir: "{{ dpdk_sdk }}" + creates: "{{ dpdk_home }}" + + - name: Load kernel modules to enable userspace IO + shell: "{{ item }}" + with_items: + - modprobe uio_pci_generic + - modprobe vfio-pci + + - name: Bind the device to the loaded kernel module(s) + shell: "{{ dpdk_home }}/sbin/dpdk-devbind --force --bind=uio_pci_generic {{ item }}" + with_items: "{{ dpdk_device }}" + + - name: Set useful environment variables + lineinfile: "dest=/root/.bash_profile line={{ item }}" + with_items: + - "export RTE_SDK={{ dpdk_sdk }}" + - "export RTE_TARGET={{ dpdk_target }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/fastcapa.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/fastcapa.yml b/metron-deployment/ansible/roles/fastcapa/tasks/fastcapa.yml new file mode 100644 index 0000000..b555668 --- /dev/null +++ b/metron-deployment/ansible/roles/fastcapa/tasks/fastcapa.yml @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Distribute fastcapa + copy: src=../../../metron-sensors/fastcapa dest={{ fastcapa_work_dir | dirname }} mode=0755 + +- name: Build fastcapa + shell: "{{ item }}" + args: + chdir: "{{ fastcapa_work_dir }}" + with_items: + - make + environment: + RTE_SDK: "{{ dpdk_sdk }}" + RTE_TARGET: "{{ dpdk_target }}" + LD_LIBRARY_PATH: "{{ fastcapa_ld_library_path }}" + +- name: Install fastcapa + shell: "cp {{ fastcapa_build_dir }}/{{ fastcapa_bin }} {{ fastcapa_prefix }}" + args: + chdir: "{{ fastcapa_work_dir }}" + creates: "{{ fastcapa_prefix }}/{{ fastcapa_bin }}" + +- name: Deploy configuration + template: src=fastcapa.conf dest={{ fastcapa_kafka_config }} mode=0755 + +- name: Deploy service + template: src=fastcapa dest=/etc/init.d/ mode=0755 + +- name: Register the service with systemd + shell: systemctl enable fastcapa + when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/kernel.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/kernel.yml b/metron-deployment/ansible/roles/fastcapa/tasks/kernel.yml new file mode 100644 index 0000000..cd4abe6 --- /dev/null +++ b/metron-deployment/ansible/roles/fastcapa/tasks/kernel.yml @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# DPDK requires specific kernel boot parameters. set the params and reboot +# the host, if the actual params differ from what is expected. +# +--- + - set_fact: + expected_kernel_params: "default_hugepagesz=1G hugepagesz=1G hugepages={{ num_huge_pages }} iommu=pt intel_iommu=on" + + - name: Check kernel boot parameters + shell: "cat /proc/cmdline" + register: actual_kernel_params + + - name: Alter kernel boot parameters + lineinfile: + dest: /etc/default/grub + regexp: '^(GRUB_CMDLINE_LINUX=\"[^\"]+)\"$' + line: '\1 {{ expected_kernel_params }}"' + backrefs: yes + when: not expected_kernel_params in actual_kernel_params.stdout + + - name: Update grub with kernel boot parameters + shell: /sbin/grub2-mkconfig -o /boot/grub2/grub.cfg + when: not expected_kernel_params in actual_kernel_params.stdout + + - name: Restart for modified kernel params + command: shutdown -r now "modified kernel params" + async: 0 + poll: 0 + ignore_errors: true + when: not expected_kernel_params in actual_kernel_params.stdout + + - name: Wait for reboot of '{{ inventory_hostname }}' + local_action: wait_for host={{ inventory_hostname }} state=started port=22 timeout=300 delay=10 + become: false + when: not expected_kernel_params in actual_kernel_params.stdout http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/main.yml b/metron-deployment/ansible/roles/fastcapa/tasks/main.yml new file mode 100644 index 0000000..b98d557 --- /dev/null +++ b/metron-deployment/ansible/roles/fastcapa/tasks/main.yml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + - include: dependencies.yml + - include: kernel.yml + - include: dpdk.yml + - include: fastcapa.yml + - include: debug.yml http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/templates/fastcapa ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/fastcapa/templates/fastcapa b/metron-deployment/ansible/roles/fastcapa/templates/fastcapa new file mode 100644 index 0000000..f34c603 --- /dev/null +++ b/metron-deployment/ansible/roles/fastcapa/templates/fastcapa @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# fastcapa daemon +# chkconfig: 345 20 80 +# description: Packet capture probe +# processname: fastcapa +# + +export LD_LIBRARY_PATH="{{ fastcapa_ld_library_path }}" + +NAME="fastcapa" +DESC="Metron network packet capture probe" +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +LOGROOT=/var/log/fastcapa +DAEMONLOG=$LOGROOT/$NAME-stdout.log +DAEMONERR=$LOGROOT/$NAME-stderr.log +NOW=`date` +DAEMON_PATH="/root" + +PORT_MASK="{{ fastcapa_portmask }}" +KAFKA_TOPIC="{{ fastcapa_topic }}" +KAFKA_CONFIG="{{ fastcapa_kafka_config }}" +RX_BURST_SIZE="{{ fastcapa_rx_burst_size }}" +TX_BURST_SIZE="{{ fastcapa_tx_burst_size }}" +NB_RX_DESC="{{ fastcapa_nb_rx_desc }}" +NB_RX_QUEUE="{{ fastcapa_nb_rx_queue }}" +TX_RING_SIZE="{{ fastcapa_tx_ring_size }}" + +DAEMON="{{ fastcapa_prefix }}/{{ fastcapa_bin }}" +DAEMONOPTS+=" " +DAEMONOPTS+=" -- " +DAEMONOPTS+="-p $PORT_MASK " +DAEMONOPTS+="-t $KAFKA_TOPIC " +DAEMONOPTS+="-c $KAFKA_CONFIG " +DAEMONOPTS+="-b $RX_BURST_SIZE " +DAEMONOPTS+="-w $TX_BURST_SIZE " +DAEMONOPTS+="-d $NB_RX_DESC " +DAEMONOPTS+="-q $NB_RX_QUEUE " +DAEMONOPTS+="-x $TX_RING_SIZE " + +case "$1" in + start) + printf "%-50s" "Starting $NAME..." + echo "$NOW: Starting $NAME..." >> $DAEMONLOG + + mkdir -p $LOGROOT + touch $DAEMONLOG + touch $DAEMONERR + + cd $DAEMON_PATH + echo "$DAEMON $DAEMONOPTS >> $DAEMONLOG 2> $DAEMONERR" >> $DAEMONLOG + + if [ -f $PIDFILE ]; then + printf "%s\n" "Already running" + else + PID=`$DAEMON $DAEMONOPTS >> $DAEMONLOG 2> $DAEMONERR & echo $!` + if [ -z $PID ]; then + printf "%s\n" "Fail" + else + echo $PID > $PIDFILE + printf "%s\n" "Ok" + fi + fi + ;; + + status) + printf "%-50s" "Checking $NAME..." + if [ -f $PIDFILE ]; then + PID=`cat $PIDFILE` + if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then + printf "%s\n" "Process dead but pidfile exists" + else + echo "Running" + fi + else + printf "%s\n" "Service not running" + fi + ;; + + stop) + printf "%-50s" "Stopping $NAME" + PID=`cat $PIDFILE` + cd $DAEMON_PATH + if [ -f $PIDFILE ]; then + while sleep 1 + echo -n "." + kill -0 $PID >/dev/null 2>&1 + do + kill -SIGINT $PID + done + printf "%s\n" "Ok" + rm -f $PIDFILE + else + printf "%s\n" "pidfile not found" + fi + ;; + + restart) + $0 stop + $0 start + ;; + + tail) + tail -F $LOGROOT/* + ;; + + kill) + printf "%-50s" "Force killing $NAME" + PID=`cat $PIDFILE` + cd $DAEMON_PATH + if [ -f $PIDFILE ]; then + while sleep 1 + echo -n "." + kill -0 $PID >/dev/null 2>&1 + do + kill -SIGTERM $PID + done + printf "%s\n" "Ok" + rm -f $PIDFILE + else + printf "%s\n" "pidfile not found" + fi + ;; + + *) + echo "Usage: $0 {status|start|stop|restart|kill|tail}" + exit 1 +esac http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/templates/fastcapa.conf ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/fastcapa/templates/fastcapa.conf b/metron-deployment/ansible/roles/fastcapa/templates/fastcapa.conf new file mode 100644 index 0000000..7d9eae4 --- /dev/null +++ b/metron-deployment/ansible/roles/fastcapa/templates/fastcapa.conf @@ -0,0 +1,67 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# kafka global settings +# +[kafka-global] + +# initial list of kafka brokers +metadata.broker.list = {{ kafka_broker_url }} + +# identifies the client to kafka +client.id = metron-fastcapa + +# max number of messages allowed on the producer queue +queue.buffering.max.messages = 1000000 + +# maximum time, in milliseconds, for buffering data on the producer queue +queue.buffering.max.ms = 3000 + +# compression codec = none, gzip or snappy +compression.codec = snappy + +# maximum number of messages batched in one MessageSet (increase for better compression) +batch.num.messages = 10 + +# max times to retry sending a failed message set +message.send.max.retries = 5 + +# backoff time before retrying a message send +retry.backoff.ms = 250 + +# how often statistics are emitted; 0 = never +statistics.interval.ms = 5000 + +# only provide delivery reports for failed messages +delivery.report.only.error = false + +# +# kafka topic settings +# +[kafka-topic] + +# broker acks { 1 = leader ack, 0 = no acks, -1 = in sync replica ack } +request.required.acks = 1 + +# local message timeout. This value is only enforced locally and limits the time a +# produced message waits for successful delivery. A time of 0 is infinite. +message.timeout.ms = 10000 + +# report offset of produced message back to application. The application must be +# use the dr_msg_cb to retrieve the offset from rd_kafka_message_t.offset +produce.offset.report = false http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/httplib2/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/httplib2/tasks/main.yml b/metron-deployment/ansible/roles/httplib2/tasks/main.yml new file mode 100644 index 0000000..5502cf4 --- /dev/null +++ b/metron-deployment/ansible/roles/httplib2/tasks/main.yml @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install python httplib2 dependency + pip: + name: httplib2 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/java_jdk/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/java_jdk/defaults/main.yml b/metron-deployment/ansible/roles/java_jdk/defaults/main.yml new file mode 100644 index 0000000..315bc0c --- /dev/null +++ b/metron-deployment/ansible/roles/java_jdk/defaults/main.yml @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +java_home: /usr/jdk64/jdk1.8.0_77 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/java_jdk/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/java_jdk/tasks/main.yml b/metron-deployment/ansible/roles/java_jdk/tasks/main.yml new file mode 100644 index 0000000..999b9c1 --- /dev/null +++ b/metron-deployment/ansible/roles/java_jdk/tasks/main.yml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Check for java at "{{ java_home }}" + stat: path="{{ java_home }}" + register: jdk_dir + +- name: Alternatives link for java + alternatives: name={{ item.name }} link={{ item.link }} path={{ item.path }} + with_items: + - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" } + - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" } + when: jdk_dir.stat.exists + +- name: Install openjdk + yum: name={{item}} + with_items: + - java-1.8.0-openjdk + - java-1.8.0-openjdk-devel + when: not jdk_dir.stat.exists http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/kafka-broker/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/kafka-broker/defaults/main.yml b/metron-deployment/ansible/roles/kafka-broker/defaults/main.yml new file mode 100644 index 0000000..b0f5a11 --- /dev/null +++ b/metron-deployment/ansible/roles/kafka-broker/defaults/main.yml @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo"; http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/kafka-broker/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/kafka-broker/meta/main.yml b/metron-deployment/ansible/roles/kafka-broker/meta/main.yml new file mode 100644 index 0000000..9587e79 --- /dev/null +++ b/metron-deployment/ansible/roles/kafka-broker/meta/main.yml @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +dependencies: + - libselinux-python http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/kafka-broker/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/kafka-broker/tasks/main.yml b/metron-deployment/ansible/roles/kafka-broker/tasks/main.yml new file mode 100644 index 0000000..db05cb0 --- /dev/null +++ b/metron-deployment/ansible/roles/kafka-broker/tasks/main.yml @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Retrieve HDP repository definition + get_url: + url: "{{ hdp_repo_def }}" + dest: /etc/yum.repos.d/hdp.repo + mode: 0644 + +- name: Install kafka + yum: name={{item}} + with_items: + - java-1.8.0-openjdk + - kafka + - zookeeper-server + +- name: Create pid directories + file: path={{ item }} state=directory mode=0755 + with_items: + - /var/run/zookeeper + - /var/run/kafka + +- name: Start zookeeper + shell: /usr/hdp/current/zookeeper-server/bin/zookeeper-server start + +- name: Start kafka + shell: /usr/hdp/current/kafka-broker/bin/kafka start http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/kafka-client/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/kafka-client/defaults/main.yml b/metron-deployment/ansible/roles/kafka-client/defaults/main.yml new file mode 100644 index 0000000..b0f5a11 --- /dev/null +++ b/metron-deployment/ansible/roles/kafka-client/defaults/main.yml @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo"; http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/kafka-client/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/kafka-client/tasks/main.yml b/metron-deployment/ansible/roles/kafka-client/tasks/main.yml new file mode 100644 index 0000000..1674225 --- /dev/null +++ b/metron-deployment/ansible/roles/kafka-client/tasks/main.yml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + - name: Retrieve HDP repository definition + get_url: + url: "{{ hdp_repo_def }}" + dest: /etc/yum.repos.d/hdp.repo + mode: 0644 + + - name: Install kafka + yum: + name: kafka + register: result + until: result.rc == 0 + retries: 5 + delay: 10 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/librdkafka/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml new file mode 100644 index 0000000..063c22f --- /dev/null +++ b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +librdkafka_version: 0.9.4 +librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz +librdkafka_home: /usr/local http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/librdkafka/tasks/dependencies.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/librdkafka/tasks/dependencies.yml b/metron-deployment/ansible/roles/librdkafka/tasks/dependencies.yml new file mode 100644 index 0000000..72ff907 --- /dev/null +++ b/metron-deployment/ansible/roles/librdkafka/tasks/dependencies.yml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install prerequisites + yum: name={{ item }} + with_items: + - cmake + - make + - gcc + - gcc-c++ + - flex + - bison + - libpcap + - libpcap-devel + - openssl-devel + - python-devel + - swig + - zlib-devel + - perl + - cyrus-sasl + - cyrus-sasl-devel + - cyrus-sasl-gssapi + register: result + until: result.rc == 0 + retries: 5 + delay: 10 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml b/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml new file mode 100644 index 0000000..a7971d6 --- /dev/null +++ b/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Download librdkafka + get_url: + url: "{{ librdkafka_url }}" + dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz" + +- name: Extract librdkafka tarball + unarchive: + src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz" + dest: /tmp + copy: no + creates: "/tmp/librdkafka-{{ librdkafka_version }}" + +- name: Compile and install librdkafka + shell: "{{ item }}" + args: + chdir: "/tmp/librdkafka-{{ librdkafka_version }}" + creates: "{{ librdkafka_home }}/lib/librdkafka.so" + with_items: + - rm -rf build/ + - "./configure --prefix={{ librdkafka_home }} --enable-sasl" + - make + - make install http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/librdkafka/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/librdkafka/tasks/main.yml b/metron-deployment/ansible/roles/librdkafka/tasks/main.yml new file mode 100644 index 0000000..2144d7f --- /dev/null +++ b/metron-deployment/ansible/roles/librdkafka/tasks/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- include: dependencies.yml +- include: librdkafka.yml http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/libselinux-python/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/libselinux-python/tasks/main.yml b/metron-deployment/ansible/roles/libselinux-python/tasks/main.yml new file mode 100644 index 0000000..9969593 --- /dev/null +++ b/metron-deployment/ansible/roles/libselinux-python/tasks/main.yml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install libselinux-python + yum: + name: libselinux-python + state: installed + register: result + until: result.rc == 0 + retries: 5 + delay: 10 + when: ansible_distribution == "CentOS" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/load_web_templates/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/load_web_templates/meta/main.yml b/metron-deployment/ansible/roles/load_web_templates/meta/main.yml new file mode 100644 index 0000000..c3d807b --- /dev/null +++ b/metron-deployment/ansible/roles/load_web_templates/meta/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +dependencies: + - ambari_gather_facts http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml b/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml new file mode 100644 index 0000000..3a91960 --- /dev/null +++ b/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Load Kibana Dashboard + command: > + curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "{{ kibana_hosts[0] }}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/requests + args: + warn: off + register: result + failed_when: "result.rc != 0 or '202' not in result.stdout" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-builder/defaults/main.yml b/metron-deployment/ansible/roles/metron-builder/defaults/main.yml new file mode 100644 index 0000000..1bdd0ff --- /dev/null +++ b/metron-deployment/ansible/roles/metron-builder/defaults/main.yml @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +metron_build_dir: "{{ playbook_dir }}/../../.." http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/tasks/build-debs.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-builder/tasks/build-debs.yml b/metron-deployment/ansible/roles/metron-builder/tasks/build-debs.yml new file mode 100644 index 0000000..4949196 --- /dev/null +++ b/metron-deployment/ansible/roles/metron-builder/tasks/build-debs.yml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Build Metron DEB Packages + shell: "{{ item }}" + args: + chdir: "{{ metron_build_dir }}/metron-deployment" + with_items: + - mvn package -DskipTests -Pbuild-debs + become: false + run_once: true + delegate_to: localhost http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/tasks/build-rpms.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-builder/tasks/build-rpms.yml b/metron-deployment/ansible/roles/metron-builder/tasks/build-rpms.yml new file mode 100644 index 0000000..c362fc2 --- /dev/null +++ b/metron-deployment/ansible/roles/metron-builder/tasks/build-rpms.yml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Build Metron RPM Packages + shell: "{{ item }}" + args: + chdir: "{{ metron_build_dir }}/metron-deployment" + with_items: + - mvn package -DskipTests -Pbuild-rpms + become: false + run_once: true + delegate_to: localhost http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/tasks/build.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-builder/tasks/build.yml b/metron-deployment/ansible/roles/metron-builder/tasks/build.yml new file mode 100644 index 0000000..b63d1ef --- /dev/null +++ b/metron-deployment/ansible/roles/metron-builder/tasks/build.yml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Build Metron + shell: "{{ item }}" + args: + chdir: "{{ metron_build_dir }}" + with_items: + - mvn package -DskipTests -T 2C -P HDP-2.5.0.0,mpack + become: false + run_once: true + delegate_to: localhost http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/tasks/clean.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-builder/tasks/clean.yml b/metron-deployment/ansible/roles/metron-builder/tasks/clean.yml new file mode 100644 index 0000000..4497d82 --- /dev/null +++ b/metron-deployment/ansible/roles/metron-builder/tasks/clean.yml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +# +# clean out build artifacts including packages +# +- name: Clean Metron + shell: "{{ item }}" + args: + chdir: "{{ metron_build_dir }}" + with_items: + - mvn clean -P HDP-2.5.0.0 + - mvn clean -P mpack + - mvn clean -P build-rpms + - mvn clean -P build-debs + become: false + run_once: true + delegate_to: localhost http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-builder/tasks/main.yml b/metron-deployment/ansible/roles/metron-builder/tasks/main.yml new file mode 100644 index 0000000..9636363 --- /dev/null +++ b/metron-deployment/ansible/roles/metron-builder/tasks/main.yml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +# +# fail if the distribution is unknown or unsupported +# +- name: Check distribution + fail: + msg: "Unsupported distribution; no packages available for {{ ansible_distribution }}" + when: (ansible_distribution != "Ubuntu") and (ansible_distribution != "CentOS") + +- include: clean.yml +- include: build.yml + +- include: build-rpms.yml + static: no + when: ansible_distribution == "CentOS" + +- include: build-debs.yml + static: no + when: ansible_distribution == "Ubuntu" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-packages/defaults/main.yml b/metron-deployment/ansible/roles/metron-packages/defaults/main.yml new file mode 100644 index 0000000..36de438 --- /dev/null +++ b/metron-deployment/ansible/roles/metron-packages/defaults/main.yml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +repo_path: "/localrepo" +metron_rpm_glob: "{{ playbook_dir }}/../../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm" +metron_deb_glob: "{{ playbook_dir }}/../../packaging/docker/deb-docker/target/*.deb" +packages_required: true http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_centos.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_centos.yml b/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_centos.yml new file mode 100644 index 0000000..cd76fa3 --- /dev/null +++ b/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_centos.yml @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Copy Metron RPMs + copy: + src: "{{ item }}" + dest: "{{ repo_path }}" + owner: root + mode: 0755 + with_fileglob: + - "{{ metron_rpm_glob }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_ubuntu.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_ubuntu.yml b/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_ubuntu.yml new file mode 100644 index 0000000..6437bb5 --- /dev/null +++ b/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_ubuntu.yml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Create local repo structure + file: + path: "{{ repo_path }}/dists/METRON/main/binary-amd64" + state: directory + mode: 0755 + +- name: Copy Metron DEBs + copy: + src: "{{ item }}" + dest: "{{ repo_path }}/dists/METRON/main/binary-amd64" + owner: root + mode: 0755 + with_fileglob: + - "{{ metron_deb_glob }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/create_directory.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/create_directory.yml b/metron-deployment/ansible/roles/metron-packages/tasks/create_directory.yml new file mode 100644 index 0000000..aae8bee --- /dev/null +++ b/metron-deployment/ansible/roles/metron-packages/tasks/create_directory.yml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Clean out local repo + file: + state: absent + path: "{{ repo_path }}" + +- name: Create local repo directory + file: + path: "{{ repo_path }}" + state: directory + mode: 0755 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_centos.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_centos.yml b/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_centos.yml new file mode 100644 index 0000000..81c265d --- /dev/null +++ b/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_centos.yml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install tooling on CentOS + package: + name: createrepo + state: installed + +- name: Create local repo with new packages on CentOS + shell: "createrepo {{ repo_path }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_ubuntu.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_ubuntu.yml b/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_ubuntu.yml new file mode 100644 index 0000000..95a214f --- /dev/null +++ b/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_ubuntu.yml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Install tooling on Ubuntu + package: + name: dpkg-dev + state: installed + +- name: Create repository index directory on Ubuntu + file: + path: "{{ repo_path }}/dists/METRON/main/binary-amd64" + state: directory + mode: 0755 + +- name: Create repository index on Ubuntu + shell: "dpkg-scanpackages . /dev/null | gzip -9c > dists/METRON/main/binary-amd64/Packages.gz" + args: + chdir: "{{ repo_path }}" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/main.yml b/metron-deployment/ansible/roles/metron-packages/tasks/main.yml new file mode 100644 index 0000000..193a64c --- /dev/null +++ b/metron-deployment/ansible/roles/metron-packages/tasks/main.yml @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- include: validate.yml +- include: create_directory.yml + +# +# copy metron packages to the remote host +# +# the repository on the remote host containing the Metron packages (by default +# located at /localrepo) MUST BE defined on every Ambari slave host. in most +# cases, this local repository can be empty. only the local repository hosting +# metron needs to contain the metron packages (either RPMs or DEBs) +# +# 'packages_required' +# +# this variable defines whether the Metron packages need to be copied to the +# repository or whether it can be left empty. copying the packages can be a +# time consuming operation on a large cluster, so it avoided unless required. +# +- include: copy_packages_centos.yml + static: no + when: ansible_distribution == "CentOS" and packages_required == true + +- include: copy_packages_ubuntu.yml + static: no + when: ansible_distribution == "Ubuntu" and packages_required == true + +# +# create the local repository +# +- include: create_repo_centos.yml + static: no + when: ansible_distribution == "CentOS" + +- include: create_repo_ubuntu.yml + static: no + when: ansible_distribution == "Ubuntu" http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/validate.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/validate.yml b/metron-deployment/ansible/roles/metron-packages/tasks/validate.yml new file mode 100644 index 0000000..8a23ab1 --- /dev/null +++ b/metron-deployment/ansible/roles/metron-packages/tasks/validate.yml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +# +# fail if the distribution is unknown or unsupported +# +- name: Validate support for distribution + fail: + msg: "Unsupported distribution; no packages available for {{ ansible_distribution }}" + when: (ansible_distribution != "CentOS") and (ansible_distribution != "Ubuntu") http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_elasticsearch_templates/files/es_templates/error_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron_elasticsearch_templates/files/es_templates/error_index.template b/metron-deployment/ansible/roles/metron_elasticsearch_templates/files/es_templates/error_index.template new file mode 100644 index 0000000..3bb4633 --- /dev/null +++ b/metron-deployment/ansible/roles/metron_elasticsearch_templates/files/es_templates/error_index.template @@ -0,0 +1,57 @@ +{ + "template": "error_index*", + "mappings": { + "error_doc": { + "_timestamp": { + "enabled": true + }, + "properties": { + "exception": { + "type": "string", + "index": "not_analyzed" + }, + "hostname": { + "type": "string", + "index": "not_analyzed" + }, + "stack": { + "type": "string", + "index": "not_analyzed" + }, + "timestamp": { + "type": "date", + "format": "epoch_millis" + }, + "message": { + "type": "string", + "index": "not_analyzed" + }, + "raw_message": { + "type": "string", + "index": "not_analyzed", + "ignore_above": 8191 + }, + "raw_message_bytes": { + "type": "binary", + "index": "no" + }, + "error_fields": { + "type": "string", + "index": "not_analyzed" + }, + "error_hash": { + "type": "string", + "index": "not_analyzed" + }, + "failed_sensor_type": { + "type": "string", + "index": "not_analyzed" + }, + "error_type": { + "type": "string", + "index": "not_analyzed" + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron_pcapservice/defaults/main.yml b/metron-deployment/ansible/roles/metron_pcapservice/defaults/main.yml new file mode 100644 index 0000000..f570602 --- /dev/null +++ b/metron-deployment/ansible/roles/metron_pcapservice/defaults/main.yml @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +metron_version: 0.4.3 +metron_directory: /usr/metron/{{ metron_version }} +pcapservice_jar_name: metron-api-{{ metron_version }}.jar +pcapservice_jar_src: "{{ playbook_dir }}/../../../metron-platform/metron-api/target/{{ pcapservice_jar_name }}" +pcapservice_jar_dst: "{{ metron_directory }}/lib/{{ pcapservice_jar_name }}" +pcapservice_port: 8081 +hbase_config_path: "/etc/hbase/conf" +query_hdfs_path: "/tmp" +pcap_hdfs_path: "/apps/metron/pcap" +metron_pcapservice_logrotate_frequency: daily +metron_pcapservice_logrotate_retention: 30 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/meta/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron_pcapservice/meta/main.yml b/metron-deployment/ansible/roles/metron_pcapservice/meta/main.yml new file mode 100644 index 0000000..ddf6aa9 --- /dev/null +++ b/metron-deployment/ansible/roles/metron_pcapservice/meta/main.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +dependencies: + - java_jdk http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/tasks/config-hbase.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron_pcapservice/tasks/config-hbase.yml b/metron-deployment/ansible/roles/metron_pcapservice/tasks/config-hbase.yml new file mode 100644 index 0000000..b77c1ec --- /dev/null +++ b/metron-deployment/ansible/roles/metron_pcapservice/tasks/config-hbase.yml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- + - name: Fetch hbase-site.xml + fetch: src=/etc/hbase/conf/hbase-site.xml dest=/tmp/hbase/conf/hbase-site.xml flat=yes + delegate_to: "{{ groups.ambari_slave[0] }}" + + - name: Create hbase conf directory + file: path=/etc/hbase/conf state=directory mode=0755 + + - name: Copy hbase-site.xml + copy: src=/tmp/hbase/conf/hbase-site.xml dest=/etc/hbase/conf/hbase-site.xml mode=0644 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron_pcapservice/tasks/main.yml b/metron-deployment/ansible/roles/metron_pcapservice/tasks/main.yml new file mode 100644 index 0000000..ed33354 --- /dev/null +++ b/metron-deployment/ansible/roles/metron_pcapservice/tasks/main.yml @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- include: pcapservice.yml + +- name: Create Logrotate Script for metron_pcapservice + template: + src: "metron-pcapservice-logrotate.yml" + dest: "/etc/logrotate.d/metron-pcapservice" + mode: 0644 + http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/tasks/pcapservice.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron_pcapservice/tasks/pcapservice.yml b/metron-deployment/ansible/roles/metron_pcapservice/tasks/pcapservice.yml new file mode 100644 index 0000000..d0ac411 --- /dev/null +++ b/metron-deployment/ansible/roles/metron_pcapservice/tasks/pcapservice.yml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Create Metron streaming directories + file: path={{ metron_directory }}/{{ item.name }} state=directory mode=0755 + with_items: + - { name: 'lib'} + - { name: 'config'} + +- name: Copy Metron pcapservice jar + copy: + src: "{{ pcapservice_jar_src }}" + dest: "{{ pcapservice_jar_dst }}" + +- name: Install service script + template: src=pcapservice dest=/etc/init.d/pcapservice mode=0755 http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml b/metron-deployment/ansible/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml new file mode 100644 index 0000000..0293ade --- /dev/null +++ b/metron-deployment/ansible/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#Metron pcapService +/var/log/metron_pcapservice.log { + {{ metron_pcapservice_logrotate_frequency }} + rotate {{ metron_pcapservice_logrotate_retention }} + missingok + notifempty + copytruncate + compress +} + http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/templates/pcapservice ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/metron_pcapservice/templates/pcapservice b/metron-deployment/ansible/roles/metron_pcapservice/templates/pcapservice new file mode 100644 index 0000000..054133d --- /dev/null +++ b/metron-deployment/ansible/roles/metron_pcapservice/templates/pcapservice @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# metron pcap service +# chkconfig: 345 20 80 +# description: Metron PCAP Service Daemon +# processname: pcapservice +# +NAME=pcapservice +DESC="Metron pcap service" +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +LOGFILE="/var/log/metron_pcapservice.log" +EXTRA_ARGS="${@:2}" +DAEMON_PATH="/" +DAEMON="/usr/bin/yarn jar" +DAEMONOPTS="{{ pcapservice_jar_dst }} org.apache.metron.pcapservice.rest.PcapService -port {{ pcapservice_port }} -query_hdfs_path {{ query_hdfs_path }} -pcap_hdfs_path {{ pcap_hdfs_path }}" + +case "$1" in + start) + printf "%-50s" "Starting $NAME..." + + # kick-off the daemon + cd $DAEMON_PATH + PID=`$DAEMON $DAEMONOPTS >> $LOGFILE 2>&1 & echo $!` + if [ -z $PID ]; then + printf "%s\n" "Fail" + else + echo $PID > $PIDFILE + printf "%s\n" "Ok" + fi + ;; + + status) + printf "%-50s" "Checking $NAME..." + if [ -f $PIDFILE ]; then + PID=`cat $PIDFILE` + if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then + printf "%s\n" "Process dead but pidfile exists" + else + echo "Running" + fi + else + printf "%s\n" "Service not running" + fi + ;; + + stop) + printf "%-50s" "Stopping $NAME" + PID=`cat $PIDFILE` + cd $DAEMON_PATH + if [ -f $PIDFILE ]; then + kill -HUP $PID + printf "%s\n" "Ok" + rm -f $PIDFILE + else + printf "%s\n" "pidfile not found" + fi + ;; + + restart) + $0 stop + $0 start + ;; + + *) + echo "Usage: $0 {status|start|stop|restart}" + exit 1 +esac
