http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html b/site/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html index eeadfe6..5cf9775 100644 --- a/site/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html +++ b/site/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html @@ -1,387 +1,230 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2018-01-03 - | Rendered using Apache Maven Fluido Skin 1.3.0 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md at 2018-06-07 + | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180103" /> + <meta name="Date-Revision-yyyymmdd" content="20180607" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – </title> - <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" /> <link rel="stylesheet" href="../../../css/site.css" /> <link rel="stylesheet" href="../../../css/print.css" media="print" /> - - - <script type="text/javascript" src="../../../js/apache-maven-fluido-1.3.0.min.js"></script> - - - -<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> - - </head> - <body class="topBarDisabled"> - - - - - <div class="container-fluid"> - <div id="banner"> - <div class="pull-left"> - <a href="http://metron.apache.org/"; id="bannerLeft"> - <img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> - </a> - </div> - <div class="pull-right"> </div> + <script type="text/javascript" src="../../../js/apache-maven-fluido-1.7.min.js"></script> +<script type="text/javascript"> + $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } ); + </script> + </head> + <body class="topBarDisabled"> + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"><a href="http://metron.apache.org/"; id="bannerLeft"><img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div> + <div class="pull-right"></div> <div class="clear"><hr/></div> </div> <div id="breadcrumbs"> <ul class="breadcrumb"> - - - <li class=""> - <a href="http://www.apache.org"; class="externalLink" title="Apache"> - Apache</a> - </li> - <li class="divider ">/</li> - <li class=""> - <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> - Metron</a> - </li> - <li class="divider ">/</li> - <li class=""> - <a href="../../../index.html" title="Documentation"> - Documentation</a> - </li> - <li class="divider ">/</li> - <li class=""></li> - - - - <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.2</li> - - </ul> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> + <li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> + <li class="active "></li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> + <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + </ul> </div> - - <div class="row-fluid"> - <div id="leftColumn" class="span3"> + <div id="leftColumn" class="span2"> <div class="well sidebar-nav"> - - - <ul class="nav nav-list"> - <li class="nav-header">User Documentation</li> - - <li> - - <a href="../../../index.html" title="Metron"> - <i class="icon-chevron-down"></i> - Metron</a> - <ul class="nav nav-list"> - - <li> - - <a href="../../../Upgrading.html" title="Upgrading"> - <i class="none"></i> - Upgrading</a> - </li> - - <li> - - <a href="../../../metron-analytics/index.html" title="Analytics"> - <i class="icon-chevron-right"></i> - Analytics</a> - </li> - - <li> - - <a href="../../../metron-contrib/metron-docker/index.html" title="Docker"> - <i class="none"></i> - Docker</a> - </li> - - <li> - - <a href="../../../metron-deployment/index.html" title="Deployment"> - <i class="icon-chevron-down"></i> - Deployment</a> - <ul class="nav nav-list"> - - <li> - - <a href="../../../metron-deployment/Kerberos-ambari-setup.html" title="Kerberos-ambari-setup"> - <i class="none"></i> - Kerberos-ambari-setup</a> - </li> - - <li> - - <a href="../../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup"> - <i class="none"></i> - Kerberos-manual-setup</a> - </li> - - <li> - - <a href="../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2"> - <i class="none"></i> - Amazon-ec2</a> - </li> - - <li> - - <a href="../../../metron-deployment/other-examples/index.html" title="Other-examples"> - <i class="icon-chevron-down"></i> - Other-examples</a> - <ul class="nav nav-list"> - - <li class="active"> - - <a href="#"><i class="none"></i>Manual_Install_CentOS6</a> - </li> - </ul> - </li> - - <li> - - <a href="../../../metron-deployment/packaging/ambari/index.html" title="Ambari"> - <i class="none"></i> - Ambari</a> - </li> - - <li> - - <a href="../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker"> - <i class="none"></i> - Ansible-docker</a> - </li> - - <li> - - <a href="../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker"> - <i class="none"></i> - Rpm-docker</a> - </li> - - <li> - - <a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"> - <i class="none"></i> - Packer-build</a> - </li> - - <li> - - <a href="../../../metron-deployment/roles/index.html" title="Roles"> - <i class="icon-chevron-right"></i> - Roles</a> - </li> - - <li> - - <a href="../../../metron-deployment/vagrant/index.html" title="Vagrant"> - <i class="icon-chevron-right"></i> - Vagrant</a> - </li> - </ul> - </li> - - <li> - - <a href="../../../metron-interface/metron-alerts/index.html" title="Alerts"> - <i class="none"></i> - Alerts</a> - </li> - - <li> - - <a href="../../../metron-interface/metron-config/index.html" title="Config"> - <i class="none"></i> - Config</a> - </li> - - <li> - - <a href="../../../metron-interface/metron-rest/index.html" title="Rest"> - <i class="none"></i> - Rest</a> - </li> - - <li> - - <a href="../../../metron-platform/index.html" title="Platform"> - <i class="icon-chevron-right"></i> - Platform</a> - </li> - - <li> - - <a href="../../../metron-sensors/index.html" title="Sensors"> - <i class="icon-chevron-right"></i> - Sensors</a> - </li> - - <li> - - <a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> - <i class="none"></i> - Stellar-3rd-party-example</a> - </li> - - <li> - - <a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"> - <i class="icon-chevron-right"></i> - Stellar-common</a> - </li> - - <li> - - <a href="../../../use-cases/index.html" title="Use-cases"> - <i class="icon-chevron-right"></i> - Use-cases</a> - </li> - </ul> - </li> - </ul> - - - - <hr class="divider" /> - - <div id="poweredBy"> - <div class="clear"></div> - <div class="clear"></div> - <div class="clear"></div> - <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> - <img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /> - </a> - </div> + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + <li><a href="../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a> + <ul class="nav nav-list"> + <li><a href="../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li> + <li><a href="../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li> + <li><a href="../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li> + <li><a href="../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li> + <li><a href="../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li> + <li><a href="../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-down"></span>Deployment</a> + <ul class="nav nav-list"> + <li><a href="../../../metron-deployment/Kerberos-ambari-setup.html" title="Kerberos-ambari-setup"><span class="none"></span>Kerberos-ambari-setup</a></li> + <li><a href="../../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup"><span class="none"></span>Kerberos-manual-setup</a></li> + <li><a href="../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2"><span class="none"></span>Amazon-ec2</a></li> + <li><a href="../../../metron-deployment/ansible/index.html" title="Ansible"><span class="icon-chevron-right"></span>Ansible</a></li> + <li><a href="../../../metron-deployment/development/index.html" title="Development"><span class="icon-chevron-right"></span>Development</a></li> + <li><a href="../../../metron-deployment/other-examples/index.html" title="Other-examples"><span class="icon-chevron-down"></span>Other-examples</a> + <ul class="nav nav-list"> + <li class="active"><a href="#"><span class="none"></span>Manual_Install_CentOS6</a></li> + </ul> +</li> + <li><a href="../../../metron-deployment/packaging/ambari/index.html" title="Ambari"><span class="icon-chevron-right"></span>Ambari</a></li> + <li><a href="../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker"><span class="none"></span>Ansible-docker</a></li> + <li><a href="../../../metron-deployment/packaging/docker/deb-docker/index.html" title="Deb-docker"><span class="none"></span>Deb-docker</a></li> + <li><a href="../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker"><span class="none"></span>Rpm-docker</a></li> + <li><a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li> + </ul> +</li> + <li><a href="../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li> + <li><a href="../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li> + <li><a href="../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li> + <li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li> + <li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li> + <li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li> + <li><a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li> + <li><a href="../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li> + <li><a href="../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li> + </ul> +</li> +</ul> + <hr /> + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> +<a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /></a> + </div> </div> </div> - - - <div id="bodyColumn" class="span9" > - - <div class="section"> + <div id="bodyColumn" class="span10" > +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<div class="section"> <h2><a name="Metron_0.4.0_with_HDP_2.5_bare-metal_install_on_Centos_6_with_MariaDB_for_Metron_REST:"></a>Metron 0.4.0 with HDP 2.5 bare-metal install on Centos 6 with MariaDB for Metron REST:</h2> <div class="section"> <h3><a name="Introduction"></a>Introduction</h3> <p>We will be installing Metron 0.4.0 with HDP 2.5 on CentOS 6. We will also install MariaDB as a database for Metron REST. Additionally, we’ll also install Apache NiFi. I installed Metron in a test environment with 3 VMs to try it out as well as a single node. I’ll try to write this guide so that the necessary steps can easily be adapted for other environments.</p></div> <div class="section"> <h3><a name="Environment"></a>Environment</h3> - <ul> - + <li> -<p>Single node: 4 CPUs, 16 GB RAM.</p></li> - + +<p>Single node: 4 CPUs, 16 GB RAM.</p> +</li> <li> + <p>Multiple nodes:</p> - <ul> - + <li>3 VMs, 2 CPUs per VM and 8 GB RAM per VM.</li> - <li>Hosts: 10.10.10.1 node1 10.10.10.2 node2 10.10.10.3 node3</li> - </ul></li> +</ul> +</li> </ul></div> <div class="section"> <h3><a name="Prerequisites:"></a>Prerequisites:</h3> - <ul> - + <li> -<p>CentOS 6</p></li> - + +<p>CentOS 6</p> +</li> <li> -<p>Add the epel repository and install tmux, vim & htop. Installing these utilities is not strictly necessary, but I install these by default for potential troubleshooting & editing of files locally):</p></li> + +<p>Add the epel repository and install tmux, vim & htop. Installing these utilities is not strictly necessary, but I install these by default for potential troubleshooting & editing of files locally):</p> +</li> </ul> -<div class="source"> -<div class="source"> -<pre># yum install epel-release -y +<div> +<div> +<pre class="source"># yum install epel-release -y # yum update -y # yum install vim tmux htop -y </pre></div></div> <ul> - + <li>Set up passwordless SSH between our nodes: If passwordless ssh has not yet been set up within the cluster, then in main node generate key:</li> </ul> -<div class="source"> -<div class="source"> -<pre># cat /dev/zero | ssh-keygen -q -N "" 2>/dev/null +<div> +<div> +<pre class="source"># cat /dev/zero | ssh-keygen -q -N "" 2>/dev/null # cd ~/.ssh # cat id_rsa.pub >> authorized_keys </pre></div></div> <ul> - + <li>If you’re not installing on a single node, add this newly generated key to all the slave nodes:</li> </ul> -<div class="source"> -<div class="source"> -<pre>ssh-copy-id -i ~/.ssh/id_rsa.pub <replace_with_node_ip> +<div> +<div> +<pre class="source">ssh-copy-id -i ~/.ssh/id_rsa.pub <replace_with_node_ip> </pre></div></div> -<p><b>Side note:</b> You might have to adapt your sshd_config file and add “PermitRootLogin yes” amongst other parameters if you want passwordless root access, but that’s outside the scope of this document.</p> +<p><b>Side note:</b> You might have to adapt your sshd_config file and add “PermitRootLogin yes” amongst other parameters if you want passwordless root access, but that’s outside the scope of this document.</p> <ul> - + <li>Increase limits for ElasticSearch and Storm on nodes where you will be installing them (if you don’t know, increase it everywhere):</li> </ul> -<div class="source"> -<div class="source"> -<pre># echo -e "elasticsearch - memlock unlimited\nstorm - nproc 257597" >> /etc/security/limits.conf +<div> +<div> +<pre class="source"># echo -e "elasticsearch - memlock unlimited\nstorm - nproc 257597" >> /etc/security/limits.conf </pre></div></div> <ul> - + <li>Adjust limits to secure level (<a class="externalLink" href="https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.0/bk_installing_manually_book/content/ref-729d1fb0-6d1b-459f-a18a-b5eba4540ab5.1.html";>link</a>):</li> </ul> -<div class="source"> -<div class="source"> -<pre># ulimit -n 32768 +<div> +<div> +<pre class="source"># ulimit -n 32768 # ulimit -u 65536 # echo -e "* - nofile 32768\n* - nproc 65536" >> /etc/security/limits.conf </pre></div></div> <ul> - + <li>Disable IPv6, leaving it enabled may force service to bind to IPv6 addresses only and thus resulting in inability to connect to it (<a class="externalLink" href="https://wiki.centos.org/FAQ/CentOS6#head-d47139912868bcb9d754441ecb6a8a10d41781df";>source link</a>): Disable for the running system:</li> </ul> -<div class="source"> -<div class="source"> -<pre># sysctl -w net.ipv6.conf.all.disable_ipv6=1 +<div> +<div> +<pre class="source"># sysctl -w net.ipv6.conf.all.disable_ipv6=1 # sysctl -w net.ipv6.conf.default.disable_ipv6=1 or # echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6 # echo 1 > /proc/sys/net/ipv6/conf/default/disable_ipv6 </pre></div></div> + <p>To survive a reboot: Add: net.ipv6.conf.all.disable_ipv6 = 1 net.ipv6.conf.default.disable_ipv6 = 1 To: /etc/sysctl.conf</p> -<div class="source"> -<div class="source"> -<pre># echo -e "\n# Disable IPv6\nnet.ipv6.conf.all.disable_ipv6 = 1\nnet.ipv6.conf.default.disable_ipv6 = 1" >> /etc/sysctl.conf +<div> +<div> +<pre class="source"># echo -e "\n# Disable IPv6\nnet.ipv6.conf.all.disable_ipv6 = 1\nnet.ipv6.conf.default.disable_ipv6 = 1" >> /etc/sysctl.conf </pre></div></div> <ul> - + <li>Disable Transparent Hugepage. Add “transparent_hugepage=never” to the end of the kernel line in /boot/grub/grub.conf and reboot. (Ambari demands it, do we need to comply?):</li> </ul> -<div class="source"> -<div class="source"> -<pre>Add "transparent_hugepage=never" in the kernel line after "quiet: +<div> +<div> +<pre class="source">Add "transparent_hugepage=never" in the kernel line after "quiet: "kernel /vmlinuz-2.6.32-696.3.1.el6.x86_64 ro root=/dev/mapper/vg_centos6-lv_root rd_NO_LUKS LANG=en_US.UTF-8 rd_LVM_LV=vg_centos6/lv_swap rd_NO_MD SYSFONT=latarcyrheb-sun16 crashkernel=128M rd_LVM_LV=vg_centos6/lv_root KEYBOARDTYPE=pc KEYTABLE=us rd_NO_DM rhgb quiet" becomes: "kernel /vmlinuz-2.6.32-696.3.1.el6.x86_64 ro root=/dev/mapper/vg_centos6-lv_root rd_NO_LUKS LANG=en_US.UTF-8 rd_LVM_LV=vg_centos6/lv_swap rd_NO_MD SYSFONT=latarcyrheb-sun16 crashkernel=128M rd_LVM_LV=vg_centos6/lv_root KEYBOARDTYPE=pc KEYTABLE=us rd_NO_DM rhgb quiet transparent_hugepage=never" @@ -391,13 +234,13 @@ Afterwards, run: </pre></div></div> <ul> - + <li>If you do not want to mess with grub/kernel parameters, add the following to /etc/rc.local:</li> </ul> -<div class="source"> -<div class="source"> -<pre>vim /etc/rc.local: +<div> +<div> +<pre class="source">vim /etc/rc.local: # Disable THP at boot time if test -f /sys/kernel/mm/redhat_transparent_hugepage/enabled; then echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled @@ -407,87 +250,90 @@ if test -f /sys/kernel/mm/redhat_transparent_hugepage/defrag; then echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag fi </pre></div></div> + <p>After reboot check that changes were applied (make sure that word “never” is selected in square-brackets):</p> -<div class="source"> -<div class="source"> -<pre># cat /sys/kernel/mm/transparent_hugepage/enabled +<div> +<div> +<pre class="source"># cat /sys/kernel/mm/transparent_hugepage/enabled always madvise [never] -</pre></div></div></div> +</pre></div></div> +</div> <div class="section"> <h3><a name="Metron_install_pre-preparation:"></a>Metron install pre-preparation:</h3> - <ul> - + <li>On all nodes Install pre-requisites for Ambari:</li> </ul> -<div class="source"> -<div class="source"> -<pre># yum install git wget curl rpm tar unzip bzip2 wget createrepo yum-utils ntp python-pip psutils python-psutil ntp libffi-devel gcc openssl-devel npm -y +<div> +<div> +<pre class="source"># yum install git wget curl rpm tar unzip bzip2 wget createrepo yum-utils ntp python-pip psutils python-psutil ntp libffi-devel gcc openssl-devel npm -y # pip install --upgrade pip # pip install requests urllib # pip install --upgrade setuptools </pre></div></div> <ul> - + <li>Install Maven 3.3.9 on main node and on Metron node install java 1.8 (if you don’t know which it is, install it everywhere):</li> </ul> -<div class="source"> -<div class="source"> -<pre># yum install java-1.8.0-openjdk java-1.8.0-openjdk-devel -y +<div> +<div> +<pre class="source"># yum install java-1.8.0-openjdk java-1.8.0-openjdk-devel -y </pre></div></div> <ul> - + <li>Set path to Java 8 if it does not exist:</li> </ul> -<div class="source"> -<div class="source"> -<pre># export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s_/jre/bin/java__") +<div> +<div> +<pre class="source"># export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s_/jre/bin/java__") </pre></div></div> <ul> - + <li>Save export for future reboots:</li> </ul> -<div class="source"> -<div class="source"> -<pre># echo 'export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s_/jre/bin/java__")' > /etc/profile.d/java_18.sh +<div> +<div> +<pre class="source"># echo 'export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s_/jre/bin/java__")' > /etc/profile.d/java_18.sh # chmod +x /etc/profile.d/java_18.sh # source /etc/profile.d/java_18.sh </pre></div></div> <ul> - + <li>Download and install Maven 3.3.9:</li> </ul> -<div class="source"> -<div class="source"> -<pre># wget http://apache.volia.net/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +<div> +<div> +<pre class="source"># wget http://apache.volia.net/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz # tar -zxf apache-maven-3.3.9-bin.tar.gz # mv apache-maven-3.3.9 /opt # PATH=/opt/apache-maven-3.3.9/bin:$PATH # echo 'export PATH=/opt/apache-maven-3.3.9/bin:$PATH' > /etc/profile.d/maven.sh # chmod +x /etc/profile.d/maven.sh </pre></div></div> + <p>Check whether Maven works:</p> -<div class="source"> -<div class="source"> -<pre># source /etc/profile.d/maven.sh +<div> +<div> +<pre class="source"># source /etc/profile.d/maven.sh # mvn -V </pre></div></div> + <p>You should see something similar to:</p> -<div class="source"> -<div class="source"> -<pre>[root@base1 ~]# mvn -V +<div> +<div> +<pre class="source">[root@base1 ~]# mvn -V Apache Maven 3.3.9 (bb52d8502b132ec0a5a3f4c09453c07478323dc5; 2015-11-10T08:41:47-08:00) Maven home: /opt/apache-maven-3.3.9 Java version: 1.8.0_131, vendor: Oracle Corporation @@ -513,54 +359,59 @@ OS name: "linux", version: "3.10.0-514.16.1.el7.x86_64", arc </pre></div></div> <ul> - + <li>On Ambari node install and enable docker (we will need it to build Metron mpack for Ambari):</li> </ul> -<div class="source"> -<div class="source"> -<pre># yum install docker-io -y +<div> +<div> +<pre class="source"># yum install docker-io -y # service docker start </pre></div></div> <ul> - + <li>Also on your build box, install npm. This is needed to build metron-config, part of the UI.</li> </ul> -<div class="source"> -<div class="source"> -<pre># yum install npm -y +<div> +<div> +<pre class="source"># yum install npm -y </pre></div></div> <ul> - + <li> -<p>Remove ipv4 ‘localhost.localdomain’ from /etc/hosts</p></li> - + +<p>Remove ipv4 ‘localhost.localdomain’ from /etc/hosts</p> +</li> <li> -<p>Remove ipv6 ‘localhost.localdomain’ from /etc/hosts</p></li> - + +<p>Remove ipv6 ‘localhost.localdomain’ from /etc/hosts</p> +</li> <li> -<p>Add “127.0.0.1 localhost” to /etc/hosts</p></li> - + +<p>Add “127.0.0.1 localhost” to /etc/hosts</p> +</li> <li> -<p>Install the database we will use for Metron REST:</p></li> + +<p>Install the database we will use for Metron REST:</p> +</li> </ul> -<div class="source"> -<div class="source"> -<pre># yum install mariadb-server mysql-connector-java -y +<div> +<div> +<pre class="source"># yum install mariadb-server mysql-connector-java -y </pre></div></div> <ul> - + <li>Configure a user and a database for Metron REST: If you haven’t run <tt>mysql_secure_installation</tt> after the database installation, do that first:</li> </ul> -<div class="source"> -<div class="source"> -<pre># service mysqld start +<div> +<div> +<pre class="source"># service mysqld start # /sbin/chkconfig --add mysqld # /sbin/chkconfig --list mysqld # /sbin/chkconfig mysqld on @@ -628,101 +479,103 @@ Thanks for using MySQL! Cleaning up... # -</pre></div></div></div> +</pre></div></div> +</div> <div class="section"> <h3><a name="Build_Metron_code"></a>Build Metron code</h3> <p>Now we are going to start to building Metron. At the time of writing, Metron 0.4.0 was in the final stages of being released.</p> - <ul> - + <li>On the main node, clone the Metron repository:</li> </ul> -<div class="source"> -<div class="source"> -<pre># git clone https://github.com/apache/metron +<div> +<div> +<pre class="source"># git clone https://github.com/apache/metron </pre></div></div> + <p>If you want to make sure you’re on the 0.4.0 release branch, do:</p> -<div class="source"> -<div class="source"> -<pre># git clone https://github.com/apache/metron +<div> +<div> +<pre class="source"># git clone https://github.com/apache/metron # cd metron # git checkout Metron_0.4.0 </pre></div></div> <ul> - + <li>Build Metron with HDP 2.5 profile:</li> </ul> -<div class="source"> -<div class="source"> -<pre># cd metron +<div> +<div> +<pre class="source"># cd metron # mvn clean package -DskipTests -T 2C -P HDP-2.5.0.0,mpack # cd metron-deployment/packaging/docker/rpm-docker # mvn clean install -DskipTests -PHDP-2.5.0.0 </pre></div></div> -<p>If for some reason, the rpm-docker fails with the message “/bin/bash: ./build.sh: Permission denied”, try disabling selinux (“setenforce 0”) and run “mvn clean install -DskipTests -PHDP-2.5.0.0” again.</p> +<p>If for some reason, the rpm-docker fails with the message “/bin/bash: ./build.sh: Permission denied”, try disabling selinux (“setenforce 0”) and run “mvn clean install -DskipTests -PHDP-2.5.0.0” again.</p> <ul> - + <li>On all nodes, create a localrepo directory and copy the RPMs from Ambari node there:</li> </ul> -<div class="source"> -<div class="source"> -<pre># mkdir /localrepo +<div> +<div> +<pre class="source"># mkdir /localrepo # cp -rp /root/metron/metron-deployment/packaging/docker/rpm-docker/RPMS/noarch/* /localrepo/ # createrepo /localrepo </pre></div></div> + <p>If you’re doing a multi node install, also create localrepo on the nodes and copy the packages to the other nodes:</p> -<div class="source"> -<div class="source"> -<pre># ssh root@node2 mkdir /localrepo +<div> +<div> +<pre class="source"># ssh root@node2 mkdir /localrepo # scp /localrepo/*\.rpm root@node2:/localrepo/. # ssh root@node2 yum install createrepo -y # ssh root@node2 createrepo /localrepo </pre></div></div> <ul> - + <li>Make sure to do the above on each node.</li> </ul> <p>Fetch & create logrotate script for Hadoop Services:</p> -<div class="source"> -<div class="source"> -<pre># wget -O /etc/logrotate.d/metron-ambari https://raw.githubusercontent.com/apache/metron/master/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml +<div> +<div> +<pre class="source"># wget -O /etc/logrotate.d/metron-ambari https://raw.githubusercontent.com/apache/metron/master/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml # sed -i 's/^ {{ hadoop_logrotate_frequency }}.*$/ daily/' /etc/logrotate.d/metron-ambari # sed -i 's/^ rotate {{ hadoop_logrotate_retention }}.*$/ rotate 30/' /etc/logrotate.d/metron-ambari # chmod 0644 /etc/logrotate.d/metron-ambari -</pre></div></div></div> +</pre></div></div> +</div> <div class="section"> <h3><a name="Ambari_2.4_with_HDP_2.5_install"></a>Ambari 2.4 with HDP 2.5 install</h3> <p>Inspired by: [http://docs.hortonworks.com/HDPDocuments/Ambari-2.4.1.0/bk_ambari-installation/content/ch_Getting_Ready.html]</p> - <ul> - + <li>Adjust limits to secure level (inspired by <a class="externalLink" href="https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.0/bk_installing_manually_book/content/ref-729d1fb0-6d1b-459f-a18a-b5eba4540ab5.1.html";>link</a>):</li> </ul> -<div class="source"> -<div class="source"> -<pre># ulimit -n 32768 +<div> +<div> +<pre class="source"># ulimit -n 32768 # ulimit -u 65536 # echo -e "* - nofile 32768\n* - nproc 65536" >> /etc/security/limits.conf </pre></div></div> <ul> - + <li>Enable time sync, disable firewall and SElinux on every node:</li> </ul> -<div class="source"> -<div class="source"> -<pre># yum install ntp -y +<div> +<div> +<pre class="source"># yum install ntp -y # service ntpd start # /sbin/chkconfig --add ntpd # /sbin/chkconfig --list ntpd @@ -731,100 +584,99 @@ Cleaning up... </pre></div></div> <ul> - + <li>Disable firewall on every node:</li> </ul> -<div class="source"> -<div class="source"> -<pre># service iptables save +<div> +<div> +<pre class="source"># service iptables save # service iptables stop # chkconfig iptables off </pre></div></div> <ul> - + <li>Disable IPv6 firewall on every node:</li> </ul> -<div class="source"> -<div class="source"> -<pre># service ip6tables save +<div> +<div> +<pre class="source"># service ip6tables save # service ip6tables stop # chkconfig ip6tables off </pre></div></div> <ul> - -<li>Disable SElinux on every node:</li> + +<li>Disable SElinux on every node:</li> </ul> -<div class="source"> -<div class="source"> -<pre># setenforce 0 (=> I know, but for the sake of simplicity, quickness & testing, I've disabled selinux.) +<div> +<div> +<pre class="source"># setenforce 0 (=> I know, but for the sake of simplicity, quickness & testing, I've disabled selinux.) </pre></div></div> <ul> - + <li>Make sure each node can resolve every other node’s hostname or add hostname of each node to <tt>/etc/hosts</tt> on every node. For example add following lines in /etc/hosts of each node: 10.10.10.1 node1 10.10.10.2 node2 10.10.10.3 node3</li> </ul> <p>Where 10.10.10.1, 10.10.10.2 and 10.10.10.3 are the IPs of your nodes and node1, node2 and node3 are hostnames.</p> - <ul> - + <li>On main node download and setup Ambari repo (you may replace the “2.4.2.0” with a newer Ambari version number):</li> </ul> -<div class="source"> -<div class="source"> -<pre># wget -nv http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.2.0/ambari.repo -O /etc/yum.repos.d/ambari.repo +<div> +<div> +<pre class="source"># wget -nv http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.2.0/ambari.repo -O /etc/yum.repos.d/ambari.repo # yum update -y </pre></div></div> <ul> - + <li>Check that it was added:</li> </ul> -<div class="source"> -<div class="source"> -<pre># yum repolist | grep ambari +<div> +<div> +<pre class="source"># yum repolist | grep ambari Updates-ambari-2.4.2.0 ambari-2.4.2.0 - Updates 12 </pre></div></div> <ul> - + <li>Install and setup Ambari server:</li> </ul> -<div class="source"> -<div class="source"> -<pre># yum install ambari-server -y +<div> +<div> +<pre class="source"># yum install ambari-server -y # ambari-server setup -s && touch /etc/ambari-server/configured </pre></div></div> <ul> - + <li>Add Metron service to Ambari by running mpack command (make sure to specify correct path to mpack in –mpack=):</li> </ul> -<div class="source"> -<div class="source"> -<pre># ambari-server install-mpack --mpack=/root/metron/metron-deployment/packaging/ambari/metron-mpack/target/metron_mpack-0.4.0.0.tar.gz --verbose +<div> +<div> +<pre class="source"># ambari-server install-mpack --mpack=/root/metron/metron-deployment/packaging/ambari/metron-mpack/target/metron_mpack-0.4.0.0.tar.gz --verbose </pre></div></div> <ul> - + <li>Start Ambari:</li> </ul> -<div class="source"> -<div class="source"> -<pre># ambari-server start +<div> +<div> +<pre class="source"># ambari-server start </pre></div></div> <ul> - + <li>Access the Ambari UI by going to the following URL in a web browser: <tt>http://<replace_with_master_node_ip>:8080/</tt>. You can use admin/admin as username/password. Start the Install Wizard.</li> </ul> <p><b>Get Started page:</b> Enter any desired cluster name.</p> @@ -834,349 +686,206 @@ Updates-ambari-2.4.2.0 ambari-2.4.2.0 - Updates 12 <p><b>Assign Masters:</b> Assign “Kafka Broker” on all nodes. Make sure move following components on one common node (Taken from previous guide, is this still necessary?): Storm UI Server Metron Indexing MySQL Server Kibana Server Elasticsearch Master Metron Parsers Metron Enrichment</p> <p><b>Assign Slaves and Clients:</b> select All for: DataNode NodeManager RegionServer Supervisor Client</p> <p><b>Customize Services:</b> Following is a list of services that need to be configured:</p> - <ul> - + <li> -<p>Set the “NameNode Java heap size” (namenode_heapsize) from the default 1024 MB to at least 4096 MB under HDFS -> Configs.</p></li> - + +<p>Set the “NameNode Java heap size” (namenode_heapsize) from the default 1024 MB to at least 4096 MB under HDFS -> Configs.</p> +</li> <li> + <p>For ElasticSearch:</p> - <ul> - + <li>Set “zen_discovery_ping_unicast_hosts” to the IP of the node where you assigned ElasticSearch Master on the Assign Master tab.</li> - <li>Under “Advanced elastic-site”: Change “network_host” to “0.0.0.0”. Do not do this if your Metron is exposed to the public internet! Is “[ <i>local</i>, <i>site</i> ]” now.</li> - </ul></li> - +</ul> +</li> <li> + <p>Kibana:</p> - <ul> - + <li>Set “kibana_es_url” to <tt>http://<replace_with_elasticsearch_master_hostname>:9200</tt>. “replace_with_elasticsearch_master_hostname” is the IP of the node where you assigned ElasticSearch Master on the Assign Master tab.</li> - -<li>Change kibana_default_application to “dashboard/Metron-Dashboard”</li> - </ul></li> - +<li>Change kibana_default_application to “dashboard/AV-YpDmwdXwc6Ua9Muh9”</li> +</ul> +</li> <li> -<p>Metron: Set “Elasticsearch Hosts” to the IP of the node where you assigned ElasticSearch Master on the Assign Master tab.</p></li> - + +<p>Metron: Set “Elasticsearch Hosts” to the IP of the node where you assigned ElasticSearch Master on the Assign Master tab.</p> +</li> <li> -<p>Storm: You might have to increase the number of supervisor.slots.ports from the default “[6700, 6701]” to [6700, 6701, 6702, 6703, 6704] if you’re only installing a single node.</p></li> - + +<p>Storm: You might have to increase the number of supervisor.slots.ports from the default “[6700, 6701]” to [6700, 6701, 6702, 6703, 6704] if you’re only installing a single node.</p> +</li> <li> + <p>For metron REST use:</p> - -<div class="source"> -<div class="source"> -<pre>Metron JDBC client path: /usr/share/java/mysql-connector-java.jar + +<div> +<div> +<pre class="source">Metron JDBC client path: /usr/share/java/mysql-connector-java.jar Metron JDBC Driver: com.mysql.jdbc.Driver Metron JDBC password: <DB PASSWORD> Metron JDBC platform: mysql Metron JDBC URL: jdbc:mysql://127.0.0.1:3306/<DB NAME> Metron JDBC username: <DB USERNAME> -</pre></div></div></li> - +</pre></div></div> +</li> <li> -<p>Set rest of the configuration values to recommended by Ambari or the ones you desire (like DB passwords) and perform install. In a 3 node cluster, I ended up with:</p></li> -</ul> +<p>Set rest of the configuration values to recommended by Ambari or the ones you desire (like DB passwords) and perform install. In a 3 node cluster, I ended up with:</p> +</li> +</ul> <table border="0" class="table table-striped"> - <thead> - +<thead> + <tr class="a"> - <th>node1 </th> - -<th>node2 </th> - -<th>node3</th> - </tr> - </thead> - <tbody> - +<th> node2 </th> +<th> node3</th></tr> +</thead><tbody> + <tr class="b"> - <td>DataNode </td> - -<td>App Timeline Server </td> - -<td>DataNode</td> - </tr> - +<td> App Timeline Server </td> +<td> DataNode</td></tr> <tr class="a"> - <td>Elasticsearch Master </td> - -<td>DataNode </td> - -<td>Elasticsearch Data Node</td> - </tr> - +<td> DataNode </td> +<td> Elasticsearch Data Node</td></tr> <tr class="b"> - <td>HBase Client </td> - -<td>DRPC Server </td> - -<td>Flume</td> - </tr> - +<td> DRPC Server </td> +<td> Flume</td></tr> <tr class="a"> - <td>HBase Master </td> - -<td>HBase Client </td> - -<td>HBase Client</td> - </tr> - +<td> HBase Client </td> +<td> HBase Client</td></tr> <tr class="b"> - <td>RegionServer </td> - -<td>RegionServer </td> - -<td>RegionServer</td> - </tr> - +<td> RegionServer </td> +<td> RegionServer</td></tr> <tr class="a"> - -<td>HCat Client </td> - <td>HCat Client </td> - -<td>HCat Client</td> - </tr> - +<td> HCat Client </td> +<td> HCat Client</td></tr> <tr class="b"> - <td>HDFS Client </td> - -<td>HDFS Client </td> - -<td>HDFS Client</td> - </tr> - +<td> HDFS Client </td> +<td> HDFS Client</td></tr> <tr class="a"> - <td>Hive Client </td> - -<td>History Server </td> - -<td>Hive Client</td> - </tr> - +<td> History Server </td> +<td> Hive Client</td></tr> <tr class="b"> - <td>Kafka Broker </td> - -<td>Hive Client </td> - -<td>Kafka Broker</td> - </tr> - +<td> Hive Client </td> +<td> Kafka Broker</td></tr> <tr class="a"> - <td>Kibana Server </td> - -<td>Hive Metastore </td> - -<td>MapReduce2 Client</td> - </tr> - +<td> Hive Metastore </td> +<td> MapReduce2 Client</td></tr> <tr class="b"> - <td>MapReduce2 Client </td> - -<td>HiveServer2 </td> - -<td>Metrics Collector</td> - </tr> - +<td> HiveServer2 </td> +<td> Metrics Collector</td></tr> <tr class="a"> - <td>Grafana </td> - -<td>Kafka Broker </td> - -<td>Metrics Monitor</td> - </tr> - +<td> Kafka Broker </td> +<td> Metrics Monitor</td></tr> <tr class="b"> - <td>Metrics Monitor </td> - -<td>MapReduce2 Client </td> - -<td>Metron Client</td> - </tr> - +<td> MapReduce2 Client </td> +<td> Metron Client</td></tr> <tr class="a"> - <td>Metron Client </td> - -<td>Metrics Monitor </td> - -<td>NodeManager</td> - </tr> - +<td> Metrics Monitor </td> +<td> NodeManager</td></tr> <tr class="b"> - <td>Metron Enrichment </td> - -<td>Metron Client </td> - -<td>Pig Client</td> - </tr> - +<td> Metron Client </td> +<td> Pig Client</td></tr> <tr class="a"> - <td>Metron Indexing </td> - -<td>MySQL Server </td> - -<td>Slider Client</td> - </tr> - +<td> MySQL Server </td> +<td> Slider Client</td></tr> <tr class="b"> - <td>Metron Parsers </td> - -<td>Nimbus </td> - -<td>Spark Client</td> - </tr> - +<td> Nimbus </td> +<td> Spark Client</td></tr> <tr class="a"> - <td>Metron REST </td> - -<td>NodeManager </td> - -<td>Supervisor</td> - </tr> - +<td> NodeManager </td> +<td> Supervisor</td></tr> <tr class="b"> - <td>NameNode </td> - -<td>Pig Client </td> - -<td>Tez Client</td> - </tr> - +<td> Pig Client </td> +<td> Tez Client</td></tr> <tr class="a"> - <td>NodeManager </td> - -<td>ResourceManager </td> - -<td>YARN Client</td> - </tr> - +<td> ResourceManager </td> +<td> YARN Client</td></tr> <tr class="b"> - <td>Pig Client </td> - -<td>SNameNode </td> - -<td>ZooKeeper Client</td> - </tr> - +<td> SNameNode </td> +<td> ZooKeeper Client</td></tr> <tr class="a"> - <td>Slider Client </td> - -<td>Slider Client </td> - -<td>ZooKeeper Server</td> - </tr> - +<td> Slider Client </td> +<td> ZooKeeper Server</td></tr> <tr class="b"> - -<td>Spark Client </td> - <td>Spark Client </td> - </tr> - +<td> Spark Client </td></tr> <tr class="a"> - <td>Spark History Server </td> - -<td>Supervisor </td> - </tr> - +<td> Supervisor </td></tr> <tr class="b"> - <td>Storm UI Server </td> - -<td>Tez Client</td> - </tr> - +<td> Tez Client</td></tr> <tr class="a"> - <td>Supervisor </td> - -<td>WebHCat Server</td> - </tr> - +<td> WebHCat Server</td></tr> <tr class="b"> - <td>Tez Client </td> - -<td>YARN Client</td> - </tr> - +<td> YARN Client</td></tr> <tr class="a"> - <td>YARN Client </td> - -<td>ZooKeeper Client</td> - </tr> - +<td> ZooKeeper Client</td></tr> <tr class="b"> - <td>Zeppelin Notebook </td> - -<td>ZooKeeper Server</td> - </tr> - +<td> ZooKeeper Server</td></tr> <tr class="a"> - -<td>ZooKeeper Client </td> - </tr> - +<td>ZooKeeper Client </td></tr> <tr class="b"> - -<td>ZooKeeper Server </td> - </tr> - </tbody> +<td>ZooKeeper Server </td></tr> +</tbody> </table> - <ul> - + <li> -<p>Install everything. Metron REST will probably not work as we still need to add a user and the database to MySQL. At this point, make sure that all the services are up. You might have to manually start a few.</p></li> - + +<p>Install everything. Metron REST will probably not work as we still need to add a user and the database to MySQL. At this point, make sure that all the services are up. You might have to manually start a few.</p> +</li> <li> -<p>Configure a user for Metron REST in MySQL. On the node where you installed the Metron REST UI, do:</p></li> + +<p>Configure a user for Metron REST in MySQL. On the node where you installed the Metron REST UI, do:</p> +</li> </ul> -<div class="source"> -<div class="source"> -<pre># mysql -u root -p +<div> +<div> +<pre class="source"># mysql -u root -p CREATE USER '<DB USERNAME>'@'localhost' IDENTIFIED BY '<DB PASSWORD>'; CREATE DATABASE IF NOT EXISTS <DB NAME>; GRANT ALL PRIVILEGES ON <DB NAME>.* TO '<DB USERNAME>'@'localhost'; </pre></div></div> + <p>For example:</p> -<div class="source"> -<div class="source"> -<pre># mysql -u root -p +<div> +<div> +<pre class="source"># mysql -u root -p > CREATE USER 'metron'@'localhost' IDENTIFIED BY 'metron'; > CREATE DATABASE IF NOT EXISTS metronrest; > GRANT ALL PRIVILEGES ON metronrest.* TO 'metron'@'localhost'; @@ -1184,11 +893,12 @@ GRANT ALL PRIVILEGES ON <DB NAME>.* TO '<DB USERNAME>'@'localhost'; Bye # </pre></div></div> + <p>Add the Metron REST username and password to the metronrest database:</p> -<div class="source"> -<div class="source"> -<pre># mysql -u <DB USERNAME> -p +<div> +<div> +<pre class="source"># mysql -u <DB USERNAME> -p > use <DB NAME>; > insert into users (username, password, enabled) values ('<USERNAME>','<PASSWORD>',1); > insert into authorities (username, authority) values ('<USERNAME>', 'ROLE_USER'); @@ -1196,11 +906,12 @@ Bye Bye # </pre></div></div> + <p>For example, to use the username ‘metron’ with password ‘metron’, do the following:</p> -<div class="source"> -<div class="source"> -<pre># mysql -u metron -p +<div> +<div> +<pre class="source"># mysql -u metron -p > use metronrest; > insert into users (username, password, enabled) values ('metron','metron',1); > insert into authorities (username, authority) values ('metron', 'ROLE_USER'); @@ -1208,12 +919,13 @@ Bye Bye # </pre></div></div> + <p>Make sure that all the services are up.</p> <p>Install metron_pcapservice:</p> -<div class="source"> -<div class="source"> -<pre># cp /root/metron/metron-platform/metron-api/target/metron-api-0.4.0.jar /usr/metron/0.4.0/lib/ +<div> +<div> +<pre class="source"># cp /root/metron/metron-platform/metron-api/target/metron-api-0.4.0.jar /usr/metron/0.4.0/lib/ # wget -O /etc/init.d/pcapservice https://raw.githubusercontent.com/apache/metron/master/metron-deployment/roles/metron_pcapservice/templates/pcapservice # sed -i 's/{{ pcapservice_jar_dst }}/\/usr\/metron\/0.4.0\/lib\/metron-api-0.4.0.jar/' /etc/init.d/pcapservice # sed -i 's/{{ pcapservice_port }}/8081/' /etc/init.d/pcapservice @@ -1225,25 +937,28 @@ Bye # sed -i 's/^ rotate {{ metron_pcapservice_logrotate_retention }}.*$/ rotate 30/' /etc/logrotate.d/metron-pcapservice # chmod 644 /etc/logrotate.d/metron-pcapservice </pre></div></div> + <p>Install tap interface:</p> -<div class="source"> -<div class="source"> -<pre># yum install tunctl -y +<div> +<div> +<pre class="source"># yum install tunctl -y # tunctl -p </pre></div></div> + <p>Bring up tap0 on 10.0.0.100:</p> -<div class="source"> -<div class="source"> -<pre># ifconfig tap0 10.0.0.100 up +<div> +<div> +<pre class="source"># ifconfig tap0 10.0.0.100 up # ip link set tap0 promisc on </pre></div></div> + <p>Install librdkafka:</p> -<div class="source"> -<div class="source"> -<pre># yum install cmake make gcc gcc-c++ flex bison libpcap libpcap-devel openssl-devel python-devel swig zlib-devel perlcyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi -y +<div> +<div> +<pre class="source"># yum install cmake make gcc gcc-c++ flex bison libpcap libpcap-devel openssl-devel python-devel swig zlib-devel perlcyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi -y # cd /tmp # wget -O /tmp/librdkafka-0.9.4.tar.gz https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz # /bin/gtar --extract -C /tmp -z -f /tmp/librdkafka-0.9.4.tar.gz @@ -1252,11 +967,12 @@ Bye # make # make install </pre></div></div> + <p>Install pycapa</p> -<div class="source"> -<div class="source"> -<pre># yum install centos-release-scl -y +<div> +<div> +<pre class="source"># yum install centos-release-scl -y # yum update -y # yum install python27 -y # scl enable python27 bash @@ -1277,11 +993,12 @@ Bye # ln -s /usr/local/lib/librdkafka.so.1 /opt/rh/python27/root/usr/lib64 # deactivate </pre></div></div> + <p>Log out and log in to make sure Python is back to version 2.6 instead of 2.7.</p> -<div class="source"> -<div class="source"> -<pre># wget -O /etc/init.d/pycapa https://raw.githubusercontent.com/apache/metron/master/metron-deployment/roles/pycapa/templates/pycapa +<div> +<div> +<pre class="source"># wget -O /etc/init.d/pycapa https://raw.githubusercontent.com/apache/metron/master/metron-deployment/roles/pycapa/templates/pycapa # sed -i 's/{{ pycapa_log }}/\/var\/log\/pycapa.log/' /etc/init.d/pycapa # sed -i 's/{{ pycapa_home }}/\/usr\/local\/pycapa/' /etc/init.d/pycapa # sed -i 's/{{ python27_home }}/\/opt\/rh\/python27\/root/' /etc/init.d/pycapa @@ -1293,47 +1010,52 @@ Bye # yum install @Development libdnet-devel rpm-build libpcap libpcap-devel pcre pcre-devel zlib zlib-devel glib2-devel -y # yum install kafka -y </pre></div></div> + <p>Install bro:</p> -<div class="source"> -<div class="source"> -<pre># wget -O /tmp/bro-2.4.2.tar.gz https://www.bro.org/downloads/bro-2.4.2.tar.gz +<div> +<div> +<pre class="source"># wget -O /tmp/bro-2.4.2.tar.gz https://www.bro.org/downloads/bro-2.4.2.tar.gz # /bin/gtar --extract -C /tmp -z -f /tmp/bro-2.4.2.tar.gz # cd /tmp/bro-2.4.2 # ./configure --prefix=/usr/local/bro # make -j4 # make install </pre></div></div> + <p>Configure bro:</p> -<div class="source"> -<div class="source"> -<pre># sed -i 's/interface=eth0/interface=tap0/' /usr/local/bro/etc/node.cfg +<div> +<div> +<pre class="source"># sed -i 's/interface=eth0/interface=tap0/' /usr/local/bro/etc/node.cfg # /usr/local/bro/bin/broctl install </pre></div></div> + <p>Edit crontab with <tt># crontab -e</tt> and add:</p> -<div class="source"> -<div class="source"> -<pre>0-59/5 * * * * /usr/local/bro/bin/broctl cron +<div> +<div> +<pre class="source">0-59/5 * * * * /usr/local/bro/bin/broctl cron 0-59/5 * * * * rm -rf /usr/local/bro/spool/tmp/* </pre></div></div> + <p>bro-kafka:</p> -<div class="source"> -<div class="source"> -<pre># git clone https://github.com/apache/metron-bro-plugin-kafka /tmp/metron-bro-plugin-kafka +<div> +<div> +<pre class="source"># git clone https://github.com/apache/metron-bro-plugin-kafka /tmp/metron-bro-plugin-kafka # cd /tmp/metron-bro-plugin-kafka # rm -rf build/ # ./configure --bro-dist=/tmp/bro-2.4.2 --install-root=/usr/local/bro/lib/bro/plugins/ --with-librdkafka=/usr/local # make -j4 # make install </pre></div></div> + <p>Configure metron-bro-plugin-kafka:</p> -<div class="source"> -<div class="source"> -<pre># cat << EOF >> /usr/local/bro/share/bro/site/local.bro +<div> +<div> +<pre class="source"># cat << EOF >> /usr/local/bro/share/bro/site/local.bro @load Bro/Kafka/logs-to-kafka.bro redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG); redef Kafka::topic_name = "bro"; @@ -1343,33 +1065,37 @@ EOF # /usr/local/bro/bin/broctl deploy # ip link set tap0 promisc on </pre></div></div> + <p>Install daq:</p> -<div class="source"> -<div class="source"> -<pre># wget -O /tmp/daq-2.0.6-1.src.rpm https://snort.org/downloads/snort/daq-2.0.6-1.src.rpm +<div> +<div> +<pre class="source"># wget -O /tmp/daq-2.0.6-1.src.rpm https://snort.org/downloads/snort/daq-2.0.6-1.src.rpm # cd /tmp # rpmbuild --rebuild daq-2.0.6-1.src.rpm </pre></div></div> + <p>This last command creates the files /root/rpmbuild/RPMS/x86_64/daq-2.0.6-1.x86_64.rpm & /root/rpmbuild/RPMS/x86_64/daq-debuginfo-2.0.6-1.x86_64.rpm. We only need to install the first rpm.</p> -<div class="source"> -<div class="source"> -<pre># yum install /root/rpmbuild/RPMS/x86_64/daq-2.0.6-1.x86_64.rpm -y +<div> +<div> +<pre class="source"># yum install /root/rpmbuild/RPMS/x86_64/daq-2.0.6-1.x86_64.rpm -y </pre></div></div> + <p>Install snort:</p> -<div class="source"> -<div class="source"> -<pre># wget -O /tmp/snort-2.9.8.0-1.src.rpm https://snort.org/downloads/archive/snort/snort-2.9.8.0-1.src.rpm +<div> +<div> +<pre class="source"># wget -O /tmp/snort-2.9.8.0-1.src.rpm https://snort.org/downloads/archive/snort/snort-2.9.8.0-1.src.rpm # cd /tmp # rpmbuild --rebuild snort-2.9.8.0-1.src.rpm </pre></div></div> + <p>This last command creates the files /root/rpmbuild/RPMS/x86_64/snort-2.9.8.0-1.x86_64.rpm & /root/rpmbuild/RPMS/x86_64/snort-debuginfo-2.9.8.0-1.x86_64.rpm. We only need to install the first rpm.</p> -<div class="source"> -<div class="source"> -<pre># yum install /root/rpmbuild/RPMS/x86_64/snort-2.9.8.0-1.x86_64.rpm -y +<div> +<div> +<pre class="source"># yum install /root/rpmbuild/RPMS/x86_64/snort-2.9.8.0-1.x86_64.rpm -y # wget -O /tmp/community-rules.tar.gz https://www.snort.org/downloads/community/community-rules.tar.gz # /bin/gtar --extract -C /tmp -z -f /tmp/community-rules.tar.gz # cp -r community-rules/community.rules /etc/snort/rules @@ -1389,9 +1115,9 @@ EOF # chmod 755 /opt/snort-producer </pre></div></div> -<div class="source"> -<div class="source"> -<pre># wget -O /opt/snort-producer/start-snort-producer.sh https://github.com/apache/metron/raw/master/metron-deployment/roles/snort/templates/start-snort-producer.sh +<div> +<div> +<pre class="source"># wget -O /opt/snort-producer/start-snort-producer.sh https://github.com/apache/metron/raw/master/metron-deployment/roles/snort/templates/start-snort-producer.sh # sed -i 's/{{ snort_alert_csv_path }}/\/var\/log\/snort\/alert.csv/' /opt/snort-producer/start-snort-producer.sh # sed -i 's/{{ kafka_prod }}/\/usr\/hdp\/current\/kafka-broker\/bin\/kafka-console-producer.sh/' /opt/snort-producer/start-snort-producer.sh # sed -i 's/{{ kafka_broker_url }}/<KAFKA_BROKER_IP>:6667/' /opt/snort-producer/start-snort-producer.sh @@ -1399,18 +1125,19 @@ EOF # chmod 755 /opt/snort-producer/start-snort-producer.sh </pre></div></div> -<div class="source"> -<div class="source"> -<pre># wget -O /etc/init.d/snort-producer https://github.com/apache/metron/raw/master/metron-deployment/roles/snort/templates/snort-producer +<div> +<div> +<pre class="source"># wget -O /etc/init.d/snort-producer https://github.com/apache/metron/raw/master/metron-deployment/roles/snort/templates/snort-producer # sed -i 's/{{ snort_producer_home }}/\/opt\/snort-producer/' /etc/init.d/snort-producer # sed -i 's/{{ snort_producer_start }}/\/opt\/snort-producer\/start-snort-producer.sh/' /etc/init.d/snort-producer # chmod 755 /etc/init.d/snort-producer </pre></div></div> + <p>Install yaf:</p> -<div class="source"> -<div class="source"> -<pre># wget -O /tmp/libfixbuf-1.7.1.tar.gz http://tools.netsa.cert.org/releases/libfixbuf-1.7.1.tar.gz +<div> +<div> +<pre class="source"># wget -O /tmp/libfixbuf-1.7.1.tar.gz http://tools.netsa.cert.org/releases/libfixbuf-1.7.1.tar.gz # /bin/gtar --extract -C /tmp -z -f /tmp/libfixbuf-1.7.1.tar.gz # cd /tmp/libfixbuf-1.7.1 # ./configure @@ -1438,11 +1165,12 @@ EOF # sed -i 's/^DAEMONOPTS=\"${@:2}\"$/DAEMONOPTS=\"${@:2} --idle-timeout 0\"/' /etc/init.d/yaf # chmod 755 /etc/init.d/yaf </pre></div></div> + <p>Install tcpreplay:</p> -<div class="source"> -<div class="source"> -<pre># wget -O /tmp/tcpreplay-4.1.1.tar.gz https://github.com/appneta/tcpreplay/releases/download/v4.1.1/tcpreplay-4.1.1.tar.gz +<div> +<div> +<pre class="source"># wget -O /tmp/tcpreplay-4.1.1.tar.gz https://github.com/appneta/tcpreplay/releases/download/v4.1.1/tcpreplay-4.1.1.tar.gz # /bin/gtar --extract -C /opt -z -f /tmp/tcpreplay-4.1.1.tar.gz # cd /opt/tcpreplay-4.1.1/ # ./configure --prefix=/opt @@ -1461,11 +1189,12 @@ EOF # sed -i 's/{{ tcpreplay_prefix }}/\/opt/' /etc/init.d/pcap-replay # chmod 755 /etc/init.d/pcap-replay </pre></div></div> + <p>Install monit</p> -<div class="source"> -<div class="source"> -<pre># yum install monit -y +<div> +<div> +<pre class="source"># yum install monit -y # wget -O /etc/monit.conf https://github.com/apache/metron/raw/master/metron-deployment/roles/monit/templates/monit/monit.conf # sed -i 's/{{ inventory_hostname }}/<IP ADDRESS>/' /etc/monit.conf @@ -1500,184 +1229,177 @@ EOF # monit stop all # monit start all # monit summary | tail -n +3 | awk -F"'" '{print $2}' -</pre></div></div></div> +</pre></div></div> +</div> <div class="section"> <h3><a name="Miscellaneous_Issues"></a>Miscellaneous Issues</h3> - <ul> - + <li> -<p>There’s currently a bug in Metron 0.4.0 where Metron REST doesn’t start when restarting the Metron services. This bug was fixed in METRON-990 (<a class="externalLink" href="https://github.com/apache/metron/pull/613/commits/1a9b19a0101ada58cb671ab224934f304df6fff8";>https://github.com/apache/metron/pull/613/commits/1a9b19a0101ada58cb671ab224934f304df6fff8</a>) but unfortunately, this fix didn’t make the 0.4.0 release. In order to fix this, edit the file “/etc/rc.d/init.d/metron-rest” and on line 148, change “$0 start” to “$0 start $2”.</p></li> - + +<p>There’s currently a bug in Metron 0.4.0 where Metron REST doesn’t start when restarting the Metron services. This bug was fixed in METRON-990 (<a class="externalLink" href="https://github.com/apache/metron/pull/613/commits/1a9b19a0101ada58cb671ab224934f304df6fff8";>https://github.com/apache/metron/pull/613/commits/1a9b19a0101ada58cb671ab224934f304df6fff8</a>) but unfortunately, this fix didn’t make the 0.4.0 release. In order to fix this, edit the file “/etc/rc.d/init.d/metron-rest” and on line 148, change “$0 start” to “$0 start $2”.</p> +</li> <li> -<p>I had a problem with Zeppelin after rebooting this machine and had to manually create the Zeppelin run directory:</p></li> + +<p>I had a problem with Zeppelin after rebooting this machine and had to manually create the Zeppelin run directory:</p> +</li> </ul> -<div class="source"> -<div class="source"> -<pre># mkdir /var/run/zeppelin +<div> +<div> +<pre class="source"># mkdir /var/run/zeppelin # chown zeppelin.hadoop zeppelin/ </pre></div></div> <ul> - + <li>Additionally, while working with Metron, I’ve noticed that at some point Zeppelin Notebook started, but immediately stopped again. In the logs, I could see “Address already in use” messages. It turns out that there was still a lingering Zeppelin process on the host. To fix it, stop Zeppelin Notebook in Ambari and then kill the latent process:</li> </ul> -<div class="source"> -<div class="source"> -<pre># ps aux | grep zeppelin +<div> +<div> +<pre class="source"># ps aux | grep zeppelin # kill <zeppelin_java_pid> </pre></div></div> -<p>Afterwards, restart Zeppelin Notebook via Ambari.</p> +<p>Afterwards, restart Zeppelin Notebook via Ambari.</p> <ul> - + <li>I had a couple of issues with Elasticsearch where it wouldn’t find a master. This was fixed by doing the following. In Ambari, set the following items: “masters_also_are_datanodes” to “true” “expected_data_nodes” = “0” “gateway_recover_after_data_nodes” = “1” Restart all services. At this point, I noticed the following in :/etc/elasticsearch/elasticsearch.yml":</li> </ul> -<div class="source"> -<div class="source"> -<pre>node: +<div> +<div> +<pre class="source">node: data: true master: true name: metron1.local </pre></div></div> + <p>After changing this to :</p> -<div class="source"> -<div class="source"> -<pre>node: +<div> +<div> +<pre class="source">node: data: true master: true name: metron </pre></div></div> -<p>and restarting elasticsearch with “service elasticsearch restart”, elasticsearch started indexing.</p> +<p>and restarting elasticsearch with “service elasticsearch restart”, elasticsearch started indexing.</p> <ul> - + <li>Another with Elasticsearch was that I saw the following error message in Kibana:</li> </ul> -<div class="source"> -<div class="source"> -<pre>plugin:elasticsearch Elasticsearch is still initializing the kibana index. +<div> +<div> +<pre class="source">plugin:elasticsearch Elasticsearch is still initializing the kibana index. </pre></div></div> + <p>This was fixed by deleting the Kibana index “.kibana”: <tt>curl -XDELETE http://localhost:9200/.kibana</tt></p></div> <div class="section"> <h3><a name="Miscellaneous_Services"></a>Miscellaneous Services</h3> - <ul> - + <li>Load the correct Elasticsearch template with:</li> </ul> -<div class="source"> -<div class="source"> -<pre>curl -s -w "%{http_code}" -u <USERNAME>:<PASSWORD> -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install ES Template from REST", "command": "ELASTICSEARCH_TEMPLATE_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "<HOSTNAME>"}]}' http://<AMBARI HOST>:8080/api/v1/clusters/<CLUSTERNAME>/requests +<div> +<div> +<pre class="source">curl -s -w "%{http_code}" -u <USERNAME>:<PASSWORD> -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install ES Template from REST", "command": "ELASTICSEARCH_TEMPLATE_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "<HOSTNAME>"}]}' http://<AMBARI HOST>:8080/api/v1/clusters/<CLUSTERNAME>/requests </pre></div></div> + <p>For example:</p> -<div class="source"> -<div class="source"> -<pre>curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install ES Template from REST", "command": "ELASTICSEARCH_TEMPLATE_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "metron"}]}' http://192.168.10.10:8080/api/v1/clusters/metron/requests +<div> +<div> +<pre class="source">curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install ES Template from REST", "command": "ELASTICSEARCH_TEMPLATE_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "metron"}]}' http://192.168.10.10:8080/api/v1/clusters/metron/requests </pre></div></div> <ul> - + <li>Load Kibana Dashboard with:</li> </ul> -<div class="source"> -<div class="source"> -<pre>curl -s -w "%{http_code}" -u <USERNAME>:<PASSWORD> -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "<HOSTNAME>"}]}' http://<AMBARI HOST>:8080/api/v1/clusters/<CLUSTERNAME>/requests +<div> +<div> +<pre class="source">curl -s -w "%{http_code}" -u <USERNAME>:<PASSWORD> -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "KIBANA_DASHBOARD_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "<HOSTNAME>"}]}' http://<AMBARI HOST>:8080/api/v1/clusters/<CLUSTERNAME>/requests </pre></div></div> + <p>For example:</p> -<div class="source"> -<div class="source"> -<pre>curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "metron"}]}' http://192.168.10.10:8080/api/v1/clusters/metron/requests +<div> +<div> +<pre class="source">curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "KIBANA_DASHBOARD_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "metron"}]}' http://192.168.10.10:8080/api/v1/clusters/metron/requests </pre></div></div> <ul> - + <li>If you installed Metron on a single node, you might have to increase the number of Storm supervisor slots from the default 2 to 5 or more. This can be done by editing the “supervisor.slots.ports” under Storm in the Ambari UI. Change:</li> </ul> -<div class="source"> -<div class="source"> -<pre>supervisor.slots.ports: [6700, 6701] +<div> +<div> +<pre class="source">supervisor.slots.ports: [6700, 6701] </pre></div></div> + <p>To:</p> -<div class="source"> -<div class="source"> -<pre>supervisor.slots.ports: [6700, 6701, 6702, 6703, 6704, 6705] +<div> +<div> +<pre class="source">supervisor.slots.ports: [6700, 6701, 6702, 6703, 6704, 6705] </pre></div></div> <ul> - + <li>Install Apache NiFi in /root (You can pretty much use any directory you want). Download nifi-1.2.0-bin.tar.gz from <a class="externalLink" href="https://nifi.apache.org/download.html";>https://nifi.apache.org/download.html</a></li> </ul> -<div class="source"> -<div class="source"> -<pre># cd /root +<div> +<div> +<pre class="source"># cd /root # wget http://apache.mirror.iweb.ca/nifi/1.2.0/nifi-1.2.0-bin.tar.gz # tar xf nifi-1.2.0-bin.tar.gz </pre></div></div> + <p>Before we run NiFi, we need to change the port as the default port collides with the Ambari port. To do this, we need to change the value “nifi.web.http.port=8080” to “nifi.web.http.port=8089” in the file “nifi-1.1.2/conf/nifi.properties”. Install and start NiFi afterwards:</p> -<div class="source"> -<div class="source"> -<pre># nifi-1.2.0/bin/nifi.sh install +<div> +<div> +<pre class="source"># nifi-1.2.0/bin/nifi.sh install # nifi-1.2.0/bin/nifi.sh start -</pre></div></div></div> +</pre></div></div> +</div> <div class="section"> <h3><a name="Exposed_Interfaces"></a>Exposed Interfaces</h3> <p>In the end, you’ll end up with a bunch of exposed UIs:</p> - <ul> - + <li>Ambari: <a class="externalLink" href="http://node1:8080/";>http://node1:8080/</a></li> - <li>Kibana: <a class="externalLink" href="http://node1:5000/";>http://node1:5000/</a></li> - <li>Sensor Status (monit): <a class="externalLink" href="http://node1:2812";>http://node1:2812</a></li> - <li>Elasticsearch: <a class="externalLink" href="http://node1:9200/";>http://node1:9200/</a></li> - <li>Storm UI: <a class="externalLink" href="http://node1:8744/";>http://node1:8744/</a></li> - <li>Metron REST interface: <a class="externalLink" href="http://node1:8082/swagger-ui.html#/";>http://node1:8082/swagger-ui.html#/</a></li> - <li>Management UI: <a class="externalLink" href="http://node1:4200/";>http://node1:4200/</a> (user/password)</li> - <li>Apache Nifi: <a class="externalLink" href="http://node1:8089/nifi/";>http://node1:8089/nifi/</a></li> - <li>Zookeeper: <a class="externalLink" href="http://node1:2181";>http://node1:2181</a></li> - <li>Kafka: <a class="externalLink" href="http://node1:6667";>http://node1:6667</a></li> </ul></div> <div class="section"> <h3><a name="TROUBLESHOOTING"></a>TROUBLESHOOTING</h3></div></div> - </div> - </div> - </div> - + </div> + </div> + </div> <hr/> - <footer> - <div class="container-fluid"> - <div class="row span12">Copyright © 2018 - <a href="https://www.apache.org";>The Apache Software Foundation</a>. - All Rights Reserved. - + <div class="container-fluid"> + <div class="row-fluid"> +é 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo, + and the Apache Metron project logo are trademarks of The Apache Software Foundation. + </div> </div> - - - - </div> </footer> </body> </html>
http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html b/site/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html new file mode 100644 index 0000000..499e89b --- /dev/null +++ b/site/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html @@ -0,0 +1,190 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/ambari/elasticsearch-mpack/index.md at 2018-06-07 + | Rendered using Apache Maven Fluido Skin 1.7 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20180607" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – </title> + <link rel="stylesheet" href="../../../../css/apache-maven-fluido-1.7.min.css" /> + <link rel="stylesheet" href="../../../../css/site.css" /> + <link rel="stylesheet" href="../../../../css/print.css" media="print" /> + <script type="text/javascript" src="../../../../js/apache-maven-fluido-1.7.min.js"></script> +<script type="text/javascript"> + $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } ); + </script> + </head> + <body class="topBarDisabled"> + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"><a href="http://metron.apache.org/"; id="bannerLeft"><img src="../../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div> + <div class="pull-right"></div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> + <li class=""><a href="../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> + <li class="active "></li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> + <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + </ul> + </div> + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + <li><a href="../../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a> + <ul class="nav nav-list"> + <li><a href="../../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li> + <li><a href="../../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li> + <li><a href="../../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li> + <li><a href="../../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li> + <li><a href="../../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li> + <li><a href="../../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-down"></span>Deployment</a> + <ul class="nav nav-list"> + <li><a href="../../../../metron-deployment/Kerberos-ambari-setup.html" title="Kerberos-ambari-setup"><span class="none"></span>Kerberos-ambari-setup</a></li> + <li><a href="../../../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup"><span class="none"></span>Kerberos-manual-setup</a></li> + <li><a href="../../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2"><span class="none"></span>Amazon-ec2</a></li> + <li><a href="../../../../metron-deployment/ansible/index.html" title="Ansible"><span class="icon-chevron-right"></span>Ansible</a></li> + <li><a href="../../../../metron-deployment/development/index.html" title="Development"><span class="icon-chevron-right"></span>Development</a></li> + <li><a href="../../../../metron-deployment/other-examples/index.html" title="Other-examples"><span class="icon-chevron-right"></span>Other-examples</a></li> + <li><a href="../../../../metron-deployment/packaging/ambari/index.html" title="Ambari"><span class="icon-chevron-down"></span>Ambari</a> + <ul class="nav nav-list"> + <li class="active"><a href="#"><span class="none"></span>Elasticsearch-mpack</a></li> + <li><a href="../../../../metron-deployment/packaging/ambari/metron-mpack/index.html" title="Mpack"><span class="none"></span>Mpack</a></li> + </ul> +</li> + <li><a href="../../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker"><span class="none"></span>Ansible-docker</a></li> + <li><a href="../../../../metron-deployment/packaging/docker/deb-docker/index.html" title="Deb-docker"><span class="none"></span>Deb-docker</a></li> + <li><a href="../../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker"><span class="none"></span>Rpm-docker</a></li> + <li><a href="../../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li> + </ul> +</li> + <li><a href="../../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li> + <li><a href="../../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li> + <li><a href="../../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li> + <li><a href="../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li> + <li><a href="../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li> + <li><a href="../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li> + <li><a href="../../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li> + <li><a href="../../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li> + <li><a href="../../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li> + </ul> +</li> +</ul> + <hr /> + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> +<a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../../images/logos/maven-feather.png" /></a> + </div> + </div> + </div> + <div id="bodyColumn" class="span10" > +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<p>This provides a Management Pack (MPack) extension for <a class="externalLink" href="https://ambari.apache.org/";>Apache Ambari</a> that simplifies the provisioning, management and monitoring of Elasticsearch and Kibana on clusters of any size.</p> +<p>This allows you to easily install Elasticsearch and Kibana using a simple, guided process. This also allows you to monitor cluster health.</p> +<div class="section"> +<div class="section"> +<h3><a name="Prerequisites"></a>Prerequisites</h3> +<ul> + +<li>Ambari 2.4.2+</li> +</ul></div> +<div class="section"> +<h3><a name="Quick_Start"></a>Quick Start</h3> +<ol style="list-style-type: decimal"> + +<li> + +<p>Build the Elasticsearch MPack. Execute the following command from the project’s root directory.</p> + +<div> +<div> +<pre class="source">mvn clean package -Pmpack -DskipTests +</pre></div></div> +</li> +<li> + +<p>This results in the Mpack being produced at the following location.</p> + +<div> +<div> +<pre class="source">metron-deployment/packaging/ambari/elasticsearch-mpack/target/elasticsearch_mpack-x.y.z.0.tar.gz +</pre></div></div> +</li> +<li> + +<p>Copy the tarball to the host where Ambari Server is installed.</p> +</li> +<li> + +<p>Ensure that Ambari Server is stopped.</p> +</li> +<li> + +<p>Install the MPack.</p> + +<div> +<div> +<pre class="source">ambari-server install-mpack --mpack=elasticsearch_mpack-x.y.z.0.tar.gz --verbose +</pre></div></div> +</li> +<li> + +<p>Elasticsearch and Kibana will now be available as an installable service within Ambari.</p> +</li> +</ol></div> +<div class="section"> +<h3><a name="Installation_Notes"></a>Installation Notes</h3> +<p>The MPack will make all Elasticsearch services available in Ambari in the same manner as any other services in a stack. These can be installed using Ambari’s user interface using “Add Services” or during an initial cluster install.</p> +<div class="section"> +<h4><a name="Kerberization"></a>Kerberization</h4> +<p>Elasticsearch does not provide free native Kerberos support.</p></div> +<div class="section"> +<h4><a name="Limitations"></a>Limitations</h4> +<p>There are a few limitations that should be addressed to improve the Elasticsearch MPack installation.</p> +<ul> + +<li>The MPack does not support upgrades.</li> +</ul></div></div></div> + </div> + </div> + </div> + <hr/> + <footer> + <div class="container-fluid"> + <div class="row-fluid"> +é 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo, + and the Apache Metron project logo are trademarks of The Apache Software Foundation. + </div> + </div> + </footer> + </body> +</html>
