http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-analytics/metron-statistics/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-analytics/metron-statistics/index.html b/site/current-book/metron-analytics/metron-statistics/index.html index c663271..4cdd493 100644 --- a/site/current-book/metron-analytics/metron-statistics/index.html +++ b/site/current-book/metron-analytics/metron-statistics/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Statistics and Mathematical Functions</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../../index.html" title="Metron"> @@ -133,7 +133,14 @@ </li> </ul> </li> - + + <li> + + <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -143,9 +150,9 @@ <li> - <a href="../../metron-docker/index.html" title="Docker"> + <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -161,7 +168,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -175,6 +182,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul>
http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-contrib/metron-docker/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-contrib/metron-docker/index.html b/site/current-book/metron-contrib/metron-docker/index.html new file mode 100644 index 0000000..bdbfa9d --- /dev/null +++ b/site/current-book/metron-contrib/metron-docker/index.html @@ -0,0 +1,509 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2017-09-15 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – Metron Docker</title> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../../css/site.css" /> + <link rel="stylesheet" href="../../css/print.css" media="print" /> + + + <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="http://metron.apache.org/"; id="bannerLeft"> + <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> + </a> + </div> + <div class="pull-right"> </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> + Metron</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="../../index.html" title="Documentation"> + Documentation</a> + </li> + <li class="divider ">/</li> + <li class="">Metron Docker</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + + <li> + + <a href="../../index.html" title="Metron"> + <i class="icon-chevron-down"></i> + Metron</a> + <ul class="nav nav-list"> + + <li> + + <a href="../../Upgrading.html" title="Upgrading"> + <i class="none"></i> + Upgrading</a> + </li> + + <li> + + <a href="../../metron-analytics/index.html" title="Analytics"> + <i class="icon-chevron-right"></i> + Analytics</a> + </li> + + <li class="active"> + + <a href="#"><i class="none"></i>Docker</a> + </li> + + <li> + + <a href="../../metron-deployment/index.html" title="Deployment"> + <i class="icon-chevron-right"></i> + Deployment</a> + </li> + + <li> + + <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> + <i class="none"></i> + Alerts</a> + </li> + + <li> + + <a href="../../metron-interface/metron-config/index.html" title="Config"> + <i class="none"></i> + Config</a> + </li> + + <li> + + <a href="../../metron-interface/metron-rest/index.html" title="Rest"> + <i class="none"></i> + Rest</a> + </li> + + <li> + + <a href="../../metron-platform/index.html" title="Platform"> + <i class="icon-chevron-right"></i> + Platform</a> + </li> + + <li> + + <a href="../../metron-sensors/index.html" title="Sensors"> + <i class="icon-chevron-right"></i> + Sensors</a> + </li> + + <li> + + <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> + </ul> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <h1>Metron Docker</h1> +<p><a name="Metron_Docker"></a></p> +<p>Metron Docker is a <a class="externalLink" href="https://docs.docker.com/compose/overview/";>Docker Compose</a> application that is intended only for development and integration testing of Metron. These images can quickly spin-up the underlying components on which Apache Metron runs.</p> +<p>None of the core Metron components are setup or launched automatically with these Docker images. You will need to manually setup and start the Metron components that you require. You should not expect to see telemetry being parsed, enriched, or indexed. If you are looking to try-out, experiment or demo Metron capabilities on a single node, then the <a href="../../metron-deployment/vagrant/full-dev-platform/index.html">Vagrant-driven VM</a> is what you need. Use this instead of Vagrant when:</p> + +<ul> + +<li>You want an environment that can be built and spun up quickly</li> + +<li>You need to frequently rebuild and restart services</li> + +<li>You only need to test, troubleshoot or develop against a subset of services</li> +</ul> +<p>Metron Docker includes these images that have been customized for Metron:</p> + +<ul> + +<li>Kafka (with Zookeeper)</li> + +<li>HBase</li> + +<li>Storm</li> + +<li>Elasticsearch</li> + +<li>Kibana</li> + +<li>HDFS</li> +</ul> +<div class="section"> +<h2><a name="Setup"></a>Setup</h2> +<p>Install <a class="externalLink" href="https://docs.docker.com/docker-for-mac/";>Docker for Mac</a> or <a class="externalLink" href="https://docs.docker.com/docker-for-windows/";>Docker for Windows</a>. The following versions have been tested:</p> + +<ul> + +<li>Docker version 1.12.0</li> + +<li>docker-machine version 0.8.0</li> + +<li>docker-compose version 1.8.0</li> +</ul> +<p>Build Metron from the top level directory with:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_HOME +$ mvn clean install -DskipTests +</pre></div></div> +<p>You are welcome to use an existing Docker host but we prefer one with more resources. You can create one of those with this script:</p> + +<div class="source"> +<div class="source"> +<pre>$ export METRON_DOCKER_HOME=$METRON_HOME/metron-contrib/metron-docker +$ cd $METRON_DOCKER_HOME +$ ./scripts/create-docker-machine.sh +</pre></div></div> +<p>This will create a host called “metron-machine”. Anytime you want to run Docker commands against this host, make sure you run this first to set the Docker environment variables:</p> + +<div class="source"> +<div class="source"> +<pre>$ eval "$(docker-machine env metron-machine)" +</pre></div></div> +<p>If you wish to use a local docker-engine install, please set an environment variable BROKER_IP_ADDR to the IP address of your host machine. This cannot be the loopback address.</p></div> +<div class="section"> +<h2><a name="Usage"></a>Usage</h2> +<p>Navigate to the compose application root:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_DOCKER_HOME/compose/ +</pre></div></div> +<p>The Metron Docker environment lifecycle is controlled by the <a class="externalLink" href="https://docs.docker.com/compose/reference/overview/";>docker-compose</a> command. The service names can be found in the docker-compose.yml file. For example, to build and start the environment run this command:</p> + +<div class="source"> +<div class="source"> +<pre>$ eval "$(docker-machine env metron-machine)" +$ docker-compose up -d +</pre></div></div> +<p>After all services have started list the containers and ensure their status is ‘Up’:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker ps --format 'table {{.Names}}\t{{.Status}}' +NAMES STATUS +metron_storm_1 Up 5 minutes +metron_hbase_1 Up 5 minutes +metron_kibana_1 Up 5 minutes +metron_kafkazk_1 Up 5 minutes +metron_elasticsearch_1 Up 5 minutes +</pre></div></div> +<p>Various services are exposed through http on the Docker host. Get the host ip from the URL property:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-machine ls +NAME ACTIVE DRIVER STATE URL SWARM DOCKER ERRORS +metron-machine * virtualbox Running tcp://192.168.99.100:2376 v1.12.5 +</pre></div></div> +<p>Then, assuming a host ip of <tt>192.168.99.100</tt>, the UIs and APIs are available at:</p> + +<ul> + +<li>Storm - <a class="externalLink" href="http://192.168.99.100:8080/";>http://192.168.99.100:8080/</a></li> + +<li>HBase - <a class="externalLink" href="http://192.168.99.100:16010/";>http://192.168.99.100:16010/</a></li> + +<li>Elasticsearch - <a class="externalLink" href="http://192.168.99.100:9200/_plugin/head/";>http://192.168.99.100:9200/_plugin/head/</a></li> + +<li>Kibana - <a class="externalLink" href="http://192.168.99.100:5601/";>http://192.168.99.100:5601/</a></li> + +<li>HDFS (Namenode) - <a class="externalLink" href="http://192.168.99.100:50070/";>http://192.168.99.100:50070/</a></li> +</ul> +<p>The Storm logs can be useful when troubleshooting topologies. They can be found on the Storm container in <tt>/usr/share/apache-storm/logs</tt>.</p> +<p>When done using the machine, shut it down with:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose down +</pre></div></div></div> +<div class="section"> +<h2><a name="Examples"></a>Examples</h2> + +<ul> + +<li><a href="#Deploy_a_new_parser_class">Deploy a new parser class</a></li> + +<li><a href="#Connect_to_a_container">Connect to a container</a></li> + +<li><a href="create-a-sensor-from-sample-data/index.html">Create a sensor from sample data</a></li> + +<li><a href="upload-configs-to-zookeeper/index.html">Upload configs to Zookeeper</a></li> + +<li><a href="manage-a-topology/index.html">Manage a topology</a></li> + +<li><a href="run-sensor-data-end-to-end/index.html">Run sensor data end to end</a></li> +</ul> +<div class="section"> +<h3><a name="Deploy_a_new_parser_class"></a>Deploy a new parser class</h3> +<p>After adding a new parser to metron-parsers, build Metron from the top level directory:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_HOME +$ mvn clean install -DskipTests +</pre></div></div> +<p>Then run these commands to redeploy the parsers to the Storm image:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_DOCKER_HOME/compose +$ docker-compose down +$ docker-compose build storm +$ docker-compose up -d +</pre></div></div></div> +<div class="section"> +<h3><a name="Connect_to_a_container"></a>Connect to a container</h3> +<p>Suppose there is a problem with Kafka and the logs are needed for further investigation. Run this command to connect and explore the running Kafka container:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_DOCKER_HOME/compose +$ docker-compose exec kafkazk bash +</pre></div></div></div> +<div class="section"> +<h3><a name="Create_a_sensor_from_sample_data"></a>Create a sensor from sample data</h3> +<p>A tool for producing test data in Kafka is included with the Kafka/Zookeeper image. It loops through lines in a test data file and outputs them to Kafka at the desired frequency. Create a test data file in <tt>./kafkazk/data/</tt> and rebuild the Kafka/Zookeeper image:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_DOCKER_HOME/compose +$ printf 'first test data\nsecond test data\nthird test data\n' > ./kafkazk/data/TestData.txt +$ docker-compose down +$ docker-compose build kafkazk +$ docker-compose up -d +</pre></div></div> +<p>This will deploy the test data file to the Kafka/Zookeeper container. Now that data can be streamed to a Kafka topic:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose exec kafkazk ./bin/produce-data.sh +Usage: produce-data.sh data_path topic [message_delay_in_seconds] + +# Stream data in TestData.txt to the 'test' Kafka topic at a frequency of 5 seconds (default is 1 second) +$ docker-compose exec kafkazk ./bin/produce-data.sh /data/TestData.txt test 5 +</pre></div></div> +<p>The Kafka/Zookeeper image comes with sample Bro and Squid data:</p> + +<div class="source"> +<div class="source"> +<pre># Stream Bro test data every 1 second +$ docker-compose exec kafkazk ./bin/produce-data.sh /data/BroExampleOutput.txt bro + +# Stream Squid test data every 0.1 seconds +$ docker-compose exec kafkazk ./bin/produce-data.sh /data/SquidExampleOutput.txt squid 0.1 +</pre></div></div></div> +<div class="section"> +<h3><a name="Upload_configs_to_Zookeeper"></a>Upload configs to Zookeeper</h3> +<p>Parser configs and a global config configured for this Docker environment are included with the Kafka/Zookeeper image. Load them with:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose exec kafkazk bash +# $METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m PUSH -i $METRON_HOME/config/zookeeper +# exit +</pre></div></div> +<p>Dump out the configs with:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose exec kafkazk bash +# $METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m DUMP +# exit +</pre></div></div></div> +<div class="section"> +<h3><a name="Manage_a_topology"></a>Manage a topology</h3> +<p>The Storm image comes with a script to easily start parser topologies:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm ./bin/start_docker_parser_topology.sh sensor_name +</pre></div></div> +<p>The enrichment topology can be started with:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm ./bin/start_enrichment_topology.sh +</pre></div></div> +<p>The indexing topology can be started with:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm ./bin/start_elasticsearch_topology.sh +</pre></div></div> +<p>Topologies can be stopped using the Storm CLI. For example, stop the enrichment topology with:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm storm kill enrichments -w 0 +</pre></div></div></div> +<div class="section"> +<h3><a name="Run_sensor_data_end_to_end"></a>Run sensor data end to end</h3> +<p>First ensure configs were uploaded as described in the previous example. Then start a sensor and leave it running:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_DOCKER_HOME/compose +$ docker-compose exec kafkazk ./bin/produce-data.sh /data/BroExampleOutput.txt bro +</pre></div></div> +<p>Open a separate console session and verify the sensor is running by consuming a message from Kafka:</p> + +<div class="source"> +<div class="source"> +<pre>$ export METRON_DOCKER_HOME=$METRON_HOME/metron-contrib/metron-docker +$ cd $METRON_DOCKER_HOME/compose +$ docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic bro +</pre></div></div> +<p>A new message should be printed every second. Now kill the consumer and start the Bro parser topology:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose exec storm ./bin/start_docker_parser_topology.sh bro +</pre></div></div> +<p>Bro data should be flowing through the bro parser topology and into the Kafka enrichments topic. The enrichments topic should be created automatically:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose exec kafkazk ./bin/kafka-topics.sh --zookeeper localhost:2181 --list +bro +enrichments +indexing +</pre></div></div> +<p>Verify parsed Bro data is in the Kafka enrichments topic:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic enrichments +</pre></div></div> +<p>Now start the enrichment topology:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm ./bin/start_enrichment_topology.sh +</pre></div></div> +<p>Parsed Bro data should be flowing through the enrichment topology and into the Kafka indexing topic. Verify enriched Bro data is in the Kafka indexing topic:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic indexing +</pre></div></div> +<p>Now start the indexing topology:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm ./bin/start_elasticsearch_topology.sh +</pre></div></div> +<p>Enriched Bro data should now be present in the Elasticsearch container:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-machine ls +NAME ACTIVE DRIVER STATE URL SWARM DOCKER ERRORS +metron-machine * virtualbox Running tcp://192.168.99.100:2376 v1.12.5 + +$ curl -XGET http://192.168.99.100:9200/_cat/indices?v +health status index pri rep docs.count docs.deleted store.size pri.store.size +yellow open .kibana 1 1 1 0 3.1kb 3.1kb +yellow open bro_index_2016.12.19.18 5 1 180 0 475kb 475kb +</pre></div></div></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2017 + <a href="https://www.apache.org";>The Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html> http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-deployment/Kerberos-ambari-setup.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-deployment/Kerberos-ambari-setup.html b/site/current-book/metron-deployment/Kerberos-ambari-setup.html index cd22919..662dca9 100644 --- a/site/current-book/metron-deployment/Kerberos-ambari-setup.html +++ b/site/current-book/metron-deployment/Kerberos-ambari-setup.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Setting Up Kerberos in Vagrant Full Dev</title> <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../index.html" title="Metron"> @@ -96,7 +96,14 @@ <i class="icon-chevron-right"></i> Analytics</a> </li> - + + <li> + + <a href="../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li> <a href="../metron-deployment/index.html" title="Deployment"> @@ -122,6 +129,20 @@ <i class="none"></i> Amazon-ec2</a> </li> + + <li> + + <a href="../metron-deployment/other-examples/index.html" title="Other-examples"> + <i class="icon-chevron-right"></i> + Other-examples</a> + </li> + + <li> + + <a href="../metron-deployment/packaging/ambari/index.html" title="Ambari"> + <i class="none"></i> + Ambari</a> + </li> <li> @@ -162,9 +183,9 @@ <li> - <a href="../metron-docker/index.html" title="Docker"> + <a href="../metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -180,7 +201,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../metron-platform/index.html" title="Platform"> @@ -194,6 +215,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul> http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-deployment/Kerberos-manual-setup.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-deployment/Kerberos-manual-setup.html b/site/current-book/metron-deployment/Kerberos-manual-setup.html index 49d003b..c07156d 100644 --- a/site/current-book/metron-deployment/Kerberos-manual-setup.html +++ b/site/current-book/metron-deployment/Kerberos-manual-setup.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Kerberos Setup</title> <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../index.html" title="Metron"> @@ -96,7 +96,14 @@ <i class="icon-chevron-right"></i> Analytics</a> </li> - + + <li> + + <a href="../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li> <a href="../metron-deployment/index.html" title="Deployment"> @@ -122,6 +129,20 @@ <i class="none"></i> Amazon-ec2</a> </li> + + <li> + + <a href="../metron-deployment/other-examples/index.html" title="Other-examples"> + <i class="icon-chevron-right"></i> + Other-examples</a> + </li> + + <li> + + <a href="../metron-deployment/packaging/ambari/index.html" title="Ambari"> + <i class="none"></i> + Ambari</a> + </li> <li> @@ -162,9 +183,9 @@ <li> - <a href="../metron-docker/index.html" title="Docker"> + <a href="../metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -180,7 +201,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../metron-platform/index.html" title="Platform"> @@ -194,6 +215,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul> @@ -262,7 +297,7 @@ export ELASTICSEARCH=node1:9200 export BROKERLIST=node1:6667 export HDP_HOME="/usr/hdp/current" export KAFKA_HOME="${HDP_HOME}/kafka-broker" -export METRON_VERSION="0.4.0" +export METRON_VERSION="0.4.1" export METRON_HOME="/usr/metron/${METRON_VERSION}" </pre></div></div></li> @@ -773,7 +808,29 @@ for sensorstub in bro snort; do service sensor-stubs stop ${sensorstub}; service sensor-stubs start ${sensorstub}; done -</pre></div></div></div></div> +</pre></div></div></div> +<div class="section"> +<h4><a name="Model_as_a_Service_on_Kerberos"></a>Model as a Service on Kerberos</h4> +<p>MaaS works with kerberos, you have to remember to kinit with the metron user. There is one small issue out of the box (particularly on vagrant), you get an error like so when running <tt>$METRON_HOME/bin/maas_service.sh</tt>:</p> + +<div class="source"> +<div class="source"> +<pre>Requested user metron is not whitelisted and has id 501,which is below the minimum allowed 1000. +</pre></div></div> +<p>In order to correct this, you should:</p> + +<ul> + +<li>Navigate to the Yarn configuration in Ambari</li> + +<li>Click on “Advanced”</li> + +<li>Scroll to “Advanced yarn-env”</li> + +<li>Adjust the “Minimum user ID for submitting job” config to 500 from 1000</li> + +<li>You should then restart Yarn to have the change take effect.</li> +</ul></div></div> <div class="section"> <h3><a name="References"></a>References</h3> http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-deployment/amazon-ec2/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-deployment/amazon-ec2/index.html b/site/current-book/metron-deployment/amazon-ec2/index.html index 0b155b0..e81f4f5 100644 --- a/site/current-book/metron-deployment/amazon-ec2/index.html +++ b/site/current-book/metron-deployment/amazon-ec2/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Apache Metron on Amazon EC2</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../../index.html" title="Metron"> @@ -96,7 +96,14 @@ <i class="icon-chevron-right"></i> Analytics</a> </li> - + + <li> + + <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -122,6 +129,20 @@ <a href="#"><i class="none"></i>Amazon-ec2</a> </li> + + <li> + + <a href="../../metron-deployment/other-examples/index.html" title="Other-examples"> + <i class="icon-chevron-right"></i> + Other-examples</a> + </li> + + <li> + + <a href="../../metron-deployment/packaging/ambari/index.html" title="Ambari"> + <i class="none"></i> + Ambari</a> + </li> <li> @@ -162,9 +183,9 @@ <li> - <a href="../../metron-docker/index.html" title="Docker"> + <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -180,7 +201,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -194,6 +215,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul> http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-deployment/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-deployment/index.html b/site/current-book/metron-deployment/index.html index 7865a52..d1071d7 100644 --- a/site/current-book/metron-deployment/index.html +++ b/site/current-book/metron-deployment/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Overview</title> <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../index.html" title="Metron"> @@ -96,7 +96,14 @@ <i class="icon-chevron-right"></i> Analytics</a> </li> - + + <li> + + <a href="../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li class="active"> <a href="#"><i class="icon-chevron-down"></i>Deployment</a> @@ -122,6 +129,20 @@ <i class="none"></i> Amazon-ec2</a> </li> + + <li> + + <a href="../metron-deployment/other-examples/index.html" title="Other-examples"> + <i class="icon-chevron-right"></i> + Other-examples</a> + </li> + + <li> + + <a href="../metron-deployment/packaging/ambari/index.html" title="Ambari"> + <i class="none"></i> + Ambari</a> + </li> <li> @@ -162,9 +183,9 @@ <li> - <a href="../metron-docker/index.html" title="Docker"> + <a href="../metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -180,7 +201,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../metron-platform/index.html" title="Platform"> @@ -194,6 +215,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul> @@ -360,7 +395,7 @@ <li>Need to handle upgrading Metron</li> </ul></div></div> <div class="section"> -<h2><a name="RPM"></a>RPM</h2> +<h2><a name="RPMs"></a>RPMs</h2> <p>RPMs can be built to install the components in metron-platform. These RPMs are built in a Docker container and placed into <tt>target</tt>.</p> <p>Components in the RPMs:</p> @@ -379,6 +414,8 @@ <li>metron-pcap</li> <li>metron-solr</li> + +<li>stellar-common</li> </ul> <div class="section"> <h3><a name="Prerequisites"></a>Prerequisites</h3> @@ -415,11 +452,11 @@ </ul></div> <div class="section"> <h3><a name="Building_RPMs"></a>Building RPMs</h3> -<p>From <tt>metron-deployment</tt> run</p> <div class="source"> <div class="source"> -<pre>mvn clean package -Pbuild-rpms +<pre>cd metron-deployment +mvn clean package -Pbuild-rpms </pre></div></div> <p>The output RPM files will land in <tt>target/RPMS/noarch</tt>. They can be installed with the standard</p> http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-deployment/other-examples/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-deployment/other-examples/index.html b/site/current-book/metron-deployment/other-examples/index.html new file mode 100644 index 0000000..7dd9888 --- /dev/null +++ b/site/current-book/metron-deployment/other-examples/index.html @@ -0,0 +1,290 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2017-09-15 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – Other Example Deployments</title> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../../css/site.css" /> + <link rel="stylesheet" href="../../css/print.css" media="print" /> + + + <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="http://metron.apache.org/"; id="bannerLeft"> + <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> + </a> + </div> + <div class="pull-right"> </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> + Metron</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="../../index.html" title="Documentation"> + Documentation</a> + </li> + <li class="divider ">/</li> + <li class="">Other Example Deployments</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + + <li> + + <a href="../../index.html" title="Metron"> + <i class="icon-chevron-down"></i> + Metron</a> + <ul class="nav nav-list"> + + <li> + + <a href="../../Upgrading.html" title="Upgrading"> + <i class="none"></i> + Upgrading</a> + </li> + + <li> + + <a href="../../metron-analytics/index.html" title="Analytics"> + <i class="icon-chevron-right"></i> + Analytics</a> + </li> + + <li> + + <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + + <li> + + <a href="../../metron-deployment/index.html" title="Deployment"> + <i class="icon-chevron-down"></i> + Deployment</a> + <ul class="nav nav-list"> + + <li> + + <a href="../../metron-deployment/Kerberos-ambari-setup.html" title="Kerberos-ambari-setup"> + <i class="none"></i> + Kerberos-ambari-setup</a> + </li> + + <li> + + <a href="../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup"> + <i class="none"></i> + Kerberos-manual-setup</a> + </li> + + <li> + + <a href="../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2"> + <i class="none"></i> + Amazon-ec2</a> + </li> + + <li class="active"> + + <a href="#"><i class="icon-chevron-down"></i>Other-examples</a> + <ul class="nav nav-list"> + + <li> + + <a href="../../metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html" title="Manual_Install_CentOS6"> + <i class="none"></i> + Manual_Install_CentOS6</a> + </li> + </ul> + </li> + + <li> + + <a href="../../metron-deployment/packaging/ambari/index.html" title="Ambari"> + <i class="none"></i> + Ambari</a> + </li> + + <li> + + <a href="../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker"> + <i class="none"></i> + Ansible-docker</a> + </li> + + <li> + + <a href="../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker"> + <i class="none"></i> + Rpm-docker</a> + </li> + + <li> + + <a href="../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"> + <i class="none"></i> + Packer-build</a> + </li> + + <li> + + <a href="../../metron-deployment/roles/index.html" title="Roles"> + <i class="icon-chevron-right"></i> + Roles</a> + </li> + + <li> + + <a href="../../metron-deployment/vagrant/index.html" title="Vagrant"> + <i class="icon-chevron-right"></i> + Vagrant</a> + </li> + </ul> + </li> + + <li> + + <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> + <i class="none"></i> + Alerts</a> + </li> + + <li> + + <a href="../../metron-interface/metron-config/index.html" title="Config"> + <i class="none"></i> + Config</a> + </li> + + <li> + + <a href="../../metron-interface/metron-rest/index.html" title="Rest"> + <i class="none"></i> + Rest</a> + </li> + + <li> + + <a href="../../metron-platform/index.html" title="Platform"> + <i class="icon-chevron-right"></i> + Platform</a> + </li> + + <li> + + <a href="../../metron-sensors/index.html" title="Sensors"> + <i class="icon-chevron-right"></i> + Sensors</a> + </li> + + <li> + + <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> + </ul> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <h1>Other Example Deployments</h1> +<p><a name="Other_Example_Deployments"></a></p> +<p>This folder contains other example deployments which are not covered under the automatic installs. These examples have been vetted and tested by the Metron community at the time of entry. However, development moves fast and things may change. If there are problems with these steps, please check the mailing lists first or log a Jira ticket.</p> + +<ul> + +<li><a href="manual-install/Manual_Install_CentOS6.html">Metron 0.4.0 on CentOS 6 Manual Install</a> (July 2017)</li> +</ul> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2017 + <a href="https://www.apache.org";>The Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html>
