METRON-1191 update public web site to point at 0.4.1 new release (mattf-horton) closes apache/metron#764
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/87ff7b73 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/87ff7b73 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/87ff7b73 Branch: refs/heads/master Commit: 87ff7b73b95f1aa1ac3d710ddab7ca67d800562c Parents: 695e904 Author: mattf-horton <[email protected]> Authored: Tue Sep 19 11:40:35 2017 -0700 Committer: mattf <[email protected]> Committed: Tue Sep 19 11:40:35 2017 -0700 ---------------------------------------------------------------------- site/current-book/Upgrading.html | 39 +- site/current-book/index.html | 42 +- site/current-book/metron-analytics/index.html | 39 +- .../metron-maas-service/index.html | 64 +- .../metron-profiler-client/index.html | 133 +- .../metron-analytics/metron-profiler/index.html | 297 +- .../metron-statistics/HLLP.html | 39 +- .../metron-statistics/index.html | 39 +- .../metron-contrib/metron-docker/index.html | 509 +++ .../Kerberos-ambari-setup.html | 53 +- .../Kerberos-manual-setup.html | 79 +- .../metron-deployment/amazon-ec2/index.html | 53 +- site/current-book/metron-deployment/index.html | 61 +- .../metron-deployment/other-examples/index.html | 290 ++ .../manual-install/Manual_Install_CentOS6.html | 1648 +++++++++ .../packaging/ambari/index.html | 786 ++++ .../packaging/docker/ansible-docker/index.html | 53 +- .../packaging/docker/rpm-docker/index.html | 53 +- .../packaging/packer-build/index.html | 53 +- .../metron-deployment/roles/index.html | 53 +- .../metron-deployment/roles/monit/index.html | 53 +- .../roles/opentaxii/index.html | 53 +- .../roles/pcap_replay/index.html | 53 +- .../roles/sensor-stubs/index.html | 53 +- .../roles/sensor-test-mode/index.html | 53 +- .../vagrant/codelab-platform/index.html | 53 +- .../vagrant/fastcapa-test-platform/index.html | 53 +- .../vagrant/full-dev-platform/index.html | 53 +- .../metron-deployment/vagrant/index.html | 53 +- .../vagrant/quick-dev-platform/index.html | 53 +- site/current-book/metron-docker/index.html | 486 --- .../metron-interface/metron-alerts/index.html | 338 ++ .../metron-interface/metron-config/index.html | 47 +- .../metron-interface/metron-rest/index.html | 429 ++- .../Performance-tuning-guide.html | 677 ++++ site/current-book/metron-platform/index.html | 60 +- .../metron-platform/metron-api/index.html | 58 +- .../metron-common/3rdPartyStellar.html | 398 -- .../metron-platform/metron-common/index.html | 2628 +------------- .../metron-data-management/index.html | 75 +- .../metron-enrichment/index.html | 88 +- .../metron-platform/metron-indexing/index.html | 94 +- .../metron-management/index.html | 82 +- .../metron-platform/metron-parsers/index.html | 210 +- .../metron-parsers/parser-testing.html | 372 ++ .../metron-pcap-backend/index.html | 58 +- .../metron-platform/metron-writer/index.html | 58 +- .../metron-sensors/bro-plugin-kafka/index.html | 39 +- .../metron-sensors/fastcapa/index.html | 86 +- site/current-book/metron-sensors/index.html | 39 +- .../metron-sensors/pycapa/index.html | 39 +- .../stellar-common/3rdPartyStellar.html | 354 ++ .../metron-stellar/stellar-common/index.html | 3410 ++++++++++++++++++ .../geographic_login_outliers/index.html | 532 +++ site/current-book/use-cases/index.html | 213 ++ site/documentation/index.md | 15 +- 56 files changed, 11879 insertions(+), 3919 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/Upgrading.html ---------------------------------------------------------------------- diff --git a/site/current-book/Upgrading.html b/site/current-book/Upgrading.html index dd68d32..00a5595 100644 --- a/site/current-book/Upgrading.html +++ b/site/current-book/Upgrading.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Upgrading</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="index.html" title="Metron"> @@ -94,7 +94,14 @@ <i class="icon-chevron-right"></i> Analytics</a> </li> - + + <li> + + <a href="metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li> <a href="metron-deployment/index.html" title="Deployment"> @@ -104,9 +111,9 @@ <li> - <a href="metron-docker/index.html" title="Docker"> + <a href="metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -122,7 +129,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="metron-platform/index.html" title="Platform"> @@ -136,6 +143,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul> http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/index.html b/site/current-book/index.html index 517703c..a213776 100644 --- a/site/current-book/index.html +++ b/site/current-book/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – </title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li class="active"> <a href="#"><i class="icon-chevron-down"></i>Metron</a> @@ -94,7 +94,14 @@ <i class="icon-chevron-right"></i> Analytics</a> </li> - + + <li> + + <a href="metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li> <a href="metron-deployment/index.html" title="Deployment"> @@ -104,9 +111,9 @@ <li> - <a href="metron-docker/index.html" title="Docker"> + <a href="metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -122,7 +129,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="metron-platform/index.html" title="Platform"> @@ -136,6 +143,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul> @@ -203,6 +224,9 @@ <li>Clone or download each repository individually</li> </ol> <p>Option 3 is more likely to have the latest code.</p> +<p><a name="Getting_Started"></a></p> +<h1>Getting Started</h1> +<p>To start exploring the capabilities of Apache Metron <a href="metron-deployment/vagrant/full-dev-platform/index.html">follow these instructions to launch Metron in a single-node VM</a> on your own hardware. </p> <p><a name="Building_Metron"></a></p> <h1>Building Metron</h1> <p>Build the full project and run tests:</p> http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-analytics/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-analytics/index.html b/site/current-book/metron-analytics/index.html index 61354bd..f1b83b4 100644 --- a/site/current-book/metron-analytics/index.html +++ b/site/current-book/metron-analytics/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Metron Analytics</title> <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../index.html" title="Metron"> @@ -124,7 +124,14 @@ </li> </ul> </li> - + + <li> + + <a href="../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li> <a href="../metron-deployment/index.html" title="Deployment"> @@ -134,9 +141,9 @@ <li> - <a href="../metron-docker/index.html" title="Docker"> + <a href="../metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -152,7 +159,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../metron-platform/index.html" title="Platform"> @@ -166,6 +173,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul> http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-analytics/metron-maas-service/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-analytics/metron-maas-service/index.html b/site/current-book/metron-analytics/metron-maas-service/index.html index b0f9162..820b149 100644 --- a/site/current-book/metron-analytics/metron-maas-service/index.html +++ b/site/current-book/metron-analytics/metron-maas-service/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Model Management Infrastructure</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../../index.html" title="Metron"> @@ -124,7 +124,14 @@ </li> </ul> </li> - + + <li> + + <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -134,9 +141,9 @@ <li> - <a href="../../metron-docker/index.html" title="Docker"> + <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -152,7 +159,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -166,6 +173,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul> @@ -314,6 +335,9 @@ usage: ModelSubmission -zr,--zk_root <arg> Zookeeper Root </pre></div></div></div> <div class="section"> +<h2><a name="Kerberos_Support"></a>Kerberos Support</h2> +<p>Model as a service will run on a kerberized cluster (see <a href="../../metron-deployment/vagrant/Kerberos-setup.html">here</a> for instructions for vagrant) with a caveat. The user who submits the service will be the user who executes the models on the cluster. That is to say that user impersonation of models deployed is not done at the moment.</p></div> +<div class="section"> <h2><a name="Stellar_Integration"></a>Stellar Integration</h2> <p>Two Stellar functions have been added to provide the ability to call out to models deployed via Model as a Service. One aimed at recovering a load balanced endpoint of a deployed model given the name and, optionally, the version. The second is aimed at calling that endpoint assuming that it is exposed as a REST endpoint.</p> @@ -345,20 +369,34 @@ usage: ModelSubmission <ul> -<li>Download the files in <a class="externalLink" href="https://gist.github.com/cestella/cba10aff0f970078a4c2c8cade3a4d1a";>this</a> gist into the <tt>/root/mock_dga</tt> directory</li> +<li>Download the files in <a class="externalLink" href="https://gist.github.com/cestella/cba10aff0f970078a4c2c8cade3a4d1a";>this</a> gist into the <tt>$HOME/mock_dga</tt> directory</li> -<li>Make <tt>rest.sh</tt> executable via <tt>chmod +x /root/mock_dga/rest.sh</tt></li> +<li>Make <tt>rest.sh</tt> executable via <tt>chmod +x $HOME/mock_dga/rest.sh</tt></li> </ul> <p>This service will treat <tt>yahoo.com</tt> and <tt>amazon.com</tt> as legit and everything else as malicious. The contract is that the REST service exposes an endpoint <tt>/apply</tt> and returns back JSON maps with a single key <tt>is_malicious</tt> which can be <tt>malicious</tt> or <tt>legit</tt>.</p></div> <div class="section"> <h2><a name="Deploy_Mock_DGA_Service_via_MaaS"></a>Deploy Mock DGA Service via MaaS</h2> +<p>The following presumes that you are a logged in as a user who has a home directory in HDFS under <tt>/user/$USER</tt>. If you do not, please create one and ensure the permissions are set appropriate:</p> + +<div class="source"> +<div class="source"> +<pre>su - hdfs -c "hadoop fs -mkdir /user/$USER" +su - hdfs -c "hadoop fs -chown $USER:$USER /user/$USER" +</pre></div></div> +<p>Or, in the common case for the <tt>metron</tt> user:</p> + +<div class="source"> +<div class="source"> +<pre>su - hdfs -c "hadoop fs -mkdir /user/metron" +su - hdfs -c "hadoop fs -chown metron:metron /user/metron" +</pre></div></div> <p>Now let’s start MaaS and deploy the Mock DGA Service:</p> <ul> <li>Start MaaS via <tt>$METRON_HOME/bin/maas_service.sh -zq node1:2181</tt></li> -<li>Start one instance of the mock DGA model with 512M of memory via <tt>$METRON_HOME/bin/maas_deploy.sh -zq node1:2181 -lmp /root/mock_dga -hmp /user/root/models -mo ADD -m 512 -n dga -v 1.0 -ni 1</tt></li> +<li>Start one instance of the mock DGA model with 512M of memory via <tt>$METRON_HOME/bin/maas_deploy.sh -zq node1:2181 -lmp $HOME/mock_dga -hmp /user/$USER/models -mo ADD -m 512 -n dga -v 1.0 -ni 1</tt></li> <li>As a sanity check: @@ -411,8 +449,6 @@ usage: ModelSubmission <div class="source"> <div class="source"> <pre>{ - "index": "squid", - "batchSize": 1, "enrichment" : { "fieldMap": {} }, http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-analytics/metron-profiler-client/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-analytics/metron-profiler-client/index.html b/site/current-book/metron-analytics/metron-profiler-client/index.html index 47cef0a..658dfe9 100644 --- a/site/current-book/metron-analytics/metron-profiler-client/index.html +++ b/site/current-book/metron-analytics/metron-profiler-client/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Metron Profiler Client</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../../index.html" title="Metron"> @@ -124,7 +124,14 @@ </li> </ul> </li> - + + <li> + + <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -134,9 +141,9 @@ <li> - <a href="../../metron-docker/index.html" title="Docker"> + <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -152,7 +159,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -166,6 +173,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul> @@ -751,7 +772,7 @@ Tuesday 11:15pm-11:45pm <div class="source"> <div class="source"> -<pre>[root@node1 0.3.1]# bin/stellar -z node1:2181 +<pre>[root@node1 0.4.1]# bin/stellar -z node1:2181 Stellar, Go! Please note that functions are loading lazily in the background and will be unavailable until loaded fully. {es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH} @@ -778,6 +799,98 @@ Returns: The selected profile measurements. [12078.0, 8921.0, 12131.0] </pre></div></div> <p>The client API call above has retrieved the past hour of the ‘test’ profile for the entity ‘192.168.138.158’.</p></div> +<div class="section"> +<h2><a name="Developing_Profiles"></a>Developing Profiles</h2> +<p>Troubleshooting issues when programming against a live stream of data can be difficult. The Stellar REPL is a powerful tool to help work out the kinds of enrichments and transformations that are needed. The Stellar REPL can also be used to help when developing profiles for the Profiler.</p> +<p>Follow these steps in the Stellar REPL to see how it can be used to help create profiles.</p> + +<ol style="list-style-type: decimal"> + +<li> +<p>Take a first pass at defining your profile. As an example, in the editor copy/paste the basic “Hello, World” profile below.</p> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> conf := SHELL_EDIT() +[Stellar]>>> conf +{ + "profiles": [ + { + "profile": "hello-world", + "onlyif": "exists(ip_src_addr)", + "foreach": "ip_src_addr", + "init": { "count": "0" }, + "update": { "count": "count + 1" }, + "result": "count" + } + ] +} +</pre></div></div></li> + +<li> +<p>Initialize the Profiler.</p> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> profiler := PROFILER_INIT(conf) +[Stellar]>>> profiler +Profiler{1 profile(s), 0 messages(s), 0 route(s)} +</pre></div></div> +<p>The profiler itself will show the number of profiles defined, the number of messages applied, and the number of routes taken. </p> +<p>A route is defined when a message is applied to a specific profile. If a message is applied and not needed by any profile, then there are no routes. If a message is needed by one profile, then one route has been defined. If a message is needed by two profiles, then two routes have been defined. </p></li> + +<li> +<p>Create a message to simulate the type of telemetry that you expect to be profiled. As an example, in the editor copy/paste the JSON below.</p> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> message := SHELL_EDIT() +[Stellar]>>> message +{ + "ip_src_addr": "10.0.0.1", + "protocol": "HTTPS", + "length": "10", + "bytes_in": "234" +} +</pre></div></div></li> + +<li> +<p>Apply some telemetry messages to your profiles. The following applies the same message 3 times.</p> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> PROFILER_APPLY(message, profiler) +Profiler{1 profile(s), 1 messages(s), 1 route(s)} +</pre></div></div> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> PROFILER_APPLY(message, profiler) +Profiler{1 profile(s), 2 messages(s), 2 route(s)} +</pre></div></div> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> PROFILER_APPLY(message, profiler) +Profiler{1 profile(s), 3 messages(s), 3 route(s)} +</pre></div></div> +<p>It is also possible to apply multiple messages at once. This is useful when testing against a larger set of data. To do this, create a string that contains a JSON array of messages and pass that to the <tt>PROFILER_APPLY</tt> function.</p></li> + +<li> +<p>Flush the Profiler to see what has been calculated. A flush is what occurs at the end of each 15 minute period in the Profiler. The result is a list of profile measurements. Each measurement is a map containing detailed information about the profile data that has been generated.</p> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> values := PROFILER_FLUSH(profiler) +[Stellar]>>> values +[{period={duration=900000, period=1669628, start=1502665200000, end=1502666100000}, + profile=hello-world, groups=[], value=3, entity=10.0.0.1}] +</pre></div></div> +<p>This profile simply counts the number of messages by IP source address. Notice that the value is ‘3’ for the entity ‘10.0.0.1’ as we applied 3 messages with an ‘ip_src_addr’ of ‘10.0.0.1’. There will always be one measurement for each [profile, entity] pair.</p></li> + +<li> +<p>If you are unhappy with the data that has been generated, then ‘wash, rinse and repeat’ this process. Once you are happy with the profile that was created, follow the <a href="../metron-profiler/index.html#Getting_Started">Getting Started</a> guide to use the profile against your live, streaming data in a Metron cluster.</p></li> +</ol></div> </div> </div> </div> http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-analytics/metron-profiler/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-analytics/metron-profiler/index.html b/site/current-book/metron-analytics/metron-profiler/index.html index 2f727dc..c35bdb6 100644 --- a/site/current-book/metron-analytics/metron-profiler/index.html +++ b/site/current-book/metron-analytics/metron-profiler/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Metron Profiler</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../../index.html" title="Metron"> @@ -124,7 +124,14 @@ </li> </ul> </li> - + + <li> + + <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -134,9 +141,9 @@ <li> - <a href="../../metron-docker/index.html" title="Docker"> + <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -152,7 +159,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -166,6 +173,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul> @@ -190,12 +211,14 @@ <h1>Metron Profiler</h1> <p><a name="Metron_Profiler"></a></p> -<p>The Profiler is a feature extraction mechanism that can generate a profile describing the behavior of an entity. An entity might be a server, user, subnet or application. Once a profile has been generated defining what normal behavior looks-like, models can be built that identify anomalous behavior. </p> +<p>The Profiler is a feature extraction mechanism that can generate a profile describing the behavior of an entity. An entity might be a server, user, subnet or application. Once a profile has been generated defining what normal behavior looks-like, models can be built that identify anomalous behavior.</p> <p>This is achieved by summarizing the streaming telemetry data consumed by Metron over sliding windows. A summary statistic is applied to the data received within a given window. Collecting this summary across many windows results in a time series that is useful for analysis.</p> <p>Any field contained within a message can be used to generate a profile. A profile can even be produced by combining fields that originate in different data sources. A user has considerable power to transform the data used in a profile by leveraging the Stellar language. A user only need configure the desired profiles and ensure that the Profiler topology is running.</p> <ul> +<li><a href="#Installation">Installation</a></li> + <li><a href="#Getting_Started">Getting Started</a></li> <li><a href="#Creating_Profiles">Creating Profiles</a></li> @@ -207,16 +230,59 @@ <li><a href="#Implementation">Implementation</a></li> </ul> <div class="section"> -<h2><a name="Getting_Started"></a>Getting Started</h2> -<p>This section will describe the steps required to get your first profile running.</p> +<h2><a name="Installation"></a>Installation</h2> +<p>Follow these instructions to install the Profiler. This assumes that core Metron has already been installed and validated. </p> <ol style="list-style-type: decimal"> <li> -<p>Stand-up a Metron environment. For this example, we will use the ‘Quick Dev’ environment. Follow the instructions included with <a href="../../metron-deployment/vagrant/quick-dev-platform/index.html">Quick Dev</a> or build your own.</p></li> +<p>Build the Metron RPMs (see Building the <a href="../../metron-deployment/index.html#RPMs">RPMs</a>).</p> +<p>You may have already built the Metron RPMs when core Metron was installed.</p> + +<div class="source"> +<div class="source"> +<pre>$ find metron-deployment/ -name "metron-profiler*.rpm" +metron-deployment//packaging/docker/rpm-docker/RPMS/noarch/metron-profiler-0.4.1-201707131420.noarch.rpm +</pre></div></div></li> <li> -<p>Create a table within HBase that will store the profile data. The table name and column family must match the <a href="#configuring-the-profiler">Profiler’s configuration</a>.</p> +<p>Copy the Profiler RPM to the installation host. </p> +<p>The installation host must be the same host on which core Metron was installed. Depending on how you installed Metron, the Profiler RPM might have already been copied to this host with the other Metron RPMs.</p> + +<div class="source"> +<div class="source"> +<pre>[root@node1 ~]# find /localrepo/ -name "metron-profiler*.rpm" +/localrepo/metron-profiler-0.4.0-201707112313.noarch.rpm +</pre></div></div></li> + +<li> +<p>Install the RPM.</p> + +<div class="source"> +<div class="source"> +<pre>[root@node1 ~]# rpm -ivh metron-profiler-*.noarch.rpm +Preparing... ########################################### [100%] + 1:metron-profiler ########################################### [100%] +</pre></div></div> + +<div class="source"> +<div class="source"> +<pre>[root@node1 ~]# rpm -ql metron-profiler +/usr/metron +/usr/metron/0.4.1 +/usr/metron/0.4.1/bin +/usr/metron/0.4.1/bin/start_profiler_topology.sh +/usr/metron/0.4.1/config +/usr/metron/0.4.1/config/profiler.properties +/usr/metron/0.4.1/flux +/usr/metron/0.4.1/flux/profiler +/usr/metron/0.4.1/flux/profiler/remote.yaml +/usr/metron/0.4.1/lib +/usr/metron/0.4.1/lib/metron-profiler-0.4.0-uber.jar +</pre></div></div></li> + +<li> +<p>Create a table within HBase that will store the profile data. By default, the table is named <tt>profiler</tt> with a column family <tt>P</tt>. The table name and column family must match the Profiler’s configuration (see <a href="#Configuring_the_Profiler">Configuring the Profiler</a>). </p> <div class="source"> <div class="source"> @@ -225,23 +291,42 @@ hbase(main):001:0> create 'profiler', 'P' </pre></div></div></li> <li> -<p>Edit the configuration file located at <tt>$METRON_HOME/config/profiler.properties</tt>. Change the kafka.zk and kafka.broker values from “node1” to the appropriate host name. Keep the same port numbers:</p> +<p>Edit the configuration file located at <tt>$METRON_HOME/config/profiler.properties</tt>. </p> <div class="source"> <div class="source"> <pre>kafka.zk=node1:2181 kafka.broker=node1:6667 +</pre></div></div> +<p>Change <tt>kafka.zk</tt> to refer to Zookeeper in your environment.<br />Change <tt>kafka.broker</tt> to refer to a Kafka Broker in your environment.</p></li> + +<li> +<p>Start the Profiler topology.</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_HOME +$ bin/start_profiler_topology.sh </pre></div></div></li> +</ol> +<p>At this point the Profiler is running and consuming telemetry messages. We have not defined any profiles yet, so it is not doing anything very useful. The next section walks you through the steps to create your very first “Hello, World!” profile.</p></div> +<div class="section"> +<h2><a name="Getting_Started"></a>Getting Started</h2> +<p>This section will describe the steps required to get your first “Hello, World!”" profile running. This assumes that you have a successful Profiler <a href="#Installation">Installation</a> and have it running.</p> + +<ol style="list-style-type: decimal"> <li> -<p>Define the profile in a file located at <tt>$METRON_HOME/config/zookeeper/profiler.json</tt>. The following example JSON will create a profile that simply counts the number of messages per <tt>ip_src_addr</tt>, during each sampling interval.</p> +<p>Create the profile definition in a file located at <tt>$METRON_HOME/config/zookeeper/profiler.json</tt>. This file will likely not exist, if you have never created Profiles before.</p> +<p>The following example will create a profile that simply counts the number of messages per <tt>ip_src_addr</tt>.</p> <div class="source"> <div class="source"> <pre>{ "profiles": [ { - "profile": "test", + "profile": "hello-world", + "onlyif": "exists(ip_src_addr)", "foreach": "ip_src_addr", "init": { "count": "0" }, "update": { "count": "count + 1" }, @@ -252,24 +337,36 @@ kafka.broker=node1:6667 </pre></div></div></li> <li> -<p>Upload the profile definition to Zookeeper. (As always, change “node1” to the actual hostname.)</p> +<p>Upload the profile definition to Zookeeper. Change <tt>node1:2181</tt> to refer the actual Zookeeper host in your environment.</p> <div class="source"> <div class="source"> <pre>$ cd $METRON_HOME $ bin/zk_load_configs.sh -m PUSH -i config/zookeeper/ -z node1:2181 -</pre></div></div></li> - -<li> -<p>Start the Profiler topology.</p> +</pre></div></div> +<p>You can validate this by reading back the Metron configuration from Zookeeper using the same script. The result should look-like the following.</p> <div class="source"> <div class="source"> -<pre>$ bin/start_profiler_topology.sh +<pre>$ bin/zk_load_configs.sh -m DUMP -z node1:2181 +... +PROFILER Config: profiler +{ + "profiles": [ + { + "profile": "hello-world", + "onlyif": "exists(ip_src_addr)", + "foreach": "ip_src_addr", + "init": { "count": "0" }, + "update": { "count": "count + 1" }, + "result": "count" + } + ] +} </pre></div></div></li> <li> -<p>Ensure that test messages are being sent to the Profiler’s input topic in Kafka. The Profiler will consume messages from the <tt>inputTopic</tt> defined in the <a href="#configuring-the-profiler">Profiler’s configuration</a>.</p></li> +<p>Ensure that test messages are being sent to the Profiler’s input topic in Kafka. The Profiler will consume messages from the input topic defined in the Profiler’s configuration (see <a href="#Configuring_the_Profiler">Configuring the Profiler</a>). By default this is the <tt>indexing</tt> topic.</p></li> <li> <p>Check the HBase table to validate that the Profiler is writing the profile. Remember that the Profiler is flushing the profile every 15 minutes. You will need to wait at least this long to start seeing profile data in HBase.</p> @@ -281,14 +378,16 @@ hbase(main):001:0> count 'profiler' </pre></div></div></li> <li> -<p>Use the Profiler Client to read the profile data. The below example <tt>PROFILE_GET</tt> command will read data written by the sample profile given above, if 10.0.0.1 is one of the input values for <tt>ip_src_addr</tt>. More information on configuring and using the client can be found <a href="../metron-profiler-client/index.html">here</a>. It is assumed that the <tt>PROFILE_GET</tt> client is correctly configured before using it.</p> +<p>Use the <a href="../metron-profiler-client/index.html">Profiler Client</a> to read the profile data. The following <tt>PROFILE_GET</tt> command will read the data written by the <tt>hello-world</tt> profile. This assumes that <tt>10.0.0.1</tt> is one of the values for <tt>ip_src_addr</tt> contained within the telemetry consumed by the Profiler.</p> <div class="source"> <div class="source"> <pre>$ bin/stellar -z node1:2181 -[Stellar]>>> PROFILE_GET( "test", "10.0.0.1", PROFILE_FIXED(30, "MINUTES")) +[Stellar]>>> PROFILE_GET( "hello-world", "10.0.0.1", PROFILE_FIXED(30, "MINUTES")) [451, 448] -</pre></div></div></li> +</pre></div></div> +<p>This result indicates that over the past 30 minutes, the Profiler stored two values related to the source IP address “10.0.0.1”. In the first 15 minute period, the IP <tt>10.0.0.1</tt> was seen in 451 telemetry messages. In the second 15 minute period, the same IP was seen in 448 telemetry messages.</p> +<p>It is assumed that the <tt>PROFILE_GET</tt> client is correctly configured to match the Profile configuration before using it to read that Profile. More information on configuring and using the Profiler client can be found <a href="../metron-profiler-client/index.html">here</a>. </p></li> </ol></div> <div class="section"> <h2><a name="Creating_Profiles"></a>Creating Profiles</h2> @@ -377,14 +476,14 @@ hbase(main):001:0> count 'profiler' <td>Optional </td> -<td>Profile data is purged after this period of time, specified in milliseconds.</td> +<td>Profile data is purged after this period of time, specified in days.</td> </tr> </tbody> </table> <div class="section"> <h3><a name="profile"></a><tt>profile</tt></h3> <p><i>Required</i></p> -<p>A unique name identifying the profile. The field is treated as a string. </p></div> +<p>A unique name identifying the profile. The field is treated as a string.</p></div> <div class="section"> <h3><a name="foreach"></a><tt>foreach</tt></h3> <p><i>Required</i></p> @@ -393,17 +492,35 @@ hbase(main):001:0> count 'profiler' <div class="section"> <h3><a name="onlyif"></a><tt>onlyif</tt></h3> <p><i>Optional</i></p> -<p>An expression that determines if a message should be applied to the profile. A Stellar expression that returns a Boolean is expected. A message is only applied to a profile if this expression is true. This allows a profile to filter the messages that get applied to it. </p></div> +<p>An expression that determines if a message should be applied to the profile. A Stellar expression that returns a Boolean is expected. A message is only applied to a profile if this expression is true. This allows a profile to filter the messages that get applied to it.</p></div> <div class="section"> <h3><a name="groupBy"></a><tt>groupBy</tt></h3> <p><i>Optional</i></p> -<p>One or more Stellar expressions used to group the profile measurements when persisted. This is intended to sort the Profile data to allow for a contiguous scan when accessing subsets of the data. </p> -<p>The ‘groupBy’ expressions can refer to any field within a <tt>org.apache.metron.profiler.ProfileMeasurement</tt>. A common use case would be grouping by day of week. This allows a contiguous scan to access all profile data for Mondays only. Using the following definition would achieve this. </p> +<p>One or more Stellar expressions used to group the profile measurements when persisted. This can be used to sort the Profile data to allow for a contiguous scan when accessing subsets of the data. This is also one way to deal with calendar effects. For example, where activity on a weekday can be very different from a weekend.</p> +<p>A common use case would be grouping by day of week. This allows a contiguous scan to access all profile data for Mondays only. Using the following definition would achieve this.</p> <div class="source"> <div class="source"> -<pre>"groupBy": [ "DAY_OF_WEEK()" ] -</pre></div></div></div> +<pre>"groupBy": [ "DAY_OF_WEEK(start)" ] +</pre></div></div> +<p>The expression can reference any of these variables.</p> + +<ul> + +<li>Any variable defined by the profile in its <tt>init</tt> or <tt>update</tt> expressions.</li> + +<li><tt>profile</tt> The name of the profile.</li> + +<li><tt>entity</tt> The name of the entity being profiled.</li> + +<li><tt>start</tt> The start time of the profile period in epoch milliseconds.</li> + +<li><tt>end</tt> The end time of the profile period in epoch milliseconds.</li> + +<li><tt>duration</tt> The duration of the profile period in milliseconds.</li> + +<li><tt>result</tt> The result of executing the <tt>result</tt> expression.</li> +</ul></div> <div class="section"> <h3><a name="init"></a><tt>init</tt></h3> <p><i>Optional</i></p> @@ -437,7 +554,7 @@ hbase(main):001:0> count 'profiler' <div class="source"> <pre>"result": "var1 + var2" </pre></div></div> -<p>For more advanced use cases, a profile can generate two types of results. A profile can define one or both of these result types at the same time. </p> +<p>For more advanced use cases, a profile can generate two types of results. A profile can define one or both of these result types at the same time.</p> <ul> @@ -446,7 +563,7 @@ hbase(main):001:0> count 'profiler' <li><tt>triage</tt>: An optional expression that defines values that are accessible within the Threat Triage process.</li> </ul> <p><b>profile</b></p> -<p>A required Stellar expression that results in a value that is persisted in the profile store for later retrieval. The expression can result in any object that is Kryo serializable. These values can be retrieved for later use with the <a href="../metron-profiler-client/index.html">Profiler Client</a>. </p> +<p>A required Stellar expression that results in a value that is persisted in the profile store for later retrieval. The expression can result in any object that is Kryo serializable. These values can be retrieved for later use with the <a href="../metron-profiler-client/index.html">Profiler Client</a>.</p> <div class="source"> <div class="source"> @@ -477,7 +594,8 @@ hbase(main):001:0> count 'profiler' <div class="section"> <h3><a name="expires"></a><tt>expires</tt></h3> <p><i>Optional</i></p> -<p>A numeric value that defines how many days the profile data is retained. After this time, the data expires and is no longer accessible. If no value is defined, the data does not expire.</p></div></div> +<p>A numeric value that defines how many days the profile data is retained. After this time, the data expires and is no longer accessible. If no value is defined, the data does not expire.</p> +<p>The REPL can be a powerful for developing profiles. Read all about <a href="../metron-profiler-client/index.html#developing_profiles">Developing Profiles</a>.</p></div></div> <div class="section"> <h2><a name="Configuring_the_Profiler"></a>Configuring the Profiler</h2> <p>The Profiler runs as an independent Storm topology. The configuration for the Profiler topology is stored in local filesystem at <tt>$METRON_HOME/config/profiler.properties</tt>. The values can be changed on disk and then the Profiler topology must be restarted.</p> @@ -496,102 +614,151 @@ hbase(main):001:0> count 'profiler' <tr class="b"> -<td>profiler.workers </td> +<td><a href="#profiler.input.topic"><tt>profiler.input.topic</tt></a> </td> -<td>The number of worker processes to create for the topology.</td> +<td>The name of the Kafka topic from which to consume data.</td> </tr> <tr class="a"> -<td>profiler.executors </td> +<td><a href="#profiler.output.topic"><tt>profiler.output.topic</tt></a> </td> -<td>The number of executors to spawn per component.</td> +<td>The name of the Kafka topic to which profile data is written. Only used with profiles that define the <a href="#result"><tt>triage</tt> result field</a>.</td> </tr> <tr class="b"> -<td>profiler.input.topic </td> +<td><a href="#profiler.period.duration"><tt>profiler.period.duration</tt></a> </td> -<td>The name of the Kafka topic from which to consume data.</td> +<td>The duration of each profile period.</td> </tr> <tr class="a"> -<td>profiler.output.topic </td> +<td><a href="#profiler.period.duration.units"><tt>profiler.period.duration.units</tt></a> </td> -<td>The name of the Kafka topic to which profile data is written. Only used with profiles that use the <a href="#result"><tt>triage</tt> result field</a>.</td> +<td>The units used to specify the <a href="#profiler.period.duration"><tt>profiler.period.duration</tt></a>.</td> </tr> <tr class="b"> -<td>profiler.period.duration </td> +<td><a href="#profiler.workers"><tt>profiler.workers</tt></a> </td> -<td>The duration of each profile period. This value should be defined along with <tt>profiler.period.duration.units</tt>.</td> +<td>The number of worker processes for the topology.</td> </tr> <tr class="a"> -<td>profiler.period.duration.units </td> +<td><a href="#profiler.executors"><tt>profiler.executors</tt></a> </td> -<td>The units used to specify the <tt>profiler.period.duration</tt>.</td> +<td>The number of executors to spawn per component.</td> </tr> <tr class="b"> -<td>profiler.ttl </td> +<td><a href="#profiler.ttl"><tt>profiler.ttl</tt></a> </td> -<td>If a message has not been applied to a Profile in this period of time, the Profile will be forgotten and its resources will be cleaned up. This value should be defined along with <tt>profiler.ttl.units</tt>.</td> +<td>If a message has not been applied to a Profile in this period of time, the Profile will be forgotten and its resources will be cleaned up.</td> </tr> <tr class="a"> -<td>profiler.ttl.units </td> +<td><a href="#profiler.ttl.units"><tt>profiler.ttl.units</tt></a> </td> <td>The units used to specify the <tt>profiler.ttl</tt>.</td> </tr> <tr class="b"> -<td>profiler.hbase.salt.divisor </td> +<td><a href="#profiler.hbase.salt.divisor"><tt>profiler.hbase.salt.divisor</tt></a> </td> -<td>A salt is prepended to the row key to help prevent hotspotting. This constant is used to generate the salt. Ideally, this constant should be roughly equal to the number of nodes in the Hbase cluster.</td> +<td>A salt is prepended to the row key to help prevent hotspotting.</td> </tr> <tr class="a"> -<td>profiler.hbase.table </td> +<td><a href="#profiler.hbase.table"><tt>profiler.hbase.table</tt></a> </td> <td>The name of the HBase table that profiles are written to.</td> </tr> <tr class="b"> -<td>profiler.hbase.column.family </td> +<td><a href="#profiler.hbase.column.family"><tt>profiler.hbase.column.family</tt></a> </td> <td>The column family used to store profiles.</td> </tr> <tr class="a"> -<td>profiler.hbase.batch </td> +<td><a href="#profiler.hbase.batch"><tt>profiler.hbase.batch</tt></a> </td> -<td>The number of puts that are written in a single batch.</td> +<td>The number of puts that are written to HBase in a single batch.</td> </tr> <tr class="b"> -<td>profiler.hbase.flush.interval.seconds </td> +<td><a href="#profiler.hbase.flush.interval.seconds"><tt>profiler.hbase.flush.interval.seconds</tt></a> </td> <td>The maximum number of seconds between batch writes to HBase.</td> </tr> </tbody> </table> -<p>After altering the configuration, start the Profiler.</p> - -<div class="source"> -<div class="source"> -<pre>$ $METRON_HOME/start_profiler_topology.sh -</pre></div></div></div> +<div class="section"> +<h3><a name="profiler.input.topic"></a><tt>profiler.input.topic</tt></h3> +<p><i>Default</i>: indexing</p> +<p>The name of the Kafka topic from which to consume data. By default, the Profiler consumes data from the <tt>indexing</tt> topic so that it has access to fully enriched telemetry.</p></div> +<div class="section"> +<h3><a name="profiler.output.topic"></a><tt>profiler.output.topic</tt></h3> +<p><i>Default</i>: enrichments</p> +<p>The name of the Kafka topic to which profile data is written. This property is only applicable to profiles that define the <a href="#result"><tt>result</tt> <tt>triage</tt> field</a>. This allows Profile data to be selectively triaged like any other source of telemetry in Metron.</p></div> +<div class="section"> +<h3><a name="profiler.period.duration"></a><tt>profiler.period.duration</tt></h3> +<p><i>Default</i>: 15</p> +<p>The duration of each profile period. This value should be defined along with <a href="#profiler.period.duration.units"><tt>profiler.period.duration.units</tt></a>.</p> +<p><i>Important</i>: To read a profile using the <a href="metron-analytics/metron-profiler-client/index.html">Profiler Client</a>, the Profiler Client’s <tt>profiler.client.period.duration</tt> property must match this value. Otherwise, the Profiler Client will be unable to read the profile data. </p></div> +<div class="section"> +<h3><a name="profiler.period.duration.units"></a><tt>profiler.period.duration.units</tt></h3> +<p><i>Default</i>: MINUTES</p> +<p>The units used to specify the <tt>profiler.period.duration</tt>. This value should be defined along with <a href="#profiler.period.duration"><tt>profiler.period.duration</tt></a>.</p> +<p><i>Important</i>: To read a profile using the Profiler Client, the Profiler Client’s <tt>profiler.client.period.duration.units</tt> property must match this value. Otherwise, the <a href="metron-analytics/metron-profiler-client/index.html">Profiler Client</a> will be unable to read the profile data.</p></div> +<div class="section"> +<h3><a name="profiler.workers"></a><tt>profiler.workers</tt></h3> +<p><i>Default</i>: 1</p> +<p>The number of worker processes to create for the Profiler topology. This property is useful for performance tuning the Profiler.</p></div> +<div class="section"> +<h3><a name="profiler.executors"></a><tt>profiler.executors</tt></h3> +<p><i>Default</i>: 0</p> +<p>The number of executors to spawn per component for the Profiler topology. This property is useful for performance tuning the Profiler.</p></div> +<div class="section"> +<h3><a name="profiler.ttl"></a><tt>profiler.ttl</tt></h3> +<p><i>Default</i>: 30</p> +<p>If a message has not been applied to a Profile in this period of time, the Profile will be terminated and its resources will be cleaned up. This value should be defined along with <a href="#profiler.ttl.units"><tt>profiler.ttl.units</tt></a>.</p> +<p>This time-to-live does not affect the persisted Profile data in HBase. It only affects the state stored in memory during the execution of the latest profile period. This state will be deleted if the time-to-live is exceeded.</p></div> +<div class="section"> +<h3><a name="profiler.ttl.units"></a><tt>profiler.ttl.units</tt></h3> +<p><i>Default</i>: MINUTES</p> +<p>The units used to specify the <a href="#profiler.ttl"><tt>profiler.ttl</tt></a>.</p></div> +<div class="section"> +<h3><a name="profiler.hbase.salt.divisor"></a><tt>profiler.hbase.salt.divisor</tt></h3> +<p><i>Default</i>: 1000</p> +<p>A salt is prepended to the row key to help prevent hotspotting. This constant is used to generate the salt. This constant should be roughly equal to the number of nodes in the Hbase cluster to ensure even distribution of data.</p></div> +<div class="section"> +<h3><a name="profiler.hbase.table"></a><tt>profiler.hbase.table</tt></h3> +<p><i>Default</i>: profiler</p> +<p>The name of the HBase table that profile data is written to. The Profiler expects that the table exists and is writable. It will not create the table.</p></div> +<div class="section"> +<h3><a name="profiler.hbase.column.family"></a><tt>profiler.hbase.column.family</tt></h3> +<p><i>Default</i>: P</p> +<p>The column family used to store profile data in HBase.</p></div> +<div class="section"> +<h3><a name="profiler.hbase.batch"></a><tt>profiler.hbase.batch</tt></h3> +<p><i>Default</i>: 10</p> +<p>The number of puts that are written to HBase in a single batch.</p></div> +<div class="section"> +<h3><a name="profiler.hbase.flush.interval.seconds"></a><tt>profiler.hbase.flush.interval.seconds</tt></h3> +<p><i>Default</i>: 30</p> +<p>The maximum number of seconds between batch writes to HBase.</p></div></div> <div class="section"> <h2><a name="Examples"></a>Examples</h2> <p>The following examples are intended to highlight the functionality provided by the Profiler. Each shows the configuration that would be required to generate the profile. </p> http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-analytics/metron-statistics/HLLP.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-analytics/metron-statistics/HLLP.html b/site/current-book/metron-analytics/metron-statistics/HLLP.html index 0017621..d1175e9 100644 --- a/site/current-book/metron-analytics/metron-statistics/HLLP.html +++ b/site/current-book/metron-analytics/metron-statistics/HLLP.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-06-27 + | Generated by Apache Maven Doxia at 2017-09-15 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta name="Date-Revision-yyyymmdd" content="20170915" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – HyperLogLogPlus</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.1</li> </ul> </div> @@ -75,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../../index.html" title="Metron"> @@ -133,7 +133,14 @@ </li> </ul> </li> - + + <li> + + <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -143,9 +150,9 @@ <li> - <a href="../../metron-docker/index.html" title="Docker"> + <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> <i class="none"></i> - Docker</a> + Alerts</a> </li> <li> @@ -161,7 +168,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -175,6 +182,20 @@ <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> </ul> </li> </ul>
