http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/metron-elasticsearch/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-elasticsearch/index.html b/site/current-book/metron-platform/metron-elasticsearch/index.html new file mode 100644 index 0000000..ff4bfb0 --- /dev/null +++ b/site/current-book/metron-platform/metron-elasticsearch/index.html @@ -0,0 +1,380 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2018-01-03 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – Elasticsearch in Metron</title> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../../css/site.css" /> + <link rel="stylesheet" href="../../css/print.css" media="print" /> + + + <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="http://metron.apache.org/"; id="bannerLeft"> + <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> + </a> + </div> + <div class="pull-right"> </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> + Metron</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="../../index.html" title="Documentation"> + Documentation</a> + </li> + <li class="divider ">/</li> + <li class="">Elasticsearch in Metron</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + + <li> + + <a href="../../index.html" title="Metron"> + <i class="icon-chevron-down"></i> + Metron</a> + <ul class="nav nav-list"> + + <li> + + <a href="../../Upgrading.html" title="Upgrading"> + <i class="none"></i> + Upgrading</a> + </li> + + <li> + + <a href="../../metron-analytics/index.html" title="Analytics"> + <i class="icon-chevron-right"></i> + Analytics</a> + </li> + + <li> + + <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + + <li> + + <a href="../../metron-deployment/index.html" title="Deployment"> + <i class="icon-chevron-right"></i> + Deployment</a> + </li> + + <li> + + <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> + <i class="none"></i> + Alerts</a> + </li> + + <li> + + <a href="../../metron-interface/metron-config/index.html" title="Config"> + <i class="none"></i> + Config</a> + </li> + + <li> + + <a href="../../metron-interface/metron-rest/index.html" title="Rest"> + <i class="none"></i> + Rest</a> + </li> + + <li> + + <a href="../../metron-platform/index.html" title="Platform"> + <i class="icon-chevron-down"></i> + Platform</a> + <ul class="nav nav-list"> + + <li> + + <a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"> + <i class="none"></i> + Performance-tuning-guide</a> + </li> + + <li> + + <a href="../../metron-platform/metron-api/index.html" title="Api"> + <i class="none"></i> + Api</a> + </li> + + <li> + + <a href="../../metron-platform/metron-common/index.html" title="Common"> + <i class="none"></i> + Common</a> + </li> + + <li> + + <a href="../../metron-platform/metron-data-management/index.html" title="Data-management"> + <i class="none"></i> + Data-management</a> + </li> + + <li class="active"> + + <a href="#"><i class="none"></i>Elasticsearch</a> + </li> + + <li> + + <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> + <i class="none"></i> + Enrichment</a> + </li> + + <li> + + <a href="../../metron-platform/metron-indexing/index.html" title="Indexing"> + <i class="none"></i> + Indexing</a> + </li> + + <li> + + <a href="../../metron-platform/metron-management/index.html" title="Management"> + <i class="none"></i> + Management</a> + </li> + + <li> + + <a href="../../metron-platform/metron-parsers/index.html" title="Parsers"> + <i class="icon-chevron-right"></i> + Parsers</a> + </li> + + <li> + + <a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"> + <i class="none"></i> + Pcap-backend</a> + </li> + + <li> + + <a href="../../metron-platform/metron-writer/index.html" title="Writer"> + <i class="none"></i> + Writer</a> + </li> + </ul> + </li> + + <li> + + <a href="../../metron-sensors/index.html" title="Sensors"> + <i class="icon-chevron-right"></i> + Sensors</a> + </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> + + <li> + + <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> + <i class="icon-chevron-right"></i> + Stellar-common</a> + </li> + + <li> + + <a href="../../use-cases/index.html" title="Use-cases"> + <i class="icon-chevron-right"></i> + Use-cases</a> + </li> + </ul> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <h1>Elasticsearch in Metron</h1> +<p><a name="Elasticsearch_in_Metron"></a></p> +<div class="section"> +<h2><a name="Introduction"></a>Introduction</h2> +<p>Elasticsearch can be used as the real-time portion of the datastore resulting from <a href="../metron-indexing/index.html">metron-indexing</a>.</p></div> +<div class="section"> +<h2><a name="Properties"></a>Properties</h2> +<div class="section"> +<h3><a name="es.clustername"></a><tt>es.clustername</tt></h3> +<p>The name of the elasticsearch Cluster. See <a class="externalLink" href="https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster.name";>here</a></p></div> +<div class="section"> +<h3><a name="es.ip"></a><tt>es.ip</tt></h3> +<p>Specifies the nodes in the elasticsearch cluster to use for writing. The format is one of the following:</p> + +<ul> + +<li>A hostname or IP address with a port (e.g. <tt>hostname1:1234</tt>), in which case <tt>es.port</tt> is ignored.</li> + +<li>A hostname or IP address without a port (e.g. <tt>hostname1</tt>), in which case <tt>es.port</tt> is used.</li> + +<li>A string containing a CSV of hostnames without ports (e.g. <tt>hostname1,hostname2,hostname3</tt>) without spaces between. <tt>es.port</tt> is assumed to be the port for each host.</li> + +<li>A string containing a CSV of hostnames with ports (e.g. <tt>hostname1:1234,hostname2:1234,hostname3:1234</tt>) without spaces between. <tt>es.port</tt> is ignored.</li> + +<li>A list of hostnames with ports (e.g. <tt>[ "hostname1:1234", "hostname2:1234"]</tt>). Note, <tt>es.port</tt> is NOT used in this construction.</li> +</ul></div> +<div class="section"> +<h3><a name="es.port"></a><tt>es.port</tt></h3> +<p>The port for the elasticsearch hosts. This will be used in accordance with the discussion of <tt>es.ip</tt>.</p></div> +<div class="section"> +<h3><a name="es.date.format"></a><tt>es.date.format</tt></h3> +<p>The date format to use when constructing the indices. For every message, the date format will be applied to the current time and that will become the last part of the index name where the message is written to.</p> +<p>For instance, an <tt>es.date.format</tt> of <tt>yyyy.MM.dd.HH</tt> would have the consequence that the indices would roll hourly, whereas an <tt>es.date.format</tt> of <tt>yyyy.MM.dd</tt> would have the consequence that the indices would roll daily.</p></div></div> +<div class="section"> +<h2><a name="Using_Metron_with_Elasticsearch_2.x"></a>Using Metron with Elasticsearch 2.x</h2> +<p>With Elasticsearch 2.x, there is a requirement that all sensors templates have a nested alert field defined. This field is a dummy field, and will be obsolete in Elasticsearch 5.x. See <a class="externalLink" href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields";>Ignoring Unmapped Fields</a> for more information</p> +<p>Without this field, an error will be thrown during ALL searches (including from UIs, resulting in no alerts being found for any sensor). This error will be found in the REST service’s logs.</p> +<p>Exception seen:</p> + +<div class="source"> +<div class="source"> +<pre>QueryParsingException[[nested] failed to find nested object under path [alert]]; +</pre></div></div> +<p>There are two steps to resolve this issue. First is to update the Elasticsearch template for each sensor, so any new indices have the field. This requires retrieving the template, removing an extraneous JSON field so we can put it back later, and adding our new field.</p> +<p>Make sure to set the ELASTICSEARCH variable appropriately. $SENSOR can contain wildcards, so if rollover has occurred, it’s not necessary to do each index individually. The example here appends <tt>index*</tt> to get all indexes for a the provided sensor.</p> + +<div class="source"> +<div class="source"> +<pre>export ELASTICSEARCH="node1" +export SENSOR="bro" +curl -XGET "http://${ELASTICSEARCH}:9200/_template/${SENSOR}_index*?pretty=true" -o "${SENSOR}.template" +sed -i '' '2d;$d' ./${SENSOR}.template +sed -i '' '/"properties" : {/ a\ +"alert": { "type": "nested"},' ${SENSOR}.template +</pre></div></div> +<p>To manually verify this, you can optionally pretty print it again with:</p> + +<div class="source"> +<div class="source"> +<pre>python -m json.tool bro.template +</pre></div></div> +<p>We’ll want to put the template back into Elasticsearch:</p> + +<div class="source"> +<div class="source"> +<pre>curl -XPUT "http://${ELASTICSEARCH}:9200/_template/${SENSOR}_index" -d @${SENSOR}.template +</pre></div></div> +<p>To update existing indexes, update Elasticsearch mappings with the new field for each sensor. </p> + +<div class="source"> +<div class="source"> +<pre>curl -XPUT "http://${ELASTICSEARCH}:9200/${SENSOR}_index*/_mapping/${SENSOR}_doc" -d ' +{ + "properties" : { + "alert" : { + "type" : "nested" + } + } +} +' +rm ${SENSOR}.template +</pre></div></div></div> +<div class="section"> +<h2><a name="Installing_Elasticsearch_Templates"></a>Installing Elasticsearch Templates</h2> +<p>The stock set of Elasticsearch templates for bro, snort, yaf, error index and meta index are installed automatically during the first time install and startup of Metron Indexing service.</p> +<p>It is possible that Elasticsearch service is not available when the Metron Indexing Service startup, in that case the Elasticsearch template will not be installed. </p> +<p>For such a scenario, an Admin can have the template installed in two ways:</p> +<p><i>Method 1</i> - Manually from the Ambari UI by following the flow: Ambari UI -> Services -> Metron -> Service Actions -> Elasticsearch Template Install</p> +<p><i>Method 2</i> - Stop the Metron Indexing service, and start it again from Ambari UI. Note that the Metron Indexing service tracks if it has successfully installed the Elasticsearch templates, and will attempt to do so each time it is Started until successful.</p> + +<blockquote> +<p>Note: If you have made any customization to your index templates, then installing Elasticsearch templates afresh will lead to overwriting your existing changes. Please exercise caution.</p> +</blockquote></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2018 + <a href="https://www.apache.org";>The Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html>
http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/metron-enrichment/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-enrichment/index.html b/site/current-book/metron-platform/metron-enrichment/index.html index 979baf1..584c36f 100644 --- a/site/current-book/metron-platform/metron-enrichment/index.html +++ b/site/current-book/metron-platform/metron-enrichment/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Enrichment</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -167,6 +167,13 @@ Data-management</a> </li> + <li> + + <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + <li class="active"> <a href="#"><i class="none"></i>Enrichment</a> @@ -208,13 +215,20 @@ </li> </ul> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -222,7 +236,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -274,7 +288,14 @@ <p>There are two types of configurations at the moment, <tt>global</tt> and <tt>sensor</tt> specific. </p></div> <div class="section"> <h2><a name="Global_Configuration"></a>Global Configuration</h2> -<p>See the “<a href="../metron-common/index.html">Global Configuration</a>” section.</p></div> +<p>There are a few enrichments which have independent configurations, such as from the global config.</p> +<p>Also, see the “<a href="../metron-common/index.html">Global Configuration</a>” section for more discussion of the global config.</p> +<div class="section"> +<h3><a name="GeoIP"></a>GeoIP</h3> +<p>Metron supports enrichment of IP information using <a class="externalLink" href="https://dev.maxmind.com/geoip/geoip2/geolite2/";>GeoLite2</a>. The location of the file is managed in the global config.</p> +<div class="section"> +<h4><a name="geo.hdfs.file"></a><tt>geo.hdfs.file</tt></h4> +<p>The location on HDFS of the GeoLite2 database file to use for GeoIP lookups. This file will be localized on the storm supervisors running the topology and used from there. This is lazy, so if this property changes in a running topology, the file will be localized from HDFS upon first time the file is used via the geo enrichment. </p></div></div></div> <div class="section"> <h2><a name="Sensor_Enrichment_Configuration"></a>Sensor Enrichment Configuration</h2> <p>The sensor specific configuration is intended to configure the individual enrichments and threat intelligence enrichments for a given sensor type (e.g. <tt>snort</tt>).</p> @@ -730,7 +751,7 @@ <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/metron-indexing/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-indexing/index.html b/site/current-book/metron-platform/metron-indexing/index.html index fc5fef2..3d79323 100644 --- a/site/current-book/metron-platform/metron-indexing/index.html +++ b/site/current-book/metron-platform/metron-indexing/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Indexing</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -169,6 +169,13 @@ <li> + <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + + <li> + <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> <i class="none"></i> Enrichment</a> @@ -208,13 +215,20 @@ </li> </ul> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -222,7 +236,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -306,6 +320,12 @@ <li><tt>enabled</tt> : Whether the writer is enabled (default <tt>true</tt>).</li> </ul> <div class="section"> +<h3><a name="Meta_Alerts"></a>Meta Alerts</h3> +<p>Alerts can be grouped, after appropriate searching, into a set of alerts called a meta alert. A meta alert is useful for maintaining the context of searching and grouping during further investigations. Standard searches can return meta alerts, but grouping and other aggregation or sorting requests will not, because there’s not a clear way to aggregate in many cases if there are multiple alerts contained in the meta alert. All meta alerts will have the source type of metaalert, regardless of the contained alert’s origins.</p></div> +<div class="section"> +<h3><a name="Elasticsearch"></a>Elasticsearch</h3> +<p>Metron comes with built-in templates for the default sensors for Elasticsearch. When adding a new sensor, it will be necessary to add a new template defining the output fields appropriately. In addition, there is a requirement for a field <tt>alert</tt> of type <tt>nested</tt> for Elasticsearch 2.x installs. This is detailed at <a href="../metron-elasticsearch/index.html#Using_Metron_with_Elasticsearch_2.x">Using Metron with Elasticsearch 2.x</a></p></div> +<div class="section"> <h3><a name="Indexing_Configuration_Examples"></a>Indexing Configuration Examples</h3> <p>For a given sensor, the following scenarios would be indicated by the following cases:</p> <div class="section"> @@ -452,9 +472,39 @@ <h2><a name="The_IndexDao_Abstraction"></a>The <tt>IndexDao</tt> Abstraction</h2> <p>The indices mentioned above as part of Update should be pluggable by the developer so that new write-ahead logs or real-time indices can be supported by providing an implementation supporting the data access patterns.</p> <p>To support a new index, one would need to implement the <tt>org.apache.metron.indexing.dao.IndexDao</tt> abstraction and provide update and search capabilities. IndexDaos may be composed and updates will be performed in parallel. This enables a flexible strategy for specifying your backing store for updates at runtime. For instance, currently the REST API supports the update functionality and may be configured with a list of IndexDao implementations to use to support the updates.</p> +<p>Updates with the IndexDao.update method replace the current object with the new object. For partial updates, use IndexDao.patch instead.</p> +<div class="section"> +<h3><a name="The_HBaseDao"></a>The <tt>HBaseDao</tt></h3> +<p>Updates will be written to HBase. The key structure includes the GUID and sensor type and for each new version, a new column is created with value as the message.</p> +<p>The HBase table and column family are configured via fields in the global configuration.</p> +<div class="section"> +<h4><a name="update.hbase.table"></a><tt>update.hbase.table</tt></h4> +<p>The HBase table to use for message updates.</p></div> +<div class="section"> +<h4><a name="update.hbase.cf"></a><tt>update.hbase.cf</tt></h4> +<p>The HBase column family to use for message updates.</p></div></div> +<div class="section"> +<h3><a name="The_MetaAlertDao"></a>The <tt>MetaAlertDao</tt></h3> +<p>The goal of meta alerts is to be able to group together a set of alerts while being able to transparently perform actions like searches, as if meta alerts were normal alerts. <tt>org.apache.metron.indexing.dao.MetaAlertDao</tt> extends <tt>IndexDao</tt> and enables several features: </p> + +<ul> + +<li>the ability to get all meta alerts associated with an alert</li> + +<li>creation of a meta alert</li> + +<li>adding alerts to a meta alert</li> + +<li>removing alerts from a meta alert</li> + +<li>changing a meta alert’s status</li> +</ul> +<p>The implementation of this is to denormalize the relationship between alerts and meta alerts, and store alerts as a nested field within a meta alert. The use of nested fields is to avoid the limitations of parent-child relationships (one-to-many) and merely linking by IDs (which causes issues with pagination as a result of being unable to join indices). A list of containing meta alerts is stored on an alert for the purpose of keeping source alerts and alerts contained in meta alerts in sync.</p> +<p>The search functionality of <tt>IndexDao</tt> is wrapped by the <tt>MetaAlertDao</tt> in order to provide both regular and meta alerts side-by-side with sorting. The updating capabilities are similarly wrapped, in order to ensure updates are carried through both the alerts and associated meta alerts. Both of these functions are handled under the hood.</p> +<p>In addition, API endpoints have been added to expose the features listed above. The denormalization handles the case of going from meta alert to alert automatically.</p> <p><a name="Notes_on_Performance_Tuning"></a></p> <h1>Notes on Performance Tuning</h1> -<p>Default installed Metron is untuned for production deployment. By far and wide, the most likely piece to require TLC from a performance perspective is the indexing layer. An index that does not keep up will back up and you will see errors in the kafka bolt. There are a few knobs to tune to get the most out of your system.</p></div> +<p>Default installed Metron is untuned for production deployment. By far and wide, the most likely piece to require TLC from a performance perspective is the indexing layer. An index that does not keep up will back up and you will see errors in the kafka bolt. There are a few knobs to tune to get the most out of your system.</p></div></div> <div class="section"> <h2><a name="Kafka_Queue"></a>Kafka Queue</h2> <p>The <tt>indexing</tt> kafka queue is a collection point from the enrichment topology. As such, make sure that the number of partitions in the kafka topic is sufficient to handle the throughput that you expect.</p></div> @@ -491,7 +541,7 @@ <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/metron-management/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-management/index.html b/site/current-book/metron-platform/metron-management/index.html index a760baf..cca14f1 100644 --- a/site/current-book/metron-platform/metron-management/index.html +++ b/site/current-book/metron-platform/metron-management/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Stellar REPL Management Utilities</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -169,6 +169,13 @@ <li> + <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + + <li> + <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> <i class="none"></i> Enrichment</a> @@ -208,13 +215,20 @@ </li> </ul> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -222,7 +236,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -272,8 +286,47 @@ <li>print structured data in a way that is easier to view (i.e. tabular)</li> </ul> <p>This functionality is exposed as a pack of Stellar functions in this project.</p> + +<ul> + +<li><a href="#Functions">Functions</a> + +<ul> + +<li><a href="#Grok_Functions">Grok Functions</a></li> + +<li><a href="#File_Functions">File Functions</a></li> + +<li><a href="#Shell_Functions">Shell Functions</a></li> + +<li><a href="#Configuration_Functions">Configuration Functions</a></li> + +<li><a href="#Parser_Functions">Parser Functions</a></li> + +<li><a href="#Indexing_Functions">Indexing Functions</a></li> + +<li><a href="#Enrichment_Functions">Enrichment Functions</a></li> + +<li><a href="#Threat_Triage_Functions">Threat Triage Functions</a></li> + </ul></li> + +<li><a href="#Examples">Examples</a> + +<ul> + +<li><a href="#Iterate_to_Find_a_Valid_Grok_Pattern">Iterate to Find a Valid Grok Pattern</a></li> + +<li><a href="#Manage_Stellar_Field_Transformations">Manage Stellar Field Transformations</a></li> + +<li><a href="#Manage_Stellar_Enrichments">Manage Stellar Enrichments</a></li> + +<li><a href="#Manage_Threat_Triage_Rules">Manage Threat Triage Rules</a></li> + +<li><a href="#Simulate_Threat_Triage_Rules">Simulate Threat Triage Rules</a></li> + </ul></li> +</ul> <div class="section"> -<h2><a name="Function_Details"></a>Function Details</h2> +<h2><a name="Functions"></a>Functions</h2> <p>The functions are split roughly into a few sections:</p> <ul> @@ -667,7 +720,7 @@ <li>Returns: The String representation of the config in zookeeper</li> </ul></li> -<li><tt>PARSER_STELLAR_TRANSFORM_PRINT</tt> +<li><tt>PARSER-STELLAR_TRANSFORM_PRINT</tt> <ul> @@ -840,37 +893,71 @@ <ul> -<li><tt>THREAT_TRIAGE_ADD</tt> +<li><tt>THREAT_TRIAGE_INIT</tt> <ul> -<li>Description: Add a threat triage rule.</li> +<li>Description: Create a threat triage engine.</li> <li>Input: <ul> -<li>sensorConfig - Sensor config to add transformation to.</li> +<li>config - the threat triage configuration (optional)</li> + </ul></li> + +<li>Returns: A threat triage engine.</li> + </ul></li> + +<li><tt>THREAT_TRIAGE_CONFIG</tt> + +<ul> + +<li>Description: Export the configuration used by a threat triage engine.</li> + +<li>Input: + +<ul> -<li>stellarTransforms - A Map associating stellar rules to scores</li> +<li>engine - threat triage engine returned by THREAT_TRIAGE_INIT.</li> + </ul></li> + +<li>Returns: The configuration used by the threat triage engine.</li> + </ul></li> + +<li><tt>THREAT_TRIAGE_SCORE</tt> + +<ul> + +<li>Description: Scores a message using a set of triage rules.</li> + +<li>Inputs: + +<ul> -<li>triageRules - Map (or list of Maps) representing a triage rule. It must contain ‘rule’ and ‘score’ keys, the stellar expression for the rule and triage score respectively. It may contain ‘name’ and ‘comment’, the name of the rule and comment associated with the rule respectively."</li> +<li>message - a string containing the message to score.</li> + +<li>engine - threat triage engine returned by THREAT_TRIAGE_INIT.</li> </ul></li> -<li>Returns: The String representation of the threat triage rules</li> +<li>Returns: A threat triage engine.</li> </ul></li> -<li><tt>THREAT_TRIAGE_PRINT</tt> +<li><tt>THREAT_TRIAGE_ADD</tt> <ul> -<li>Description: Retrieve stellar enrichment transformations.</li> +<li>Description: Add a threat triage rule.</li> <li>Input: <ul> <li>sensorConfig - Sensor config to add transformation to.</li> + +<li>stellarTransforms - A Map associating stellar rules to scores</li> + +<li>triageRules - Map (or list of Maps) representing a triage rule. It must contain ‘rule’ and ‘score’ keys, the stellar expression for the rule and triage score respectively. It may contain ‘name’ and ‘comment’, the name of the rule and comment associated with the rule respectively."</li> </ul></li> <li>Returns: The String representation of the threat triage rules</li> @@ -894,6 +981,22 @@ <li>Returns: The String representation of the enrichment config</li> </ul></li> +<li><tt>THREAT_TRIAGE_PRINT</tt> + +<ul> + +<li>Description: Retrieve stellar enrichment transformations.</li> + +<li>Input: + +<ul> + +<li>sensorConfig - Sensor config to add transformation to.</li> + </ul></li> + +<li>Returns: The String representation of the threat triage rules</li> + </ul></li> + <li><tt>THREAT_TRIAGE_SET_AGGREGATOR</tt> <ul> @@ -937,7 +1040,7 @@ <h2><a name="Examples"></a>Examples</h2> <p>Included for description and education purposes are a couple example Stellar REPL transcripts with helpful comments to illustrate some common operations.</p> <div class="section"> -<h3><a name="Iterate_in_finding_a_valid_Grok_pattern"></a>Iterate in finding a valid Grok pattern</h3> +<h3><a name="Iterate_to_Find_a_Valid_Grok_pattern"></a>Iterate to Find a Valid Grok pattern</h3> <div class="source"> <div class="source"> @@ -1644,7 +1747,133 @@ SION('is_both') ] ) "configuration" : { } } [Stellar]>>> -</pre></div></div></div></div> +</pre></div></div></div> +<div class="section"> +<h3><a name="Simulate_Threat_Triage_Rules"></a>Simulate Threat Triage Rules</h3> + +<ol style="list-style-type: decimal"> + +<li> +<p>Create a threat triage engine.</p> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> t := THREAT_TRIAGE_INIT() +[Stellar]>>> t +ThreatTriage{0 rule(s)} +</pre></div></div></li> + +<li> +<p>Add a few triage rules.</p> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule1", "rule":"value>10", +</pre></div></div> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule2", "rule":"value>20", "score":20}) +</pre></div></div> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule3", "rule":"value>30", "score":30}) +</pre></div></div></li> + +<li> +<p>Review the rules that you have created.</p> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> THREAT_TRIAGE_PRINT(t) +╔═══════╤═════════╤═════════════╤═══════╤════════╗ +║ Name │ Comment │ Triage Rule │ Score │ Reason ║ +╠═══════╪═════════╪═════════════╪═══════╪════════╣ +║ rule1 │ │ value>10 │ 10 │ ║ +╟───────┼─────────┼─────────────┼───────┼────────╢ +║ rule2 │ │ value>20 │ 20 │ ║ +╟───────┼─────────┼─────────────┼───────┼────────╢ +║ rule3 │ │ value>30 │ 30 │ ║ +╚═══════╧═════════╧═════════════╧═══════╧════════╝ +</pre></div></div></li> + +<li> +<p>Create a few test messages to simulate your telemetry.</p> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> msg1 := "{ \"value\":22 }" +[Stellar]>>> msg1 +{ "value":22 } +</pre></div></div> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> msg2 := "{ \"value\":44 }" +[Stellar]>>> msg2 +{ "value":44 } +</pre></div></div></li> + +<li> +<p>Score a message based on the rules that have been defined. The result allows you to see the total score, the aggregator, along with details about each rule that fired.</p> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> THREAT_TRIAGE_SCORE( msg1, t) +{score=20.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value>10}, {score=20.0, name=rule2, rule=value>20}]} +</pre></div></div> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> THREAT_TRIAGE_SCORE( msg2, t) +{score=30.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value>10}, {score=20.0, name=rule2, rule=value>20}, {score=30.0, name=rule3, rule=value>30}]} +</pre></div></div></li> + +<li> +<p>From here you can iterate on your rule set until it does exactly what you need it to do. Once you have a working rule set, extract the configuration and push it into your live, Metron cluster.</p> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> conf := THREAT_TRIAGE_CONFIG( t) +[Stellar]>>> conf +{ + "enrichment" : { + "fieldMap" : { }, + "fieldToTypeMap" : { }, + "config" : { } + }, + "threatIntel" : { + "fieldMap" : { }, + "fieldToTypeMap" : { }, + "config" : { }, + "triageConfig" : { + "riskLevelRules" : [ { + "name" : "rule1", + "rule" : "value>10", + "score" : 10.0 + }, { + "name" : "rule2", + "rule" : "value>20", + "score" : 20.0 + }, { + "name" : "rule3", + "rule" : "value>30", + "score" : 30.0 + }], + "aggregator" : "MAX", + "aggregationConfig" : { } + } + }, + "configuration" : { } +} +</pre></div></div> + +<div class="source"> +<div class="source"> +<pre>[Stellar]>>> CONFIG_PUT("ENRICHMENT", conf, "bro") +</pre></div></div></li> +</ol></div></div> </div> </div> </div> @@ -1653,7 +1882,7 @@ SION('is_both') ] ) <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/metron-parsers/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-parsers/index.html b/site/current-book/metron-platform/metron-parsers/index.html index 529badc..23955ac 100644 --- a/site/current-book/metron-platform/metron-parsers/index.html +++ b/site/current-book/metron-platform/metron-parsers/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Parsers</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -169,6 +169,13 @@ <li> + <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + + <li> + <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> <i class="none"></i> Enrichment</a> @@ -217,13 +224,20 @@ </li> </ul> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -231,7 +245,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -373,7 +387,29 @@ </pre></div></div></div> <div class="section"> <h2><a name="Global_Configuration"></a>Global Configuration</h2> -<p>See the “<a href="../metron-common/index.html">Global Configuration</a>” section.</p></div> +<p>There are a few properties which can be managed in the global configuration that have pertinence to parsers and parsing in general.</p> +<div class="section"> +<h3><a name="parser.error.topic"></a><tt>parser.error.topic</tt></h3> +<p>The topic where messages which were unable to be parsed due to error are sent. Error messages will be indexed under a sensor type of <tt>error</tt> and the messages will have the following fields:</p> + +<ul> + +<li><tt>sensor.type</tt>: <tt>error</tt></li> + +<li><tt>failed_sensor_type</tt> : The sensor type of the message which wasn’t able to be parsed</li> + +<li><tt>error_type</tt> : The error type, in this case <tt>parser</tt>.</li> + +<li><tt>stack</tt> : The stack trace of the error</li> + +<li><tt>hostname</tt> : The hostname of the node where the error happened</li> + +<li><tt>raw_message</tt> : The raw message in string form</li> + +<li><tt>raw_message_bytes</tt> : The raw message bytes</li> + +<li><tt>error_hash</tt> : A hash of the error message</li> +</ul></div></div> <div class="section"> <h2><a name="Parser_Configuration"></a>Parser Configuration</h2> <p>The configuration for the various parser topologies is defined by JSON documents stored in zookeeper.</p> @@ -709,37 +745,41 @@ HH:mm:ss', MAP_GET(dc, dc2tz, 'UTC') )" <div class="source"> <div class="source"> <pre>usage: start_parser_topology.sh - -e,--extra_topology_options <JSON_FILE> Extra options in the form - of a JSON file with a map - for content. - -esc,--extra_kafka_spout_config <JSON_FILE> Extra spout config options - in the form of a JSON file - with a map for content. - Possible keys are: - retryDelayMaxMs,retryDelay - Multiplier,retryInitialDel - ayMs,stateUpdateIntervalMs - ,bufferSizeBytes,fetchMaxW - ait,fetchSizeBytes,maxOffs - etBehind,metricsTimeBucket - SizeInSecs,socketTimeoutMs - -ewnt,--error_writer_num_tasks <NUM_TASKS> Error Writer Num Tasks - -ewp,--error_writer_p <PARALLELISM_HINT> Error Writer Parallelism - Hint - -h,--help This screen - -k,--kafka <BROKER_URL> Kafka Broker URL - -mt,--message_timeout <TIMEOUT_IN_SECS> Message Timeout in Seconds - -mtp,--max_task_parallelism <MAX_TASK> Max task parallelism - -na,--num_ackers <NUM_ACKERS> Number of Ackers - -nw,--num_workers <NUM_WORKERS> Number of Workers - -pnt,--parser_num_tasks <NUM_TASKS> Parser Num Tasks - -pp,--parser_p <PARALLELISM_HINT> Parser Parallelism Hint - -s,--sensor <SENSOR_TYPE> Sensor Type - -snt,--spout_num_tasks <NUM_TASKS> Spout Num Tasks - -sp,--spout_p <SPOUT_PARALLELISM_HINT> Spout Parallelism Hint - -t,--test <TEST> Run in Test Mode - -z,--zk <ZK_QUORUM> Zookeeper Quroum URL - (zk1:2181,zk2:2181,... + -e,--extra_topology_options <JSON_FILE> Extra options in the form + of a JSON file with a map + for content. + -esc,--extra_kafka_spout_config <JSON_FILE> Extra spout config options + in the form of a JSON file + with a map for content. + Possible keys are: + retryDelayMaxMs,retryDelay + Multiplier,retryInitialDel + ayMs,stateUpdateIntervalMs + ,bufferSizeBytes,fetchMaxW + ait,fetchSizeBytes,maxOffs + etBehind,metricsTimeBucket + SizeInSecs,socketTimeoutMs + -ewnt,--error_writer_num_tasks <NUM_TASKS> Error Writer Num Tasks + -ewp,--error_writer_p <PARALLELISM_HINT> Error Writer Parallelism + Hint + -h,--help This screen + -iwnt,--invalid_writer_num_tasks <NUM_TASKS> Invalid Writer Num Tasks + -iwp,--invalid_writer_p <PARALLELISM_HINT> Invalid Message Writer Parallelism Hint + -k,--kafka <BROKER_URL> Kafka Broker URL + -ksp,--kafka_security_protocol <SECURITY_PROTOCOL> Kafka Security Protocol + -mt,--message_timeout <TIMEOUT_IN_SECS> Message Timeout in Seconds + -mtp,--max_task_parallelism <MAX_TASK> Max task parallelism + -na,--num_ackers <NUM_ACKERS> Number of Ackers + -nw,--num_workers <NUM_WORKERS> Number of Workers + -ot,--output_topic <KAFKA_TOPIC> Output Kafka Topic + -pnt,--parser_num_tasks <NUM_TASKS> Parser Num Tasks + -pp,--parser_p <PARALLELISM_HINT> Parser Parallelism Hint + -s,--sensor <SENSOR_TYPE> Sensor Type + -snt,--spout_num_tasks <NUM_TASKS> Spout Num Tasks + -sp,--spout_p <SPOUT_PARALLELISM_HINT> Spout Parallelism Hint + -t,--test <TEST> Run in Test Mode + -z,--zk <ZK_QUORUM> Zookeeper Quroum URL + (zk1:2181,zk2:2181,... </pre></div></div></div></div> <div class="section"> <h2><a name="The_--extra_kafka_spout_config_Option"></a>The <tt>--extra_kafka_spout_config</tt> Option</h2> @@ -793,7 +833,11 @@ HH:mm:ss', MAP_GET(dc, dc2tz, 'UTC') )" <p>and pass <tt>--extra_topology_options custom_config.json</tt> to <tt>start_parser_topology.sh</tt>.</p> <p><a name="Notes_on_Performance_Tuning"></a></p> <h1>Notes on Performance Tuning</h1> -<p>Default installed Metron is untuned for production deployment. There are a few knobs to tune to get the most out of your system.</p></div> +<p>Default installed Metron is untuned for production deployment. There are a few knobs to tune to get the most out of your system.</p> +<p><a name="Notes_on_Adding_a_New_Sensor"></a></p> +<h1>Notes on Adding a New Sensor</h1> +<p>In order to allow for meta alerts to be queries alongside regular alerts in Elasticsearch 2.x, it is necessary to add an additional field to the templates and mapping for existing sensors.</p> +<p>Please see a description of the steps necessary to make this change in the metron-elasticsearch <a href="../../metron-platform/metron-elasticsearch/index.html#Using_Metron_with_Elasticsearch_2.x">Using Metron with Elasticsearch 2.x</a></p></div> <div class="section"> <h2><a name="Kafka_Queue"></a>Kafka Queue</h2> <p>The kafka queue associated with your parser is a collection point for all of the data sent to your parser. As such, make sure that the number of partitions in the kafka topic is sufficient to handle the throughput that you expect from your parser topology.</p></div> @@ -857,7 +901,7 @@ HH:mm:ss', MAP_GET(dc, dc2tz, 'UTC') )" <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/metron-parsers/parser-testing.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-parsers/parser-testing.html b/site/current-book/metron-platform/metron-parsers/parser-testing.html index 25288a3..0edfd53 100644 --- a/site/current-book/metron-platform/metron-parsers/parser-testing.html +++ b/site/current-book/metron-platform/metron-parsers/parser-testing.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Parser Contribution and Testing</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -169,6 +169,13 @@ <li> + <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + + <li> + <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> <i class="none"></i> Enrichment</a> @@ -217,13 +224,20 @@ </li> </ul> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -231,7 +245,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -358,7 +372,7 @@ <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/metron-pcap-backend/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-pcap-backend/index.html b/site/current-book/metron-platform/metron-pcap-backend/index.html index 601fac8..01961eb 100644 --- a/site/current-book/metron-platform/metron-pcap-backend/index.html +++ b/site/current-book/metron-platform/metron-pcap-backend/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Metron PCAP Backend</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -169,6 +169,13 @@ <li> + <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + + <li> + <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> <i class="none"></i> Enrichment</a> @@ -208,13 +215,20 @@ </li> </ul> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -222,7 +236,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -648,7 +662,7 @@ dfs.blocksize <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/metron-writer/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-writer/index.html b/site/current-book/metron-platform/metron-writer/index.html index 9464a4d..b9c2125 100644 --- a/site/current-book/metron-platform/metron-writer/index.html +++ b/site/current-book/metron-platform/metron-writer/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Writer</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -169,6 +169,13 @@ <li> + <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + + <li> + <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> <i class="none"></i> Enrichment</a> @@ -208,13 +215,20 @@ </li> </ul> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -222,7 +236,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -335,7 +349,7 @@ limitations under the License. --><h1>Writer</h1> <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved.
