http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-interface/metron-config/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-interface/metron-config/index.html b/site/current-book/metron-interface/metron-config/index.html index e3bf4b6..e231b51 100644 --- a/site/current-book/metron-interface/metron-config/index.html +++ b/site/current-book/metron-interface/metron-config/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Metron Management UI</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -129,20 +129,27 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> <i class="icon-chevron-right"></i> Platform</a> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -150,7 +157,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -189,7 +196,7 @@ <li>A network accessible Metron REST application</li> -<li>nodejs v6.9+ (nodejs can be installed on quick dev with <tt>curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - && yum install -y nodejs</tt>)</li> +<li>nodejs v6.9+ (nodejs can be installed on Full Dev with <tt>curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - && yum install -y nodejs</tt>)</li> </ul></div> <div class="section"> <h2><a name="Installation"></a>Installation</h2> @@ -327,7 +334,7 @@ npm install <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved.
http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-interface/metron-rest/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-interface/metron-rest/index.html b/site/current-book/metron-interface/metron-rest/index.html index 8f549d5..f673ca6 100644 --- a/site/current-book/metron-interface/metron-rest/index.html +++ b/site/current-book/metron-interface/metron-rest/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Metron REST</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -129,20 +129,27 @@ <a href="#"><i class="none"></i>Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> <i class="icon-chevron-right"></i> Platform</a> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -150,7 +157,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -332,17 +339,6 @@ lib <tr class="b"> -<td>METRON_USER </td> - -<td>Run the application as this user </td> - -<td>Optional </td> - -<td>metron</td> - </tr> - -<tr class="a"> - <td>METRON_LOG_DIR </td> <td>Directory where the log file is written </td> @@ -352,18 +348,18 @@ lib <td>/var/log/metron/</td> </tr> -<tr class="b"> +<tr class="a"> -<td>METRON_PID_DIR </td> +<td>METRON_PID_FILE </td> -<td>Directory where the pid file is written </td> +<td>File where the pid is written </td> <td>Optional </td> <td>/var/run/metron/</td> </tr> -<tr class="a"> +<tr class="b"> <td>METRON_REST_PORT </td> @@ -374,7 +370,7 @@ lib <td>8082</td> </tr> -<tr class="b"> +<tr class="a"> <td>METRON_JDBC_CLIENT_PATH </td> @@ -385,7 +381,7 @@ lib <td>H2 is bundled</td> </tr> -<tr class="a"> +<tr class="b"> <td>METRON_TEMP_GROK_PATH </td> @@ -396,7 +392,7 @@ lib <td>./patterns/temp</td> </tr> -<tr class="b"> +<tr class="a"> <td>METRON_DEFAULT_GROK_PATH </td> @@ -407,7 +403,7 @@ lib <td>/apps/metron/patterns</td> </tr> -<tr class="a"> +<tr class="b"> <td>SECURITY_ENABLED </td> @@ -482,27 +478,31 @@ lib </tr> </tbody> </table> -<p>These are set in the <tt>/etc/sysconfig/metron</tt> file.</p></div></div> +<p>These are set in the <tt>/etc/default/metron</tt> file.</p></div></div> <div class="section"> <h2><a name="Database_setup"></a>Database setup</h2> -<p>The REST application persists data in a relational database and requires a dedicated database user and database (see <a class="externalLink" href="https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html";>https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html</a> for more detail).</p> +<p>The REST application persists data in a relational database and requires a dedicated database user and database (see <a class="externalLink" href="https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html";>https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html</a> for more detail).<br />Spring uses Hibernate as the default ORM framework but another framework is needed becaused Hibernate is not compatible with the Apache 2 license. For this reason Metron uses <a class="externalLink" href="https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html#boot-features-embedded-database-support";>EclipseLink</a>. See the <a class="externalLink" href="https://github.com/spring-projects/spring-data-examples/tree/master/jpa/eclipselink";>Spring Data JPA - EclipseLink</a> project for an example on how to configure EclipseLink in Spring.</p> <div class="section"> <h3><a name="Development"></a>Development</h3> -<p>The REST application comes with embedded database support for development purposes (<a class="externalLink" href="https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html#boot-features-embedded-database-support)">https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html#boot-features-embedded-database-support)</a>.</p> -<p>For example, edit these variables in <tt>/etc/sysconfig/metron</tt> before starting the application to configure H2:</p> +<p>The REST application comes with <a class="externalLink" href="https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html#boot-features-embedded-database-support";>embedded database support</a> for development purposes.</p> +<p>For example, edit these variables in <tt>/etc/default/metron</tt> before starting the application to configure H2:</p> <div class="source"> <div class="source"> <pre>METRON_JDBC_DRIVER="org.h2.Driver" METRON_JDBC_URL="jdbc:h2:file:~/metrondb" METRON_JDBC_USERNAME="root" -METRON_JDBC_PASSWORD='root" METRON_JDBC_PLATFORM="h2" </pre></div></div></div> <div class="section"> <h3><a name="Production"></a>Production</h3> <p>The REST application should be configured with a production-grade database outside of development.</p> -<p>For example, the following configures the application for MySQL:</p> +<div class="section"> +<h4><a name="Ambari_Install"></a>Ambari Install</h4> +<p>Installing with Ambari is recommended for production deployments. Ambari handles setup, configuration, and management of the REST component. This includes managing the PID file, directing logging, etc.</p></div> +<div class="section"> +<h4><a name="Manual_Install"></a>Manual Install</h4> +<p>The following configures the application for MySQL:</p> <ol style="list-style-type: decimal"> @@ -530,7 +530,7 @@ tar xf mysql-connector-java-5.1.41.tar.gz </pre></div></div></li> <li> -<p>Edit these variables in <tt>/etc/sysconfig/metron</tt> to configure the REST application for MySQL:</p> +<p>Edit these variables in <tt>/etc/default/metron</tt> to configure the REST application for MySQL:</p> <div class="source"> <div class="source"> @@ -540,21 +540,30 @@ METRON_JDBC_USERNAME="metron" METRON_JDBC_PLATFORM="mysql" METRON_JDBC_CLIENT_PATH=$METRON_HOME/lib/mysql-connector-java-5.1.41/mysql-connector-java-5.1.41-bin.jar </pre></div></div></li> -</ol></div></div> -<div class="section"> -<h2><a name="Usage"></a>Usage</h2> -<p>After configuration is complete, the REST application can be managed as a service:</p> - + +<li> +<p>Switch to the metron user</p> + <div class="source"> <div class="source"> -<pre>service metron-rest start -</pre></div></div> -<p>If a production database is configured, the JDBC password should be passed in as the first argument on startup:</p> - +<pre>sudo su - metron +</pre></div></div></li> + +<li> +<p>Start the REST API. Adjust the password as necessary.</p> + <div class="source"> <div class="source"> -<pre>service metron-rest start Myp@ssw0rd -</pre></div></div> +<pre>set -o allexport; +source /etc/default/metron; +set +o allexport; +export METRON_JDBC_PASSWORD='Myp@ssw0rd'; +$METRON_HOME/bin/metron-rest.sh +unset METRON_JDBC_PASSWORD; +</pre></div></div></li> +</ol></div></div></div> +<div class="section"> +<h2><a name="Usage"></a>Usage</h2> <p>The REST application can be accessed with the Swagger UI at <a class="externalLink" href="http://host:port/swagger-ui.html#/";>http://host:port/swagger-ui.html#/</a>. The default port is 8082.</p></div> <div class="section"> <h2><a name="Security"></a>Security</h2> @@ -570,7 +579,7 @@ insert into authorities (username, authority) values ('your_username', 'ROLE_USE </pre></div></div></div> <div class="section"> <h3><a name="Kerberos"></a>Kerberos</h3> -<p>Metron REST can be configured for a cluster with Kerberos enabled. A client JAAS file is required for Kafka and Zookeeper and a Kerberos keytab for the metron user principal is required for all other services. Configure these settings in the <tt>/etc/sysconfig/metron</tt> file:</p> +<p>Metron REST can be configured for a cluster with Kerberos enabled. A client JAAS file is required for Kafka and Zookeeper and a Kerberos keytab for the metron user principal is required for all other services. Configure these settings in the <tt>/etc/default/metron</tt> file:</p> <div class="source"> <div class="source"> @@ -624,7 +633,7 @@ METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab" </tr> </tbody> </table> -<p>Setting active profiles is done with the METRON_SPRING_PROFILES_ACTIVE variable. For example, set this variable in <tt>/etc/sysconfig/metron</tt> to configure the REST application for the Vagrant environment and add a test user:</p> +<p>Setting active profiles is done with the METRON_SPRING_PROFILES_ACTIVE variable. For example, set this variable in <tt>/etc/default/metron</tt> to configure the REST application for the Vagrant environment and add a test user:</p> <div class="source"> <div class="source"> @@ -651,6 +660,26 @@ METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab" <tr class="a"> +<td><a href="#GET_apiv1alertprofile"> <tt>GET /api/v1/alert/profile</tt></a></td> + </tr> + +<tr class="b"> + +<td><a href="#GET_apiv1alertprofileall"> <tt>GET /api/v1/alert/profile/all</tt></a></td> + </tr> + +<tr class="a"> + +<td><a href="#DELETE_apiv1alertprofile"> <tt>DELETE /api/v1/alert/profile</tt></a></td> + </tr> + +<tr class="b"> + +<td><a href="#POST_apiv1alertprofile"> <tt>POST /api/v1/alert/profile</tt></a></td> + </tr> + +<tr class="a"> + <td><a href="#GET_apiv1globalconfig"> <tt>GET /api/v1/global/config</tt></a></td> </tr> @@ -726,22 +755,52 @@ METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab" <tr class="b"> +<td><a href="#GET_apiv1metaalertsearchByAlert"> <tt>GET /api/v1/metaalert/searchByAlert</tt></a></td> + </tr> + +<tr class="a"> + +<td><a href="#GET_apiv1metaalertcreate"> <tt>GET /api/v1/metaalert/create</tt></a></td> + </tr> + +<tr class="b"> + +<td><a href="#GET_apiv1metaalertaddalert"> <tt>GET /api/v1/metaalert/add/alert</tt></a></td> + </tr> + +<tr class="a"> + +<td><a href="#GET_apiv1metaalertremovealert"> <tt>GET /api/v1/metaalert/remove/alert</tt></a></td> + </tr> + +<tr class="b"> + +<td><a href="#GET_apiv1metaalertupdatestatusguidstatus"> <tt>GET /api/v1/metaalert/update/status/{guid}/{status}</tt></a></td> + </tr> + +<tr class="a"> + <td><a href="#GET_apiv1searchsearch"> <tt>GET /api/v1/search/search</tt></a></td> </tr> +<tr class="b"> + +<td><a href="#get-apiv1searchsearch"> <tt>POST /api/v1/search/search</tt></a></td> + </tr> + <tr class="a"> -<td><a href="#GET_apiv1searchfindOne"> <tt>GET /api/v1/search/findOne</tt></a></td> +<td><a href="#get-apiv1searchgroup"> <tt>POST /api/v1/search/group</tt></a></td> </tr> <tr class="b"> -<td><a href="#get-apiv1searchcolumnmetadata"> <tt>GET /api/v1/search/search</tt></a></td> +<td><a href="#GET_apiv1searchfindOne"> <tt>GET /api/v1/search/findOne</tt></a></td> </tr> <tr class="a"> -<td><a href="#get-apiv1searchcolumnmetadatacommon"> <tt>GET /api/v1/search/search</tt></a></td> +<td><a href="#GET_apiv1searchcolumnmetadata"> <tt>GET /api/v1/search/column/metadata</tt></a></td> </tr> <tr class="b"> @@ -982,6 +1041,86 @@ METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab" </ul></li> </ul></div> <div class="section"> +<h3><a name="GET_apiv1alertprofile"></a><tt>GET /api/v1/alert/profile</tt></h3> + +<ul> + +<li>Description: Retrieves the current user’s alerts profile</li> + +<li>Returns: + +<ul> + +<li>200 - Alerts profile</li> + +<li>404 - The current user does not have an alerts profile</li> + </ul></li> +</ul></div> +<div class="section"> +<h3><a name="GET_apiv1alertprofileall"></a><tt>GET /api/v1/alert/profile/all</tt></h3> + +<ul> + +<li>Description: Retrieves all users’ alerts profiles. Only users that are part of the “ROLE_ADMIN” role are allowed to get all alerts profiles.</li> + +<li>Returns: + +<ul> + +<li>200 - List of all alerts profiles</li> + +<li>403 - The current user does not have permission to get all alerts profiles</li> + </ul></li> +</ul></div> +<div class="section"> +<h3><a name="DELETE_apiv1alertprofile"></a><tt>DELETE /api/v1/alert/profile</tt></h3> + +<ul> + +<li>Description: Deletes a user’s alerts profile. Only users that are part of the “ROLE_ADMIN” role are allowed to delete user alerts profiles.</li> + +<li>Input: + +<ul> + +<li>user - The user whose prolife will be deleted</li> + </ul></li> + +<li>Returns: + +<ul> + +<li>200 - Alerts profile was deleted</li> + +<li>403 - The current user does not have permission to delete alerts profiles</li> + +<li>404 - Alerts profile could not be found</li> + </ul></li> +</ul></div> +<div class="section"> +<h3><a name="POST_apiv1alertprofile"></a><tt>POST /api/v1/alert/profile</tt></h3> + +<ul> + +<li>Description: Creates or updates the current user’s alerts profile</li> + +<li>Input: + +<ul> + +<li>alertsProfile - The alerts profile to be saved</li> + </ul></li> + +<li>Returns: + +<ul> + +<li>200 - Alerts profile updated. Returns saved alerts profile.</li> + +<li>201 - Alerts profile created. Returns saved alerts profile.</li> + </ul></li> +</ul></div> +<div class="section"> <h3><a name="GET_apiv1globalconfig"></a><tt>GET /api/v1/global/config</tt></h3> <ul> @@ -1097,7 +1236,7 @@ METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab" <ul> -<li>Description: Writes contents to an HDFS file. Warning: this will overwrite the contents of a file if it already exists.</li> +<li>Description: Writes contents to an HDFS file. Warning: this will overwrite the contents of a file if it already exists. Permissions must be set for all three groups if they are to be set. If any are missing, the default permissions will be used, and if any are invalid an exception will be thrown.</li> <li>Input: @@ -1106,6 +1245,12 @@ METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab" <li>path - Path to HDFS file</li> <li>contents - File contents</li> + +<li>userMode - [optional] symbolic permission string for user portion of the permissions to be set on the file written. For example ‘rwx’ or read, write, execute. The symbol ‘-’ is used to exclude that permission such as ‘rw-’ for read, write, no execute</li> + +<li>groupMode - [optional] symbolic permission string for group portion of the permissions to be set on the file written. For example ‘rwx’ or read, write, execute. The symbol ‘-’ is used to exclude that permission such as ‘rw-’ for read, write, no execute</li> + +<li>otherMode - [optional] symbolic permission string for other portion of the permissions to be set on the file written. For example ‘rwx’ or read, write, execute. The symbol ‘-’ is used to exclude that permission such as ‘rw-’ for read, write, no execute</li> </ul></li> <li>Returns: @@ -1287,54 +1432,118 @@ METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab" </ul></li> </ul></div> <div class="section"> -<h3><a name="GET_apiv1searchfindOne"></a><tt>GET /api/v1/search/findOne</tt></h3> +<h3><a name="POST_apiv1metaalertsearchByAlert"></a><tt>POST /api/v1/metaalert/searchByAlert</tt></h3> <ul> -<li>Description: Returns latest document for a guid and sensor</li> +<li>Description: Get all meta alerts that contain an alert.</li> <li>Input: <ul> -<li>getRequest - Get request +<li>guid - GUID of the alert</li> + </ul></li> + +<li>Returns: + +<ul> +<li>200 - Search results</li> + </ul></li> +</ul></div> +<div class="section"> +<h3><a name="POST_apiv1metaalertcreate"></a><tt>POST /api/v1/metaalert/create</tt></h3> + +<ul> + +<li>Description: Creates a new meta alert from a list of existing alerts. The meta alert status will initially be set to ‘ACTIVE’ and summary statistics will be computed from the list of alerts. A list of groups included in the request are also added to the meta alert.</li> + +<li>Input: + <ul> - -<li>guid - message UUID</li> - -<li>sensorType - Sensor Type</li> - </ul></li> -<li>Example: Return <tt>bro</tt> document with UUID of <tt>000-000-0000</tt></li> +<li>request - Meta alert create request which includes a list of alert get requests and a list of custom groups used to annotate a meta alert.</li> </ul></li> -</ul> + +<li>Returns: + +<ul> + +<li>200 - The GUID of the new meta alert</li> + </ul></li> +</ul></div> +<div class="section"> +<h3><a name="POST_apiv1metaalertaddalert"></a><tt>POST /api/v1/metaalert/add/alert</tt></h3> -<div class="source"> -<div class="source"> -<pre>{ - "guid" : "000-000-0000", - "sensorType" : "bro" -} -</pre></div></div> +<ul> + +<li>Description: Adds an alert to an existing meta alert. An alert will not be added if it is already contained in a meta alert.</li> + +<li>Input: + +<ul> + +<li>request - Meta alert add request which includes a meta alert GUID and list of alert get requests</li> + </ul></li> + +<li>Returns: + +<ul> + +<li>200 - Returns ‘true’ if the alert was added and ‘false’ if the meta alert did not change.</li> + </ul></li> +</ul></div> +<div class="section"> +<h3><a name="POST_apiv1metaalertremovealert"></a><tt>POST /api/v1/metaalert/remove/alert</tt></h3> <ul> +<li>Description: Removes an alert from an existing meta alert. If the alert to be removed is not in a meta alert, ‘false’ will be returned.</li> + +<li>Input: + +<ul> + +<li>request - Meta alert remove request which includes a meta alert GUID and list of alert get requests</li> + </ul></li> + <li>Returns: <ul> -<li>200 - Document representing the output</li> +<li>200 - Returns ‘true’ if the alert was removed and ‘false’ if the meta alert did not change.</li> + </ul></li> +</ul></div> +<div class="section"> +<h3><a name="POST_apiv1metaalertupdatestatusguidstatus"></a><tt>POST /api/v1/metaalert/update/status/{guid}/{status}</tt></h3> + +<ul> + +<li>Description: Updates the status of a meta alert to either ‘ACTIVE’ or ‘INACTIVE’.</li> + +<li>Input: + +<ul> -<li>404 - Document with UUID and sensor type not found</li> +<li>guid - Meta alert GUID</li> + +<li>status - Meta alert status with a value of either ‘ACTIVE’ or ‘INACTIVE’</li> + </ul></li> + +<li>Returns: + +<ul> + +<li>200 - Returns ‘true’ if the status changed and ‘false’ if it did not.</li> </ul></li> </ul></div> <div class="section"> -<h3><a name="GET_apiv1searchsearch"></a><tt>GET /api/v1/search/search</tt></h3> +<h3><a name="POST_apiv1searchsearch"></a><tt>POST /api/v1/search/search</tt></h3> <ul> -<li>Description: Searches the indexing store</li> +<li>Description: Searches the indexing store. GUIDs must be quoted to ensure correct results.</li> <li>Input: @@ -1347,49 +1556,103 @@ METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab" <ul> -<li>200 - Search results</li> +<li>200 - Search response</li> </ul></li> </ul></div> <div class="section"> -<h3><a name="GET_apiv1searchcolumnmetadata"></a><tt>GET /api/v1/search/column/metadata</tt></h3> +<h3><a name="POST_apiv1searchgroup"></a><tt>POST /api/v1/search/group</tt></h3> <ul> -<li>Description: Get column metadata for each index in the list of indicies</li> +<li>Description: Searches the indexing store and returns field groups. GUIDs must be quoted to ensure correct results. Groups are hierarchical and nested in the order the fields appear in the ‘groups’ request parameter. The default sorting within groups is by count descending. A groupOrder type of count will sort based on then number of documents in a group while a groupType of term will sort by the groupBy term.</li> <li>Input: <ul> -<li>indices - Indices</li> +<li>groupRequest - Group request + +<ul> + +<li>indices - list of indices to search</li> + +<li>query - lucene query</li> + +<li>scoreField - field used to compute a total score for each group</li> + +<li>groups - List of groups (field name and sort order)</li> + </ul></li> </ul></li> <li>Returns: <ul> -<li>200 - Column Metadata</li> +<li>200 - Group response</li> + </ul></li> +</ul></div> +<div class="section"> +<h3><a name="GET_apiv1searchfindOne"></a><tt>GET /api/v1/search/findOne</tt></h3> + +<ul> + +<li>Description: Returns latest document for a guid and sensor</li> + +<li>Input: + +<ul> + +<li>getRequest - Get request + +<ul> + +<li>guid - message UUID</li> + +<li>sensorType - Sensor Type</li> + </ul></li> + +<li>Example: Return <tt>bro</tt> document with UUID of <tt>000-000-0000</tt></li> + </ul></li> +</ul> + +<div class="source"> +<div class="source"> +<pre>{ + "guid" : "000-000-0000", + "sensorType" : "bro" +} +</pre></div></div> + +<ul> + +<li>Returns: + +<ul> + +<li>200 - Document representing the output</li> + +<li>404 - Document with UUID and sensor type not found</li> </ul></li> </ul></div> <div class="section"> -<h3><a name="GET_apiv1searchcolumnmetadatacommon"></a><tt>GET /api/v1/search/column/metadata/common</tt></h3> +<h3><a name="GET_apiv1searchcolumnmetadata"></a><tt>GET /api/v1/search/column/metadata</tt></h3> <ul> -<li>Description: Get metadata for columns shared by the list of indices</li> +<li>Description: Get index column metadata for a list of sensor types with duplicates removed. Column names and types for each sensor are retrieved from the most recent index. Columns that exist in multiple indices with different types will default to type ‘other’.</li> <li>Input: <ul> -<li>indices - Indices</li> +<li>sensorTypes - Sensor Types</li> </ul></li> <li>Returns: <ul> -<li>200 - Common Column Metadata</li> +<li>200 - Column Metadata</li> </ul></li> </ul></div> <div class="section"> @@ -2234,7 +2497,7 @@ METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab" </ul></div></div> <div class="section"> <h2><a name="Testing"></a>Testing</h2> -<p>Profiles are includes for both the metron-docker and Quick Dev environments.</p> +<p>Profiles are includes for both the metron-docker and Full Dev environments.</p> <div class="section"> <h3><a name="metron-docker"></a>metron-docker</h3> <p>Start the <a href="../../metron-docker/index.html">metron-docker</a> environment. Build the metron-rest module and start it with the Spring Boot Maven plugin:</p> @@ -2246,8 +2509,8 @@ mvn spring-boot:run -Drun.profiles=docker,dev </pre></div></div> <p>The metron-rest application will be available at <a class="externalLink" href="http://localhost:8080/swagger-ui.html#/";>http://localhost:8080/swagger-ui.html#/</a>.</p></div> <div class="section"> -<h3><a name="Quick_Dev"></a>Quick Dev</h3> -<p>Start the <a href="../../metron-deployment/vagrant/quick-dev-platform/index.html">Quick Dev</a> environment. Build the metron-rest module and start it with the Spring Boot Maven plugin:</p> +<h3><a name="Full_Dev"></a>Full Dev</h3> +<p>Start the <a href="../../metron-deployment/vagrant/full-dev-platform/index.html">Full Dev</a> environment. Build the metron-rest module and start it with the Spring Boot Maven plugin:</p> <div class="source"> <div class="source"> @@ -2255,7 +2518,7 @@ mvn spring-boot:run -Drun.profiles=docker,dev mvn spring-boot:run -Drun.profiles=vagrant,dev </pre></div></div> <p>The metron-rest application will be available at <a class="externalLink" href="http://localhost:8080/swagger-ui.html#/";>http://localhost:8080/swagger-ui.html#/</a>.</p> -<p>To run the application locally on the Quick Dev host (node1), follow the <a href="#Installation">Installation</a> instructions above. Then set the METRON_SPRING_PROFILES_ACTIVE variable in <tt>/etc/sysconfig/metron</tt>:</p> +<p>To run the application locally on the Full Dev host (node1), follow the <a href="#Installation">Installation</a> instructions above. Then set the METRON_SPRING_PROFILES_ACTIVE variable in <tt>/etc/default/metron</tt>:</p> <div class="source"> <div class="source"> @@ -2267,7 +2530,7 @@ mvn spring-boot:run -Drun.profiles=vagrant,dev <div class="source"> <pre>service metron-rest start </pre></div></div> -<p>In a cluster with Kerberos enabled, update the security settings in <tt>/etc/sysconfig/metron</tt>. Security is disabled by default in the <tt>vagrant</tt> Spring profile so that setting must be overriden with the METRON_SPRING_OPTIONS variable:</p> +<p>In a cluster with Kerberos enabled, update the security settings in <tt>/etc/default/metron</tt>. Security is disabled by default in the <tt>vagrant</tt> Spring profile so that setting must be overriden with the METRON_SPRING_OPTIONS variable:</p> <div class="source"> <div class="source"> @@ -2287,7 +2550,7 @@ METRON_SPRING_OPTIONS="--kerberos.enabled=true" <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/Performance-tuning-guide.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/Performance-tuning-guide.html b/site/current-book/metron-platform/Performance-tuning-guide.html index e985bdd..941e2d7 100644 --- a/site/current-book/metron-platform/Performance-tuning-guide.html +++ b/site/current-book/metron-platform/Performance-tuning-guide.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Metron Performance Tuning Guide</title> <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../metron-platform/index.html" title="Platform"> @@ -167,6 +167,13 @@ <li> + <a href="../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + + <li> + <a href="../metron-platform/metron-enrichment/index.html" title="Enrichment"> <i class="none"></i> Enrichment</a> @@ -208,13 +215,20 @@ </li> </ul> </li> - + <li> <a href="../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -222,7 +236,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../use-cases/index.html" title="Use-cases"> @@ -471,39 +485,50 @@ enrichments enrichments 43 29754331 297 <div class="source"> <div class="source"> -<pre>/usr/metron/0.4.0/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s bro -ksp SASL_PLAINTEXT - -ot enrichments +<pre>/usr/metron/0.4.2/bin/start_parser_topology.sh \ -e ~metron/.storm/storm-bro.config \ -esc ~/.storm/spout-bro.config \ - -sp 24 \ - -snt 24 \ + -k $BROKERLIST \ + -ksp SASL_PLAINTEXT \ -nw 1 \ + -ot enrichments \ -pnt 24 \ -pp 24 \ + -s bro \ + -snt 24 \ + -sp 24 \ + -z $ZOOKEEPER \ </pre></div></div> -<p>From the usage docs, here are the options we’ve used. The full reference can be found here - <a class="externalLink" href="https://github.com/apache/metron/blob/master/metron-platform/metron-parsers/README.md";>https://github.com/apache/metron/blob/master/metron-platform/metron-parsers/README.md</a></p> - -<div class="source"> -<div class="source"> -<pre>-e,--extra_topology_options <JSON_FILE> Extra options in the form - of a JSON file with a map - for content. --esc,--extra_kafka_spout_config <JSON_FILE> Extra spout config options - in the form of a JSON file - with a map for content. - Possible keys are: - retryDelayMaxMs,retryDelay - Multiplier,retryInitialDel - ayMs,stateUpdateIntervalMs - ,bufferSizeBytes,fetchMaxW - ait,fetchSizeBytes,maxOffs - etBehind,metricsTimeBucket - SizeInSecs,socketTimeoutMs --sp,--spout_p <SPOUT_PARALLELISM_HINT> Spout Parallelism Hint --snt,--spout_num_tasks <NUM_TASKS> Spout Num Tasks --nw,--num_workers <NUM_WORKERS> Number of Workers --pnt,--parser_num_tasks <NUM_TASKS> Parser Num Tasks --pp,--parser_p <PARALLELISM_HINT> Parser Parallelism Hint +<p>From the usage docs, here are the options we’ve used. The full reference can be found <a href="../metron-platform/metron-parsers/index.html#Starting_the_Parser_Topology">here</a>.</p> + +<div class="source"> +<div class="source"> +<pre>usage: start_parser_topology.sh + -e,--extra_topology_options <JSON_FILE> Extra options in the form + of a JSON file with a map + for content. + -esc,--extra_kafka_spout_config <JSON_FILE> Extra spout config options + in the form of a JSON file + with a map for content. + Possible keys are: + retryDelayMaxMs,retryDelay + Multiplier,retryInitialDel + ayMs,stateUpdateIntervalMs + ,bufferSizeBytes,fetchMaxW + ait,fetchSizeBytes,maxOffs + etBehind,metricsTimeBucket + SizeInSecs,socketTimeoutMs + -k,--kafka <BROKER_URL> Kafka Broker URL + -ksp,--kafka_security_protocol <SECURITY_PROTOCOL> Kafka Security Protocol + -nw,--num_workers <NUM_WORKERS> Number of Workers + -ot,--output_topic <KAFKA_TOPIC> Output Kafka Topic + -pnt,--parser_num_tasks <NUM_TASKS> Parser Num Tasks + -pp,--parser_p <PARALLELISM_HINT> Parser Parallelism Hint + -s,--sensor <SENSOR_TYPE> Sensor Type + -snt,--spout_num_tasks <NUM_TASKS> Spout Num Tasks + -sp,--spout_p <SPOUT_PARALLELISM_HINT> Spout Parallelism Hint + -z,--zk <ZK_QUORUM> Zookeeper Quroum URL + (zk1:2181,zk2:2181,... </pre></div></div></div> <div class="section"> <h3><a name="Enrichment_Tuning"></a>Enrichment Tuning</h3> @@ -663,7 +688,7 @@ session timeout or by reducing the maximum size of batches returned in poll() wi <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/index.html b/site/current-book/metron-platform/index.html index b58ddcc..ee219ed 100644 --- a/site/current-book/metron-platform/index.html +++ b/site/current-book/metron-platform/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Current Build</title> <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li class="active"> <a href="#"><i class="icon-chevron-down"></i>Platform</a> @@ -167,6 +167,13 @@ <li> + <a href="../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + + <li> + <a href="../metron-platform/metron-enrichment/index.html" title="Enrichment"> <i class="none"></i> Enrichment</a> @@ -208,13 +215,20 @@ </li> </ul> </li> - + <li> <a href="../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -222,7 +236,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../use-cases/index.html" title="Use-cases"> @@ -266,7 +280,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --><h1>Current Build</h1> <p><a name="Current_Build"></a></p> -<p>The latest build of metron-platform is 0.4.1.</p> +<p>The latest build of metron-platform is 0.4.2.</p> <p>We are still in the process of merging/porting additional features from our production code base into this open source release. This release will be followed by a number of additional beta releases until the port is complete. We will also work on getting additional documentation and user/developer guides to the community as soon as we can. At this time we offer no support for the beta software, but will try to respond to requests as promptly as we can.</p> <p><a name="metron-platform"></a></p> <h1>metron-platform</h1> @@ -282,7 +296,7 @@ limitations under the License. --><h1>Current Build</h1> <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/metron-api/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-api/index.html b/site/current-book/metron-platform/metron-api/index.html index ddc7e78..690ab94 100644 --- a/site/current-book/metron-platform/metron-api/index.html +++ b/site/current-book/metron-platform/metron-api/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Metron PCAP Service</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -167,6 +167,13 @@ <li> + <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + + <li> + <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> <i class="none"></i> Enrichment</a> @@ -208,13 +215,20 @@ </li> </ul> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -222,7 +236,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -318,7 +332,7 @@ <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/metron-common/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-common/index.html b/site/current-book/metron-platform/metron-common/index.html index 92bf3a2..7111111 100644 --- a/site/current-book/metron-platform/metron-common/index.html +++ b/site/current-book/metron-platform/metron-common/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Contents</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -167,6 +167,13 @@ <li> + <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + + <li> + <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> <i class="none"></i> Enrichment</a> @@ -208,13 +215,20 @@ </li> </ul> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -222,7 +236,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -315,7 +329,7 @@ "es.ip": "node1", "es.port": "9300", "es.date.format": "yyyy.MM.dd.HH", - "parser.error.topic": "indexing" + "parser.error.topic": "indexing", "fieldValidations" : [ { "input" : [ "ip_src_addr", "ip_dst_addr" ], @@ -327,6 +341,182 @@ ] } </pre></div></div> +<p>Various parts of our stack uses the global config are documented throughout the Metron documentation, but a convenient index is provided here:</p> + +<table border="0" class="table table-striped"> + <thead> + +<tr class="a"> + +<th>Property Name </th> + +<th>Subsystem </th> + +<th>Type </th> + +<th>Ambari Property </th> + </tr> + </thead> + <tbody> + +<tr class="b"> + +<td><a href="../metron-elasticsearch/index.html#es.clustername"><tt>es.clustername</tt></a> </td> + +<td>Indexing </td> + +<td>String </td> + +<td><tt>es_cluster_name</tt> </td> + </tr> + +<tr class="a"> + +<td><a href="../metron-elasticsearch/index.html#es.ip"><tt>es.ip</tt></a> </td> + +<td>Indexing </td> + +<td>String </td> + +<td><tt>es_hosts</tt> </td> + </tr> + +<tr class="b"> + +<td><a href="../metron-elasticsearch/index.html#es.port"><tt>es.port</tt></a> </td> + +<td>Indexing </td> + +<td>String </td> + +<td><tt>es_port</tt> </td> + </tr> + +<tr class="a"> + +<td><a href="../metron-elasticsearch/index.html#es.date.format"><tt>es.date.format</tt></a> </td> + +<td>Indexing </td> + +<td>String </td> + +<td><tt>es_date_format</tt> </td> + </tr> + +<tr class="b"> + +<td><a href="#validation-framework"><tt>fieldValidations</tt></a> </td> + +<td>Parsing </td> + +<td>Object </td> + +<td>N/A </td> + </tr> + +<tr class="a"> + +<td><a href="../metron-parsers/index.html#parser.error.topic"><tt>parser.error.topic</tt></a> </td> + +<td>Parsing </td> + +<td>String </td> + +<td>N/A </td> + </tr> + +<tr class="b"> + +<td><a href="../../metron-stellar/stellar-common/index.html#stellar.function.paths"><tt>stellar.function.paths</tt></a> </td> + +<td>Stellar </td> + +<td>CSV String </td> + +<td>N/A </td> + </tr> + +<tr class="a"> + +<td><a href="../../metron-stellar/stellar-common/index.html#stellarfunctionresolverincludesexcludes"><tt>stellar.function.resolver.includes</tt></a> </td> + +<td>Stellar </td> + +<td>CSV String </td> + +<td>N/A </td> + </tr> + +<tr class="b"> + +<td><a href="../../metron-stellar/stellar-common/index.html#stellarfunctionresolverincludesexcludes"><tt>stellar.function.resolver.excludes</tt></a> </td> + +<td>Stellar </td> + +<td>CSV String </td> + +<td>N/A </td> + </tr> + +<tr class="a"> + +<td><a href="../../metron-analytics/metron-profiler/index.html#profiler.period.duration"><tt>profiler.period.duration</tt></a> </td> + +<td>Profiler </td> + +<td>Integer </td> + +<td><tt>profiler_period_duration</tt> </td> + </tr> + +<tr class="b"> + +<td><a href="../../metron-analytics/metron-profiler/index.html#profiler.period.duration.units"><tt>profiler.period.duration.units</tt></a> </td> + +<td>Profiler </td> + +<td>String </td> + +<td><tt>profiler_period_units</tt> </td> + </tr> + +<tr class="a"> + +<td><a href="../metron-indexing/index.html#update.hbase.table"><tt>update.hbase.table</tt></a> </td> + +<td>REST/Indexing </td> + +<td>String </td> + +<td><tt>update_hbase_table</tt> </td> + </tr> + +<tr class="b"> + +<td><a href="../metron-indexing/index.html#update.hbase.cf"><tt>update.hbase.cf</tt></a> </td> + +<td>REST/Indexing </td> + +<td>String </td> + +<td><tt>update_hbase_cf</tt> </td> + </tr> + +<tr class="a"> + +<td><a href="../metron-enrichment/index.html#geo.hdfs.file"><tt>geo.hdfs.file</tt></a> </td> + +<td>Enrichment </td> + +<td>String </td> + +<td><tt>geo_hdfs_file</tt> </td> + </tr> + </tbody> +</table> +<div class="section"> +<h2><a name="Note_Configs_in_Ambari"></a>Note Configs in Ambari</h2> +<p>If a field is managed via ambari, you should change the field via ambari. Otherwise, upon service restarts, you may find your update overwritten.</p> <p><a name="Validation_Framework"></a></p> <h1>Validation Framework</h1> <p>Inside of the global configuration, there is a validation framework in place that enables the validation that messages coming from all parsers are valid. This is done in the form of validation plugins where assertions about fields or whole messages can be made. </p> @@ -377,16 +567,28 @@ <div class="source"> <div class="source"> -<pre> -f,--force Force operation +<pre> -c,--config_type <CONFIG_TYPE> The configuration type: GLOBAL, + PARSER, ENRICHMENT, INDEXING, + PROFILER + -f,--force Force operation -h,--help Generate Help screen -i,--input_dir <DIR> The input directory containing the configuration files named like "$source.json" -m,--mode <MODE> The mode of operation: DUMP, - PULL, PUSH + PULL, PUSH, PATCH + -n,--config_name <CONFIG_NAME> The configuration name: bro, + yaf, snort, squid, etc. -o,--output_dir <DIR> The output directory which will store the JSON configuration from Zookeeper + -pk,--patch_key <PATCH_KEY> The key to modify + -pm,--patch_mode <PATCH_MODE> One of: ADD, REMOVE - relevant + only for key/value patches, + i.e. when a patch file is not + used. + -pf,--patch_file <PATCH_FILE> Path to the patch file. + -pv,--patch_value <PATCH_VALUE> Value to use in the patch. -z,--zk_quorum <host:port,[host:port]*> Zookeeper Quorum URL (zk1:port,zk2:port,...) </pre></div></div> @@ -396,10 +598,163 @@ <li>To dump the existing configs from zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m DUMP</tt></li> +<li>To dump the existing GLOBAL configs from zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m DUMP -c GLOBAL</tt></li> + <li>To push the configs into zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper</tt></li> -<li>To pull the configs from zookeeper to the singlenode vagrant machine disk: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PULL -o $METRON_HOME/config/zookeeper -f</tt></li> +<li>To push only the GLOBAL configs into zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper -c GLOBAL</tt></li> + +<li>To push only the PARSER configs into zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper -c PARSER</tt></li> + +<li>To push only the PARSER ‘bro’ configs into zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper -c PARSER -n bro</tt></li> + +<li>To pull all configs from zookeeper to the singlenode vagrant machine disk: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PULL -o $METRON_HOME/config/zookeeper -f</tt></li> +</ul></div> +<div class="section"> +<h2><a name="Patching_mechanism"></a>Patching mechanism</h2> +<p>The configuration management utility leverages a JSON patching library that conforms to <a class="externalLink" href="https://tools.ietf.org/html/rfc6902";>RFC-6902 spec</a>. We’re using the zjsonpatch library implementation from here - <a class="externalLink" href="https://github.com/flipkart-incubator/zjsonpatch";>https://github.com/flipkart-incubator/zjsonpatch</a>. There are a couple options for leveraging patching. You can choose to patch the Zookeeper config via patch file:</p> +<p><tt>$METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pf /tmp/mypatch.txt</tt></p> +<p>or key/value pair:</p> +<p><tt>$METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pm ADD -pk foo -pv \"\"bar\"\"</tt></p> +<p>The options exposed via patch file are the full range of options from RFC-6902:</p> + +<ul> + +<li>ADD</li> + +<li>REMOVE</li> + +<li>REPLACE</li> + +<li>MOVE</li> + +<li>COPY</li> + +<li>TEST</li> </ul> +<p>whereas with key/value patching, we only current expose ADD and REMOVE. Note that ADD will function as a REPLACE when the key already exists.</p> +<div class="section"> +<h3><a name="Patch_File"></a>Patch File</h3> +<p>Let’s say we want to add a complex JSON object to our configuration with a patch file. e.g.</p> + +<div class="source"> +<div class="source"> +<pre>"foo" : { + "bar" : { + "baz" : [ "bazval1", "bazval2" ] + } + } +</pre></div></div> +<p>We would write a patch file “/tmp/mypatch.txt” with contents:</p> + +<div class="source"> +<div class="source"> +<pre>[ + { + "op": "add", + "path": "/foo", + "value": { "bar" : { "baz" : [ "bazval1", "bazval2" ] } } + } +] +</pre></div></div> +<p>And submit via zk_load_configs as follows:</p> + +<div class="source"> +<div class="source"> +<pre> $METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pf /tmp/mypatch.txt +</pre></div></div></div> +<div class="section"> +<h3><a name="Patch_KeyValue"></a>Patch Key/Value</h3> +<p>Now let’s try the same without using a patch file, instead using the patch_key and patch_value options right from the command line utility. This would like like the following.</p> + +<div class="source"> +<div class="source"> +<pre>$METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pm ADD -pk "/foo" -pv "{ \"bar\" : { \"baz\" : [ \"bazval1\", \"bazval2\" ] } }" +</pre></div></div></div> +<div class="section"> +<h3><a name="Applying_Multiple_Patches"></a>Applying Multiple Patches</h3> +<p>Applying multiple patches is also pretty straightforward. You can achieve this in a single command using patch files, or simply execute multiple commands in sequence using the patch_key/value approach.</p> +<p>Let’s say we wanted to add the following to our global config:</p> + +<div class="source"> +<div class="source"> +<pre>"apache" : "metron", +"is" : "the best", +"streaming" : "analytics platform" +</pre></div></div> +<p>and remove the /foo key from the previous example.</p> +<p>Create a patch file /tmp/mypatch.txt with four separate patch operations.</p> + +<div class="source"> +<div class="source"> +<pre>[ + { + "op": "remove", + "path": "/foo" + }, + { + "op": "add", + "path": "/apache", + "value": "metron" + }, + { + "op": "add", + "path": "/is", + "value": "the best" + }, + { + "op": "add", + "path": "/streaming", + "value": "analytics platform" + } +] +</pre></div></div> +<p>Now submit again and you should see a Global config with the “foo” key removed and three new keys added.</p> + +<div class="source"> +<div class="source"> +<pre> $METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pf /tmp/mypatch.txt +</pre></div></div></div> +<div class="section"> +<h3><a name="Notes_On_Patching"></a>Notes On Patching</h3> +<p>For any given patch key, the last/leaf node in the key’s parent <i>must</i> exist, otherwise an exception will be thrown. For example, if you want to add the following:</p> + +<div class="source"> +<div class="source"> +<pre>"foo": { + "bar": "baz" +} +</pre></div></div> +<p>It is not sufficient to use /foo/bar as a key if foo does not already exist. You would either need to incrementally build the JSON and make this a two step process</p> + +<div class="source"> +<div class="source"> +<pre>[ + { + "op": "add", + "path": "/foo", + "value": { } + }, + { + "op": "add", + "path": "/foo/bar", + "value": "baz" + } +] +</pre></div></div> +<p>Or provide the value as a complete JSON object.</p> + +<div class="source"> +<div class="source"> +<pre>[ + { + "op": "add", + "path": "/foo", + "value": { "bar" : "baz" } + } +] +</pre></div></div> +<p>The REMOVE operation is idempotent. Running the remove command on the same key multiple times will not fail once the key has been removed.</p> <p><a name="Topology_Errors"></a></p> <h1>Topology Errors</h1> <p>Errors generated in Metron topologies are transformed into JSON format and follow this structure:</p> @@ -499,7 +854,7 @@ if (perfLog.isDebugEnabled()) { } </pre></div></div> <p><b>Side Effects</b></p> -<p>Calling the mark() method multiple times simply resets the start time to the current nano time. Calling log() with a non-existent mark name will log 0 ns elapsed time with a warning indicating that log has been invoked for a mark name that does not exist. The class is not thread-safe and makes no attempt at keeping multiple threads from modifying the same markers.</p> +<p>Calling the mark() method multiple times simply resets the start time to the current nano time. Calling log() with a non-existent mark name will log 0 ns elapsed time with a warning indicating that log has been invoked for a mark name that does not exist. The class is not thread-safe and makes no attempt at keeping multiple threads from modifying the same markers.</p></div></div> </div> </div> </div> @@ -508,7 +863,7 @@ if (perfLog.isDebugEnabled()) { <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/metron/blob/3381b853/site/current-book/metron-platform/metron-data-management/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-data-management/index.html b/site/current-book/metron-platform/metron-data-management/index.html index b04a6f4..df34389 100644 --- a/site/current-book/metron-platform/metron-data-management/index.html +++ b/site/current-book/metron-platform/metron-data-management/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-09-15 + | Generated by Apache Maven Doxia at 2018-01-03 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170915" /> + <meta name="Date-Revision-yyyymmdd" content="20180103" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Resource Data Management</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -61,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.1</li> + <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.2</li> </ul> </div> @@ -103,7 +103,7 @@ <i class="none"></i> Docker</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -131,7 +131,7 @@ <i class="none"></i> Rest</a> </li> - + <li> <a href="../../metron-platform/index.html" title="Platform"> @@ -167,6 +167,13 @@ <li> + <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> + <i class="none"></i> + Elasticsearch</a> + </li> + + <li> + <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> <i class="none"></i> Enrichment</a> @@ -208,13 +215,20 @@ </li> </ul> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> <i class="icon-chevron-right"></i> Sensors</a> </li> + + <li> + + <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> + <i class="none"></i> + Stellar-3rd-party-example</a> + </li> <li> @@ -222,7 +236,7 @@ <i class="icon-chevron-right"></i> Stellar-common</a> </li> - + <li> <a href="../../use-cases/index.html" title="Use-cases"> @@ -1041,7 +1055,7 @@ <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017 + <div class="row span12">Copyright © 2018 <a href="https://www.apache.org";>The Apache Software Foundation</a>. All Rights Reserved.
