http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/39059e5b/site/current-book/metron-platform/metron-common/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-platform/metron-common/index.html
b/site/current-book/metron-platform/metron-common/index.html
new file mode 100644
index 0000000..c116276
--- /dev/null
+++ b/site/current-book/metron-platform/metron-common/index.html
@@ -0,0 +1,2233 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-02-23
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20170223" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Contents</title>
+ <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css"
/>
+ <link rel="stylesheet" href="../../css/site.css" />
+ <link rel="stylesheet" href="../../css/print.css" media="print" />
+
+
+ <script type="text/javascript"
src="../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+
+<script type="text/javascript">$( document ).ready( function() { $(
'.carousel' ).carousel( { interval: 3500 } ) } );</script>
+
+ </head>
+ <body class="topBarDisabled">
+
+
+
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a
href="http://metron.incubator.apache.org/"; id="bannerLeft">
+
<img src="../../images/metron-logo.png" alt="Apache Metron -
Incubating" width="148px" height="48px"/>
+ </a>
+ </div>
+ <div class="pull-right"> <a
href="http://incubator.apache.org/"; id="bannerRight">
+
<img src="../../images/ApacheIncubating_Logo.png" alt="Apache
Incubating" width="192px" height="48px"/>
+ </a>
+ </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org"; class="externalLink"
title="Apache">
+ Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://metron.incubator.apache.org/";
class="externalLink" title="Metron-Incubating">
+ Metron-Incubating</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="../../index.html" title="Documentation">
+ Documentation</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Contents</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published:
2017-02-23</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.3.1</li>
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span3">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+
+ <li>
+
+ <a href="../../index.html" title="Metron">
+ <i class="icon-chevron-down"></i>
+ Metron</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../Upgrading.html" title="Upgrading">
+ <i class="none"></i>
+ Upgrading</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-analytics/index.html"
title="Analytics">
+ <i class="icon-chevron-right"></i>
+ Analytics</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/index.html"
title="Deployment">
+ <i class="icon-chevron-right"></i>
+ Deployment</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-docker/index.html"
title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/index.html"
title="Platform">
+ <i class="icon-chevron-down"></i>
+ Platform</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a
href="../../metron-platform/metron-api/index.html" title="Api">
+ <i class="none"></i>
+ Api</a>
+ </li>
+
+ <li class="active">
+
+ <a href="#"><i class="none"></i>Common</a>
+ </li>
+
+ <li>
+
+ <a
href="../../metron-platform/metron-data-management/index.html"
title="Data-management">
+ <i class="none"></i>
+ Data-management</a>
+ </li>
+
+ <li>
+
+ <a
href="../../metron-platform/metron-enrichment/index.html" title="Enrichment">
+ <i class="none"></i>
+ Enrichment</a>
+ </li>
+
+ <li>
+
+ <a
href="../../metron-platform/metron-indexing/index.html" title="Indexing">
+ <i class="none"></i>
+ Indexing</a>
+ </li>
+
+ <li>
+
+ <a
href="../../metron-platform/metron-management/index.html" title="Management">
+ <i class="none"></i>
+ Management</a>
+ </li>
+
+ <li>
+
+ <a
href="../../metron-platform/metron-parsers/index.html" title="Parsers">
+ <i class="none"></i>
+ Parsers</a>
+ </li>
+
+ <li>
+
+ <a
href="../../metron-platform/metron-pcap-backend/index.html"
title="Pcap-backend">
+ <i class="none"></i>
+ Pcap-backend</a>
+ </li>
+ </ul>
+ </li>
+
+ <li>
+
+ <a href="../../metron-sensors/index.html"
title="Sensors">
+ <i class="icon-chevron-right"></i>
+ Sensors</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/"; title="Built
by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven"
src="../../images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span9" >
+
+ <h1>Contents</h1>
+<p><a name="Contents"></a></p>
+
+<ul>
+
+<li><a href="#Stellar_Language">Stellar Language</a>
+
+<ul>
+
+<li><a href="#Stellar_Language_Keywords">Stellar Language Keywords</a></li>
+
+<li><a href="#Stellar_Core_Functions">Stellar Core Functions</a></li>
+
+<li><a href="#Stellar_Shell">Stellar Shell</a></li>
+ </ul></li>
+
+<li><a href="#Global_Configuration">Global Configuration</a></li>
+
+<li><a href="#Management_Utility">Management Utility</a></li>
+</ul>
+<p><a name="Stellar_Language"></a></p>
+<h1>Stellar Language</h1>
+<p>For a variety of components (threat intelligence triage and field
transformations) we have the need to do simple computation and transformation
using the data from messages as variables.<br />For those purposes, there
exists a simple, scaled down DSL created to do simple computation and
transformation.</p>
+<p>The query language supports the following:</p>
+
+<ul>
+
+<li>Referencing fields in the enriched JSON</li>
+
+<li>Simple boolean operations: <tt>and</tt>, <tt>not</tt>, <tt>or</tt></li>
+
+<li>Simple arithmetic operations: <tt>*</tt>, <tt>/</tt>, <tt>+</tt>,
<tt>-</tt> on real numbers or integers</li>
+
+<li>Simple comparison operations <tt><</tt>, <tt>></tt>, <tt><=</tt>,
<tt>>=</tt></li>
+
+<li>Simple equality comparison operations <tt>==</tt>, <tt>!=</tt></li>
+
+<li>if/then/else comparisons (i.e. <tt>if var1 < 10 then 'less than 10'
else '10 or more'</tt>)</li>
+
+<li>Determining whether a field exists (via <tt>exists</tt>)</li>
+
+<li>An <tt>in</tt> operator that works like the <tt>in</tt> in Python</li>
+
+<li>The ability to have parenthesis to make order of operations explicit</li>
+
+<li>User defined functions</li>
+</ul>
+<div class="section">
+<h2><a name="Stellar_Language_Keywords"></a>Stellar Language Keywords</h2>
+<p>The following keywords need to be single quote escaped in order to be used
in Stellar expressions:</p>
+
+<table border="0" class="table table-striped">
+ <thead>
+
+<tr class="a">
+
+<th align="center"> </th>
+
+<th align="center"> </th>
+
+<th align="center"> </th>
+
+<th align="center"> </th>
+
+<th align="center"> </th>
+ </tr>
+ </thead>
+ <tbody>
+
+<tr class="b">
+
+<td align="center">not </td>
+
+<td align="center">else </td>
+
+<td align="center">exists </td>
+
+<td align="center">if </td>
+
+<td align="center">then </td>
+ </tr>
+
+<tr class="a">
+
+<td align="center">and </td>
+
+<td align="center">or </td>
+
+<td align="center">in </td>
+
+<td align="center">== </td>
+
+<td align="center">!= </td>
+ </tr>
+
+<tr class="b">
+
+<td align="center"><= </td>
+
+<td align="center">> </td>
+
+<td align="center">>= </td>
+
+<td align="center">+ </td>
+
+<td align="center">- </td>
+ </tr>
+
+<tr class="a">
+
+<td align="center">< </td>
+
+<td align="center">? </td>
+
+<td align="center">* </td>
+
+<td align="center">/ </td>
+
+<td align="center">, </td>
+ </tr>
+ </tbody>
+</table>
+<p>Using parens such as: “foo” : “<ok>”
requires escaping; “foo”: “'<ok>'”</p></div>
+<div class="section">
+<h2><a name="Stellar_Language_Inclusion_Checks_in_and_not_in"></a>Stellar
Language Inclusion Checks (<tt>in</tt> and <tt>not in</tt>)</h2>
+
+<ol style="list-style-type: decimal">
+
+<li><tt>in</tt> supports string contains. e.g. <tt>'foo' in 'foobar' ==
true</tt></li>
+
+<li><tt>in</tt> supports collection contains. e.g. <tt>'foo' in [ 'foo', 'bar'
] == true</tt></li>
+
+<li><tt>in</tt> supports map key contains. e.g. <tt>'foo' in { 'foo' : 5} ==
true</tt></li>
+
+<li><tt>not in</tt> is the negation of the in expression. e.g. <tt>'grok' not
in 'foobar' == true</tt></li>
+</ol></div>
+<div class="section">
+<h2><a name="Stellar_Language_Comparisons____"></a>Stellar Language
Comparisons (<tt><</tt>, <tt><=</tt>, <tt>></tt>, <tt>>=</tt>)</h2>
+
+<ol style="list-style-type: decimal">
+
+<li>If either side of the comparison is null then return false.</li>
+
+<li>If both values being compared implement number then the following:
+
+<ul>
+
+<li>If either side is a double then get double value from both sides and
compare using given operator.</li>
+
+<li>Else if either side is a float then get float value from both sides and
compare using given operator.</li>
+
+<li>Else if either side is a long then get long value from both sides and
compare using given operator.</li>
+
+<li>Otherwise get the int value from both sides and compare using given
operator.</li>
+ </ul></li>
+
+<li>If both sides are of the same type and are comparable then use the
compareTo method to compare values.</li>
+
+<li>If none of the above are met then an exception is thrown.</li>
+</ol></div>
+<div class="section">
+<h2><a name="Stellar_Language_Equality_Check__"></a>Stellar Language Equality
Check (<tt>==</tt>, <tt>!=</tt>)</h2>
+<p>Below is how the <tt>==</tt> operator is expected to work:</p>
+
+<ol style="list-style-type: decimal">
+
+<li>If either side of the expression is null then check equality using
Java’s <tt>==</tt> expression.</li>
+
+<li>Else if both sides of the expression are of Java’s type Number then:
+
+<ul>
+
+<li>If either side of the expression is a double then use the double value of
both sides to test equality.</li>
+
+<li>Else if either side of the expression is a float then use the float value
of both sides to test equality.</li>
+
+<li>Else if either side of the expression is a long then use long value of
both sides to test equality.</li>
+
+<li>Otherwise use int value of both sides to test equality</li>
+ </ul></li>
+
+<li>Otherwise use equals method compare the left side with the right side.</li>
+</ol>
+<p>The <tt>!=</tt> operator is the negation of the above.</p></div>
+<div class="section">
+<h2><a name="Stellar_Core_Functions"></a>Stellar Core Functions</h2>
+
+<table border="0" class="table table-striped">
+ <thead>
+
+<tr class="a">
+
+<th> </th>
+ </tr>
+ </thead>
+ <tbody>
+
+<tr class="b">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#ABS">
<tt>ABS</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#BIN">
<tt>BIN</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#BLOOM_ADD"> <tt>BLOOM_ADD</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#BLOOM_EXISTS"> <tt>BLOOM_EXISTS</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#BLOOM_INIT"> <tt>BLOOM_INIT</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#BLOOM_MERGE"> <tt>BLOOM_MERGE</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#DAY_OF_MONTH"> <tt>DAY_OF_MONTH</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#DAY_OF_WEEK"> <tt>DAY_OF_WEEK</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#DAY_OF_YEAR"> <tt>DAY_OF_YEAR</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#DOMAIN_REMOVE_SUBDOMAINS"> <tt>DOMAIN_REMOVE_SUBDOMAINS</tt></a>
</td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#DOMAIN_REMOVE_TLD"> <tt>DOMAIN_REMOVE_TLD</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#DOMAIN_TO_TLD"> <tt>DOMAIN_TO_TLD</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#ENDS_WITH"> <tt>ENDS_WITH</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#ENRICHMENT_EXISTS"> <tt>ENRICHMENT_EXISTS</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#ENRICHMENT_GET"> <tt>ENRICHMENT_GET</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#FILL_LEFT"> <tt>FILL_LEFT</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#FILL_RIGHT"> <tt>FILL_RIGHT</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#FORMAT"> <tt>FORMAT</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a
href="../../metron-analytics/metron-statistics/index.html#HLLP_CARDINALITY">
<tt>HLLP_CARDINALITY</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#HLLP_INIT">
<tt>HLLP_INIT</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#HLLP_MERGE">
<tt>HLLP_MERGE</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#HLLP_OFFER">
<tt>HLLP_OFFER</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#GEO_GET"> <tt>GEO_GET</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#GET"> <tt>GET</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#GET_FIRST"> <tt>GET_FIRST</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#GET_LAST"> <tt>GET_LAST</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#IN_SUBNET"> <tt>IN_SUBNET</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#IS_DATE"> <tt>IS_DATE</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#IS_DOMAIN"> <tt>IS_DOMAIN</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#IS_EMAIL"> <tt>IS_EMAIL</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#IS_EMPTY"> <tt>IS_EMPTY</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#IS_INTEGER"> <tt>IS_INTEGER</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#IS_IP"> <tt>IS_IP</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#IS_URL"> <tt>IS_URL</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#JOIN"> <tt>JOIN</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#KAFKA_GET"> <tt>KAFKA_GET</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#KAFKA_PROPS"> <tt>KAFKA_PROPS</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#KAFKA_PUT"> <tt>KAFKA_PUT</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#KAFKA_TAIL"> <tt>KAFKA_TAIL</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#LENGTH"> <tt>LENGTH</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#MAAS_GET_ENDPOINT"> <tt>MAAS_GET_ENDPOINT</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#MAAS_MODEL_APPLY"> <tt>MAAS_MODEL_APPLY</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#MAP_EXISTS"> <tt>MAP_EXISTS</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#MONTH"> <tt>MONTH</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#PROFILE_GET"> <tt>PROFILE_GET</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#PROFILE_FIXED"> <tt>PROFILE_FIXED</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#PROTOCOL_TO_NAME"> <tt>PROTOCOL_TO_NAME</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#REGEXP_MATCH"> <tt>REGEXP_MATCH</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#SPLIT"> <tt>SPLIT</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#STARTS_WITH"> <tt>STARTS_WITH</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#STATS_ADD">
<tt>STATS_ADD</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#STATS_BIN">
<tt>STATS_BIN</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#STATS_COUNT">
<tt>STATS_COUNT</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a
href="../../metron-analytics/metron-statistics/index.html#STATS_GEOMETRIC_MEAN">
<tt>STATS_GEOMETRIC_MEAN</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#STATS_INIT">
<tt>STATS_INIT</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a
href="../../metron-analytics/metron-statistics/index.html#STATS_KURTOSIS">
<tt>STATS_KURTOSIS</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#STATS_MAX">
<tt>STATS_MAX</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#STATS_MEAN">
<tt>STATS_MEAN</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#STATS_MERGE">
<tt>STATS_MERGE</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#STATS_MIN">
<tt>STATS_MIN</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a
href="../../metron-analytics/metron-statistics/index.html#STATS_PERCENTILE">
<tt>STATS_PERCENTILE</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a
href="../../metron-analytics/metron-statistics/index.html#STATS_POPULATION_VARIANCE">
<tt>STATS_POPULATION_VARIANCE</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a
href="../../metron-analytics/metron-statistics/index.html#STATS_QUADRATIC_MEAN">
<tt>STATS_QUADRATIC_MEAN</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#STATS_SD">
<tt>STATS_SD</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a
href="../../metron-analytics/metron-statistics/index.html#STATS_SKEWNESS">
<tt>STATS_SKEWNESS</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="../../metron-analytics/metron-statistics/index.html#STATS_SUM">
<tt>STATS_SUM</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a
href="../../metron-analytics/metron-statistics/index.html#STATS_SUM_LOGS">
<tt>STATS_SUM_LOGS</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a
href="../../metron-analytics/metron-statistics/index.html#STATS_SUM_SQUARES">
<tt>STATS_SUM_SQUARES</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a
href="../../metron-analytics/metron-statistics/index.html#STATS_VARIANCE">
<tt>STATS_VARIANCE</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#STRING_ENTROPY"> <tt>STRING_ENTROPY</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#SYSTEM_ENV_GET"> <tt>SYSTEM_ENV_GET</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#SYSTEM_PROPERTY_GET"> <tt>SYSTEM_PROPERTY_GET</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#TO_DOUBLE"> <tt>TO_DOUBLE</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#TO_EPOCH_TIMESTAMP"> <tt>TO_EPOCH_TIMESTAMP</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#TO_FLOAT"> <tt>TO_FLOAT</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#TO_INTEGER"> <tt>TO_INTEGER</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#TO_LONG"> <tt>TO_LONG</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#TO_LOWER"> <tt>TO_LOWER</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#TO_STRING"> <tt>TO_STRING</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#TO_UPPER"> <tt>TO_UPPER</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#TRIM"> <tt>TRIM</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#URL_TO_HOST"> <tt>URL_TO_HOST</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#URL_TO_PATH"> <tt>URL_TO_PATH</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#URL_TO_PORT"> <tt>URL_TO_PORT</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#URL_TO_PROTOCOL"> <tt>URL_TO_PROTOCOL</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#WEEK_OF_MONTH"> <tt>WEEK_OF_MONTH</tt></a> </td>
+ </tr>
+
+<tr class="b">
+
+<td><a href="#WEEK_OF_YEAR"> <tt>WEEK_OF_YEAR</tt></a> </td>
+ </tr>
+
+<tr class="a">
+
+<td><a href="#YEAR"> <tt>YEAR</tt></a> </td>
+ </tr>
+ </tbody>
+</table>
+<div class="section">
+<h3><a name="BLOOM_ADD"></a><tt>BLOOM_ADD</tt></h3>
+
+<ul>
+
+<li>Description: Adds an element to the bloom filter passed in</li>
+
+<li>Input:
+
+<ul>
+
+<li>bloom - The bloom filter</li>
+
+<li>value* - The values to add</li>
+ </ul></li>
+
+<li>Returns: Bloom Filter</li>
+</ul></div>
+<div class="section">
+<h3><a name="BLOOM_EXISTS"></a><tt>BLOOM_EXISTS</tt></h3>
+
+<ul>
+
+<li>Description: If the bloom filter contains the value</li>
+
+<li>Input:
+
+<ul>
+
+<li>bloom - The bloom filter</li>
+
+<li>value - The value to check</li>
+ </ul></li>
+
+<li>Returns: True if the filter might contain the value and false
otherwise</li>
+</ul></div>
+<div class="section">
+<h3><a name="BLOOM_INIT"></a><tt>BLOOM_INIT</tt></h3>
+
+<ul>
+
+<li>Description: Returns an empty bloom filter</li>
+
+<li>Input:
+
+<ul>
+
+<li>expectedInsertions - The expected insertions</li>
+
+<li>falsePositiveRate - The false positive rate you are willing to
tolerate</li>
+ </ul></li>
+
+<li>Returns: Bloom Filter</li>
+</ul></div>
+<div class="section">
+<h3><a name="BLOOM_MERGE"></a><tt>BLOOM_MERGE</tt></h3>
+
+<ul>
+
+<li>Description: Returns a merged bloom filter</li>
+
+<li>Input:
+
+<ul>
+
+<li>bloomfilters - A list of bloom filters to merge</li>
+ </ul></li>
+
+<li>Returns: Bloom Filter or null if the list is empty</li>
+</ul></div>
+<div class="section">
+<h3><a name="DAY_OF_MONTH"></a><tt>DAY_OF_MONTH</tt></h3>
+
+<ul>
+
+<li>Description: The numbered day within the month. The first day within the
month has a value of 1.</li>
+
+<li>Input:
+
+<ul>
+
+<li>dateTime - The datetime as a long representing the milliseconds since unix
epoch</li>
+ </ul></li>
+
+<li>Returns: The numbered day within the month.</li>
+</ul></div>
+<div class="section">
+<h3><a name="DAY_OF_WEEK"></a><tt>DAY_OF_WEEK</tt></h3>
+
+<ul>
+
+<li>Description: The numbered day within the week. The first day of the week,
Sunday, has a value of 1.</li>
+
+<li>Input:
+
+<ul>
+
+<li>dateTime - The datetime as a long representing the milliseconds since unix
epoch</li>
+ </ul></li>
+
+<li>Returns: The numbered day within the week.</li>
+</ul></div>
+<div class="section">
+<h3><a name="DAY_OF_YEAR"></a><tt>DAY_OF_YEAR</tt></h3>
+
+<ul>
+
+<li>Description: The day number within the year. The first day of the year has
value of 1.</li>
+
+<li>Input:
+
+<ul>
+
+<li>dateTime - The datetime as a long representing the milliseconds since unix
epoch</li>
+ </ul></li>
+
+<li>Returns: The day number within the year.</li>
+</ul></div>
+<div class="section">
+<h3><a
name="DOMAIN_REMOVE_SUBDOMAINS"></a><tt>DOMAIN_REMOVE_SUBDOMAINS</tt></h3>
+
+<ul>
+
+<li>Description: Removes the subdomains from a domain.</li>
+
+<li>Input:
+
+<ul>
+
+<li>domain - Fully qualified domain name</li>
+ </ul></li>
+
+<li>Returns: The domain without the subdomains. (for example,
DOMAIN_REMOVE_SUBDOMAINS(‘mail.yahoo.com’) yields
‘yahoo.com’)</li>
+</ul></div>
+<div class="section">
+<h3><a name="DOMAIN_REMOVE_TLD"></a><tt>DOMAIN_REMOVE_TLD</tt></h3>
+
+<ul>
+
+<li>Description: Removes the top level domain (TLD) suffix from a domain.</li>
+
+<li>Input:
+
+<ul>
+
+<li>domain - Fully qualified domain name</li>
+ </ul></li>
+
+<li>Returns: The domain without the TLD. (for example,
DOMAIN_REMOVE_TLD(‘mail.yahoo.co.uk’) yields
‘mail.yahoo’)</li>
+</ul></div>
+<div class="section">
+<h3><a name="DOMAIN_TO_TLD"></a><tt>DOMAIN_TO_TLD</tt></h3>
+
+<ul>
+
+<li>Description: Extracts the top level domain from a domain</li>
+
+<li>Input:
+
+<ul>
+
+<li>domain - Fully qualified domain name</li>
+ </ul></li>
+
+<li>Returns: The TLD of the domain. (for example,
DOMAIN_TO_TLD(‘mail.yahoo.co.uk’) yields
‘co.uk’)</li>
+</ul></div>
+<div class="section">
+<h3><a name="ENDS_WITH"></a><tt>ENDS_WITH</tt></h3>
+
+<ul>
+
+<li>Description: Determines whether a string ends with a specified suffix</li>
+
+<li>Input:
+
+<ul>
+
+<li>string - The string to test</li>
+
+<li>suffix - The proposed suffix</li>
+ </ul></li>
+
+<li>Returns: True if the string ends with the specified suffix and false if
otherwise</li>
+</ul></div>
+<div class="section">
+<h3><a name="ENRICHMENT_EXISTS"></a><tt>ENRICHMENT_EXISTS</tt></h3>
+
+<ul>
+
+<li>Description: Interrogates the HBase table holding the simple hbase
enrichment data and returns whether the enrichment type and indicator are in
the table.</li>
+
+<li>Input:
+
+<ul>
+
+<li>enrichment_type - The enrichment type</li>
+
+<li>indicator - The string indicator to look up</li>
+
+<li>nosql_table - The NoSQL Table to use</li>
+
+<li>column_family - The Column Family to use</li>
+ </ul></li>
+
+<li>Returns: True if the enrichment indicator exists and false otherwise</li>
+</ul></div>
+<div class="section">
+<h3><a name="ENRICHMENT_GET"></a><tt>ENRICHMENT_GET</tt></h3>
+
+<ul>
+
+<li>Description: Interrogates the HBase table holding the simple hbase
enrichment data and retrieves the tabular value associated with the enrichment
type and indicator.</li>
+
+<li>Input:
+
+<ul>
+
+<li>enrichment_type - The enrichment type</li>
+
+<li>indicator - The string indicator to look up</li>
+
+<li>nosql_table - The NoSQL Table to use</li>
+
+<li>column_family - The Column Family to use</li>
+ </ul></li>
+
+<li>Returns: A Map associated with the indicator and enrichment type. Empty
otherwise.</li>
+</ul></div>
+<div class="section">
+<h3><a name="FILL_LEFT"></a><tt>FILL_LEFT</tt></h3>
+
+<ul>
+
+<li>Description: Fills or pads a given string with a given character, to a
given length on the left</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - string</li>
+
+<li>fill - the fill character</li>
+
+<li>len - the required length</li>
+ </ul></li>
+
+<li>Returns: the filled string</li>
+</ul></div>
+<div class="section">
+<h3><a name="FILL_RIGHT"></a><tt>FILL_RIGHT</tt></h3>
+
+<ul>
+
+<li>Description: Fills or pads a given string with a given character, to a
given length on the right</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - string</li>
+
+<li>fill - the fill character string</li>
+
+<li>len - the required length</li>
+ </ul></li>
+
+<li>Returns: Last element of the list</li>
+</ul></div>
+<div class="section">
+<h3><a name="FORMAT"></a><tt>FORMAT</tt></h3>
+
+<ul>
+
+<li>Description: Returns a formatted string using the specified format string
and arguments. Uses Java’s string formatting conventions.</li>
+
+<li>Input:
+
+<ul>
+
+<li>format - string</li>
+
+<li>arguments… - object(s)</li>
+ </ul></li>
+
+<li>Returns: A formatted string.</li>
+</ul></div>
+<div class="section">
+<h3><a name="GEO_GET"></a><tt>GEO_GET</tt></h3>
+
+<ul>
+
+<li>Description: Look up an IPV4 address and returns geographic information
about it</li>
+
+<li>Input:
+
+<ul>
+
+<li>ip - The IPV4 address to lookup</li>
+
+<li>fields - Optional list of GeoIP fields to grab. Options are locID,
country, city postalCode, dmaCode, latitude, longitude, location_point</li>
+ </ul></li>
+
+<li>Returns: If a Single field is requested a string of the field, If multiple
fields a map of string of the fields, and null otherwise</li>
+</ul></div>
+<div class="section">
+<h3><a name="GET"></a><tt>GET</tt></h3>
+
+<ul>
+
+<li>Description: Returns the i’th element of the list</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - List</li>
+
+<li>i - The index (0-based)</li>
+ </ul></li>
+
+<li>Returns: First element of the list</li>
+</ul></div>
+<div class="section">
+<h3><a name="GET_FIRST"></a><tt>GET_FIRST</tt></h3>
+
+<ul>
+
+<li>Description: Returns the first element of the list</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - List</li>
+ </ul></li>
+
+<li>Returns: First element of the list</li>
+</ul></div>
+<div class="section">
+<h3><a name="GET_LAST"></a><tt>GET_LAST</tt></h3>
+
+<ul>
+
+<li>Description: Returns the last element of the list</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - List</li>
+ </ul></li>
+
+<li>Returns: Last element of the list</li>
+</ul></div>
+<div class="section">
+<h3><a name="IN_SUBNET"></a><tt>IN_SUBNET</tt></h3>
+
+<ul>
+
+<li>Description: Returns true if an IP is within a subnet range.</li>
+
+<li>Input:
+
+<ul>
+
+<li>ip - The IP address in string form</li>
+
+<li>cidr+ - One or more IP ranges specified in CIDR notation (for example
192.168.0.0/24)</li>
+ </ul></li>
+
+<li>Returns: True if the IP address is within at least one of the network
ranges and false if otherwise</li>
+</ul></div>
+<div class="section">
+<h3><a name="IS_DATE"></a><tt>IS_DATE</tt></h3>
+
+<ul>
+
+<li>Description: Determines if the date contained in the string conforms to
the specified format.</li>
+
+<li>Input:
+
+<ul>
+
+<li>date - The date in string form</li>
+
+<li>format - The format of the date</li>
+ </ul></li>
+
+<li>Returns: True if the date is in the specified format and false if
otherwise.</li>
+</ul></div>
+<div class="section">
+<h3><a name="IS_DOMAIN"></a><tt>IS_DOMAIN</tt></h3>
+
+<ul>
+
+<li>Description: Tests if a string refers to a valid domain name. Domain names
are evaluated according to the standards RFC1034 section 3, and RFC1123 section
2.1.</li>
+
+<li>Input:
+
+<ul>
+
+<li>address - The string to test</li>
+ </ul></li>
+
+<li>Returns: True if the string refers to a valid domain name and false if
otherwise</li>
+</ul></div>
+<div class="section">
+<h3><a name="IS_EMAIL"></a><tt>IS_EMAIL</tt></h3>
+
+<ul>
+
+<li>Description: Tests if a string is a valid email address</li>
+
+<li>Input:
+
+<ul>
+
+<li>address - The string to test</li>
+ </ul></li>
+
+<li>Returns: True if the string is a valid email address and false if
otherwise.</li>
+</ul></div>
+<div class="section">
+<h3><a name="IS_EMPTY"></a><tt>IS_EMPTY</tt></h3>
+
+<ul>
+
+<li>Description: Returns true if string or collection is empty or null and
false if otherwise.</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - Object of string or collection type (for example, list)</li>
+ </ul></li>
+
+<li>Returns: True if the string or collection is empty or null and false if
otherwise.</li>
+</ul></div>
+<div class="section">
+<h3><a name="IS_INTEGER"></a><tt>IS_INTEGER</tt></h3>
+
+<ul>
+
+<li>Description: Determines whether or not an object is an integer.</li>
+
+<li>Input:
+
+<ul>
+
+<li>x - The object to test</li>
+ </ul></li>
+
+<li>Returns: True if the object can be converted to an integer and false if
otherwise.</li>
+</ul></div>
+<div class="section">
+<h3><a name="IS_IP"></a><tt>IS_IP</tt></h3>
+
+<ul>
+
+<li>Description: Determine if an string is an IP or not.</li>
+
+<li>Input:
+
+<ul>
+
+<li>ip - An object which we wish to test is an ip</li>
+
+<li>type (optional) - Object of string or collection type (e.g. list) one of
IPV4 or IPV6 or both. The default is IPV4.</li>
+ </ul></li>
+
+<li>Returns: True if the string is an IP and false otherwise.</li>
+</ul></div>
+<div class="section">
+<h3><a name="IS_URL"></a><tt>IS_URL</tt></h3>
+
+<ul>
+
+<li>Description: Tests if a string is a valid URL</li>
+
+<li>Input:
+
+<ul>
+
+<li>url - The string to test</li>
+ </ul></li>
+
+<li>Returns: True if the string is a valid URL and false if otherwise.</li>
+</ul></div>
+<div class="section">
+<h3><a name="JOIN"></a><tt>JOIN</tt></h3>
+
+<ul>
+
+<li>Description: Joins the components in the list of strings with the
specified delimiter.</li>
+
+<li>Input:
+
+<ul>
+
+<li>list - List of strings</li>
+
+<li>delim - String delimiter</li>
+ </ul></li>
+
+<li>Returns: String</li>
+</ul></div>
+<div class="section">
+<h3><a name="KAFKA_GET"></a><tt>KAFKA_GET</tt></h3>
+
+<ul>
+
+<li>Description: Retrieves messages from a Kafka topic. Subsequent calls will
continue retrieving messages sequentially from the original offset.</li>
+
+<li>Input:
+
+<ul>
+
+<li>topic - The name of the Kafka topic.</li>
+
+<li>count - The number of Kafka messages to retrieve.</li>
+
+<li>config - Optional map of key/values that override any global
properties.</li>
+ </ul></li>
+
+<li>Returns: List of String</li>
+</ul></div>
+<div class="section">
+<h3><a name="KAFKA_PROPS"></a><tt>KAFKA_PROPS</tt></h3>
+
+<ul>
+
+<li>Description: Retrieves the Kafka properties that are used by other KAFKA_*
functions like KAFKA_GET and KAFKA_PUT. The Kafka properties are compiled from
a set of default properties, the global properties, and any overrides.</li>
+
+<li>Input:
+
+<ul>
+
+<li>config - An optional map of key/values that override any global
properties.</li>
+ </ul></li>
+
+<li>Returns: Map of key/value pairs</li>
+</ul></div>
+<div class="section">
+<h3><a name="KAFKA_PUT"></a><tt>KAFKA_PUT</tt></h3>
+
+<ul>
+
+<li>Description: Sends messages to a Kafka topic.</li>
+
+<li>Input:
+
+<ul>
+
+<li>topic - The name of the Kafka topic.</li>
+
+<li>messages - A list of messages to write.</li>
+
+<li>config - Optional map of key/values that override any global
properties.</li>
+ </ul></li>
+
+<li>Returns: n/a</li>
+</ul></div>
+<div class="section">
+<h3><a name="KAFKA_TAIL"></a><tt>KAFKA_TAIL</tt></h3>
+
+<ul>
+
+<li>Description: etrieves messages from a Kafka topic always starting with the
most recent message first.</li>
+
+<li>Input:
+
+<ul>
+
+<li>topic - The name of the Kafka topic.</li>
+
+<li>count - The number of Kafka messages to retrieve.</li>
+
+<li>config - Optional map of key/values that override any global
properties.</li>
+ </ul></li>
+
+<li>Returns: List of String</li>
+</ul></div>
+<div class="section">
+<h3><a name="LENGTH"></a><tt>LENGTH</tt></h3>
+
+<ul>
+
+<li>Description: Returns the length of a string or size of a collection.
Returns 0 for empty or null Strings</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - Object of string or collection type (e.g. list)</li>
+ </ul></li>
+
+<li>Returns: Integer</li>
+</ul></div>
+<div class="section">
+<h3><a name="MAAS_GET_ENDPOINT"></a><tt>MAAS_GET_ENDPOINT</tt></h3>
+
+<ul>
+
+<li>Description: Inspects ZooKeeper and returns a map containing the name,
version and url for the model referred to by the input parameters.</li>
+
+<li>Input:
+
+<ul>
+
+<li>model_name - The name of the model</li>
+
+<li>model_version - The optional version of the model. If the model version is
not specified, the most current version is used.</li>
+ </ul></li>
+
+<li>Returns: A map containing the name, version, and url for the REST endpoint
(fields named name, version and url). Note that the output of this function is
suitable for input into the first argument of MAAS_MODEL_APPLY.</li>
+</ul></div>
+<div class="section">
+<h3><a name="MAAS_MODEL_APPLY"></a><tt>MAAS_MODEL_APPLY</tt></h3>
+
+<ul>
+
+<li>Description: Returns the output of a model deployed via Model as a
Service. NOTE: Results are cached locally for 10 minutes.</li>
+
+<li>Input:
+
+<ul>
+
+<li>endpoint - A map containing the name, version, and url for the REST
endpoint</li>
+
+<li>function - The optional endpoint path; default is
‘apply’</li>
+
+<li>model_args - A Dictionary of arguments for the model (these become request
params)</li>
+ </ul></li>
+
+<li>Returns: The output of the model deployed as a REST endpoint in Map form.
Assumes REST endpoint returns a JSON Map.</li>
+</ul></div>
+<div class="section">
+<h3><a name="MAP_EXISTS"></a><tt>MAP_EXISTS</tt></h3>
+
+<ul>
+
+<li>Description: Checks for existence of a key in a map.</li>
+
+<li>Input:
+
+<ul>
+
+<li>key - The key to check for existence</li>
+
+<li>map - The map to check for existence of the key</li>
+ </ul></li>
+
+<li>Returns: True if the key is found in the map and false if otherwise.</li>
+</ul></div>
+<div class="section">
+<h3><a name="MAP_GET"></a><tt>MAP_GET</tt></h3>
+
+<ul>
+
+<li>Description: Gets the value associated with a key from a map</li>
+
+<li>Input:
+
+<ul>
+
+<li>key - The key</li>
+
+<li>map - The map</li>
+
+<li>default - Optionally the default value to return if the key is not in the
map.</li>
+ </ul></li>
+
+<li>Returns: The object associated with the key in the map. If no value is
associated with the key and default is specified, then default is returned. If
no value is associated with the key or default, then null is returned.</li>
+</ul></div>
+<div class="section">
+<h3><a name="MONTH"></a><tt>MONTH</tt></h3>
+
+<ul>
+
+<li>Description: The number representing the month. The first month, January,
has a value of 0.</li>
+
+<li>Input:
+
+<ul>
+
+<li>dateTime - The datetime as a long representing the milliseconds since unix
epoch</li>
+ </ul></li>
+
+<li>Returns: The current month (0-based).</li>
+</ul></div>
+<div class="section">
+<h3><a name="PROFILE_GET"></a><tt>PROFILE_GET</tt></h3>
+
+<ul>
+
+<li>Description: Retrieves a series of values from a stored profile.</li>
+
+<li>Input:
+
+<ul>
+
+<li>profile - The name of the profile.</li>
+
+<li>entity - The name of the entity.</li>
+
+<li>periods - The list of profile periods to grab. These are ProfilePeriod
objects.</li>
+
+<li>groups_list - Optional, must correspond to the ‘groupBy’
list used in profile creation - List (in square brackets) of groupBy values
used to filter the profile. Default is the empty list, meaning groupBy was not
used when creating the profile.</li>
+
+<li>config_overrides - Optional - Map (in curly braces) of name:value pairs,
each overriding the global config parameter of the same name. Default is the
empty Map, meaning no overrides.</li>
+ </ul></li>
+
+<li>Returns: The selected profile measurements.</li>
+</ul></div>
+<div class="section">
+<h3><a name="PROFILE_FIXED"></a><tt>PROFILE_FIXED</tt></h3>
+
+<ul>
+
+<li>Description: The profile periods associated with a fixed lookback starting
from now</li>
+
+<li>Input:
+
+<ul>
+
+<li>durationAgo - How long ago should values be retrieved from?</li>
+
+<li>units - The units of ‘durationAgo’.</li>
+
+<li>config_overrides - Optional - Map (in curly braces) of name:value pairs,
each overriding the global config parameter of the same name. Default is the
empty Map, meaning no overrides.</li>
+ </ul></li>
+
+<li>Returns: The selected profile measurement timestamps. These are
ProfilePeriod objects.</li>
+</ul></div>
+<div class="section">
+<h3><a name="PROTOCOL_TO_NAME"></a><tt>PROTOCOL_TO_NAME</tt></h3>
+
+<ul>
+
+<li>Description: Converts the IANA protocol number to the protocol name</li>
+
+<li>Input:
+
+<ul>
+
+<li>IANA Number</li>
+ </ul></li>
+
+<li>Returns: The protocol name associated with the IANA number.</li>
+</ul></div>
+<div class="section">
+<h3><a name="REGEXP_MATCH"></a><tt>REGEXP_MATCH</tt></h3>
+
+<ul>
+
+<li>Description: Determines whether a regex matches a string</li>
+
+<li>Input:
+
+<ul>
+
+<li>string - The string to test</li>
+
+<li>pattern - The proposed regex pattern</li>
+ </ul></li>
+
+<li>Returns: True if the regex pattern matches the string and false if
otherwise.</li>
+</ul></div>
+<div class="section">
+<h3><a name="STRING_ENTROPY"></a><tt>STRING_ENTROPY</tt></h3>
+
+<ul>
+
+<li>Description: Computes the base-2 shannon entropy of a string.</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - String</li>
+ </ul></li>
+
+<li>Returns: The base-2 shannon entropy of the string (<a class="externalLink"
href="https://en.wikipedia.org/wiki/Entropy_(information_theory)#Definition)">https://en.wikipedia.org/wiki/Entropy_(information_theory)#Definition)</a>.
The unit of this is bits.</li>
+</ul></div>
+<div class="section">
+<h3><a name="SPLIT"></a><tt>SPLIT</tt></h3>
+
+<ul>
+
+<li>Description: Splits the string by the delimiter.</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - String to split</li>
+
+<li>delim - String delimiter</li>
+ </ul></li>
+
+<li>Returns: List of strings</li>
+</ul></div>
+<div class="section">
+<h3><a name="STARTS_WITH"></a><tt>STARTS_WITH</tt></h3>
+
+<ul>
+
+<li>Description: Determines whether a string starts with a prefix</li>
+
+<li>Input:
+
+<ul>
+
+<li>string - The string to test</li>
+
+<li>prefix - The proposed prefix</li>
+ </ul></li>
+
+<li>Returns: True if the string starts with the specified prefix and false if
otherwise</li>
+</ul></div>
+<div class="section">
+<h3><a name="SYSTEM_ENV_GET"></a><tt>SYSTEM_ENV_GET</tt></h3>
+
+<ul>
+
+<li>Description: Returns the value associated with an environment variable</li>
+
+<li>Input:
+
+<ul>
+
+<li>env_var - Environment variable name to get the value for</li>
+ </ul></li>
+
+<li>Returns: String</li>
+</ul></div>
+<div class="section">
+<h3><a name="SYSTEM_PROPERTY_GET"></a><tt>SYSTEM_PROPERTY_GET</tt></h3>
+
+<ul>
+
+<li>Description: Returns the value associated with a Java system property</li>
+
+<li>Input:
+
+<ul>
+
+<li>key - Property to get the value for</li>
+ </ul></li>
+
+<li>Returns: String</li>
+</ul></div>
+<div class="section">
+<h3><a name="TO_DOUBLE"></a><tt>TO_DOUBLE</tt></h3>
+
+<ul>
+
+<li>Description: Transforms the first argument to a double precision
number</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - Object of string or numeric type</li>
+ </ul></li>
+
+<li>Returns: Double version of the first argument</li>
+</ul></div>
+<div class="section">
+<h3><a name="TO_EPOCH_TIMESTAMP"></a><tt>TO_EPOCH_TIMESTAMP</tt></h3>
+
+<ul>
+
+<li>Description: Returns the epoch timestamp of the dateTime in the specified
format. If the format does not have a timestamp and you wish to assume a given
timestamp, you may specify the timezone optionally.</li>
+
+<li>Input:
+
+<ul>
+
+<li>dateTime - DateTime in String format</li>
+
+<li>format - DateTime format as a String</li>
+
+<li>timezone - Optional timezone in String format</li>
+ </ul></li>
+
+<li>Returns: Epoch timestamp</li>
+</ul></div>
+<div class="section">
+<h3><a name="TO_FOAT"></a><tt>TO_FOAT</tt></h3>
+
+<ul>
+
+<li>Description: Transforms the first argument to a float</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - Object of string or numeric type</li>
+ </ul></li>
+
+<li>Returns: Float version of the first argument</li>
+</ul></div>
+<div class="section">
+<h3><a name="TO_INTEGER"></a><tt>TO_INTEGER</tt></h3>
+
+<ul>
+
+<li>Description: Transforms the first argument to an integer</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - Object of string or numeric type</li>
+ </ul></li>
+
+<li>Returns: Integer version of the first argument</li>
+</ul></div>
+<div class="section">
+<h3><a name="TO_LONG"></a><tt>TO_LONG</tt></h3>
+
+<ul>
+
+<li>Description: Transforms the first argument to a long integer</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - Object of string or numeric type</li>
+ </ul></li>
+
+<li>Returns: Long version of the first argument</li>
+</ul></div>
+<div class="section">
+<h3><a name="TO_LOWER"></a><tt>TO_LOWER</tt></h3>
+
+<ul>
+
+<li>Description: Transforms the first argument to a lowercase string</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - String</li>
+ </ul></li>
+
+<li>Returns: Lowercase string</li>
+</ul></div>
+<div class="section">
+<h3><a name="TO_STRING"></a><tt>TO_STRING</tt></h3>
+
+<ul>
+
+<li>Description: Transforms the first argument to a string</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - Object</li>
+ </ul></li>
+
+<li>Returns: String</li>
+</ul></div>
+<div class="section">
+<h3><a name="TO_UPPER"></a><tt>TO_UPPER</tt></h3>
+
+<ul>
+
+<li>Description: Transforms the first argument to an uppercase string</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - String</li>
+ </ul></li>
+
+<li>Returns: Uppercase string</li>
+</ul></div>
+<div class="section">
+<h3><a name="TRIM"></a><tt>TRIM</tt></h3>
+
+<ul>
+
+<li>Description: Trims whitespace from both sides of a string.</li>
+
+<li>Input:
+
+<ul>
+
+<li>input - String</li>
+ </ul></li>
+
+<li>Returns: String</li>
+</ul></div>
+<div class="section">
+<h3><a name="URL_TO_HOST"></a><tt>URL_TO_HOST</tt></h3>
+
+<ul>
+
+<li>Description: Extract the hostname from a URL.</li>
+
+<li>Input:
+
+<ul>
+
+<li>url - URL in String form</li>
+ </ul></li>
+
+<li>Returns: The hostname from the URL as a String. e.g.
URL_TO_HOST(’<a class="externalLink"
href="http://www.yahoo.com/foo'";>http://www.yahoo.com/foo'</a>) would
yield ‘www.yahoo.com’</li>
+</ul></div>
+<div class="section">
+<h3><a name="URL_TO_PATH"></a><tt>URL_TO_PATH</tt></h3>
+
+<ul>
+
+<li>Description: Extract the path from a URL.</li>
+
+<li>Input:
+
+<ul>
+
+<li>url - URL in String form</li>
+ </ul></li>
+
+<li>Returns: The path from the URL as a String. e.g. URL_TO_PATH(’<a
class="externalLink"
href="http://www.yahoo.com/foo'";>http://www.yahoo.com/foo'</a>) would
yield ‘foo’</li>
+</ul></div>
+<div class="section">
+<h3><a name="URL_TO_PORT"></a><tt>URL_TO_PORT</tt></h3>
+
+<ul>
+
+<li>Description: Extract the port from a URL. If the port is not explicitly
stated in the URL, then an implicit port is inferred based on the protocol.</li>
+
+<li>Input:
+
+<ul>
+
+<li>url - URL in string form</li>
+ </ul></li>
+
+<li>Returns: The port used in the URL as an integer (for example,
URL_TO_PORT(’<a class="externalLink"
href="http://www.yahoo.com/foo'";>http://www.yahoo.com/foo'</a>) would
yield 80)</li>
+</ul></div>
+<div class="section">
+<h3><a name="URL_TO_PROTOCOL"></a><tt>URL_TO_PROTOCOL</tt></h3>
+
+<ul>
+
+<li>Description: Extract the protocol from a URL.</li>
+
+<li>Input:
+
+<ul>
+
+<li>url - URL in String form</li>
+ </ul></li>
+
+<li>Returns: The protocol from the URL as a String. e.g.
URL_TO_PROTOCOL(’<a class="externalLink"
href="http://www.yahoo.com/foo'";>http://www.yahoo.com/foo'</a>) would
yield ‘http’</li>
+</ul></div>
+<div class="section">
+<h3><a name="WEEK_OF_MONTH"></a><tt>WEEK_OF_MONTH</tt></h3>
+
+<ul>
+
+<li>Description: The numbered week within the month. The first week within the
month has a value of 1.</li>
+
+<li>Input:
+
+<ul>
+
+<li>dateTime - The datetime as a long representing the milliseconds since unix
epoch</li>
+ </ul></li>
+
+<li>Returns: The numbered week within the month.</li>
+</ul></div>
+<div class="section">
+<h3><a name="WEEK_OF_YEAR"></a><tt>WEEK_OF_YEAR</tt></h3>
+
+<ul>
+
+<li>Description: The numbered week within the year. The first week in the year
has a value of 1.</li>
+
+<li>Input:
+
+<ul>
+
+<li>dateTime - The datetime as a long representing the milliseconds since unix
epoch</li>
+ </ul></li>
+
+<li>Returns: The numbered week within the year.</li>
+</ul></div>
+<div class="section">
+<h3><a name="YEAR"></a><tt>YEAR</tt></h3>
+
+<ul>
+
+<li>Description: The number representing the year.</li>
+
+<li>Input:
+
+<ul>
+
+<li>dateTime - The datetime as a long representing the milliseconds since unix
epoch</li>
+ </ul></li>
+
+<li>Returns: The current year</li>
+</ul>
+<p>The following is an example query (i.e. a function which returns a boolean)
which would be seen possibly in threat triage:</p>
+<p><tt>IN_SUBNET( ip, '192.168.0.0/24') or ip in [ '10.0.0.1', '10.0.0.2' ] or
exists(is_local)</tt></p>
+<p>This evaluates to true precisely when one of the following is true:</p>
+
+<ul>
+
+<li>The value of the <tt>ip</tt> field is in the <tt>192.168.0.0/24</tt>
subnet</li>
+
+<li>The value of the <tt>ip</tt> field is <tt>10.0.0.1</tt> or
<tt>10.0.0.2</tt></li>
+
+<li>The field <tt>is_local</tt> exists</li>
+</ul>
+<p>The following is an example transformation which might be seen in a field
transformation:</p>
+<p><tt>TO_EPOCH_TIMESTAMP(timestamp, 'yyyy-MM-dd HH:mm:ss', MAP_GET(dc, dc2tz,
'UTC'))</tt></p>
+<p>For a message with a <tt>timestamp</tt> and <tt>dc</tt> field, we want to
set the transform the timestamp to an epoch timestamp given a timezone which we
will lookup in a separate map, called <tt>dc2tz</tt>.</p>
+<p>This will convert the timestamp field to an epoch timestamp based on the
</p>
+
+<ul>
+
+<li>Format <tt>yyyy-MM-dd HH:mm:ss</tt></li>
+
+<li>The value in <tt>dc2tz</tt> associated with the value associated with
field <tt>dc</tt>, defaulting to <tt>UTC</tt></li>
+</ul></div></div>
+<div class="section">
+<h2><a name="Stellar_Shell"></a>Stellar Shell</h2>
+<p>A REPL (Read Eval Print Loop) for the Stellar language that helps in
debugging, troubleshooting and learning Stellar. The Stellar DSL (domain
specific language) is used to act upon streaming data within Apache Storm. It
is difficult to troubleshoot Stellar when it can only be executed within a
Storm topology. This REPL is intended to help mitigate that problem by allowing
a user to replicate data encountered in production, isolate initialization
errors, or understand function resolution problems.</p>
+<p>The shell supports customization via <tt>~/.inputrc</tt> as it is backed by
a proper readline implementation. </p>
+<p>Shell-like operations are supported such as </p>
+
+<ul>
+
+<li>reverse search via ctrl-r</li>
+
+<li>autocomplete of Stellar functions and variables via tab
+
+<ul>
+
+<li>NOTE: Stellar functions are read via a classpath search which happens in
the background. Until that happens, autocomplete will not include function
names.</li>
+ </ul></li>
+
+<li>emacs or vi keybindings for edit mode</li>
+</ul>
+<div class="section">
+<h3><a name="Getting_Started"></a>Getting Started</h3>
+
+<div class="source">
+<div class="source">
+<pre>$ $METRON_HOME/bin/stellar
+
+Stellar, Go!
+{es.clustername=metron, es.ip=node1, es.port=9300,
es.date.format=yyyy.MM.dd.HH}
+
+[Stellar]>>> %functions
+BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, DAY_OF_MONTH, DAY_OF_WEEK,
DAY_OF_YEAR, ...
+
+[Stellar]>>> ?PROTOCOL_TO_NAME
+PROTOCOL_TO_NAME
+ desc: Convert the IANA protocol number to the protocol name
+ args: IANA Number
+ ret: The protocol name associated with the IANA number.
+
+[Stellar]>>> ip.protocol := 6
+6
+[Stellar]>>> PROTOCOL_TO_NAME(ip.protocol)
+TCP
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Command_Line_Options"></a>Command Line Options</h3>
+
+<div class="source">
+<div class="source">
+<pre>$ $METRON_HOME/bin/stellar -h
+usage: stellar
+ -h,--help Print help
+ -irc,--inputrc <arg> File containing the inputrc if not the default
+ ~/.inputrc
+ -v,--variables <arg> File containing a JSON Map of variables
+ -z,--zookeeper <arg> Zookeeper URL
+ -na,--no_ansi Make the input prompt not use ANSI colors.
+</pre></div></div>
+<div class="section">
+<h4><a name="a-v_--variables"></a><tt>-v, --variables</tt></h4>
+<p><i>Optional</i></p>
+<p>Optionally load a JSON map which contains variable assignments. This is
intended to give you the ability to save off a message from Metron and work on
it via the REPL.</p></div>
+<div class="section">
+<h4><a name="a-z_--zookeeper"></a><tt>-z, --zookeeper</tt></h4>
+<p><i>Optional</i></p>
+<p>Attempts to connect to Zookeeper and read the Metron global configuration.
Stellar functions may require the global configuration to work properly. If
found, the global configuration values are printed to the console.</p>
+
+<div class="source">
+<div class="source">
+<pre>$ $METRON_HOME/bin/stellar -z node1:2181
+Stellar, Go!
+{es.clustername=metron, es.ip=node1, es.port=9300,
es.date.format=yyyy.MM.dd.HH}
+[Stellar]>>>
+</pre></div></div></div></div>
+<div class="section">
+<h3><a name="Variable_Assignment"></a>Variable Assignment</h3>
+<p>Stellar has no concept of variable assignment. For testing and debugging
purposes, it is important to be able to create variables that simulate data
contained within incoming messages. The REPL has created a means for a user to
perform variable assignment outside of the core Stellar language. This is done
via the <tt>:=</tt> operator, such as <tt>foo := 1 + 1</tt> would assign the
result of the stellar expression <tt>1 + 1</tt> to the variable
<tt>foo</tt>.</p>
+
+<div class="source">
+<div class="source">
+<pre>[Stellar]>>> foo := 2 + 2
+4.0
+[Stellar]>>> 2 + 2
+4.0
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Magic_Commands"></a>Magic Commands</h3>
+<p>The REPL has a set of magic commands that provide the REPL user with
information about the Stellar execution environment. The following magic
commands are supported.</p>
+<div class="section">
+<h4><a name="afunctions"></a><tt>%functions</tt></h4>
+<p>This command lists all functions resolvable in the Stellar environment.
Stellar searches the classpath for Stellar functions. This can make it
difficult in some cases to understand which functions are resolvable. </p>
+
+<div class="source">
+<div class="source">
+<pre>[Stellar]>>> %functions
+BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, DAY_OF_MONTH, DAY_OF_WEEK,
DAY_OF_YEAR,
+DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENDS_WITH, GET,
GET_FIRST,
+GET_LAST, IN_SUBNET, IS_DATE, IS_DOMAIN, IS_EMAIL, IS_EMPTY, IS_INTEGER,
IS_IP, IS_URL,
+JOIN, LENGTH, MAAS_GET_ENDPOINT, MAAS_MODEL_APPLY, MAP_EXISTS, MAP_GET, MONTH,
PROTOCOL_TO_NAME,
+REGEXP_MATCH, SPLIT, STARTS_WITH, STATS_ADD, STATS_COUNT,
STATS_GEOMETRIC_MEAN, STATS_INIT,
+STATS_KURTOSIS, STATS_MAX, STATS_MEAN, STATS_MERGE, STATS_MIN,
STATS_PERCENTILE,
+STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS,
STATS_SUM,
+STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, TO_DOUBLE,
TO_EPOCH_TIMESTAMP, TO_FLOAT,
+TO_INTEGER, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH,
URL_TO_PORT,
+URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR
+[Stellar]>>>
+</pre></div></div></div>
+<div class="section">
+<h4><a name="avars"></a><tt>%vars</tt></h4>
+<p>Lists all variables in the Stellar environment.</p>
+
+<div class="source">
+<div class="source">
+<pre>Stellar, Go!
+{es.clustername=metron, es.ip=node1, es.port=9300,
es.date.format=yyyy.MM.dd.HH}
+[Stellar]>>> %vars
+[Stellar]>>> foo := 2 + 2
+4.0
+[Stellar]>>> %vars
+foo = 4.0
+</pre></div></div></div>
+<div class="section">
+<h4><a name="afunction"></a><tt>?<function></tt></h4>
+<p>Returns formatted documentation of the Stellar function. Provides the
description of the function along with the expected arguments.</p>
+
+<div class="source">
+<div class="source">
+<pre>[Stellar]>>> ?BLOOM_ADD
+BLOOM_ADD
+ desc: Adds an element to the bloom filter passed in
+ args: bloom - The bloom filter, value* - The values to add
+ ret: Bloom Filter
+[Stellar]>>> ?IS_EMAIL
+IS_EMAIL
+ desc: Tests if a string is a valid email address
+ args: address - The String to test
+ ret: True if the string is a valid email address and false otherwise.
+[Stellar]>>>
+</pre></div></div>
+<p><a name="Global_Configuration"></a></p>
+<h1>Global Configuration</h1>
+<p>The format of the global enrichment is a JSON String to Object map. This is
intended for configuration which is non sensor specific configuration.</p>
+<p>This configuration is stored in zookeeper, but looks something like</p>
+
+<div class="source">
+<div class="source">
+<pre>{
+ "es.clustername": "metron",
+ "es.ip": "node1",
+ "es.port": "9300",
+ "es.date.format": "yyyy.MM.dd.HH",
+ "fieldValidations" : [
+ {
+ "input" : [ "ip_src_addr",
"ip_dst_addr" ],
+ "validation" : "IP",
+ "config" : {
+ "type" : "IPV4"
+ }
+ }
+ ]
+}
+</pre></div></div></div></div></div>
+<div class="section">
+<h2><a name="Validation_Framework"></a>Validation Framework</h2>
+<p>Inside of the global configuration, there is a validation framework in
place that enables the validation that messages coming from all parsers are
valid. This is done in the form of validation plugins where assertions about
fields or whole messages can be made. </p>
+<p>The format for this is a <tt>fieldValidations</tt> field inside of global
config. This is associated with an array of field validation objects structured
like so:</p>
+
+<ul>
+
+<li><tt>input</tt> : An array of input fields or a single field. If this is
omitted, then the whole messages is passed to the validator.</li>
+
+<li><tt>config</tt> : A String to Object map for validation configuration.
This is optional if the validation function requires no configuration.</li>
+
+<li><tt>validation</tt> : The validation function to be used. This is one of
+
+<ul>
+
+<li><tt>STELLAR</tt> : Execute a Stellar Language statement. Expects the query
string in the <tt>condition</tt> field of the config.</li>
+
+<li><tt>IP</tt> : Validates that the input fields are an IP address. By
default, if no configuration is set, it assumes <tt>IPV4</tt>, but you can
specify the type by passing in the config by passing in <tt>type</tt> with
either <tt>IPV6</tt> or <tt>IPV4</tt> or by passing in a list
[<tt>IPV4</tt>,<tt>IPV6</tt>] in which case the input(s) will be validated
against both.</li>
+
+<li><tt>DOMAIN</tt> : Validates that the fields are all domains.</li>
+
+<li><tt>EMAIL</tt> : Validates that the fields are all email addresses</li>
+
+<li><tt>URL</tt> : Validates that the fields are all URLs</li>
+
+<li><tt>DATE</tt> : Validates that the fields are a date. Expects
<tt>format</tt> in the config.</li>
+
+<li><tt>INTEGER</tt> : Validates that the fields are an integer. String
representation of an integer is allowed.</li>
+
+<li><tt>REGEX_MATCH</tt> : Validates that the fields match a regex. Expects
<tt>pattern</tt> in the config.</li>
+
+<li><tt>NOT_EMPTY</tt> : Validates that the fields exist and are not empty
(after trimming.)</li>
+ </ul></li>
+</ul>
+<p><a name="Management_Utility"></a></p>
+<h1>Management Utility</h1>
+<p>Configurations should be stored on disk in the following structure starting
at <tt>$BASE_DIR</tt>:</p>
+
+<ul>
+
+<li>global.json : The global config</li>
+
+<li><tt>sensors</tt> : The subdirectory containing sensor enrichment
configuration JSON (e.g. <tt>snort.json</tt>, <tt>bro.json</tt>)</li>
+</ul>
+<p>By default, this directory as deployed by the ansible infrastructure is at
<tt>$METRON_HOME/config/zookeeper</tt></p>
+<p>While the configs are stored on disk, they must be loaded into Zookeeper to
be used. To this end, there is a utility program to assist in this called
<tt>$METRON_HOME/bin/zk_load_config.sh</tt></p>
+<p>This has the following options:</p>
+
+<div class="source">
+<div class="source">
+<pre> -f,--force Force operation
+ -h,--help Generate Help screen
+ -i,--input_dir <DIR> The input directory containing
+ the configuration files named
+ like "$source.json"
+ -m,--mode <MODE> The mode of operation: DUMP,
+ PULL, PUSH
+ -o,--output_dir <DIR> The output directory which
will
+ store the JSON configuration
+ from Zookeeper
+ -z,--zk_quorum <host:port,[host:port]*> Zookeeper Quorum URL
+ (zk1:port,zk2:port,...)
+</pre></div></div>
+<p>Usage examples:</p>
+
+<ul>
+
+<li>To dump the existing configs from zookeeper on the singlenode vagrant
machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m DUMP</tt></li>
+
+<li>To push the configs into zookeeper on the singlenode vagrant machine:
<tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i
$METRON_HOME/config/zookeeper</tt></li>
+
+<li>To pull the configs from zookeeper to the singlenode vagrant machine disk:
<tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PULL -o
$METRON_HOME/config/zookeeper -f</tt></li>
+</ul></div>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2017.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>
