metron git commit: METRON-1619: Stellar empty collections should be considered false in boolean expressions closes apache/incubator-metron#1064
Repository: metron
Updated Branches:
refs/heads/master 81282de28 -> c4c790dbe
METRON-1619: Stellar empty collections should be considered false in boolean
expressions closes apache/incubator-metron#1064
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c4c790db
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c4c790db
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c4c790db
Branch: refs/heads/master
Commit: c4c790dbe4c481c8f19293ef9d6134539674d253
Parents: 81282de
Author: cstella
Authored: Thu Jun 28 15:57:08 2018 -0400
Committer: cstella
Committed: Thu Jun 28 15:57:08 2018 -0400
--
metron-stellar/stellar-common/README.md | 14
.../metron/stellar/common/StellarCompiler.java | 82 +---
.../org/apache/metron/stellar/dsl/Token.java| 1 +
.../stellar/dsl/functions/BasicStellarTest.java | 25 ++
.../metron/stellar/dsl/functions/MatchTest.java | 35 +
use-cases/forensic_clustering/README.md | 2 +-
use-cases/geographic_login_outliers/README.md | 5 +-
use-cases/typosquat_detection/README.md | 2 +-
8 files changed, 149 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/c4c790db/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md
b/metron-stellar/stellar-common/README.md
index d8d09b4..0dd8e16 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -54,6 +54,20 @@ The Stellar language supports the following:
* The ability to have parenthesis to make order of operations explicit
* User defined functions, including Lambda expressions
+### Boolean Expressions
+
+Variables may be used in boolean expressions and variables which are not
+explicitly boolean may be interpreted as booleans subject to the
+following rules:
+* Similar to python and javascript, empty collections (e.g. `[]`) will be
+ interpreted as `false`
+* Similar to python and javascript, missing variables will be
+ interpreted as `false`
+* Variables set to `null` will be interpreted as `false`
+
+Otherwise, boolean variables will be interpreted as their values
+reflect.
+
### Stellar Language Keywords
The following keywords need to be single quote escaped in order to be used in
Stellar expressions:
http://git-wip-us.apache.org/repos/asf/metron/blob/c4c790db/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
index 72f0d0a..8a328a2 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
@@ -29,7 +29,10 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
+
+import com.google.common.collect.Iterables;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.metron.stellar.common.evaluators.ArithmeticEvaluator;
@@ -104,6 +107,52 @@ public class StellarCompiler extends StellarBaseListener {
return tokenDeque;
}
+/**
+ * When treating empty or missing values as false, we need to ensure we
ONLY do so in a conditional context.
+ * @param tokenValueType
+ * @return
+ */
+private boolean isConditionalContext(Class tokenValueType) {
+ return tokenValueType != null && (
+ tokenValueType == BooleanArg.class
+|| tokenValueType == IfExpr.class
+|| tokenValueType == MatchClauseCheckExpr.class
+ );
+}
+
+/**
+ * Determine if a token and value is an empty list in the appropriate
conditional context
+ * @param token
+ * @param value
+ * @return
+ */
+private boolean isEmptyList(Token token, Object value) {
+ if(value != null && isConditionalContext(token.getUnderlyingType())) {
+if (value instanceof Iterable) {
+ return Iterables.isEmpty((Iterable) value);
+} else if (value instanceof Map) {
+ return ((Map) value).isEmpty();
+}
+else {
+ return false;
+}
+ }else {
+return false;
+ }
+}
+
+/**
+ * Determine if a token is missing in a conditional context.
+ * @param token
+ * @return
+ */
+
metron git commit: METRON-1646 Sensor Stubs should work when kerberized (nickwallen) closes apache/metron#1087
Repository: metron
Updated Branches:
refs/heads/master 3a6fb38e5 -> 81282de28
METRON-1646 Sensor Stubs should work when kerberized (nickwallen) closes
apache/metron#1087
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/81282de2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/81282de2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/81282de2
Branch: refs/heads/master
Commit: 81282de288d94b1fa8c57917ff6a34079eca77e1
Parents: 3a6fb38
Author: nickwallen
Authored: Thu Jun 28 14:07:32 2018 -0400
Committer: nickallen
Committed: Thu Jun 28 14:07:32 2018 -0400
--
.../roles/sensor-stubs/templates/start-bro-stub | 25 ++-
.../sensor-stubs/templates/start-snort-stub | 25 ++-
.../roles/sensor-stubs/templates/start-yaf-stub | 26 ++--
3 files changed, 57 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/81282de2/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
--
diff --git
a/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
index 979de3d..24027b3 100644
--- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
@@ -21,14 +21,20 @@
# simulates the behavior of a sensor by sending canned telemetry data
# to a Kafka topic.
#
-# a subset of the canned data is randomly selected and is sent in
+# a subset of the canned data is randomly selected and is sent in
# batches. the timestamp of the message is altered to match current
-# system time. the number of messages sent in each batch, along with
+# system time. the number of messages sent in each batch, along with
# the time delay between batches can be configured.
#
# start-bro-stub
#
+METRON_SYSCONFIG="/etc/default/metron"
+if [ -f "$METRON_SYSCONFIG" ]; then
+ set -a
+ . "$METRON_SYSCONFIG"
+fi
+
#
# how long to delay between each 'batch' in seconds.
#
@@ -36,20 +42,27 @@ DELAY=${1:-{{ sensor_stubs_delay }}}
#
# how many messages to send in each 'batch'. the messages are drawn randomly
-# from the entire set of canned data.
+# from the entire set of canned data.
#
COUNT=${2:-{{ sensor_stubs_count }}}
INPUT="{{ sensor_stubs_data }}/bro.out"
PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
TOPIC="bro"
+SECURITY_ENABLED=${SECURITY_ENABLED:-false}
+KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
+
+if [ ${SECURITY_ENABLED,,} == 'true' ]; then
+ echo "Security enabled"
+ kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME
+fi
while true; do
-
+
# transform the bro timestamp and push to kafka
SEARCH="\"ts\"\:[0-9]\+\."
REPLACE="\"ts\"\:`date +%s`\."
- shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER
--broker-list {{ kafka_broker_url }} --topic $TOPIC
-
+ shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER
--broker-list $BROKERLIST --topic $TOPIC --security-protocol
$KAFKA_SECURITY_PROTOCOL
+
sleep $DELAY
done
http://git-wip-us.apache.org/repos/asf/metron/blob/81282de2/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
--
diff --git
a/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
index 3123782..c60c002 100644
--- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
@@ -21,14 +21,20 @@
# simulates the behavior of a sensor by sending canned telemetry data
# to a Kafka topic.
#
-# a subset of the canned data is randomly selected and is sent in
+# a subset of the canned data is randomly selected and is sent in
# batches. the timestamp of the message is altered to match current
-# system time. the number of messages sent in each batch, along with
+# system time. the number of messages sent in each batch, along with
# the time delay between batches can be configured.
#
# start-snort-stub
#
+METRON_SYSCONFIG="/etc/default/metron"
+if [ -f "$METRON_SYSCONFIG" ]; then
+ set -a
+ . "$METRON_SYSCONFIG"
+fi
+
#
# how long to delay between each 'batch' in seconds.
#
@@ -36,20 +42,27 @@ DELAY=${1:-{{ sensor_stubs_delay }}}
#
# how many messages to send in each 'batch'. the messages are drawn randomly
-# from the entire set of canned data.
+# from the entire set of canned data.
#
COUNT=${2:-{{ sensor_stubs_count }}}
INPUT="{{ sensor_stubs_data }}/snort.out"
metron git commit: METRON-1645: Check wether the Solr management pack is installed before configuring the solr principal name. closes apache/incubator-metron#1086
Repository: metron
Updated Branches:
refs/heads/master e0fc47582 -> 3a6fb38e5
METRON-1645: Check wether the Solr management pack is installed before
configuring the solr principal name. closes apache/incubator-metron#1086
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3a6fb38e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3a6fb38e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3a6fb38e
Branch: refs/heads/master
Commit: 3a6fb38e55c1b8b9c1fa9ec74ed233c68a18d076
Parents: e0fc475
Author: MohanDV
Authored: Thu Jun 28 09:33:21 2018 -0400
Committer: cstella
Committed: Thu Jun 28 09:33:21 2018 -0400
--
.../METRON/CURRENT/package/scripts/params/params_linux.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3a6fb38e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 07132a0..0ab9bcf 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -258,8 +258,9 @@ if security_enabled:
kafka_keytab_path = config['configurations']['kafka-env']['kafka_keytab']
nimbus_seeds = config['configurations']['storm-site']['nimbus.seeds']
-
-solr_principal_name = solr_principal_name.replace('_HOST',
hostname_lowercase)
+# Check wether Solr mpack is installed
+if 'solr-config-env' in config['configurations']:
+solr_principal_name = solr_principal_name.replace('_HOST',
hostname_lowercase)
# Management UI
metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", [hostname])[0]
