Neha Sinha created METRON-440:
---------------------------------
Summary: DSL parse exception seen for Bro Topology
Key: METRON-440
URL: https://issues.apache.org/jira/browse/METRON-440
Project: Metron
Issue Type: Bug
Affects Versions: 0.2.2BETA
Reporter: Neha Sinha
I updated the bro parser to the following in my environment and uploaded to
zookeeper.
Post that i am seeing dsl parse exception messages for Bro topology.
Bro Parser
=========================================
PARSER Config: bro
{
"parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
"sensorTopic":"bro",
"parserConfig": {},
"fieldTransformations" : [
{
"transformation" : "STELLAR"
,"output" : [ "full_hostname", "domain_without_subdomains", "is_alert" ]
,"config" : {
"full_hostname" : "URL_TO_HOST(url)"
,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"
,"is_alert" :"true"
}
}
]
}
==================================================
Bro logs
===================================================
2016-08-23 10:54:45.108 b.s.d.executor [ERROR]
org.apache.metron.common.dsl.ParseException: Unable to pop an empty stack
at
org.apache.metron.common.stellar.StellarCompiler.popStack(StellarCompiler.java:397)
~[stormjar.jar:?]
at
org.apache.metron.common.stellar.StellarCompiler.exitTransformationFunc(StellarCompiler.java:250)
~[stormjar.jar:?]
at
org.apache.metron.common.stellar.generated.StellarParser$TransformationFuncContext.exitRule(StellarParser.java:1634)
~[stormjar.jar:?]
at org.antlr.v4.runtime.Parser.triggerExitRuleEvent(Parser.java:422)
~[stormjar.jar:?]
at org.antlr.v4.runtime.Parser.exitRule(Parser.java:632)
~[stormjar.jar:?]
at
org.apache.metron.common.stellar.generated.StellarParser.transformation(StellarParser.java:158)
~[stormjar.jar:?]
at
org.apache.metron.common.stellar.BaseStellarProcessor.parse(BaseStellarProcessor.java:57)
~[stormjar.jar:?]
at
org.apache.metron.common.field.transformation.StellarTransformation.map(StellarTransformation.java:46)
~[stormjar.jar:?]
at
org.apache.metron.common.configuration.FieldTransformer.transform(FieldTransformer.java:111)
~[stormjar.jar:?]
at
org.apache.metron.common.configuration.FieldTransformer.transformAndUpdate(FieldTransformer.java:123)
~[stormjar.jar:?]
at
org.apache.metron.parsers.bolt.ParserBolt.execute(ParserBolt.java:116)
[stormjar.jar:?]
at
backtype.storm.daemon.executor$fn__5492$tuple_action_fn__5494.invoke(executor.clj:684)
[storm-core-0.10.0.2.4.2.0-258.jar:0.10.0.2.4.2.0-258]
at
backtype.storm.daemon.executor$mk_task_receiver$fn__5415.invoke(executor.clj:431)
[storm-core-0.10.0.2.4.2.0-258.jar:0.10.0.2.4.2.0-258]
at
backtype.storm.disruptor$clojure_handler$reify__4991.onEvent(disruptor.clj:58)
[storm-core-0.10.0.2.4.2.0-258.jar:0.10.0.2.4.2.0-258]
at
backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:125)
[storm-core-0.10.0.2.4.2.0-258.jar:0.10.0.2.4.2.0-258]
at
backtype.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:99)
[storm-core-0.10.0.2.4.2.0-258.jar:0.10.0.2.4.2.0-258]
at
backtype.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:80)
[storm-core-0.10.0.2.4.2.0-258.jar:0.10.0.2.4.2.0-258]
at
backtype.storm.daemon.executor$fn__5492$fn__5505$fn__5556.invoke(executor.clj:813)
[storm-core-0.10.0.2.4.2.0-258.jar:0.10.0.2.4.2.0-258]
at backtype.storm.util$async_loop$fn__644.invoke(util.clj:479)
[storm-core-0.10.0.2.4.2.0-258.jar:0.10.0.2.4.2.0-258]
at clojure.lang.AFn.run(AFn.java:22) [clojure-1.6.0.jar:?]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_60]
===================================================
Zookeeper Dump
===================================================
[root@metron-test1-3 parsers]# /usr/metron/0.2.0BETA/bin/zk_load_configs.sh -z
metron-test1-3.openstacklocal:2181 -m DUMP -i
/usr/metron/0.2.0BETA/config/zookeeper/
log4j:WARN No appenders could be found for logger
(org.apache.curator.framework.imps.CuratorFrameworkImpl).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more
info.
GLOBAL Config: global
{
"es.clustername": "metron",
"es.ip": "metron-test1-10.openstacklocal",
"es.port": "9300",
"es.date.format": "yyyy.MM.dd.HH"
}
PARSER Config: bluecoat
{
"parserClassName":"org.apache.metron.parsers.bluecoat.BasicBluecoatParser",
"sensorTopic":"bluecoat",
"parserConfig": {}
}
PARSER Config: websphere
{
"parserClassName":"org.apache.metron.parsers.websphere.GrokWebSphereParser",
"sensorTopic":"websphere",
"parserConfig":
{
"grokPath":"/patterns/websphere",
"patternLabel":"WEBSPHERE",
"timestampField":"timestamp_string",
"dateFormat":"yyyy MMM dd HH:mm:ss"
}
}
PARSER Config: squid
{
"parserClassName": "org.apache.metron.parsers.GrokParser",
"sensorTopic": "squid",
"parserConfig": {
"grokPath": "/patterns/squid",
"patternLabel": "SQUID_DELIMITED",
"timestampField": "timestamp"
},
"fieldTransformations" : [
{
"transformation" : "STELLAR"
,"output" : [ "full_hostname", "domain_without_subdomains" ]
,"config" : {
"full_hostname" : "URL_TO_HOST(url)"
,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"
}
}
]
}
PARSER Config: bro
{
"parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
"sensorTopic":"bro",
"parserConfig": {},
"fieldTransformations" : [
{
"transformation" : "STELLAR"
,"output" : [ "full_hostname", "domain_without_subdomains", "is_alert" ]
,"config" : {
"full_hostname" : "URL_TO_HOST(url)"
,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"
,"is_alert" :"true"
}
}
]
}
PARSER Config: snort
{
"parserClassName":"org.apache.metron.parsers.snort.BasicSnortParser",
"sensorTopic":"snort",
"parserConfig": {}
}
PARSER Config: yaf
{
"parserClassName":"org.apache.metron.parsers.GrokParser",
"sensorTopic":"yaf",
"fieldTransformations" : [
{
"input" : "protocol"
,"transformation": "IP_PROTOCOL"
}
],
"parserConfig":
{
"grokPath":"/patterns/yaf",
"patternLabel":"YAF_DELIMITED",
"timestampField":"start_time",
"timeFields": ["start_time", "end_time"],
"dateFormat":"yyyy-MM-dd HH:mm:ss.S"
}
}
ENRICHMENT Config: websphere
{
"index": "websphere",
"batchSize": 5,
"enrichment": {
"fieldMap": {
"geo": [
"ip_src_addr"
],
"host": [
"ip_src_addr"
]
},
"fieldToTypeMap": {
"ip_src_addr": [
"playful_classification"
]
}
}
}
ENRICHMENT Config: bro
{
"index": "bro",
"batchSize": 5,
"enrichment" : {
"fieldMap": {
"geo": ["ip_dst_addr", "ip_src_addr"],
"host": ["host"]
}
},
"threatIntel": {
"fieldMap": {
"hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
},
"fieldToTypeMap": {
"ip_src_addr" : ["malicious_ip"],
"ip_dst_addr" : ["malicious_ip"]
}
}
}
ENRICHMENT Config: snort
{
"index": "snort",
"batchSize": 1,
"enrichment" : {
"fieldMap":
{
"geo": ["ip_dst_addr", "ip_src_addr"],
"host": ["host"]
}
},
"threatIntel" : {
"fieldMap":
{
"hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
},
"fieldToTypeMap":
{
"ip_src_addr" : ["malicious_ip"],
"ip_dst_addr" : ["malicious_ip"]
},
"triageConfig" : {
"riskLevelRules" : {
"not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24'))" : 10
},
"aggregator" : "MAX"
}
}
}
ENRICHMENT Config: yaf
{
"index": "yaf",
"batchSize": 5,
"enrichment" : {
"fieldMap":
{
"geo": ["ip_dst_addr", "ip_src_addr"],
"host": ["host"]
}
},
"threatIntel": {
"fieldMap":
{
"hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
},
"fieldToTypeMap":
{
"ip_src_addr" : ["malicious_ip"],
"ip_dst_addr" : ["malicious_ip"]
}
}
}
===================================================
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
