[
https://issues.apache.org/jira/browse/METRON-439?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15552821#comment-15552821
]
ASF GitHub Bot commented on METRON-439:
---------------------------------------
Github user mmiklavc commented on the issue:
https://github.com/apache/incubator-metron/pull/296
@ottobackwards The Stellar validation currently runs by passing an empty
member to the apply function. The original reason for the validation was to
help keep bad configs from ending up in zookeeper. Now with the Stellar REPL,
users should be able to more easily do their own client-side validations.
> Stellar : IS_EMPTY(host) throws exception
> -----------------------------------------
>
> Key: METRON-439
> URL: https://issues.apache.org/jira/browse/METRON-439
> Project: Metron
> Issue Type: Bug
> Affects Versions: 0.2.2BETA
> Reporter: Neha Sinha
> Assignee: Michael Miklavcic
>
> Hi,
> I am getting the following exception message when i try to use the "IS_EMPTY"
> stellar function.
> [root@metron-test1-3 enrichments]#
> /usr/metron/0.2.0BETA/bin/zk_load_configs.sh -z
> metron-test1-3.openstacklocal:2181 -m DUMP -i
> /usr/metron/0.2.0BETA/config/zookeeper/
> log4j:WARN No appenders could be found for logger
> (org.apache.curator.framework.imps.CuratorFrameworkImpl).
> log4j:WARN Please initialize the log4j system properly.
> log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more
> info.
> GLOBAL Config: global
> {
> "es.clustername": "metron",
> "es.ip": "metron-test1-10.openstacklocal",
> "es.port": "9300",
> "es.date.format": "yyyy.MM.dd.HH"
> }
> PARSER Config: bluecoat
> {
> "parserClassName":"org.apache.metron.parsers.bluecoat.BasicBluecoatParser",
> "sensorTopic":"bluecoat",
> "parserConfig": {}
> }
> PARSER Config: websphere
> {
> "parserClassName":"org.apache.metron.parsers.websphere.GrokWebSphereParser",
> "sensorTopic":"websphere",
> "parserConfig":
> {
> "grokPath":"/patterns/websphere",
> "patternLabel":"WEBSPHERE",
> "timestampField":"timestamp_string",
> "dateFormat":"yyyy MMM dd HH:mm:ss"
> }
> }
> PARSER Config: squid
> {
> "parserClassName": "org.apache.metron.parsers.GrokParser",
> "sensorTopic": "squid",
> "parserConfig": {
> "grokPath": "/patterns/squid",
> "patternLabel": "SQUID_DELIMITED",
> "timestampField": "timestamp"
> },
> "fieldTransformations" : [
> {
> "transformation" : "STELLAR"
> ,"output" : [ "full_hostname", "domain_without_subdomains" ]
> ,"config" : {
> "full_hostname" : "URL_TO_HOST(url)"
> ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"
> }
> }
> ]
> }
> PARSER Config: bro
> {
> "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
> "sensorTopic":"bro",
> "parserConfig": {},
> "fieldTransformations" : [
> {
> "transformation" : "STELLAR"
> ,"output" : [ "is_alert", "new_field" ]
> ,"config" : {
> "is_alert" :"true",
> "new_field" : "SPLIT(ip_dst_addr,'.')"
> }
> }
> ]
> }
> PARSER Config: snort
> {
> "parserClassName":"org.apache.metron.parsers.snort.BasicSnortParser",
> "sensorTopic":"snort",
> "parserConfig": {}
> }
> PARSER Config: yaf
> {
> "parserClassName":"org.apache.metron.parsers.GrokParser",
> "sensorTopic":"yaf",
> "fieldTransformations" : [
> {
> "input" : "protocol"
> ,"transformation": "IP_PROTOCOL"
> }
> ],
> "parserConfig":
> {
> "grokPath":"/patterns/yaf",
> "patternLabel":"YAF_DELIMITED",
> "timestampField":"start_time",
> "timeFields": ["start_time", "end_time"],
> "dateFormat":"yyyy-MM-dd HH:mm:ss.S"
> }
> }
> ENRICHMENT Config: websphere
> {
> "index": "websphere",
> "batchSize": 5,
> "enrichment": {
> "fieldMap": {
> "geo": [
> "ip_src_addr"
> ],
> "host": [
> "ip_src_addr"
> ]
> },
> "fieldToTypeMap": {
> "ip_src_addr": [
> "playful_classification"
> ]
> }
> }
> }
> Exception in thread "main" java.lang.RuntimeException: Unable to load {
> "index": "bro",
> "batchSize": 5,
> "enrichment" : {
> "fieldMap": {
> "geo": ["ip_dst_addr", "ip_src_addr"],
> "host": ["host"]
> }
> },
> "threatIntel": {
> "fieldMap": {
> "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
> },
> "fieldToTypeMap": {
> "ip_src_addr" : ["malicious_ip"],
> "ip_dst_addr" : ["malicious_ip"]
> },
> "triageConfig" : {
> "riskLevelRules" : {
> "exists(ip_dst_addr)" : 0.10,
> "IS_EMPTY(host)" : 0.91,
> "exists(ip_dst_port)" : 0.20,
> "exists(ip_src_port)" : 0.30000000000
> },
> "aggregator" : "MAX",
> "aggregationConfig":
> {
> "NEGATIVE_VALUES_TRUMP_CONF" : "false"
> }
> }
> }
> }
> at
> org.apache.metron.common.configuration.ConfigurationType.lambda$static$2(ConfigurationType.java:54)
> at
> org.apache.metron.common.configuration.ConfigurationType.deserialize(ConfigurationType.java:87)
> at
> org.apache.metron.common.configuration.ConfigurationsUtils.lambda$dumpConfigs$0(ConfigurationsUtils.java:331)
> at
> org.apache.metron.common.configuration.ConfigurationsUtils.visitConfigs(ConfigurationsUtils.java:323)
> at
> org.apache.metron.common.configuration.ConfigurationsUtils.visitConfigs(ConfigurationsUtils.java:306)
> at
> org.apache.metron.common.configuration.ConfigurationsUtils.dumpConfigs(ConfigurationsUtils.java:330)
> at
> org.apache.metron.common.cli.ConfigurationManager.dump(ConfigurationManager.java:115)
> at
> org.apache.metron.common.cli.ConfigurationManager.run(ConfigurationManager.java:177)
> at
> org.apache.metron.common.cli.ConfigurationManager.run(ConfigurationManager.java:161)
> at
> org.apache.metron.common.cli.ConfigurationManager.main(ConfigurationManager.java:198)
> Caused by: com.fasterxml.jackson.databind.JsonMappingException: Unable to pop
> an empty stack
> at [Source: {
> "index": "bro",
> "batchSize": 5,
> "enrichment" : {
> "fieldMap": {
> "geo": ["ip_dst_addr", "ip_src_addr"],
> "host": ["host"]
> }
> },
> "threatIntel": {
> "fieldMap": {
> "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
> },
> "fieldToTypeMap": {
> "ip_src_addr" : ["malicious_ip"],
> "ip_dst_addr" : ["malicious_ip"]
> },
> "triageConfig" : {
> "riskLevelRules" : {
> "exists(ip_dst_addr)" : 0.10,
> "IS_EMPTY(host)" : 0.91,
> "exists(ip_dst_port)" : 0.20,
> "exists(ip_src_port)" : 0.30000000000
> },
> "aggregator" : "MAX",
> "aggregationConfig":
> {
> "NEGATIVE_VALUES_TRUMP_CONF" : "false"
> }
> }
> {
> }
> }
> ; line: 24, column: 7] (through reference chain:
> org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig["threatIntel"]->org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig["triageConfig"]->org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig["riskLevelRules"])
> at
> com.fasterxml.jackson.databind.JsonMappingException.from(JsonMappingException.java:262)
> at
> com.fasterxml.jackson.databind.deser.SettableBeanProperty._throwAsIOE(SettableBeanProperty.java:537)
> at
> com.fasterxml.jackson.databind.deser.SettableBeanProperty._throwAsIOE(SettableBeanProperty.java:518)
> at
> com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:99)
> at
> com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:260)
> at
> com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:125)
> at
> com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize(SettableBeanProperty.java:490)
> at
> com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:95)
> at
> com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:260)
> at
> com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:125)
> at
> com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize(SettableBeanProperty.java:490)
> at
> com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:95)
> at
> com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:260)
> at
> com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:125)
> at
> com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:3807)
> at
> com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2797)
> at
> org.apache.metron.common.utils.JSONUtils.load(JSONUtils.java:79)
> at
> org.apache.metron.common.configuration.ConfigurationType.lambda$static$2(ConfigurationType.java:52)
> ... 9 more
> Caused by: org.apache.metron.common.dsl.ParseException: Unable to pop an
> empty stack
> at
> org.apache.metron.common.stellar.StellarCompiler.popStack(StellarCompiler.java:397)
> at
> org.apache.metron.common.stellar.StellarCompiler.exitTransformationFunc(StellarCompiler.java:250)
> at
> org.apache.metron.common.stellar.generated.StellarParser$TransformationFuncContext.exitRule(StellarParser.java:1634)
> at
> org.antlr.v4.runtime.Parser.triggerExitRuleEvent(Parser.java:422)
> at org.antlr.v4.runtime.Parser.exitRule(Parser.java:632)
> at
> org.apache.metron.common.stellar.generated.StellarParser.transformation(StellarParser.java:158)
> at
> org.apache.metron.common.stellar.BaseStellarProcessor.parse(BaseStellarProcessor.java:57)
> at
> org.apache.metron.common.stellar.StellarPredicateProcessor.parse(StellarPredicateProcessor.java:53)
> at
> org.apache.metron.common.stellar.StellarPredicateProcessor.parse(StellarPredicateProcessor.java:37)
> at
> org.apache.metron.common.stellar.BaseStellarProcessor.validate(BaseStellarProcessor.java:67)
> at
> org.apache.metron.common.stellar.BaseStellarProcessor.validate(BaseStellarProcessor.java:62)
> at
> org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig.setRiskLevelRules(ThreatTriageConfig.java:42)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at
> com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:97)
> ... 23 more
> I get the same error when i use :-
> IS_EMPTY(ip_src_addr)
> IS_EMPTY(protocol)
> However, IS_EMPTY('') and IS_EMPTY('someString') don't throw the above error
> message.
> Also, TO_LOWER(protocol) and TO_LOWER(host) don't throw any error msg.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
