Neha Sinha created METRON-439: --------------------------------- Summary: Stellar : IS_EMPTY(host) throws exception Key: METRON-439 URL: https://issues.apache.org/jira/browse/METRON-439 Project: Metron Issue Type: Bug Affects Versions: 0.2.2BETA Reporter: Neha Sinha
Hi, I am getting the following exception message when i try to use the "IS_EMPTY" stellar function. [root@metron-test1-3 enrichments]# /usr/metron/0.2.0BETA/bin/zk_load_configs.sh -z metron-test1-3.openstacklocal:2181 -m DUMP -i /usr/metron/0.2.0BETA/config/zookeeper/ log4j:WARN No appenders could be found for logger (org.apache.curator.framework.imps.CuratorFrameworkImpl). log4j:WARN Please initialize the log4j system properly. log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info. GLOBAL Config: global { "es.clustername": "metron", "es.ip": "metron-test1-10.openstacklocal", "es.port": "9300", "es.date.format": "yyyy.MM.dd.HH" } PARSER Config: bluecoat { "parserClassName":"org.apache.metron.parsers.bluecoat.BasicBluecoatParser", "sensorTopic":"bluecoat", "parserConfig": {} } PARSER Config: websphere { "parserClassName":"org.apache.metron.parsers.websphere.GrokWebSphereParser", "sensorTopic":"websphere", "parserConfig": { "grokPath":"/patterns/websphere", "patternLabel":"WEBSPHERE", "timestampField":"timestamp_string", "dateFormat":"yyyy MMM dd HH:mm:ss" } } PARSER Config: squid { "parserClassName": "org.apache.metron.parsers.GrokParser", "sensorTopic": "squid", "parserConfig": { "grokPath": "/patterns/squid", "patternLabel": "SQUID_DELIMITED", "timestampField": "timestamp" }, "fieldTransformations" : [ { "transformation" : "STELLAR" ,"output" : [ "full_hostname", "domain_without_subdomains" ] ,"config" : { "full_hostname" : "URL_TO_HOST(url)" ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" } } ] } PARSER Config: bro { "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser", "sensorTopic":"bro", "parserConfig": {}, "fieldTransformations" : [ { "transformation" : "STELLAR" ,"output" : [ "is_alert", "new_field" ] ,"config" : { "is_alert" :"true", "new_field" : "SPLIT(ip_dst_addr,'.')" } } ] } PARSER Config: snort { "parserClassName":"org.apache.metron.parsers.snort.BasicSnortParser", "sensorTopic":"snort", "parserConfig": {} } PARSER Config: yaf { "parserClassName":"org.apache.metron.parsers.GrokParser", "sensorTopic":"yaf", "fieldTransformations" : [ { "input" : "protocol" ,"transformation": "IP_PROTOCOL" } ], "parserConfig": { "grokPath":"/patterns/yaf", "patternLabel":"YAF_DELIMITED", "timestampField":"start_time", "timeFields": ["start_time", "end_time"], "dateFormat":"yyyy-MM-dd HH:mm:ss.S" } } ENRICHMENT Config: websphere { "index": "websphere", "batchSize": 5, "enrichment": { "fieldMap": { "geo": [ "ip_src_addr" ], "host": [ "ip_src_addr" ] }, "fieldToTypeMap": { "ip_src_addr": [ "playful_classification" ] } } } Exception in thread "main" java.lang.RuntimeException: Unable to load { "index": "bro", "batchSize": 5, "enrichment" : { "fieldMap": { "geo": ["ip_dst_addr", "ip_src_addr"], "host": ["host"] } }, "threatIntel": { "fieldMap": { "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] }, "fieldToTypeMap": { "ip_src_addr" : ["malicious_ip"], "ip_dst_addr" : ["malicious_ip"] }, "triageConfig" : { "riskLevelRules" : { "exists(ip_dst_addr)" : 0.10, "IS_EMPTY(host)" : 0.91, "exists(ip_dst_port)" : 0.20, "exists(ip_src_port)" : 0.30000000000 }, "aggregator" : "MAX", "aggregationConfig": { "NEGATIVE_VALUES_TRUMP_CONF" : "false" } } } } at org.apache.metron.common.configuration.ConfigurationType.lambda$static$2(ConfigurationType.java:54) at org.apache.metron.common.configuration.ConfigurationType.deserialize(ConfigurationType.java:87) at org.apache.metron.common.configuration.ConfigurationsUtils.lambda$dumpConfigs$0(ConfigurationsUtils.java:331) at org.apache.metron.common.configuration.ConfigurationsUtils.visitConfigs(ConfigurationsUtils.java:323) at org.apache.metron.common.configuration.ConfigurationsUtils.visitConfigs(ConfigurationsUtils.java:306) at org.apache.metron.common.configuration.ConfigurationsUtils.dumpConfigs(ConfigurationsUtils.java:330) at org.apache.metron.common.cli.ConfigurationManager.dump(ConfigurationManager.java:115) at org.apache.metron.common.cli.ConfigurationManager.run(ConfigurationManager.java:177) at org.apache.metron.common.cli.ConfigurationManager.run(ConfigurationManager.java:161) at org.apache.metron.common.cli.ConfigurationManager.main(ConfigurationManager.java:198) Caused by: com.fasterxml.jackson.databind.JsonMappingException: Unable to pop an empty stack at [Source: { "index": "bro", "batchSize": 5, "enrichment" : { "fieldMap": { "geo": ["ip_dst_addr", "ip_src_addr"], "host": ["host"] } }, "threatIntel": { "fieldMap": { "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] }, "fieldToTypeMap": { "ip_src_addr" : ["malicious_ip"], "ip_dst_addr" : ["malicious_ip"] }, "triageConfig" : { "riskLevelRules" : { "exists(ip_dst_addr)" : 0.10, "IS_EMPTY(host)" : 0.91, "exists(ip_dst_port)" : 0.20, "exists(ip_src_port)" : 0.30000000000 }, "aggregator" : "MAX", "aggregationConfig": { "NEGATIVE_VALUES_TRUMP_CONF" : "false" } } { } } ; line: 24, column: 7] (through reference chain: org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig["threatIntel"]->org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig["triageConfig"]->org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig["riskLevelRules"]) at com.fasterxml.jackson.databind.JsonMappingException.from(JsonMappingException.java:262) at com.fasterxml.jackson.databind.deser.SettableBeanProperty._throwAsIOE(SettableBeanProperty.java:537) at com.fasterxml.jackson.databind.deser.SettableBeanProperty._throwAsIOE(SettableBeanProperty.java:518) at com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:99) at com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:260) at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:125) at com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize(SettableBeanProperty.java:490) at com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:95) at com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:260) at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:125) at com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize(SettableBeanProperty.java:490) at com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:95) at com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:260) at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:125) at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:3807) at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2797) at org.apache.metron.common.utils.JSONUtils.load(JSONUtils.java:79) at org.apache.metron.common.configuration.ConfigurationType.lambda$static$2(ConfigurationType.java:52) ... 9 more Caused by: org.apache.metron.common.dsl.ParseException: Unable to pop an empty stack at org.apache.metron.common.stellar.StellarCompiler.popStack(StellarCompiler.java:397) at org.apache.metron.common.stellar.StellarCompiler.exitTransformationFunc(StellarCompiler.java:250) at org.apache.metron.common.stellar.generated.StellarParser$TransformationFuncContext.exitRule(StellarParser.java:1634) at org.antlr.v4.runtime.Parser.triggerExitRuleEvent(Parser.java:422) at org.antlr.v4.runtime.Parser.exitRule(Parser.java:632) at org.apache.metron.common.stellar.generated.StellarParser.transformation(StellarParser.java:158) at org.apache.metron.common.stellar.BaseStellarProcessor.parse(BaseStellarProcessor.java:57) at org.apache.metron.common.stellar.StellarPredicateProcessor.parse(StellarPredicateProcessor.java:53) at org.apache.metron.common.stellar.StellarPredicateProcessor.parse(StellarPredicateProcessor.java:37) at org.apache.metron.common.stellar.BaseStellarProcessor.validate(BaseStellarProcessor.java:67) at org.apache.metron.common.stellar.BaseStellarProcessor.validate(BaseStellarProcessor.java:62) at org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig.setRiskLevelRules(ThreatTriageConfig.java:42) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at com.fasterxml.jackson.databind.deser.impl.MethodProperty.deserializeAndSet(MethodProperty.java:97) ... 23 more I get the same error when i use :- IS_EMPTY(ip_src_addr) IS_EMPTY(protocol) However, IS_EMPTY('') and IS_EMPTY('someString') don't throw the above error message. Also, TO_LOWER(protocol) and TO_LOWER(host) don't throw any error msg. -- This message was sent by Atlassian JIRA (v6.3.4#6332)