[ 
https://issues.apache.org/jira/browse/METRON-1466?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16383944#comment-16383944
 ] 

Michael Miklavcic commented on METRON-1466:
-------------------------------------------

Looks like trouble with the maxmind DB

{code}
2018-03-02 17:36:56.357 o.a.m.e.a.g.GeoLiteDatabase 
Thread-39-threatIntelJoinBolt-executor[15 15] [ERROR] [Metron] Unable to open 
new database file /apps/metron/geo/default/GeoLite2-City.mmdb.gz
java.util.zip.ZipException: Not in GZIP format
        at java.util.zip.GZIPInputStream.readHeader(GZIPInputStream.java:165) 
~[?:1.8.0_161]
        at java.util.zip.GZIPInputStream.<init>(GZIPInputStream.java:79) 
~[?:1.8.0_161]
        at java.util.zip.GZIPInputStream.<init>(GZIPInputStream.java:91) 
~[?:1.8.0_161]
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.update(GeoLiteDatabase.java:136)
 [stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.prepare(ThreatIntelJoinBolt.java:110)
 [stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.JoinBolt.prepare(JoinBolt.java:101) 
[stormjar.jar:?]
        at 
org.apache.storm.daemon.executor$fn__10193$fn__10206.invoke(executor.clj:794) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at org.apache.storm.util$async_loop$fn__1221.invoke(util.clj:482) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at clojure.lang.AFn.run(AFn.java:22) [clojure-1.7.0.jar:?]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.357 o.a.m.e.a.g.GeoLiteDatabase 
Thread-19-geoEnrichmentBolt-executor[6 6] [ERROR] [Metron] Unable to open new 
database file /apps/metron/geo/default/GeoLite2-City.mmdb.gz
java.util.zip.ZipException: Not in GZIP format
        at java.util.zip.GZIPInputStream.readHeader(GZIPInputStream.java:165) 
~[?:1.8.0_161]
        at java.util.zip.GZIPInputStream.<init>(GZIPInputStream.java:79) 
~[?:1.8.0_161]
        at java.util.zip.GZIPInputStream.<init>(GZIPInputStream.java:91) 
~[?:1.8.0_161]
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.update(GeoLiteDatabase.java:136)
 [stormjar.jar:?]
        at 
org.apache.metron.enrichment.adapters.geo.GeoAdapter.initializeAdapter(GeoAdapter.java:59)
 [stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.GenericEnrichmentBolt.prepare(GenericEnrichmentBolt.java:157)
 [stormjar.jar:?]
        at 
org.apache.storm.daemon.executor$fn__10193$fn__10206.invoke(executor.clj:794) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at org.apache.storm.util$async_loop$fn__1221.invoke(util.clj:482) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at clojure.lang.AFn.run(AFn.java:22) [clojure-1.7.0.jar:?]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.357 o.a.m.e.a.g.GeoLiteDatabase Curator-TreeCache-11 
[ERROR] [Metron] Unable to open new database file 
/apps/metron/geo/default/GeoLite2-City.mmdb.gz
java.util.zip.ZipException: Not in GZIP format
        at java.util.zip.GZIPInputStream.readHeader(GZIPInputStream.java:165) 
~[?:1.8.0_161]
        at java.util.zip.GZIPInputStream.<init>(GZIPInputStream.java:79) 
~[?:1.8.0_161]
        at java.util.zip.GZIPInputStream.<init>(GZIPInputStream.java:91) 
~[?:1.8.0_161]
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.update(GeoLiteDatabase.java:136)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.updateIfNecessary(GeoLiteDatabase.java:114)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.adapters.geo.GeoAdapter.updateAdapter(GeoAdapter.java:65)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.GenericEnrichmentBolt.reloadCallback(GenericEnrichmentBolt.java:133)
 ~[stormjar.jar:?]
        at 
org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater.reloadCallback(ConfigurationsUpdater.java:145)
 ~[stormjar.jar:?]
        at 
org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater.update(ConfigurationsUpdater.java:74)
 ~[stormjar.jar:?]
        at 
org.apache.metron.zookeeper.SimpleEventListener.childEvent(SimpleEventListener.java:118)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$2.apply(TreeCache.java:685)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$2.apply(TreeCache.java:679)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:92)
 [stormjar.jar:?]
        at 
org.apache.metron.guava.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:84)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache.callListeners(TreeCache.java:678)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache.access$1400(TreeCache.java:69)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$4.run(TreeCache.java:790) 
[stormjar.jar:?]
        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 
[?:1.8.0_161]
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) 
[?:1.8.0_161]
        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 
[?:1.8.0_161]
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) 
[?:1.8.0_161]
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
[?:1.8.0_161]
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
[?:1.8.0_161]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.363 o.a.c.f.r.c.TreeCache Curator-TreeCache-11 [ERROR] 
java.lang.IllegalStateException: [Metron] Unable to update MaxMind database
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.update(GeoLiteDatabase.java:150)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.updateIfNecessary(GeoLiteDatabase.java:114)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.adapters.geo.GeoAdapter.updateAdapter(GeoAdapter.java:65)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.GenericEnrichmentBolt.reloadCallback(GenericEnrichmentBolt.java:133)
 ~[stormjar.jar:?]
        at 
org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater.reloadCallback(ConfigurationsUpdater.java:145)
 ~[stormjar.jar:?]
        at 
org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater.update(ConfigurationsUpdater.java:74)
 ~[stormjar.jar:?]
        at 
org.apache.metron.zookeeper.SimpleEventListener.childEvent(SimpleEventListener.java:118)
 ~[stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$2.apply(TreeCache.java:685)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$2.apply(TreeCache.java:679)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:92)
 [stormjar.jar:?]
        at 
org.apache.metron.guava.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:84)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache.callListeners(TreeCache.java:678)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache.access$1400(TreeCache.java:69)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$4.run(TreeCache.java:790) 
[stormjar.jar:?]
        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 
[?:1.8.0_161]
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) 
[?:1.8.0_161]
        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 
[?:1.8.0_161]
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) 
[?:1.8.0_161]
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
[?:1.8.0_161]
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
[?:1.8.0_161]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.364 o.a.s.util Thread-39-threatIntelJoinBolt-executor[15 
15] [ERROR] Async loop died!
java.lang.IllegalStateException: [Metron] Unable to update MaxMind database
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.update(GeoLiteDatabase.java:150)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.prepare(ThreatIntelJoinBolt.java:110)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.JoinBolt.prepare(JoinBolt.java:101) 
~[stormjar.jar:?]
        at 
org.apache.storm.daemon.executor$fn__10193$fn__10206.invoke(executor.clj:794) 
~[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at org.apache.storm.util$async_loop$fn__1221.invoke(util.clj:482) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at clojure.lang.AFn.run(AFn.java:22) [clojure-1.7.0.jar:?]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.364 o.a.s.util Thread-19-geoEnrichmentBolt-executor[6 6] 
[ERROR] Async loop died!
java.lang.IllegalStateException: [Metron] Unable to update MaxMind database
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.update(GeoLiteDatabase.java:150)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.adapters.geo.GeoAdapter.initializeAdapter(GeoAdapter.java:59)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.GenericEnrichmentBolt.prepare(GenericEnrichmentBolt.java:157)
 ~[stormjar.jar:?]
        at 
org.apache.storm.daemon.executor$fn__10193$fn__10206.invoke(executor.clj:794) 
~[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at org.apache.storm.util$async_loop$fn__1221.invoke(util.clj:482) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at clojure.lang.AFn.run(AFn.java:22) [clojure-1.7.0.jar:?]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.366 o.a.s.d.executor 
Thread-39-threatIntelJoinBolt-executor[15 15] [ERROR] 
java.lang.IllegalStateException: [Metron] Unable to update MaxMind database
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.update(GeoLiteDatabase.java:150)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.prepare(ThreatIntelJoinBolt.java:110)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.JoinBolt.prepare(JoinBolt.java:101) 
~[stormjar.jar:?]
        at 
org.apache.storm.daemon.executor$fn__10193$fn__10206.invoke(executor.clj:794) 
~[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at org.apache.storm.util$async_loop$fn__1221.invoke(util.clj:482) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at clojure.lang.AFn.run(AFn.java:22) [clojure-1.7.0.jar:?]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.366 o.a.s.d.executor Thread-19-geoEnrichmentBolt-executor[6 
6] [ERROR] 
java.lang.IllegalStateException: [Metron] Unable to update MaxMind database
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.update(GeoLiteDatabase.java:150)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.adapters.geo.GeoAdapter.initializeAdapter(GeoAdapter.java:59)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.GenericEnrichmentBolt.prepare(GenericEnrichmentBolt.java:157)
 ~[stormjar.jar:?]
        at 
org.apache.storm.daemon.executor$fn__10193$fn__10206.invoke(executor.clj:794) 
~[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at org.apache.storm.util$async_loop$fn__1221.invoke(util.clj:482) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at clojure.lang.AFn.run(AFn.java:22) [clojure-1.7.0.jar:?]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.414 o.a.m.e.a.g.GeoLiteDatabase Curator-TreeCache-7 [ERROR] 
[Metron] Unable to open new database file 
/apps/metron/geo/default/GeoLite2-City.mmdb.gz
java.util.zip.ZipException: Not in GZIP format
        at java.util.zip.GZIPInputStream.readHeader(GZIPInputStream.java:165) 
~[?:1.8.0_161]
        at java.util.zip.GZIPInputStream.<init>(GZIPInputStream.java:79) 
~[?:1.8.0_161]
        at java.util.zip.GZIPInputStream.<init>(GZIPInputStream.java:91) 
~[?:1.8.0_161]
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.update(GeoLiteDatabase.java:136)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.updateIfNecessary(GeoLiteDatabase.java:114)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.reloadCallback(ThreatIntelJoinBolt.java:206)
 ~[stormjar.jar:?]
        at 
org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater.reloadCallback(ConfigurationsUpdater.java:145)
 ~[stormjar.jar:?]
        at 
org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater.update(ConfigurationsUpdater.java:74)
 ~[stormjar.jar:?]
        at 
org.apache.metron.zookeeper.SimpleEventListener.childEvent(SimpleEventListener.java:118)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$2.apply(TreeCache.java:685)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$2.apply(TreeCache.java:679)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:92)
 [stormjar.jar:?]
        at 
org.apache.metron.guava.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:84)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache.callListeners(TreeCache.java:678)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache.access$1400(TreeCache.java:69)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$4.run(TreeCache.java:790) 
[stormjar.jar:?]
        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 
[?:1.8.0_161]
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) 
[?:1.8.0_161]
        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 
[?:1.8.0_161]
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) 
[?:1.8.0_161]
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
[?:1.8.0_161]
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
[?:1.8.0_161]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.415 o.a.c.f.r.c.TreeCache Curator-TreeCache-7 [ERROR] 
java.lang.IllegalStateException: [Metron] Unable to update MaxMind database
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.update(GeoLiteDatabase.java:150)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase.updateIfNecessary(GeoLiteDatabase.java:114)
 ~[stormjar.jar:?]
        at 
org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.reloadCallback(ThreatIntelJoinBolt.java:206)
 ~[stormjar.jar:?]
        at 
org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater.reloadCallback(ConfigurationsUpdater.java:145)
 ~[stormjar.jar:?]
        at 
org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater.update(ConfigurationsUpdater.java:74)
 ~[stormjar.jar:?]
        at 
org.apache.metron.zookeeper.SimpleEventListener.childEvent(SimpleEventListener.java:118)
 ~[stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$2.apply(TreeCache.java:685)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$2.apply(TreeCache.java:679)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:92)
 [stormjar.jar:?]
        at 
org.apache.metron.guava.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:84)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache.callListeners(TreeCache.java:678)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache.access$1400(TreeCache.java:69)
 [stormjar.jar:?]
        at 
org.apache.curator.framework.recipes.cache.TreeCache$4.run(TreeCache.java:790) 
[stormjar.jar:?]
        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 
[?:1.8.0_161]
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) 
[?:1.8.0_161]
        at 
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 
[?:1.8.0_161]
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) 
[?:1.8.0_161]
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
[?:1.8.0_161]
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
[?:1.8.0_161]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.430 o.a.s.util Thread-19-geoEnrichmentBolt-executor[6 6] 
[ERROR] Halting process: ("Worker died")
java.lang.RuntimeException: ("Worker died")
        at org.apache.storm.util$exit_process_BANG_.doInvoke(util.clj:341) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at clojure.lang.RestFn.invoke(RestFn.java:423) [clojure-1.7.0.jar:?]
        at 
org.apache.storm.daemon.worker$fn__10795$fn__10796.invoke(worker.clj:763) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at 
org.apache.storm.daemon.executor$mk_executor_data$fn__10011$fn__10012.invoke(executor.clj:276)
 [storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at org.apache.storm.util$async_loop$fn__1221.invoke(util.clj:494) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at clojure.lang.AFn.run(AFn.java:22) [clojure-1.7.0.jar:?]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.430 o.a.s.util Thread-39-threatIntelJoinBolt-executor[15 
15] [ERROR] Halting process: ("Worker died")
java.lang.RuntimeException: ("Worker died")
        at org.apache.storm.util$exit_process_BANG_.doInvoke(util.clj:341) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at clojure.lang.RestFn.invoke(RestFn.java:423) [clojure-1.7.0.jar:?]
        at 
org.apache.storm.daemon.worker$fn__10795$fn__10796.invoke(worker.clj:763) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at 
org.apache.storm.daemon.executor$mk_executor_data$fn__10011$fn__10012.invoke(executor.clj:276)
 [storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at org.apache.storm.util$async_loop$fn__1221.invoke(util.clj:494) 
[storm-core-1.1.0.2.6.4.0-91.jar:1.1.0.2.6.4.0-91]
        at clojure.lang.AFn.run(AFn.java:22) [clojure-1.7.0.jar:?]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
2018-03-02 17:36:56.432 o.a.s.d.worker Thread-42 [INFO] Shutting down worker 
enrichment-1-1519983445 90a573d2-c442-4150-9a01-a8f099a4fd5a 6700
2018-03-02 17:36:56.433 o.a.s.d.worker Thread-42 [INFO] Terminating messaging 
context
2018-03-02 17:36:56.433 o.a.s.d.worker Thread-42 [INFO] Shutting down executors
2018-03-02 17:36:56.434 o.a.s.d.executor Thread-42 [INFO] Shutting down 
executor kafkaSpout:[8 8]
{code}

> The Elastic Search index are not getting generated in the latest metron 
> deployments for all the topologies
> ----------------------------------------------------------------------------------------------------------
>
>                 Key: METRON-1466
>                 URL: https://issues.apache.org/jira/browse/METRON-1466
>             Project: Metron
>          Issue Type: Bug
>         Environment: Metron secured cluster 
>            Reporter: Mohan
>            Priority: Major
>         Attachments: bro_worker.log, enrichmnts_worker.log, 
> indexing_worker.log
>
>
> The Elastic Search index are not getting generated in the latest metron 
> deployments for all the topologies , 
> I see that the data is getting published to the respective topics and also 
> getting enriched .
> below are the console consumer output for topics "bro" and "enrichments" I 
> dont get anything for "indexing" topic .
> {code:java}
> ./kafka-console-consumer.sh --zookeeper 
> ctr-e138-1518143905142-61778-01-000002.hwx.site:2181 --security-protocol 
> PLAINTEXTSASL --topic bro 
> {"http":{"id.orig_p":49210,"status_code":200,"method":"GET","request_body_len":0,"id.resp_p":80,"uri":"\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764","tags":[],"uid":"CRDObQRKAmoHCQq1a","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","resp_mime_types":["image\/png"],"trans_depth":3,"host":"7oqnsnzwwnm6zb7y.gigapaysun.com","status_msg":"OK","id.orig_h":"12.172.138.158","response_body_len":1823,"user_agent":"Mozilla\/4.0
>  (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 
> 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 
> 6.0)","ts":1.469686182899762E9,"id.resp_h":"13.163.121.204","resp_fuids":["FQbZZfax5TLVX6M42"]}}
>  
> {"http":{"id.orig_p":49210,"status_code":200,"method":"GET","request_body_len":0,"id.resp_p":80,"uri":"\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764","tags":[],"uid":"CRDObQRKAmoHCQq1a","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","resp_mime_types":["image\/png"],"trans_depth":3,"host":"7oqnsnzwwnm6zb7y.gigapaysun.com","status_msg":"OK","id.orig_h":"12.172.138.158","response_body_len":1823,"user_agent":"Mozilla\/4.0
>  (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 
> 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 
> 6.0)","ts":1.469686182899762E9,"id.resp_h":"13.163.121.204","resp_fuids":["FQbZZfax5TLVX6M42"]}}
>  
> {"http":{"id.orig_p":49210,"status_code":200,"method":"GET","request_body_len":0,"id.resp_p":80,"uri":"\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764","tags":[],"uid":"CRDObQRKAmoHCQq1a","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","resp_mime_types":["image\/png"],"trans_depth":3,"host":"7oqnsnzwwnm6zb7y.gigapaysun.com","status_msg":"OK","id.orig_h":"12.172.138.158","response_body_len":1823,"user_agent":"Mozilla\/4.0
>  (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 
> 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 
> 6.0)","ts":1.469686182899762E9,"id.resp_h":"13.163.121.204","resp_fuids":["FQbZZfax5TLVX6M42"]}}
>  
> {"http":{"id.orig_p":49210,"status_code":200,"method":"GET","request_body_len":0,"id.resp_p":80,"uri":"\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764","tags":[],"uid":"CRDObQRKAmoHCQq1a","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","resp_mime_types":["image\/png"],"trans_depth":3,"host":"7oqnsnzwwnm6zb7y.gigapaysun.com","status_msg":"OK","id.orig_h":"12.172.138.158","response_body_len":1823,"user_agent":"Mozilla\/4.0
>  (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 
> 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 
> 6.0)","ts":1.469686182899762E9,"id.resp_h":"13.163.121.204","resp_fuids":["FQbZZfax5TLVX6M42"]}}
>  
> {"http":{"id.orig_p":49210,"status_code":200,"method":"GET","request_body_len":0,"id.resp_p":80,"uri":"\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764","tags":[],"uid":"CRDObQRKAmoHCQq1a","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","resp_mime_types":["image\/png"],"trans_depth":3,"host":"7oqnsnzwwnm6zb7y.gigapaysun.com","status_msg":"OK","id.orig_h":"12.172.138.158","response_body_len":1823,"user_agent":"Mozilla\/4.0
>  (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 
> 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 
> 6.0)","ts":1.469686182899762E9,"id.resp_h":"13.163.121.204","resp_fuids":["FQbZZfax5TLVX6M42"]}}
>  ./kafka-console-consumer.sh --zookeeper 
> ctr-e138-1518143905142-61778-01-000002.hwx.site:2181 --security-protocol 
> PLAINTEXTSASL --topic enrichments 
> {"bro_timestamp":"1469686182.899762","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764","tags":[],"source.type":"bro","uid":"CRDObQRKAmoHCQq1a","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","resp_mime_types":["image\/png"],"trans_depth":3,"protocol":"http","original_string":"HTTP
>  | id.orig_p:49210 status_code:200 method:GET request_body_len:0 id.resp_p:80 
> uri:\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764 tags:[] 
> uid:CRDObQRKAmoHCQq1a 
> referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg 
> resp_mime_types:[\"image\\\/png\"] trans_depth:3 
> host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:12.172.138.158 
> response_body_len:1823 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows 
> NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; 
> .NET CLR 3.0.30729; Media Center PC 6.0) ts:1469686182.899762 
> id.resp_h:13.163.121.204 
> resp_fuids:[\"FQbZZfax5TLVX6M42\"]","ip_dst_addr":"13.163.121.204","ip_src_port":49210,"host":"7oqnsnzwwnm6zb7y.gigapaysun.com","status_msg":"OK","guid":"897e534c-ed6b-458c-9b2d-8749a68ed447","response_body_len":1823,"ip_src_addr":"12.172.138.158","user_agent":"Mozilla\/4.0
>  (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 
> 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 
> 6.0)","resp_fuids":["FQbZZfax5TLVX6M42"],"timestamp":1469686182899} 
> {"bro_timestamp":"1469686182.899762","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764","tags":[],"source.type":"bro","uid":"CRDObQRKAmoHCQq1a","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","resp_mime_types":["image\/png"],"trans_depth":3,"protocol":"http","original_string":"HTTP
>  | id.orig_p:49210 status_code:200 method:GET request_body_len:0 id.resp_p:80 
> uri:\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764 tags:[] 
> uid:CRDObQRKAmoHCQq1a 
> referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg 
> resp_mime_types:[\"image\\\/png\"] trans_depth:3 
> host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:12.172.138.158 
> response_body_len:1823 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows 
> NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; 
> .NET CLR 3.0.30729; Media Center PC 6.0) ts:1469686182.899762 
> id.resp_h:13.163.121.204 
> resp_fuids:[\"FQbZZfax5TLVX6M42\"]","ip_dst_addr":"13.163.121.204","ip_src_port":49210,"host":"7oqnsnzwwnm6zb7y.gigapaysun.com","status_msg":"OK","guid":"6caaab54-e78d-4d31-9b2c-5c2bb25d609f","response_body_len":1823,"ip_src_addr":"12.172.138.158","user_agent":"Mozilla\/4.0
>  (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 
> 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 
> 6.0)","resp_fuids":["FQbZZfax5TLVX6M42"],"timestamp":1469686182899} 
> {"bro_timestamp":"1469686182.899762","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764","tags":[],"source.type":"bro","uid":"CRDObQRKAmoHCQq1a","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","resp_mime_types":["image\/png"],"trans_depth":3,"protocol":"http","original_string":"HTTP
>  | id.orig_p:49210 status_code:200 method:GET request_body_len:0 id.resp_p:80 
> uri:\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764 tags:[] 
> uid:CRDObQRKAmoHCQq1a 
> referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg 
> resp_mime_types:[\"image\\\/png\"] trans_depth:3 
> host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:12.172.138.158 
> response_body_len:1823 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows 
> NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; 
> .NET CLR 3.0.30729; Media Center PC 6.0) ts:1469686182.899762 
> id.resp_h:13.163.121.204 
> resp_fuids:[\"FQbZZfax5TLVX6M42\"]","ip_dst_addr":"13.163.121.204","ip_src_port":49210,"host":"7oqnsnzwwnm6zb7y.gigapaysun.com","status_msg":"OK","guid":"a7475614-789a-4221-88cb-0734ef5fdc0a","response_body_len":1823,"ip_src_addr":"12.172.138.158","user_agent":"Mozilla\/4.0
>  (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 
> 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 
> 6.0)","resp_fuids":["FQbZZfax5TLVX6M42"],"timestamp":1469686182899} 
> {"bro_timestamp":"1469686182.899762","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764","tags":[],"source.type":"bro","uid":"CRDObQRKAmoHCQq1a","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","resp_mime_types":["image\/png"],"trans_depth":3,"protocol":"http","original_string":"HTTP
>  | id.orig_p:49210 status_code:200 method:GET request_body_len:0 id.resp_p:80 
> uri:\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764 tags:[] 
> uid:CRDObQRKAmoHCQq1a 
> referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg 
> resp_mime_types:[\"image\\\/png\"] trans_depth:3 
> host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:12.172.138.158 
> response_body_len:1823 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows 
> NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; 
> .NET CLR 3.0.30729; Media Center PC 6.0) ts:1469686182.899762 
> id.resp_h:13.163.121.204 
> resp_fuids:[\"FQbZZfax5TLVX6M42\"]","ip_dst_addr":"13.163.121.204","ip_src_port":49210,"host":"7oqnsnzwwnm6zb7y.gigapaysun.com","status_msg":"OK","guid":"f05fa5de-5549-4293-85cf-006c66dba5d4","response_body_len":1823,"ip_src_addr":"12.172.138.158","user_agent":"Mozilla\/4.0
>  (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 
> 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 
> 6.0)","resp_fuids":["FQbZZfax5TLVX6M42"],"timestamp":1469686182899} 
> {"bro_timestamp":"1469686182.899762","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764","tags":[],"source.type":"bro","uid":"CRDObQRKAmoHCQq1a","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","resp_mime_types":["image\/png"],"trans_depth":3,"protocol":"http","original_string":"HTTP
>  | id.orig_p:49210 status_code:200 method:GET request_body_len:0 id.resp_p:80 
> uri:\/picture.php?k=11iqmfg&b7f2a994c3eaaf014608b272c46cf764 tags:[] 
> uid:CRDObQRKAmoHCQq1a 
> referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg 
> resp_mime_types:[\"image\\\/png\"] trans_depth:3 
> host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:12.172.138.158 
> response_body_len:1823 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows 
> NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; 
> .NET CLR 3.0.30729; Media Center PC 6.0) ts:1469686182.899762 
> id.resp_h:13.163.121.204 
> resp_fuids:[\"FQbZZfax5TLVX6M42\"]","ip_dst_addr":"13.163.121.204","ip_src_port":49210,"host":"7oqnsnzwwnm6zb7y.gigapaysun.com","status_msg":"OK","guid":"116cb86a-2f41-48a8-8cc0-ee15d25d8db6","response_body_len":1823,"ip_src_addr":"12.172.138.158","user_agent":"Mozilla\/4.0
>  (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 
> 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 
> 6.0)","resp_fuids":["FQbZZfax5TLVX6M42"],"timestamp":1469686182899}
> {code}



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Reply via email to