Repository: incubator-metron Updated Branches: refs/heads/master 84d347195 -> 898d2360c
METRON-733: Remove Geo database from ParserBolt (justinleet via cestella) closes apache/incubator-metron#461 Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/898d2360 Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/898d2360 Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/898d2360 Branch: refs/heads/master Commit: 898d2360c6344eb88698f17974e7a17760b316a3 Parents: 84d3471 Author: justinleet <justinjl...@gmail.com> Authored: Fri Feb 24 15:59:55 2017 -0500 Committer: cstella <ceste...@gmail.com> Committed: Fri Feb 24 15:59:55 2017 -0500 ---------------------------------------------------------------------- .../apache/metron/parsers/bolt/ParserBolt.java | 3 --- .../metron/parsers/bolt/ParserBoltTest.java | 27 ------------------- .../integration/WriterBoltIntegrationTest.java | 1 - .../resources/GeoLite/GeoIP2-City-Test.mmdb.gz | Bin 10011 -> 0 bytes 4 files changed, 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/898d2360/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 416d3ed..abbd9d8 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -17,7 +17,6 @@ */ package org.apache.metron.parsers.bolt; -import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase; import org.apache.commons.lang3.StringUtils; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -73,8 +72,6 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable { @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { super.prepare(stormConf, context, collector); - String hdfsFile = (String) getConfigurations().getGlobalConfig().get(GeoLiteDatabase.GEO_HDFS_FILE); - GeoLiteDatabase.INSTANCE.update(hdfsFile); this.collector = collector; initializeStellar(); if(getSensorParserConfig() != null && filter == null) { http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/898d2360/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java index de98a64..6b00998 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java @@ -19,7 +19,6 @@ package org.apache.metron.parsers.bolt; import org.apache.metron.common.configuration.*; -import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase; import org.apache.metron.test.utils.UnitTestHelper; import org.apache.storm.task.OutputCollector; import org.apache.storm.tuple.Tuple; @@ -139,8 +138,6 @@ public class ParserBoltTest extends BaseBoltTest { }; - buildGlobalConfig(parserBolt); - parserBolt.setCuratorFramework(client); parserBolt.setTreeCache(cache); parserBolt.prepare(new HashMap(), topologyContext, outputCollector); @@ -179,8 +176,6 @@ public class ParserBoltTest extends BaseBoltTest { }; - buildGlobalConfig(parserBolt); - parserBolt.setCuratorFramework(client); parserBolt.setTreeCache(cache); parserBolt.prepare(new HashMap(), topologyContext, outputCollector); @@ -238,8 +233,6 @@ public void testImplicitBatchOfOne() throws Exception { } }; - buildGlobalConfig(parserBolt); - parserBolt.setCuratorFramework(client); parserBolt.setTreeCache(cache); parserBolt.prepare(new HashMap(), topologyContext, outputCollector); @@ -286,8 +279,6 @@ public void testImplicitBatchOfOne() throws Exception { } }; - buildGlobalConfig(parserBolt); - parserBolt.setCuratorFramework(client); parserBolt.setTreeCache(cache); parserBolt.prepare(new HashMap(), topologyContext, outputCollector); @@ -325,8 +316,6 @@ public void testImplicitBatchOfOne() throws Exception { } }; - buildGlobalConfig(parserBolt); - parserBolt.setCuratorFramework(client); parserBolt.setTreeCache(cache); parserBolt.prepare(new HashMap(), topologyContext, outputCollector); @@ -396,8 +385,6 @@ public void testImplicitBatchOfOne() throws Exception { } }; - buildGlobalConfig(parserBolt); - parserBolt.setCuratorFramework(client); parserBolt.setTreeCache(cache); parserBolt.prepare(new HashMap(), topologyContext, outputCollector); @@ -432,8 +419,6 @@ public void testImplicitBatchOfOne() throws Exception { } }; - buildGlobalConfig(parserBolt); - parserBolt.setCuratorFramework(client); parserBolt.setTreeCache(cache); parserBolt.prepare(new HashMap(), topologyContext, outputCollector); @@ -473,8 +458,6 @@ public void testImplicitBatchOfOne() throws Exception { } }; - buildGlobalConfig(parserBolt); - parserBolt.setCuratorFramework(client); parserBolt.setTreeCache(cache); parserBolt.prepare(new HashMap(), topologyContext, outputCollector); @@ -524,8 +507,6 @@ public void testImplicitBatchOfOne() throws Exception { } }; - buildGlobalConfig(parserBolt); - parserBolt.setCuratorFramework(client); parserBolt.setTreeCache(cache); parserBolt.prepare(new HashMap(), topologyContext, outputCollector); @@ -550,14 +531,6 @@ public void testImplicitBatchOfOne() throws Exception { } - protected void buildGlobalConfig(ParserBolt parserBolt) { - HashMap<String, Object> globalConfig = new HashMap<>(); - String baseDir = UnitTestHelper.findDir("GeoLite"); - File geoHdfsFile = new File(new File(baseDir), "GeoIP2-City-Test.mmdb.gz"); - globalConfig.put(GeoLiteDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath()); - parserBolt.getConfigurations().updateGlobalConfig(globalConfig); - } - private static void writeNonBatch(OutputCollector collector, ParserBolt bolt, Tuple t) { bolt.execute(t); } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/898d2360/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java index eaeaeb5..f37b1fc 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java @@ -61,7 +61,6 @@ public class WriterBoltIntegrationTest extends BaseIntegrationTest { } /** { - "geo.hdfs.file": "src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz", "fieldValidations" : [ { "validation" : "org.apache.metron.writers.integration.WriterBoltIntegrationTest$MockValidator" http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/898d2360/metron-platform/metron-parsers/src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz b/metron-platform/metron-parsers/src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz deleted file mode 100644 index 406c656..0000000 Binary files a/metron-platform/metron-parsers/src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz and /dev/null differ
