Repository: metron Updated Branches: refs/heads/master 46bc63dbc -> 3ba9ae251
METRON-1504: Enriching missing values does not match the semantics between the new enrichment topology and old closes apache/incubator-metron#976 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3ba9ae25 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3ba9ae25 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3ba9ae25 Branch: refs/heads/master Commit: 3ba9ae25126bc393e3b9307f4ffe63ac66a909f4 Parents: 46bc63d Author: cstella <ceste...@gmail.com> Authored: Mon Apr 9 11:47:51 2018 -0400 Committer: cstella <ceste...@gmail.com> Committed: Mon Apr 9 11:47:51 2018 -0400 ---------------------------------------------------------------------- .../enrichment/parallel/ParallelEnricher.java | 3 + .../integration/EnrichmentIntegrationTest.java | 12 +++- .../parallel/ParallelEnricherTest.java | 59 +++++++++++++++++++- 3 files changed, 70 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java index 6ddb892..b10c148 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java @@ -170,6 +170,9 @@ public class ParallelEnricher { for(Object o : m.keySet()) { String field = (String) o; Object value = m.get(o); + if(value == null) { + continue; + } CacheKey cacheKey = new CacheKey(field, value, config); String prefix = adapter.getOutputPrefix(cacheKey); Supplier<JSONObject> supplier = () -> { http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java index 267ca62..3c55c95 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java @@ -95,7 +95,15 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest { private static List<byte[]> getInputMessages(String path){ try{ - return TestUtils.readSampleData(path); + List<byte[]> ret = TestUtils.readSampleData(path); + { + //we want one of the fields without a destination IP to ensure that enrichments can function + Map<String, Object> sansDestinationIp = JSONUtils.INSTANCE.load(new String(ret.get(ret.size() -1)) + , JSONUtils.MAP_SUPPLIER); + sansDestinationIp.remove(Constants.Fields.DST_ADDR.getName()); + ret.add(JSONUtils.INSTANCE.toJSONPretty(sansDestinationIp)); + } + return ret; }catch(IOException ioe){ return null; } @@ -262,8 +270,6 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest { //ensure we always have a source ip and destination ip Assert.assertNotNull(jsonDoc.get(SRC_IP)); - Assert.assertNotNull(jsonDoc.get(DST_IP)); - Assert.assertNotNull(jsonDoc.get("ALL_CAPS")); Assert.assertNotNull(jsonDoc.get("map.blah")); Assert.assertNull(jsonDoc.get("map")); http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java index 4a4573b..d4fcdf4 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java @@ -79,7 +79,39 @@ public class ParallelEnricherTest { } }.ofType("ENRICHMENT"); adapter.initializeAdapter(new HashMap<>()); - enrichmentsByType = ImmutableMap.of("stellar", adapter); + EnrichmentAdapter<CacheKey> dummy = new EnrichmentAdapter<CacheKey>() { + @Override + public void logAccess(CacheKey value) { + + } + + @Override + public JSONObject enrich(CacheKey value) { + return null; + } + + @Override + public boolean initializeAdapter(Map<String, Object> config) { + return false; + } + + @Override + public void updateAdapter(Map<String, Object> config) { + + } + + @Override + public void cleanup() { + + } + + @Override + public String getOutputPrefix(CacheKey value) { + return null; + } + }; + + enrichmentsByType = ImmutableMap.of("stellar", adapter, "dummy", dummy); enricher = new ParallelEnricher(enrichmentsByType, infrastructure, false); } @@ -115,6 +147,31 @@ public class ParallelEnricherTest { Assert.assertEquals("TEST", ret.get("ALL_CAPS")); Assert.assertEquals(0, result.getEnrichmentErrors().size()); } +/** + * { + "enrichment": { + "fieldMap": { + "dummy" : ["notthere"] + } + ,"fieldToTypeMap": { } + }, + "threatIntel": { } +} + */ + @Multiline + public static String nullConfig; + + @Test + public void testNullEnrichment() throws Exception { + SensorEnrichmentConfig config = JSONUtils.INSTANCE.load(nullConfig, SensorEnrichmentConfig.class); + config.getConfiguration().putIfAbsent("stellarContext", stellarContext); + JSONObject message = new JSONObject() {{ + put(Constants.SENSOR_TYPE, "test"); + }}; + ParallelEnricher.EnrichmentResult result = enricher.apply(message, EnrichmentStrategies.ENRICHMENT, config, null); + JSONObject ret = result.getResult(); + Assert.assertEquals("Got the wrong result count: " + ret, 4, ret.size()); + } /** * {