http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java index 36863e3..fcc8050 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java @@ -17,18 +17,13 @@ */ package org.apache.metron.common.configuration; -import org.apache.commons.io.FilenameUtils; -import org.apache.curator.RetryPolicy; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.metron.common.Constants; -import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.StellarFunctions; -import org.apache.metron.common.utils.JSONUtils; -import org.apache.zookeeper.KeeperException; +import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; +import static org.apache.metron.common.configuration.ConfigurationType.GLOBAL; +import static org.apache.metron.common.configuration.ConfigurationType.INDEXING; +import static org.apache.metron.common.configuration.ConfigurationType.PARSER; +import static org.apache.metron.common.configuration.ConfigurationType.PROFILER; +import com.fasterxml.jackson.databind.JsonNode; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; @@ -38,8 +33,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; - -import static org.apache.metron.common.configuration.ConfigurationType.*; +import org.apache.commons.io.FilenameUtils; +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.zookeeper.KeeperException; public class ConfigurationsUtils { @@ -55,7 +59,7 @@ public class ConfigurationsUtils { } } public static void writeGlobalConfigToZookeeper(Map<String, Object> globalConfig, CuratorFramework client) throws Exception { - writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSON(globalConfig), client); + writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSONPretty(globalConfig), client); } public static void writeGlobalConfigToZookeeper(byte[] globalConfig, String zookeeperUrl) throws Exception { @@ -76,7 +80,7 @@ public class ConfigurationsUtils { } public static void writeSensorParserConfigToZookeeper(String sensorType, SensorParserConfig sensorParserConfig, String zookeeperUrl) throws Exception { - writeSensorParserConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSON(sensorParserConfig), zookeeperUrl); + writeSensorParserConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSONPretty(sensorParserConfig), zookeeperUrl); } public static void writeSensorParserConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception { @@ -93,7 +97,7 @@ public class ConfigurationsUtils { } public static void writeSensorIndexingConfigToZookeeper(String sensorType, Map<String, Object> sensorIndexingConfig, String zookeeperUrl) throws Exception { - writeSensorIndexingConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSON(sensorIndexingConfig), zookeeperUrl); + writeSensorIndexingConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSONPretty(sensorIndexingConfig), zookeeperUrl); } public static void writeSensorIndexingConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception { @@ -109,7 +113,7 @@ public class ConfigurationsUtils { } public static void writeSensorEnrichmentConfigToZookeeper(String sensorType, SensorEnrichmentConfig sensorEnrichmentConfig, String zookeeperUrl) throws Exception { - writeSensorEnrichmentConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSON(sensorEnrichmentConfig), zookeeperUrl); + writeSensorEnrichmentConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSONPretty(sensorEnrichmentConfig), zookeeperUrl); } public static void writeSensorEnrichmentConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception { @@ -125,13 +129,37 @@ public class ConfigurationsUtils { } public static void writeConfigToZookeeper(String name, Map<String, Object> config, String zookeeperUrl) throws Exception { - writeConfigToZookeeper(name, JSONUtils.INSTANCE.toJSON(config), zookeeperUrl); + writeConfigToZookeeper(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name, JSONUtils.INSTANCE.toJSONPretty(config), zookeeperUrl); } - public static void writeConfigToZookeeper(String name, byte[] config, String zookeeperUrl) throws Exception { - try(CuratorFramework client = getClient(zookeeperUrl)) { + public static void writeConfigToZookeeper(ConfigurationType configType, byte[] configData, + String zookeeperUrl) throws Exception { + writeConfigToZookeeper(configType, Optional.empty(), configData, zookeeperUrl); + } + + public static void writeConfigToZookeeper(ConfigurationType configType, + Optional<String> configName, byte[] configData, String zookeeperUrl) throws Exception { + writeConfigToZookeeper(getConfigZKPath(configType, configName), configData, zookeeperUrl); + } + + public static void writeConfigToZookeeper(ConfigurationType configType,Optional<String> configName, + byte[] configData, CuratorFramework client) throws Exception { + writeToZookeeper(getConfigZKPath(configType, configName), configData, client); + } + + private static String getConfigZKPath(ConfigurationType configType, Optional<String> configName) { + String pathSuffix = configName.isPresent() && configType != GLOBAL ? "/" + configName : ""; + return configType.getZookeeperRoot() + pathSuffix; + } + + /** + * Writes config to path in Zookeeper, /metron/topology/$CONFIG_TYPE/$CONFIG_NAME + */ + public static void writeConfigToZookeeper(String configPath, byte[] config, String zookeeperUrl) + throws Exception { + try (CuratorFramework client = getClient(zookeeperUrl)) { client.start(); - writeToZookeeper(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name, config, client); + writeToZookeeper(configPath, config, client); } } @@ -203,8 +231,30 @@ public class ConfigurationsUtils { return readFromZookeeper(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name, client); } + public static byte[] readConfigBytesFromZookeeper(ConfigurationType configType, + String zookeeperUrl) throws Exception { + return readConfigBytesFromZookeeper(configType, Optional.empty(), zookeeperUrl); + } + + public static byte[] readConfigBytesFromZookeeper(ConfigurationType configType, Optional<String> configName, + CuratorFramework client) throws Exception { + return readFromZookeeper(getConfigZKPath(configType, configName), client); + } + + public static byte[] readConfigBytesFromZookeeper(ConfigurationType configType, Optional<String> configName, + String zookeeperUrl) throws Exception { + return readFromZookeeper(getConfigZKPath(configType, configName), zookeeperUrl); + } + + public static byte[] readFromZookeeper(String path, String zookeeperUrl) throws Exception { + try (CuratorFramework client = getClient(zookeeperUrl)) { + client.start(); + return readFromZookeeper(path, client); + } + } + public static byte[] readFromZookeeper(String path, CuratorFramework client) throws Exception { - if(client != null && client.getData() != null && path != null) { + if (client != null && client.getData() != null && path != null) { return client.getData().forPath(path); } return new byte[]{}; @@ -216,7 +266,7 @@ public class ConfigurationsUtils { String indexingConfigPath, String profilerConfigPath, String zookeeperUrl) throws Exception { - try(CuratorFramework client = getClient(zookeeperUrl)) { + try (CuratorFramework client = getClient(zookeeperUrl)) { client.start(); uploadConfigsToZookeeper(globalConfigPath, parsersConfigPath, enrichmentsConfigPath, indexingConfigPath, profilerConfigPath, client); } @@ -226,6 +276,61 @@ public class ConfigurationsUtils { uploadConfigsToZookeeper(rootFilePath, rootFilePath, rootFilePath, rootFilePath, rootFilePath, client); } + /** + * Uploads config to Zookeeper based on the specified rootPath and configuration type. The local + * file and zookeeper paths are dynamically calculated based on the rootPath and config type. + * When grabbing files from the local FS, the rootPath is used. When reading/writing to Zookeeper, + * the path returned by + * {@link org.apache.metron.common.configuration.ConfigurationType#getZookeeperRoot()} is used. + * For example, when grabbing GLOBAL config from the local FS, the path is based on 'rootPath/.' + * whereas PARSER would be based on 'rootPath/parsers'. + * + * @param rootFilePath base configuration path on the local FS + * @param client zk client + * @param type config type to upload configs for + */ + public static void uploadConfigsToZookeeper(String rootFilePath, CuratorFramework client, + ConfigurationType type) throws Exception { + uploadConfigsToZookeeper(rootFilePath, client, type, Optional.empty()); + } + + /** + * Does the same as + * {@link org.apache.metron.common.configuration.ConfigurationsUtils#uploadConfigsToZookeeper( + * java.lang.String, org.apache.curator.framework.CuratorFramework, + * org.apache.metron.common.configuration.ConfigurationType)} + * with the addition of being able to specify a specific config name for the given configuration + * type. e.g. config type=PARSER, config name=bro + * + * @param rootFilePath base configuration path on the local FS + * @param client zk client + * @param type config type to upload configs for + * @param configName specific config under the specified config type + */ + public static void uploadConfigsToZookeeper(String rootFilePath, CuratorFramework client, + ConfigurationType type, Optional<String> configName) throws Exception { + switch (type) { + case GLOBAL: + final byte[] globalConfig = readGlobalConfigFromFile(rootFilePath); + if (globalConfig.length > 0) { + setupStellarStatically(client, Optional.of(new String(globalConfig))); + writeGlobalConfigToZookeeper(globalConfig, client); + } + break; + case PARSER: // intentional pass-through + case ENRICHMENT: // intentional pass-through + case INDEXING: + Map<String, byte[]> sensorIndexingConfigs = readSensorConfigsFromFile(rootFilePath, type, + configName); + for (String sensorType : sensorIndexingConfigs.keySet()) { + writeConfigToZookeeper(type, configName, sensorIndexingConfigs.get(sensorType), client); + } + break; + default: + throw new IllegalArgumentException("Configuration type not found: " + type); + } + } + public static void uploadConfigsToZookeeper(String globalConfigPath, String parsersConfigPath, String enrichmentsConfigPath, @@ -311,7 +416,7 @@ public class ConfigurationsUtils { public static byte[] readGlobalConfigFromFile(String rootPath) throws IOException { byte[] globalConfig = new byte[0]; - File configPath = new File(rootPath, GLOBAL.getName() + ".json"); + File configPath = new File(rootPath, GLOBAL.getTypeName() + ".json"); if (configPath.exists()) { globalConfig = Files.readAllBytes(configPath.toPath()); } @@ -319,15 +424,15 @@ public class ConfigurationsUtils { } public static Map<String, byte[]> readSensorParserConfigsFromFile(String rootPath) throws IOException { - return readSensorConfigsFromFile(rootPath, PARSER); + return readSensorConfigsFromFile(rootPath, PARSER, Optional.empty()); } public static Map<String, byte[]> readSensorEnrichmentConfigsFromFile(String rootPath) throws IOException { - return readSensorConfigsFromFile(rootPath, ENRICHMENT); + return readSensorConfigsFromFile(rootPath, ENRICHMENT, Optional.empty()); } public static Map<String, byte[]> readSensorIndexingConfigsFromFile(String rootPath) throws IOException { - return readSensorConfigsFromFile(rootPath, INDEXING); + return readSensorConfigsFromFile(rootPath, INDEXING, Optional.empty()); } /** @@ -337,7 +442,7 @@ public class ConfigurationsUtils { public static byte[] readProfilerConfigFromFile(String rootPath) throws IOException { byte[] config = new byte[0]; - File configPath = new File(rootPath, PROFILER.getName() + ".json"); + File configPath = new File(rootPath, PROFILER.getTypeName() + ".json"); if (configPath.exists()) { config = Files.readAllBytes(configPath.toPath()); } @@ -346,19 +451,101 @@ public class ConfigurationsUtils { } public static Map<String, byte[]> readSensorConfigsFromFile(String rootPath, ConfigurationType configType) throws IOException { + return readSensorConfigsFromFile(rootPath, configType, Optional.empty()); + } + + /** + * Will read configs from local disk at the specified rootPath. Will read all configs for a given + * configuration type. If an optional specific config name is also provided, it will only read + * configs for that configuration type and name combo. e.g. PARSER, bro + * @param rootPath root FS location to read configs from + * @param configType e.g. GLOBAL, PARSER, ENRICHMENT, etc. + * @param configName a specific config, for instance a sensor name like bro, yaf, snort, etc. + * @return map of file names to the contents of that file as a byte array + * @throws IOException + */ + public static Map<String, byte[]> readSensorConfigsFromFile(String rootPath, + ConfigurationType configType, Optional<String> configName) throws IOException { Map<String, byte[]> sensorConfigs = new HashMap<>(); File configPath = new File(rootPath, configType.getDirectory()); - if (configPath.exists()) { + if (configPath.exists() && configPath.isDirectory()) { File[] children = configPath.listFiles(); - if (children != null) { + if (!configName.isPresent()) { for (File file : children) { - sensorConfigs.put(FilenameUtils.removeExtension(file.getName()), Files.readAllBytes(file.toPath())); + sensorConfigs.put(FilenameUtils.removeExtension(file.getName()), + Files.readAllBytes(file.toPath())); + } + } else { + for (File file : children) { + if (FilenameUtils.removeExtension(file.getName()).equals(configName.get())) { + sensorConfigs.put(FilenameUtils.removeExtension(file.getName()), + Files.readAllBytes(file.toPath())); + } + } + if (sensorConfigs.isEmpty()) { + throw new RuntimeException("Unable to find configuration for " + configName.get()); } } } return sensorConfigs; } + /** + * Reads Json data for the specified config type from zookeeper, + * applies the patch from patchData, and writes it back to Zookeeper in a pretty print format. + * Patching JSON flattens existing formatting, so this will keep configs readable. + * Starts up curatorclient based on zookeeperUrl. + * + * @param configurationType GLOBAL, PARSER, etc. + * @param configName e.g. bro, yaf, snort + * @param patchData a JSON patch in the format specified by RFC 6902 + * @param zookeeperUrl configs are here + */ + public static void applyConfigPatchToZookeeper(ConfigurationType configurationType, + byte[] patchData, String zookeeperUrl) throws Exception { + applyConfigPatchToZookeeper(configurationType, Optional.empty(), patchData, zookeeperUrl); + } + + /** + * Reads Json data for the specified config type and config name (if applicable) from zookeeper, + * applies the patch from patchData, and writes it back to Zookeeper in a pretty print format. + * Patching JSON flattens existing formatting, so this will keep configs readable. + * Starts up curatorclient based on zookeeperUrl. + * + * @param configurationType GLOBAL, PARSER, etc. + * @param configName e.g. bro, yaf, snort + * @param patchData a JSON patch in the format specified by RFC 6902 + * @param zookeeperUrl configs are here + */ + public static void applyConfigPatchToZookeeper(ConfigurationType configurationType, + Optional<String> configName, byte[] patchData, String zookeeperUrl) throws Exception { + try (CuratorFramework client = getClient(zookeeperUrl)) { + client.start(); + applyConfigPatchToZookeeper(configurationType, configName, patchData, client); + } + } + + /** + * Reads Json data for the specified config type and config name (if applicable) from zookeeper, + * applies the patch from patchData, and writes it back to Zookeeper in a pretty print format. + * Patching JSON flattens existing formatting, so this will keep configs readable. The + * curatorclient should be started already. + * + * @param configurationType GLOBAL, PARSER, etc. + * @param configName e.g. bro, yaf, snort + * @param patchData a JSON patch in the format specified by RFC 6902 + * @param client access to zookeeeper + */ + public static void applyConfigPatchToZookeeper(ConfigurationType configurationType, + Optional<String> configName, + byte[] patchData, CuratorFramework client) throws Exception { + byte[] configData = readConfigBytesFromZookeeper(configurationType, configName, client); + JsonNode source = JSONUtils.INSTANCE.readTree(configData); + JsonNode patch = JSONUtils.INSTANCE.readTree(patchData); + JsonNode patchedConfig = JSONUtils.INSTANCE.applyPatch(patch, source); + writeConfigToZookeeper(configurationType, configName, + JSONUtils.INSTANCE.toJSONPretty(patchedConfig), client); + } public interface ConfigurationVisitor{ void visit(ConfigurationType configurationType, String name, String data); @@ -368,14 +555,14 @@ public class ConfigurationsUtils { visitConfigs(client, (type, name, data) -> { setupStellarStatically(client, Optional.ofNullable(data)); callback.visit(type, name, data); - }, GLOBAL); - visitConfigs(client, callback, PARSER); - visitConfigs(client, callback, INDEXING); - visitConfigs(client, callback, ENRICHMENT); - visitConfigs(client, callback, PROFILER); + }, GLOBAL, Optional.empty()); + visitConfigs(client, callback, PARSER, Optional.empty()); + visitConfigs(client, callback, INDEXING, Optional.empty()); + visitConfigs(client, callback, ENRICHMENT, Optional.empty()); + visitConfigs(client, callback, PROFILER, Optional.empty()); } - public static void visitConfigs(CuratorFramework client, ConfigurationVisitor callback, ConfigurationType configType) throws Exception { + public static void visitConfigs(CuratorFramework client, ConfigurationVisitor callback, ConfigurationType configType, Optional<String> configName) throws Exception { if (client.checkExists().forPath(configType.getZookeeperRoot()) != null) { @@ -388,20 +575,52 @@ public class ConfigurationsUtils { callback.visit(configType, "profiler", new String(profilerConfigData)); } else if (configType.equals(PARSER) || configType.equals(ENRICHMENT) || configType.equals(INDEXING)) { - List<String> children = client.getChildren().forPath(configType.getZookeeperRoot()); - for (String child : children) { - - byte[] data = client.getData().forPath(configType.getZookeeperRoot() + "/" + child); - callback.visit(configType, child, new String(data)); + if (configName.isPresent()) { + byte[] data = readConfigBytesFromZookeeper(configType, configName, client); + callback.visit(configType, configName.get(), new String(data)); + } else { + List<String> children = client.getChildren().forPath(configType.getZookeeperRoot()); + for (String child : children) { + byte[] data = client.getData().forPath(configType.getZookeeperRoot() + "/" + child); + callback.visit(configType, child, new String(data)); + } } } } } + /** + * Writes all config content to the provided print stream. + * + * @param out stream to use as output + * @param client zk client + * @throws Exception + */ public static void dumpConfigs(PrintStream out, CuratorFramework client) throws Exception { ConfigurationsUtils.visitConfigs(client, (type, name, data) -> { type.deserialize(data); - out.println(type + " Config: " + name + "\n" + data); + out.println(type + " Config: " + name + System.lineSeparator() + data); }); } + + /** + * Writes config content for a specific config type to the provided print stream. Optionally + * provide a config name in addition to the config type and it will only print the json for a + * specific config, e.g. bro, yaf, snort, etc. + * + * @param out stream to use as output + * @param client zk client + * @param configType GLOBAL, PARSER, ENRICHMENT, etc. + * @param configName Typically a sensor name like bro, snort, yaf, etc. + * @throws Exception + */ + public static void dumpConfigs(PrintStream out, CuratorFramework client, + ConfigurationType configType, Optional<String> configName) throws Exception { + ConfigurationsUtils.visitConfigs(client, (type, name, data) -> { + setupStellarStatically(client, Optional.ofNullable(data)); + type.deserialize(data); + out.println(type + " Config: " + name + System.lineSeparator() + data); + }, configType, configName); + } } +
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java index bf5b856..a28b15c 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java @@ -44,6 +44,6 @@ public class EnrichmentConfigurations extends Configurations { } private String getKey(String sensorType) { - return ConfigurationType.ENRICHMENT.getName() + "." + sensorType; + return ConfigurationType.ENRICHMENT.getTypeName() + "." + sensorType; } } http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java index 1d60084..3bd7645 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java @@ -59,7 +59,7 @@ public class IndexingConfigurations extends Configurations { } private String getKey(String sensorType) { - return ConfigurationType.INDEXING.getName() + "." + sensorType; + return ConfigurationType.INDEXING.getTypeName() + "." + sensorType; } public boolean isDefault(String sensorName, String writerName) { http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java index ab7463f..0ec0ed4 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java @@ -44,6 +44,6 @@ public class ParserConfigurations extends Configurations { } private String getKey(String sensorType) { - return ConfigurationType.PARSER.getName() + "." + sensorType; + return ConfigurationType.PARSER.getTypeName() + "." + sensorType; } } http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfigurations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfigurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfigurations.java index c098787..e001d74 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfigurations.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfigurations.java @@ -48,6 +48,6 @@ public class ProfilerConfigurations extends Configurations { } private String getKey() { - return ConfigurationType.PROFILER.getName(); + return ConfigurationType.PROFILER.getTypeName(); } } http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java index 60d009f..280b167 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,22 +21,26 @@ package org.apache.metron.common.utils; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.zjsonpatch.JsonPatch; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; -import java.io.*; -import java.util.Map; - public enum JSONUtils { INSTANCE; private static ThreadLocal<JSONParser> _parser = ThreadLocal.withInitial(() -> - new JSONParser()); + new JSONParser()); private static ThreadLocal<ObjectMapper> _mapper = ThreadLocal.withInitial(() -> - new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL)); + new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL)); public <T> T convert(Object original, Class<T> targetClass) { return _mapper.get().convertValue(original, targetClass); @@ -83,8 +87,12 @@ public enum JSONUtils { } } - public byte[] toJSON(Object config) throws JsonProcessingException { - return _mapper.get().writeValueAsBytes(config); + public byte[] toJSONPretty(String config) throws IOException { + return toJSONPretty(readTree(config)); + } + + public byte[] toJSONPretty(Object config) throws JsonProcessingException { + return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsBytes(config); } /** @@ -93,4 +101,51 @@ public enum JSONUtils { public JSONObject toJSONObject(Object o) throws JsonProcessingException, ParseException { return (JSONObject) _parser.get().parse(toJSON(o, false)); } + + /** + * Reads a JSON string into a JsonNode Object + * + * @param json JSON value to deserialize + * @return deserialized JsonNode Object + */ + public JsonNode readTree(String json) throws IOException { + return _mapper.get().readTree(json); + } + + /** + * Reads a JSON byte array into a JsonNode Object + * + * @param json JSON value to deserialize + * @return deserialized JsonNode Object + */ + public JsonNode readTree(byte[] json) throws IOException { + return _mapper.get().readTree(json); + } + + /** + * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902) + * Operations: + * <ul> + * <li>add</li> + * <li>remove</li> + * <li>replace</li> + * <li>move</li> + * <li>copy</li> + * <li>test</li> + * </ul> + * + * @param patch Array of JSON patches, e.g. [{ "op": "move", "from": "/a", "path": "/c" }] + * @param source Source JSON to apply patch to + * @return new json after applying the patch + */ + public JsonNode applyPatch(String patch, String source) throws IOException { + JsonNode patchNode = readTree(patch); + JsonNode sourceNode = readTree(source); + return applyPatch(patchNode, sourceNode); + } + + public JsonNode applyPatch(JsonNode patch, JsonNode source) throws IOException { + return JsonPatch.apply(patch, source); + } + } http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java index 9b0d77a..0882788 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java @@ -19,8 +19,6 @@ package org.apache.metron.common.utils; import com.google.common.base.Joiner; -import com.google.common.collect.Iterables; - import java.util.Arrays; import java.util.Optional; @@ -33,4 +31,16 @@ public class StringUtils { .toArray() ); } + + /** + * Strips specified number of lines from beginning for String val + * @param val + * @param numLines + */ + public static String stripLines(String val, int numLines) { + int start = org.apache.commons.lang3.StringUtils.ordinalIndexOf(val, System.lineSeparator(), numLines); + start = start >= 0 ? start : 0; + return val.substring(start); + } + } http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java index 68ef604..44612cd 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java @@ -66,7 +66,7 @@ public class ConfiguredEnrichmentBoltTest extends BaseConfiguredBoltTest { this.zookeeperUrl = testZkServer.getConnectString(); byte[] globalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH); ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig, zookeeperUrl); - enrichmentConfigurationTypes.add(ConfigurationType.GLOBAL.getName()); + enrichmentConfigurationTypes.add(ConfigurationType.GLOBAL.getTypeName()); Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); for (String sensorType : sensorEnrichmentConfigs.keySet()) { ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, sensorEnrichmentConfigs.get(sensorType), zookeeperUrl); @@ -105,13 +105,13 @@ public class ConfiguredEnrichmentBoltTest extends BaseConfiguredBoltTest { Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig(); sampleGlobalConfig.put("newGlobalField", "newGlobalValue"); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); - waitForConfigUpdate(ConfigurationType.GLOBAL.getName()); + waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig()); configsUpdated = new HashSet<>(); sampleGlobalConfig.remove("newGlobalField"); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); - waitForConfigUpdate(ConfigurationType.GLOBAL.getName()); + waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Remove global config field", sampleConfigurations, configuredBolt.getConfigurations()); configsUpdated = new HashSet<>(); @@ -133,4 +133,4 @@ public class ConfiguredEnrichmentBoltTest extends BaseConfiguredBoltTest { Assert.assertEquals("Add new sensor config", sampleConfigurations, configuredBolt.getConfigurations()); configuredBolt.cleanup(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java index db56892..06603a3 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java @@ -67,7 +67,7 @@ public class ConfiguredParserBoltTest extends BaseConfiguredBoltTest { this.zookeeperUrl = testZkServer.getConnectString(); byte[] globalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH); ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig, zookeeperUrl); - parserConfigurationTypes.add(ConfigurationType.GLOBAL.getName()); + parserConfigurationTypes.add(ConfigurationType.GLOBAL.getTypeName()); Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); for (String sensorType : sensorEnrichmentConfigs.keySet()) { ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, sensorEnrichmentConfigs.get(sensorType), zookeeperUrl); @@ -106,13 +106,13 @@ public class ConfiguredParserBoltTest extends BaseConfiguredBoltTest { Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig(); sampleGlobalConfig.put("newGlobalField", "newGlobalValue"); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); - waitForConfigUpdate(ConfigurationType.GLOBAL.getName()); + waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig()); configsUpdated = new HashSet<>(); sampleGlobalConfig.remove("newGlobalField"); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); - waitForConfigUpdate(ConfigurationType.GLOBAL.getName()); + waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Remove global config field", sampleConfigurations, configuredBolt.getConfigurations()); configsUpdated = new HashSet<>(); @@ -129,4 +129,4 @@ public class ConfiguredParserBoltTest extends BaseConfiguredBoltTest { Assert.assertEquals("Add new sensor config", sampleConfigurations, configuredBolt.getConfigurations()); configuredBolt.cleanup(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java index c9a7d22..ed7f4bc 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,33 +18,57 @@ package org.apache.metron.common.cli; +import static org.apache.metron.common.cli.ConfigurationManager.PatchMode.ADD; +import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; +import static org.apache.metron.common.configuration.ConfigurationType.GLOBAL; +import static org.apache.metron.common.configuration.ConfigurationType.INDEXING; +import static org.apache.metron.common.configuration.ConfigurationType.PARSER; +import static org.apache.metron.common.utils.StringUtils.stripLines; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.fail; + import com.google.common.base.Splitter; import com.google.common.collect.Collections2; import com.google.common.collect.Iterables; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.nio.file.DirectoryNotEmptyException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.cli.PosixParser; +import org.apache.commons.lang3.ArrayUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.test.TestingServer; import org.apache.hadoop.io.BooleanWritable; import org.apache.metron.TestConstants; +import org.apache.metron.common.cli.ConfigurationManager.PatchMode; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.integration.utils.TestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.io.File; -import java.io.IOException; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.*; - public class ConfigurationManagerIntegrationTest { private TestingServer testZkServer; private CuratorFramework client; private String zookeeperUrl; private String outDir = "target/configs"; + private File tmpDir; + private File configDir; + private File parsersDir; + private File enrichmentsDir; + private File indexingDir; private Set<String> sensors = new HashSet<>(); private void cleanDir(File rootDir) throws IOException { @@ -69,12 +93,17 @@ public class ConfigurationManagerIntegrationTest { zookeeperUrl = testZkServer.getConnectString(); client = ConfigurationsUtils.getClient(zookeeperUrl); client.start(); - File sensorDir = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), ConfigurationType.ENRICHMENT.getDirectory()); + File sensorDir = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), ENRICHMENT.getDirectory()); sensors.addAll(Collections2.transform( Arrays.asList(sensorDir.list()) ,s -> Iterables.getFirst(Splitter.on('.').split(s), "null") ) ); + tmpDir = TestUtils.createTempDir(this.getClass().getName()); + configDir = TestUtils.createDir(tmpDir, "config"); + parsersDir = TestUtils.createDir(configDir, "parsers"); + enrichmentsDir = TestUtils.createDir(configDir, "enrichments"); + indexingDir = TestUtils.createDir(configDir, "indexing"); pushConfigs(); } @@ -112,11 +141,11 @@ public class ConfigurationManagerIntegrationTest { public void validateConfigsOnDisk(File configDir) throws IOException { File globalConfigFile = new File(configDir, "global.json"); Assert.assertTrue("Global config does not exist", globalConfigFile.exists()); - validateConfig("global", ConfigurationType.GLOBAL, new String(Files.readAllBytes(Paths.get(globalConfigFile.toURI())))); + validateConfig("global", GLOBAL, new String(Files.readAllBytes(Paths.get(globalConfigFile.toURI())))); for(String sensor : sensors) { - File sensorFile = new File(configDir, ConfigurationType.ENRICHMENT.getDirectory() + "/" + sensor + ".json"); + File sensorFile = new File(configDir, ENRICHMENT.getDirectory() + "/" + sensor + ".json"); Assert.assertTrue(sensor + " config does not exist", sensorFile.exists()); - validateConfig(sensor, ConfigurationType.ENRICHMENT, new String(Files.readAllBytes(Paths.get(sensorFile.toURI())))); + validateConfig(sensor, ENRICHMENT, new String(Files.readAllBytes(Paths.get(sensorFile.toURI())))); } } @@ -128,7 +157,7 @@ public class ConfigurationManagerIntegrationTest { try { //second time without force should pullConfigs(false); - Assert.fail("Should have failed to pull configs in a directory structure that already exists."); + fail("Should have failed to pull configs in a directory structure that already exists."); } catch(IllegalStateException t) { //make sure we didn't bork anything @@ -142,7 +171,7 @@ public class ConfigurationManagerIntegrationTest { try { type.deserialize(data); } catch (Exception e) { - Assert.fail("Unable to load config " + name + ": " + data); + fail("Unable to load config " + name + ": " + data); } } @Test @@ -155,7 +184,7 @@ public class ConfigurationManagerIntegrationTest { public void visit(ConfigurationType configurationType, String name, String data) { Assert.assertTrue(data.length() > 0); validateConfig(name, configurationType, data); - if(configurationType == ConfigurationType.GLOBAL) { + if(configurationType == GLOBAL) { validateConfig(name, configurationType, data); foundGlobal.set(true); } @@ -168,6 +197,303 @@ public class ConfigurationManagerIntegrationTest { Assert.assertEquals(sensorsInZookeeper, sensors); } + /** + * { "a": "b" } + */ + @Multiline + private static String someConfig; + + @Test + public void writes_global_config_to_zookeeper() throws Exception { + File configFile = new File(configDir, "global.json"); + TestUtils.write(configFile, someConfig); + pushConfigs(GLOBAL, configDir); + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(someConfig); + byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(GLOBAL), 1)); + Assert.assertThat(actual, equalTo(expected)); + } + + private void pushConfigs(ConfigurationType type, File configPath) throws Exception { + pushConfigs(type, configPath, Optional.empty()); + } + + private void pushConfigs(ConfigurationType type, File configPath, Optional<String> configName) throws Exception { + String[] args = new String[]{ + "-z", zookeeperUrl, + "--mode", "PUSH", + "--config_type", type.toString(), + "--input_dir", configPath.getAbsolutePath() + }; + if (configName.isPresent()) { + args = ArrayUtils.addAll(args, "--config_name", configName.get()); + } + ConfigurationManager manager = new ConfigurationManager(); + manager.run(ConfigurationManager.ConfigurationOptions.parse(new PosixParser(), args)); + } + + private String dumpConfigs(ConfigurationType type) throws Exception { + return dumpConfigs(type, Optional.empty()); + } + + private String dumpConfigs(ConfigurationType type, Optional<String> configName) throws Exception { + String[] args = new String[]{ + "-z", zookeeperUrl, + "--mode", "DUMP", + "--config_type", type.toString() + }; + if (configName.isPresent()) { + args = ArrayUtils.addAll(args, "--config_name", configName.get()); + } + ConfigurationManager manager = new ConfigurationManager(); + return redirectSystemOut(args, (a) -> { + manager.run(ConfigurationManager.ConfigurationOptions.parse(new PosixParser(), a)); + }); + } + + public interface RedirectCallback { + + void call(String[] args) throws Exception; + } + + private String redirectSystemOut(final String[] args, RedirectCallback callback) + throws Exception { + PrintStream os = System.out; + try (OutputStream baos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(baos)) { + System.setOut(ps); + callback.call(args); + System.out.flush(); + System.setOut(os); + return baos.toString(); + } finally { + System.setOut(os); + } + } + + /** + *{ + "parserClassName": "org.apache.metron.parsers.GrokParser", + "sensorTopic": "squid", + "parserConfig": { + "grokPath": "/patterns/squid", + "patternLabel": "SQUID_DELIMITED", + "timestampField": "timestamp" + }, + "fieldTransformations" : [ + { + "transformation" : "STELLAR" + ,"output" : [ "full_hostname", "domain_without_subdomains" ] + ,"config" : { + "full_hostname" : "URL_TO_HOST(url)" + ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" + } + } + ] + } + */ + @Multiline + private static String squidParserConfig; + + @Test + public void writes_single_parser_config_to_zookeeper() throws Exception { + File configFile = new File(parsersDir, "myparser.json"); + TestUtils.write(configFile, squidParserConfig); + pushConfigs(PARSER, configDir, Optional.of("myparser")); + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(squidParserConfig); + byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(PARSER, Optional.of("myparser")), 1)); + Assert.assertThat(actual, equalTo(expected)); + } + + /** + * { + "enrichment" : { + "fieldMap": { + "geo": ["ip_dst_addr", "ip_src_addr"], + "host": ["host"] + } + }, + "threatIntel": { + "fieldMap": { + "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] + }, + "fieldToTypeMap": { + "ip_src_addr" : ["malicious_ip"], + "ip_dst_addr" : ["malicious_ip"] + } + } + } + */ + @Multiline + private static String someEnrichmentConfig; + + @Test + public void writes_single_enrichment_config_to_zookeeper() throws Exception { + File configFile = new File(enrichmentsDir, "myenrichment.json"); + TestUtils.write(configFile, someEnrichmentConfig); + pushConfigs(ENRICHMENT, configDir, Optional.of("myenrichment")); + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(someEnrichmentConfig); + byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(ENRICHMENT, Optional.of("myenrichment")), 1)); + Assert.assertThat(actual, equalTo(expected)); + } + + /** + * { + "hdfs" : { + "index": "myindex", + "batchSize": 5, + "enabled" : true + }, + "elasticsearch" : { + "index": "myindex", + "batchSize": 5, + "enabled" : true + }, + "solr" : { + "index": "myindex", + "batchSize": 5, + "enabled" : true + } + } + */ + @Multiline + private static String someIndexingConfig; + + @Test + public void writes_single_indexing_config_to_zookeeper() throws Exception { + File configFile = new File(indexingDir, "myindex.json"); + TestUtils.write(configFile, someIndexingConfig); + pushConfigs(INDEXING, configDir, Optional.of("myindex")); + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(someIndexingConfig); + byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(INDEXING, Optional.of("myindex")), 1)); + Assert.assertThat(actual, equalTo(expected)); + } + + /** + * [ { "op": "replace", "path": "/a", "value": [ "new1", "new2" ] } ] + */ + @Multiline + private static String somePatchConfig; + + /** + * { "a": [ "new1", "new2" ] } + */ + @Multiline + private static String expectedSomeConfig; + + @Test + public void patches_global_config_from_file() throws Exception { + File patchFile = new File(tmpDir, "global-config-patch.json"); + TestUtils.write(patchFile, somePatchConfig); + File configFile = new File(configDir, "global.json"); + TestUtils.write(configFile, someConfig); + pushConfigs(GLOBAL, configDir, Optional.of("global")); + patchConfigs(GLOBAL, Optional.of(patchFile), Optional.of("global"), Optional.empty(), Optional.empty(), Optional.empty()); + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedSomeConfig); + byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(GLOBAL, Optional.of("global")), 1)); + Assert.assertThat(actual, equalTo(expected)); + } + + private void patchConfigs(ConfigurationType type, Optional<File> patchPath, + Optional<String> configName, Optional<PatchMode> patchMode, Optional<String> key, + Optional<String> value) throws Exception { + String[] args = new String[]{ + "-z", zookeeperUrl, + "--mode", "PATCH", + "--config_type", type.toString() + }; + if (configName.isPresent()) { + args = ArrayUtils.addAll(args, "--config_name", configName.get()); + } + if (patchPath.isPresent()) { + args = ArrayUtils.addAll(args, "--patch_file", patchPath.get().getAbsolutePath()); + } else if (patchMode.isPresent()) { + args = ArrayUtils.addAll(args, + "--patch_mode", patchMode.get().toString(), + "--patch_key", key.get(), + "--patch_value", value.get()); + } + ConfigurationManager manager = new ConfigurationManager(); + manager.run(ConfigurationManager.ConfigurationOptions.parse(new PosixParser(), args)); + } + + /** + * [ { "op": "replace", "path": "/parserConfig/timestampField", "value": "heyjoe" } ] + */ + @Multiline + public static String someParserPatch; + + /** + *{ + "parserClassName": "org.apache.metron.parsers.GrokParser", + "sensorTopic": "squid", + "parserConfig": { + "grokPath": "/patterns/squid", + "patternLabel": "SQUID_DELIMITED", + "timestampField": "heyjoe" + }, + "fieldTransformations" : [ + { + "transformation" : "STELLAR" + ,"output" : [ "full_hostname", "domain_without_subdomains" ] + ,"config" : { + "full_hostname" : "URL_TO_HOST(url)" + ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" + } + } + ] + } + */ + @Multiline + public static String expectedPatchedParser; + + @Test + public void patches_parser_config_from_file() throws Exception { + File patchFile = new File(tmpDir, "parser-patch.json"); + TestUtils.write(patchFile, someParserPatch); + File configFile = new File(parsersDir, "myparser.json"); + TestUtils.write(configFile, squidParserConfig); + pushConfigs(PARSER, configDir, Optional.of("myparser")); + patchConfigs(PARSER, Optional.of(patchFile), Optional.of("myparser"), Optional.empty(), Optional.empty(), Optional.empty()); + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedPatchedParser); + byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(PARSER, Optional.of("myparser")), 1)); + Assert.assertThat(actual, equalTo(expected)); + } + + @Test + public void patches_parser_config_from_key_value() throws Exception { + File configFile = new File(parsersDir, "myparser.json"); + TestUtils.write(configFile, squidParserConfig); + pushConfigs(PARSER, configDir, Optional.of("myparser")); + patchConfigs(PARSER, Optional.empty(), Optional.of("myparser"), Optional.of(ADD), Optional.of("/parserConfig/timestampField"), Optional.of("\"\"heyjoe\"\"")); + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedPatchedParser); + byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(PARSER, Optional.of("myparser")), 1)); + Assert.assertThat(actual, equalTo(expected)); + } + + /** + * { + * "a": "b", + * "foo": { + * "bar": { + * "baz": [ "bazval1", "bazval2" ] + * } + * } + * } + */ + @Multiline + private static String expectedComplexConfig; + + @Test + public void patches_global_config_from_complex_key_value() throws Exception { + File configFile = new File(configDir, "global.json"); + TestUtils.write(configFile, someConfig); + pushConfigs(GLOBAL, configDir, Optional.of("global")); + patchConfigs(GLOBAL, Optional.empty(), Optional.of("global"), Optional.of(ADD), Optional.of("/foo"), Optional.of("{ \"bar\" : { \"baz\" : [ \"bazval1\", \"bazval2\" ] } }")); + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedComplexConfig); + byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(GLOBAL, Optional.of("global")), 1)); + Assert.assertThat(actual, equalTo(expected)); + } + @After public void tearDown() throws IOException { client.close(); http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java deleted file mode 100644 index 0c72183..0000000 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.common.cli; - -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.test.TestingServer; -import org.apache.metron.TestConstants; -import org.apache.metron.common.configuration.ConfigurationsUtils; -import org.apache.metron.common.utils.JSONUtils; -import org.junit.Assert; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -public class ConfigurationsUtilsTest { - - private TestingServer testZkServer; - private String zookeeperUrl; - private CuratorFramework client; - private byte[] expectedGlobalConfig; - private Map<String, byte[]> expectedSensorParserConfigMap; - private Map<String, byte[]> expectedSensorEnrichmentConfigMap; - - @Before - public void setup() throws Exception { - testZkServer = new TestingServer(true); - zookeeperUrl = testZkServer.getConnectString(); - client = ConfigurationsUtils.getClient(zookeeperUrl); - client.start(); - expectedGlobalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH); - expectedSensorParserConfigMap = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH); - expectedSensorEnrichmentConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); - } - - @Test - public void test() throws Exception { - Assert.assertTrue(expectedGlobalConfig.length > 0); - ConfigurationsUtils.writeGlobalConfigToZookeeper(expectedGlobalConfig, zookeeperUrl); - byte[] actualGlobalConfigBytes = ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client); - Assert.assertTrue(Arrays.equals(expectedGlobalConfig, actualGlobalConfigBytes)); - - Assert.assertTrue(expectedSensorParserConfigMap.size() > 0); - String testSensorType = "yaf"; - byte[] expectedSensorParserConfigBytes = expectedSensorParserConfigMap.get(testSensorType); - ConfigurationsUtils.writeSensorParserConfigToZookeeper(testSensorType, expectedSensorParserConfigBytes, zookeeperUrl); - byte[] actualSensorParserConfigBytes = ConfigurationsUtils.readSensorParserConfigBytesFromZookeeper(testSensorType, client); - Assert.assertTrue(Arrays.equals(expectedSensorParserConfigBytes, actualSensorParserConfigBytes)); - - Assert.assertTrue(expectedSensorEnrichmentConfigMap.size() > 0); - byte[] expectedSensorEnrichmentConfigBytes = expectedSensorEnrichmentConfigMap.get(testSensorType); - ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(testSensorType, expectedSensorEnrichmentConfigBytes, zookeeperUrl); - byte[] actualSensorEnrichmentConfigBytes = ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(testSensorType, client); - Assert.assertTrue(Arrays.equals(expectedSensorEnrichmentConfigBytes, actualSensorEnrichmentConfigBytes)); - - String name = "testConfig"; - Map<String, Object> testConfig = new HashMap<>(); - testConfig.put("stringField", "value"); - testConfig.put("intField", 1); - testConfig.put("doubleField", 1.1); - ConfigurationsUtils.writeConfigToZookeeper(name, testConfig, zookeeperUrl); - byte[] readConfigBytes = ConfigurationsUtils.readConfigBytesFromZookeeper(name, client); - Assert.assertTrue(Arrays.equals(JSONUtils.INSTANCE.toJSON(testConfig), readConfigBytes)); - - } - - @After - public void tearDown() throws IOException { - client.close(); - testZkServer.close(); - testZkServer.stop(); - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java new file mode 100644 index 0000000..145d7ce --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.configuration; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.test.TestingServer; +import org.apache.metron.TestConstants; +import org.apache.metron.common.utils.JSONUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class ConfigurationsUtilsTest { + + private TestingServer testZkServer; + private String zookeeperUrl; + private CuratorFramework client; + private byte[] expectedGlobalConfig; + private Map<String, byte[]> expectedSensorParserConfigMap; + private Map<String, byte[]> expectedSensorEnrichmentConfigMap; + + @Before + public void setup() throws Exception { + testZkServer = new TestingServer(true); + zookeeperUrl = testZkServer.getConnectString(); + client = ConfigurationsUtils.getClient(zookeeperUrl); + client.start(); + expectedGlobalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH); + expectedSensorParserConfigMap = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH); + expectedSensorEnrichmentConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); + } + + @Test + public void test() throws Exception { + Assert.assertTrue(expectedGlobalConfig.length > 0); + ConfigurationsUtils.writeGlobalConfigToZookeeper(expectedGlobalConfig, zookeeperUrl); + byte[] actualGlobalConfigBytes = ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client); + Assert.assertTrue(Arrays.equals(expectedGlobalConfig, actualGlobalConfigBytes)); + + Assert.assertTrue(expectedSensorParserConfigMap.size() > 0); + String testSensorType = "yaf"; + byte[] expectedSensorParserConfigBytes = expectedSensorParserConfigMap.get(testSensorType); + ConfigurationsUtils.writeSensorParserConfigToZookeeper(testSensorType, expectedSensorParserConfigBytes, zookeeperUrl); + byte[] actualSensorParserConfigBytes = ConfigurationsUtils.readSensorParserConfigBytesFromZookeeper(testSensorType, client); + Assert.assertTrue(Arrays.equals(expectedSensorParserConfigBytes, actualSensorParserConfigBytes)); + + Assert.assertTrue(expectedSensorEnrichmentConfigMap.size() > 0); + byte[] expectedSensorEnrichmentConfigBytes = expectedSensorEnrichmentConfigMap.get(testSensorType); + ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(testSensorType, expectedSensorEnrichmentConfigBytes, zookeeperUrl); + byte[] actualSensorEnrichmentConfigBytes = ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(testSensorType, client); + Assert.assertTrue(Arrays.equals(expectedSensorEnrichmentConfigBytes, actualSensorEnrichmentConfigBytes)); + + String name = "testConfig"; + Map<String, Object> testConfig = new HashMap<>(); + testConfig.put("stringField", "value"); + testConfig.put("intField", 1); + testConfig.put("doubleField", 1.1); + ConfigurationsUtils.writeConfigToZookeeper(name, testConfig, zookeeperUrl); + byte[] readConfigBytes = ConfigurationsUtils.readConfigBytesFromZookeeper(name, client); + Assert.assertTrue(Arrays.equals(JSONUtils.INSTANCE.toJSONPretty(testConfig), readConfigBytes)); + + } + + /** + * { + * "foo" : "bar" + * } + */ + @Multiline + private static String someConfig; + + @Test + public void modifies_global_configuration() throws Exception { + ConfigurationType type = ConfigurationType.GLOBAL; + ConfigurationsUtils + .writeConfigToZookeeper(type, JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl); + byte[] actual = ConfigurationsUtils.readConfigBytesFromZookeeper(type, zookeeperUrl); + assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(someConfig))); + } + + @Test + public void modifies_single_parser_configuration() throws Exception { + ConfigurationType type = ConfigurationType.PARSER; + String parserName = "a-happy-metron-parser"; + ConfigurationsUtils.writeConfigToZookeeper(type, Optional.of(parserName), + JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl); + byte[] actual = ConfigurationsUtils + .readConfigBytesFromZookeeper(type, Optional.of(parserName), zookeeperUrl); + assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(someConfig))); + } + + /** + * [ + * { + * "op": "replace", + * "path": "/foo", + * "value": "baz" + * } + * ] + */ + @Multiline + private static String patchSomeConfig; + + /** + * { + * "foo" : "baz" + * } + */ + @Multiline + private static String modifiedSomeConfig; + + /** + * Note: the current configuration structure mixes abstractions based on the configuration type + * and requires testing each type. GLOBAL is a actually representative of the final node name, + * whereas the other types, e.g. PARSER, represent a directory/path and not a ZK node where values + * are stored. The semantics are similar but slightly different. + */ + @Test + public void patches_global_configuration_via_patch_json() throws Exception { + ConfigurationType type = ConfigurationType.GLOBAL; + String parserName = "patched-metron-global-config"; + ConfigurationsUtils.writeConfigToZookeeper(type, JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl); + ConfigurationsUtils.applyConfigPatchToZookeeper(type, JSONUtils.INSTANCE.toJSONPretty(patchSomeConfig), zookeeperUrl); + byte[] actual = ConfigurationsUtils.readConfigBytesFromZookeeper(type, zookeeperUrl); + assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(modifiedSomeConfig))); + } + + @Test + public void patches_parser_configuration_via_patch_json() throws Exception { + ConfigurationType type = ConfigurationType.PARSER; + String parserName = "patched-metron-parser"; + ConfigurationsUtils.writeConfigToZookeeper(type, Optional.of(parserName), JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl); + ConfigurationsUtils.applyConfigPatchToZookeeper(type, Optional.of(parserName), JSONUtils.INSTANCE.toJSONPretty(patchSomeConfig), zookeeperUrl); + byte[] actual = ConfigurationsUtils.readConfigBytesFromZookeeper(type, Optional.of(parserName), zookeeperUrl); + assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(modifiedSomeConfig))); + } + + @After + public void tearDown() throws IOException { + client.close(); + testZkServer.close(); + testZkServer.stop(); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java index c478de5..7f4846e 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,29 +15,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.common.utils; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.test.utils.UnitTestHelper; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import java.io.File; -import java.util.HashMap; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.equalTo; - public class JSONUtilsTest { + private static File tmpDir; /** - { - "a" : "hello", - "b" : "world" - } + * { "a" : "hello", "b" : "world" } */ @Multiline private static String config; @@ -55,9 +55,10 @@ public class JSONUtilsTest { put("a", "hello"); put("b", "world"); }}; - Map<String, Object> actual = JSONUtils.INSTANCE.load(configFile, new TypeReference<Map<String, Object>>() { - }); - Assert.assertThat("config not equal", actual, equalTo(expected)); + Map<String, Object> actual = JSONUtils.INSTANCE + .load(configFile, new TypeReference<Map<String, Object>>() { + }); + assertThat("config not equal", actual, equalTo(expected)); } @Test @@ -67,18 +68,19 @@ public class JSONUtilsTest { put("b", "world"); }}; Map<String, Object> actual = JSONUtils.INSTANCE.load(configFile, Map.class); - Assert.assertThat("config not equal", actual, equalTo(expected)); + assertThat("config not equal", actual, equalTo(expected)); } @Test public void loads_file_with_custom_class() throws Exception { TestConfig expected = new TestConfig().setA("hello").setB("world"); TestConfig actual = JSONUtils.INSTANCE.load(configFile, TestConfig.class); - Assert.assertThat("a not equal", actual.getA(), equalTo(expected.getA())); - Assert.assertThat("b not equal", actual.getB(), equalTo(expected.getB())); + assertThat("a not equal", actual.getA(), equalTo(expected.getA())); + assertThat("b not equal", actual.getB(), equalTo(expected.getB())); } public static class TestConfig { + private String a; private String b; @@ -100,4 +102,67 @@ public class JSONUtilsTest { return this; } } + + /** + * { "a": "b" } + */ + @Multiline + public static String sourceJson; + + /** + * [{ "op": "move", "from": "/a", "path": "/c" }] + */ + @Multiline + public static String patchJson; + + /** + * { "c": "b" } + */ + @Multiline + public static String expectedJson; + + @Test + public void applyPatch_modifies_source_json_doc() throws IOException { + JsonNode actual = JSONUtils.INSTANCE.applyPatch(patchJson, sourceJson); + JsonNode expected = JSONUtils.INSTANCE.readTree(expectedJson); + assertThat(actual, equalTo(expected)); + } + + /** + * { + * "foo" : { + * "bar" : { + * "baz" : [ "val1", "val2" ] + * } + * } + * } + */ + @Multiline + public static String complexJson; + + /** + * [{ "op": "add", "path": "/foo/bar/baz", "value": [ "new1", "new2" ] }] + */ + @Multiline + public static String patchComplexJson; + + /** + * { + * "foo" : { + * "bar" : { + * "baz" : [ "new1", "new2" ] + * } + * } + * } + */ + @Multiline + public static String expectedComplexJson; + + @Test + public void applyPatch_modifies_complex_source_json_doc() throws IOException { + JsonNode actual = JSONUtils.INSTANCE.applyPatch(patchComplexJson, complexJson); + JsonNode expected = JSONUtils.INSTANCE.readTree(expectedComplexJson); + assertThat(actual, equalTo(expected)); + } + } http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 b/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 index f8b9b66..133f9c5 100755 --- a/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 +++ b/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 @@ -42,15 +42,15 @@ threat.intel.join.cache.size={{threatintel_join_cache_size}} ##### Enrichment ##### hbase.provider.impl={{enrichment_hbase_provider_impl}} -enrichment.simple.hbase.table={{enrichment_table}} -enrichment.simple.hbase.cf={{enrichment_cf}} +enrichment.simple.hbase.table={{enrichment_hbase_table}} +enrichment.simple.hbase.cf={{enrichment_hbase_cf}} enrichment.host.known_hosts={{enrichment_host_known_hosts}} ##### Threat Intel ##### -threat.intel.tracker.table={{threatintel_table}} -threat.intel.tracker.cf={{threatintel_cf}} -threat.intel.simple.hbase.table={{threatintel_table}} -threat.intel.simple.hbase.cf={{threatintel_cf}} +threat.intel.tracker.table={{threatintel_hbase_table}} +threat.intel.tracker.cf={{threatintel_hbase_cf}} +threat.intel.simple.hbase.table={{threatintel_hbase_table}} +threat.intel.simple.hbase.cf={{threatintel_hbase_cf}} ##### Parallelism ##### kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}} http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java index f77f16e..c457e86 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java @@ -17,6 +17,13 @@ */ package org.apache.metron.enrichment.integration; +import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_RULES_KEY; +import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_RULE_COMMENT; +import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_RULE_NAME; +import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_RULE_REASON; +import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_RULE_SCORE; +import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_SCORE_KEY; + import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; import com.google.common.base.Joiner; @@ -24,6 +31,18 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Stream; +import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; import org.apache.metron.TestConstants; import org.apache.metron.common.Constants; @@ -36,8 +55,8 @@ import org.apache.metron.enrichment.integration.components.ConfigUploadComponent import org.apache.metron.enrichment.lookup.LookupKV; import org.apache.metron.enrichment.lookup.accesstracker.PersistentBloomTrackerCreator; import org.apache.metron.enrichment.stellar.SimpleHBaseEnrichmentFunctions; -import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.hbase.mock.MockHBaseTableProvider; +import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.integration.BaseIntegrationTest; import org.apache.metron.integration.ComponentRunner; import org.apache.metron.integration.ProcessorResult; @@ -53,21 +72,6 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import javax.annotation.Nullable; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Stream; - -import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.*; - public class EnrichmentIntegrationTest extends BaseIntegrationTest { private static final String ERROR_TOPIC = "enrichment_error"; private static final String SRC_IP = "ip_src_addr"; @@ -129,15 +133,15 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest { setProperty("enrichment_join_cache_size", "1000"); setProperty("threatintel_join_cache_size", "1000"); setProperty("enrichment_hbase_provider_impl", "" + MockHBaseTableProvider.class.getName()); - setProperty("enrichment_table", enrichmentsTableName); - setProperty("enrichment_cf", cf); + setProperty("enrichment_hbase_table", enrichmentsTableName); + setProperty("enrichment_hbase_cf", cf); setProperty("enrichment_host_known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"}," + "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"}," + "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}," + "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}]"); - setProperty("threatintel_table", threatIntelTableName); - setProperty("threatintel_cf", cf); + setProperty("threatintel_hbase_table", threatIntelTableName); + setProperty("threatintel_hbase_cf", cf); setProperty("enrichment_kafka_spout_parallelism", "1"); http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-indexing/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml index 7d07665..55fe455 100644 --- a/metron-platform/metron-indexing/pom.xml +++ b/metron-platform/metron-indexing/pom.xml @@ -74,25 +74,6 @@ </exclusions> <scope>provided</scope> </dependency> - <dependency> - <groupId>com.flipkart.zjsonpatch</groupId> - <artifactId>zjsonpatch</artifactId> - <version>0.3.1</version> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - </exclusions> - </dependency> <dependency> <groupId>org.apache.storm</groupId> http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java index c890544..32bfab9 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GroupRequest; @@ -125,7 +124,7 @@ public class HBaseDao implements IndexDao { Put put = new Put(update.getGuid().getBytes()); long ts = update.getTimestamp() == null?System.currentTimeMillis():update.getTimestamp(); byte[] columnQualifier = Bytes.toBytes(ts); - byte[] doc = JSONUtils.INSTANCE.toJSON(update.getDocument()); + byte[] doc = JSONUtils.INSTANCE.toJSONPretty(update.getDocument()); put.addColumn(cf, columnQualifier, doc); getTableInterface().put(put); } http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java index 745dccd..be5d4fe 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java @@ -19,8 +19,12 @@ package org.apache.metron.indexing.dao; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; -import com.flipkart.zjsonpatch.JsonPatch; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; import org.apache.metron.indexing.dao.search.GroupResponse; @@ -28,16 +32,9 @@ import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.metron.indexing.dao.update.ReplaceRequest; -import org.apache.metron.indexing.dao.update.OriginalNotFoundException; - -import java.io.IOException; -import org.apache.metron.indexing.dao.search.FieldType; - -import java.util.List; -import java.util.Map; -import java.util.Optional; public interface IndexDao { @@ -115,7 +112,7 @@ public interface IndexDao { } } JsonNode originalNode = JSONUtils.INSTANCE.convert(latest, JsonNode.class); - JsonNode patched = JsonPatch.apply(request.getPatch(), originalNode); + JsonNode patched = JSONUtils.INSTANCE.applyPatch(request.getPatch(), originalNode); Map<String, Object> updated = JSONUtils.INSTANCE.getMapper() .convertValue(patched, new TypeReference<Map<String, Object>>() {}); Document d = new Document( updated http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java index 67acb33..e67d3c9 100644 --- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java @@ -18,8 +18,15 @@ package org.apache.metron.integration.utils; import java.io.BufferedReader; +import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.List; @@ -34,4 +41,79 @@ public class TestUtils { br.close(); return ret; } + + public static void write(File file, String[] contents) throws IOException { + StringBuilder b = new StringBuilder(); + for (String line : contents) { + b.append(line); + b.append(System.lineSeparator()); + } + write(file, b.toString()); + } + + /** + * Returns file passed in after writing + * + * @param file + * @param contents + * @return + * @throws IOException + */ + public static File write(File file, String contents) throws IOException { + com.google.common.io.Files.createParentDirs(file); + com.google.common.io.Files.write(contents, file, StandardCharsets.UTF_8); + return file; + } + + /** + * Cleans up after test run via runtime shutdown hooks + */ + public static File createTempDir(String prefix) throws IOException { + final Path tmpDir = Files.createTempDirectory(prefix); + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + try { + cleanDir(tmpDir); + } catch (IOException e) { + System.out.println("Warning: Unable to clean tmp folder."); + } + } + + }); + return tmpDir.toFile(); + } + + public static void cleanDir(Path dir) throws IOException { + Files.walkFileTree(dir, new SimpleFileVisitor<Path>() { + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (exc == null) { + return FileVisitResult.CONTINUE; + } else { + throw exc; + } + } + }); + } + + public static File createDir(File parent, String child) { + File newDir = new File(parent, child); + newDir.mkdirs(); + return newDir; + } + }
