Repository: metron Updated Branches: refs/heads/master 0caa9cd2b -> b43b3cc3c
METRON-1336 Patching Can Result in Bad Configuration (nickwallen) closes apache/metron#851 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b43b3cc3 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b43b3cc3 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b43b3cc3 Branch: refs/heads/master Commit: b43b3cc3cd78fcd35351650d895655488c230554 Parents: 0caa9cd Author: nickwallen <[email protected]> Authored: Wed Dec 6 18:26:13 2017 -0500 Committer: nickallen <[email protected]> Committed: Wed Dec 6 18:26:13 2017 -0500 ---------------------------------------------------------------------- .../metron/common/cli/ConfigurationManager.java | 9 +- .../configuration/ConfigurationOperations.java | 35 ++ .../common/configuration/ConfigurationType.java | 83 +-- .../configuration/ConfigurationsUtils.java | 69 ++- .../EnrichmentConfigurationOperations.java | 44 ++ .../GlobalConfigurationOperations.java | 51 ++ .../IndexingConfigurationOperations.java | 46 ++ .../ParserConfigurationOperations.java | 43 ++ .../ProfilerConfigurationOperations.java | 49 ++ .../ConfigurationManagerIntegrationTest.java | 500 +++++++++++++++---- .../configuration/ConfigurationsUtilsTest.java | 127 +++-- 11 files changed, 830 insertions(+), 226 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/b43b3cc3/metron-platform/metron-common/src/main/java/org/apache/metron/common/cli/ConfigurationManager.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/cli/ConfigurationManager.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/cli/ConfigurationManager.java index e5a4426..f42831c 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/cli/ConfigurationManager.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/cli/ConfigurationManager.java @@ -122,6 +122,7 @@ public class ConfigurationManager { ; Option option; String shortCode; + ConfigurationOptions(String shortCode, Function<String, Option> optionHandler) { this.shortCode = shortCode; this.option = optionHandler.apply(shortCode); @@ -295,19 +296,23 @@ public class ConfigurationManager { private void patch(CuratorFramework client, ConfigurationType configType, Optional<String> configName, Optional<String> patchMode, Optional<String> patchPath, - Optional<String> patchKey, Optional<String> patchValue) { + Optional<String> patchKey, Optional<String> patchValue) throws Exception { try { - byte[] patchData = null; + byte[] patchData; if (patchKey.isPresent()) { patchData = buildPatch(patchMode, patchKey, patchValue).getBytes(StandardCharsets.UTF_8); } else { patchData = java.nio.file.Files.readAllBytes(Paths.get(patchPath.get())); } ConfigurationsUtils.applyConfigPatchToZookeeper(configType, configName, patchData, client); + } catch (IOException e) { LOG.error("Unable to load patch file '%s'", patchPath, e); + throw e; + } catch (Exception e) { LOG.error("Unable to apply patch to Zookeeper config", e); + throw e; } } http://git-wip-us.apache.org/repos/asf/metron/blob/b43b3cc3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationOperations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationOperations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationOperations.java new file mode 100644 index 0000000..4de9889 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationOperations.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.configuration; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.Constants; + +import java.io.IOException; + +public interface ConfigurationOperations { + String getTypeName(); + default String getDirectory() { + return getTypeName(); + } + default String getZookeeperRoot() { + return Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + getTypeName(); + } + Object deserialize(String s) throws IOException; + void writeSensorConfigToZookeeper(String sensorType, byte[] configData, CuratorFramework client) throws Exception; +} http://git-wip-us.apache.org/repos/asf/metron/blob/b43b3cc3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java index e803341..6a03cb6 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java @@ -18,79 +18,38 @@ package org.apache.metron.common.configuration; -import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; -import org.apache.metron.common.Constants; -import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; -import org.apache.metron.common.configuration.profiler.ProfilerConfig; -import org.apache.metron.common.utils.JSONUtils; - import java.io.IOException; -import java.util.Map; - -public enum ConfigurationType implements Function<String, Object> { +import org.apache.curator.framework.CuratorFramework; - GLOBAL("global",".", s -> { - try { - return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() { - }); - } catch (IOException e) { - throw new RuntimeException("Unable to load " + s, e); - } - }), +public enum ConfigurationType implements Function<String, Object>, ConfigurationOperations { - PARSER("parsers","parsers", s -> { - try { - return JSONUtils.INSTANCE.load(s, SensorParserConfig.class); - } catch (IOException e) { - throw new RuntimeException("Unable to load " + s, e); - } - }), - - ENRICHMENT("enrichments","enrichments", s -> { - try { - return JSONUtils.INSTANCE.load(s, SensorEnrichmentConfig.class); - } catch (IOException e) { - throw new RuntimeException("Unable to load " + s, e); - } - }), - INDEXING("indexing","indexing", s -> { - try { - return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() { }); - } catch (IOException e) { - throw new RuntimeException("Unable to load " + s, e); - } - }), - PROFILER("profiler",".", s -> { - try { - return JSONUtils.INSTANCE.load(s, ProfilerConfig.class); - } catch (IOException e) { - throw new RuntimeException("Unable to load " + s, e); - } - }); + GLOBAL(new GlobalConfigurationOperations()), + PARSER(new ParserConfigurationOperations()), + ENRICHMENT(new EnrichmentConfigurationOperations()), + INDEXING(new IndexingConfigurationOperations()), + PROFILER(new ProfilerConfigurationOperations()); - String typeName; - String directory; - String zookeeperRoot; - Function<String,?> deserializer; + ConfigurationOperations ops; - ConfigurationType(String typeName, String directory, Function<String, ?> deserializer) { - this.typeName = typeName; - this.directory = directory; - this.zookeeperRoot = Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + typeName; - this.deserializer = deserializer; + ConfigurationType(ConfigurationOperations ops) { + this.ops = ops; } public String getTypeName() { - return typeName; + return ops.getTypeName(); } public String getDirectory() { - return directory; + return ops.getDirectory(); } public Object deserialize(String s) { - return deserializer.apply(s); + try { + return ops.deserialize(s); + } catch (IOException e) { + throw new RuntimeException("Unable to load " + s, e); + } } @Override @@ -98,8 +57,14 @@ public enum ConfigurationType implements Function<String, Object> { return deserialize(s); } + @Override + public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, + CuratorFramework client) throws Exception { + ops.writeSensorConfigToZookeeper(sensorType, configData, client); + } + public String getZookeeperRoot() { - return zookeeperRoot; + return ops.getZookeeperRoot(); } } http://git-wip-us.apache.org/repos/asf/metron/blob/b43b3cc3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java index ae21152..5183788 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java @@ -156,7 +156,7 @@ public class ConfigurationsUtils { } private static String getConfigZKPath(ConfigurationType configType, Optional<String> configName) { - String pathSuffix = configName.isPresent() && configType != GLOBAL ? "/" + configName : ""; + String pathSuffix = configName.isPresent() && configType != GLOBAL ? "/" + configName.get() : ""; return configType.getZookeeperRoot() + pathSuffix; } @@ -343,9 +343,14 @@ public class ConfigurationsUtils { * @param type config type to upload configs for * @param configName specific config under the specified config type */ - public static void uploadConfigsToZookeeper(String rootFilePath, CuratorFramework client, - ConfigurationType type, Optional<String> configName) throws Exception { + public static void uploadConfigsToZookeeper( + String rootFilePath, + CuratorFramework client, + ConfigurationType type, + Optional<String> configName) throws Exception { + switch (type) { + case GLOBAL: final byte[] globalConfig = readGlobalConfigFromFile(rootFilePath); if (globalConfig.length > 0) { @@ -353,15 +358,27 @@ public class ConfigurationsUtils { writeGlobalConfigToZookeeper(globalConfig, client); } break; - case PARSER: // intentional pass-through - case ENRICHMENT: // intentional pass-through - case INDEXING: - Map<String, byte[]> sensorIndexingConfigs = readSensorConfigsFromFile(rootFilePath, type, - configName); - for (String sensorType : sensorIndexingConfigs.keySet()) { - writeConfigToZookeeper(type, configName, sensorIndexingConfigs.get(sensorType), client); + + case PARSER: //pass through intentional + case ENRICHMENT: //pass through intentional + case INDEXING: //pass through intentional + { + Map<String, byte[]> configs = readSensorConfigsFromFile(rootFilePath, type, configName); + for (String sensorType : configs.keySet()) { + byte[] configData = configs.get(sensorType); + type.writeSensorConfigToZookeeper(sensorType, configData, client); } break; + } + + case PROFILER: { + byte[] configData = readProfilerConfigFromFile(rootFilePath); + if (configData.length > 0) { + ConfigurationsUtils.writeProfilerConfigToZookeeper(configData, client); + } + break; + } + default: throw new IllegalArgumentException("Configuration type not found: " + type); } @@ -437,14 +454,16 @@ public class ConfigurationsUtils { In order to validate stellar functions, the function resolver must be initialized. Otherwise, those utilities that require validation cannot validate the stellar expressions necessarily. */ - Context.Builder builder = new Context.Builder().with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) - ; + Context.Builder builder = new Context.Builder() + .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client); + if(globalConfig.isPresent()) { - builder = builder.with(Context.Capabilities.GLOBAL_CONFIG, () -> GLOBAL.deserialize(globalConfig.get())) + builder = builder + .with(Context.Capabilities.GLOBAL_CONFIG, () -> GLOBAL.deserialize(globalConfig.get())) .with(Context.Capabilities.STELLAR_CONFIG, () -> GLOBAL.deserialize(globalConfig.get())); - } - else { - builder = builder.with(Context.Capabilities.STELLAR_CONFIG, () -> new HashMap<>()); + } else { + builder = builder + .with(Context.Capabilities.STELLAR_CONFIG, () -> new HashMap<>()); } Context stellarContext = builder.build(); StellarFunctions.FUNCTION_RESOLVER().initialize(stellarContext); @@ -533,7 +552,6 @@ public class ConfigurationsUtils { * Starts up curatorclient based on zookeeperUrl. * * @param configurationType GLOBAL, PARSER, etc. - * @param configName e.g. bro, yaf, snort * @param patchData a JSON patch in the format specified by RFC 6902 * @param zookeeperUrl configs are here */ @@ -572,15 +590,22 @@ public class ConfigurationsUtils { * @param patchData a JSON patch in the format specified by RFC 6902 * @param client access to zookeeeper */ - public static void applyConfigPatchToZookeeper(ConfigurationType configurationType, - Optional<String> configName, - byte[] patchData, CuratorFramework client) throws Exception { + public static void applyConfigPatchToZookeeper( + ConfigurationType configurationType, + Optional<String> configName, + byte[] patchData, CuratorFramework client) throws Exception { + byte[] configData = readConfigBytesFromZookeeper(configurationType, configName, client); JsonNode source = JSONUtils.INSTANCE.readTree(configData); JsonNode patch = JSONUtils.INSTANCE.readTree(patchData); JsonNode patchedConfig = JSONUtils.INSTANCE.applyPatch(patch, source); - writeConfigToZookeeper(configurationType, configName, - JSONUtils.INSTANCE.toJSONPretty(patchedConfig), client); + byte[] prettyPatchedConfig = JSONUtils.INSTANCE.toJSONPretty(patchedConfig); + + // ensure the patch produces a valid result; otherwise exception thrown during deserialization + String prettyPatchedConfigStr = new String(prettyPatchedConfig); + configurationType.deserialize(prettyPatchedConfigStr); + + writeConfigToZookeeper(configurationType, configName, prettyPatchedConfig, client); } public interface ConfigurationVisitor{ http://git-wip-us.apache.org/repos/asf/metron/blob/b43b3cc3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurationOperations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurationOperations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurationOperations.java new file mode 100644 index 0000000..f422be9 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurationOperations.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.configuration; + +import java.io.IOException; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.common.utils.JSONUtils; + +public class EnrichmentConfigurationOperations implements ConfigurationOperations { + + @Override + public String getTypeName() { + return "enrichments"; + } + + @Override + public Object deserialize(String s) throws IOException { + return JSONUtils.INSTANCE.load(s, SensorEnrichmentConfig.class); + } + + @Override + public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, + CuratorFramework client) throws Exception { + ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, configData, client); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/b43b3cc3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java new file mode 100644 index 0000000..4842c91 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.configuration; + +import com.fasterxml.jackson.core.type.TypeReference; +import java.io.IOException; +import java.util.Map; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.utils.JSONUtils; + +public class GlobalConfigurationOperations implements ConfigurationOperations { + + @Override + public String getTypeName() { + return "global"; + } + + @Override + public String getDirectory() { + return "."; + } + + @Override + public Object deserialize(String s) throws IOException { + return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() { + }); + } + + @Override + public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, + CuratorFramework client) { + throw new UnsupportedOperationException("Global configs are not per-sensor"); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/b43b3cc3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java new file mode 100644 index 0000000..a75c84e --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.configuration; + +import com.fasterxml.jackson.core.type.TypeReference; +import java.io.IOException; +import java.util.Map; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.utils.JSONUtils; + +public class IndexingConfigurationOperations implements ConfigurationOperations { + + @Override + public String getTypeName() { + return "indexing"; + } + + @Override + public Object deserialize(String s) throws IOException { + return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() { + }); + } + + @Override + public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, + CuratorFramework client) throws Exception { + ConfigurationsUtils.writeSensorIndexingConfigToZookeeper(sensorType, configData, client); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/b43b3cc3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurationOperations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurationOperations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurationOperations.java new file mode 100644 index 0000000..8380e78 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurationOperations.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.configuration; + +import java.io.IOException; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.utils.JSONUtils; + +public class ParserConfigurationOperations implements ConfigurationOperations { + + @Override + public String getTypeName() { + return "parsers"; + } + + @Override + public Object deserialize(String s) throws IOException { + return JSONUtils.INSTANCE.load(s, SensorParserConfig.class); + } + + @Override + public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, + CuratorFramework client) throws Exception { + ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorType, configData, client); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/b43b3cc3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ProfilerConfigurationOperations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ProfilerConfigurationOperations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ProfilerConfigurationOperations.java new file mode 100644 index 0000000..7d8c2f3 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ProfilerConfigurationOperations.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.configuration; + +import java.io.IOException; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.configuration.profiler.ProfilerConfig; +import org.apache.metron.common.utils.JSONUtils; + +public class ProfilerConfigurationOperations implements ConfigurationOperations { + + @Override + public String getTypeName() { + return "profiler"; + } + + @Override + public String getDirectory() { + return "."; + } + + @Override + public Object deserialize(String s) throws IOException { + return JSONUtils.INSTANCE.load(s, ProfilerConfig.class); + } + + @Override + public void writeSensorConfigToZookeeper(String sensorType, byte[] configData, + CuratorFramework client) throws Exception { + throw new UnsupportedOperationException("Profiler configs are not per-sensor"); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/b43b3cc3/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java index ed7f4bc..438da3c 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java @@ -23,6 +23,7 @@ import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMEN import static org.apache.metron.common.configuration.ConfigurationType.GLOBAL; import static org.apache.metron.common.configuration.ConfigurationType.INDEXING; import static org.apache.metron.common.configuration.ConfigurationType.PARSER; +import static org.apache.metron.common.configuration.ConfigurationType.PROFILER; import static org.apache.metron.common.utils.StringUtils.stripLines; import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.Assert.fail; @@ -96,28 +97,30 @@ public class ConfigurationManagerIntegrationTest { File sensorDir = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), ENRICHMENT.getDirectory()); sensors.addAll(Collections2.transform( Arrays.asList(sensorDir.list()) - ,s -> Iterables.getFirst(Splitter.on('.').split(s), "null") - ) - ); + , s -> Iterables.getFirst(Splitter.on('.').split(s), "null"))); tmpDir = TestUtils.createTempDir(this.getClass().getName()); configDir = TestUtils.createDir(tmpDir, "config"); parsersDir = TestUtils.createDir(configDir, "parsers"); enrichmentsDir = TestUtils.createDir(configDir, "enrichments"); indexingDir = TestUtils.createDir(configDir, "indexing"); - pushConfigs(); + pushAllConfigs(); } - private void pushConfigs() throws Exception { + private void pushAllConfigs() throws Exception { + pushAllConfigs(TestConstants.SAMPLE_CONFIG_PATH); + } + + private void pushAllConfigs(String inputDir) throws Exception { String[] args = new String[]{ "-z", zookeeperUrl , "--mode", "PUSH" - , "--input_dir", TestConstants.SAMPLE_CONFIG_PATH + , "--input_dir", inputDir }; ConfigurationManager manager = new ConfigurationManager(); manager.run(ConfigurationManager.ConfigurationOptions.parse(new PosixParser(), args)); } - public void pullConfigs(boolean force) throws Exception { + private void pullConfigs(boolean force) throws Exception { String[] args = null; if(force) { args = new String[]{ @@ -126,8 +129,8 @@ public class ConfigurationManagerIntegrationTest { , "--output_dir", outDir , "--force" }; - } - else { + + } else { args = new String[]{ "-z", zookeeperUrl , "--mode", "PULL" @@ -138,7 +141,7 @@ public class ConfigurationManagerIntegrationTest { manager.run(ConfigurationManager.ConfigurationOptions.parse(new PosixParser(), args)); } - public void validateConfigsOnDisk(File configDir) throws IOException { + private void validateConfigsOnDisk(File configDir) throws IOException { File globalConfigFile = new File(configDir, "global.json"); Assert.assertTrue("Global config does not exist", globalConfigFile.exists()); validateConfig("global", GLOBAL, new String(Files.readAllBytes(Paths.get(globalConfigFile.toURI())))); @@ -158,25 +161,30 @@ public class ConfigurationManagerIntegrationTest { //second time without force should pullConfigs(false); fail("Should have failed to pull configs in a directory structure that already exists."); - } - catch(IllegalStateException t) { + + } catch(IllegalStateException t) { //make sure we didn't bork anything validateConfigsOnDisk(new File(outDir)); } pullConfigs(true); validateConfigsOnDisk(new File(outDir)); } - public void validateConfig(String name, ConfigurationType type, String data) - { + + private void validateConfig(String name, ConfigurationType type, String data) { try { type.deserialize(data); } catch (Exception e) { fail("Unable to load config " + name + ": " + data); } } + @Test - public void testPush() throws Exception { - pushConfigs(); + public void testPushAll() throws Exception { + + // push all configs; parser, enrichment, indexing, etc + pushAllConfigs(); + + // validate final Set<String> sensorsInZookeeper = new HashSet<>(); final BooleanWritable foundGlobal = new BooleanWritable(false); ConfigurationsUtils.visitConfigs(client, new ConfigurationsUtils.ConfigurationVisitor() { @@ -197,22 +205,63 @@ public class ConfigurationManagerIntegrationTest { Assert.assertEquals(sensorsInZookeeper, sensors); } + @Test(expected = RuntimeException.class) + public void testPushAllWithBadConfig() throws Exception { + + // create a bad global config + File globalConfigFile = new File(configDir, "global.json"); + TestUtils.write(globalConfigFile, badGlobalConfig); + + // create a parser config + File squidConfigFile = new File(parsersDir, "squid.json"); + TestUtils.write(squidConfigFile, badParserConfig); + + pushAllConfigs(configDir.getAbsolutePath()); + + // exception expected as the global and parser config is invalid + } + /** * { "a": "b" } */ @Multiline - private static String someConfig; + private static String globalConfig; @Test - public void writes_global_config_to_zookeeper() throws Exception { + public void testPushGlobal() throws Exception { + + // create the config File configFile = new File(configDir, "global.json"); - TestUtils.write(configFile, someConfig); + TestUtils.write(configFile, globalConfig); + + // push the global config pushConfigs(GLOBAL, configDir); - byte[] expected = JSONUtils.INSTANCE.toJSONPretty(someConfig); + + // validate + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(globalConfig); byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(GLOBAL), 1)); Assert.assertThat(actual, equalTo(expected)); } + /** + * { "invalid as needs to be key/values" } + */ + @Multiline + private static String badGlobalConfig; + + @Test(expected = RuntimeException.class) + public void testPushGlobalWithBadConfig() throws Exception { + + // create the config + File configFile = new File(configDir, "global.json"); + TestUtils.write(configFile, badGlobalConfig); + + // push the global config + pushConfigs(GLOBAL, configDir); + + // exception expected as the global config is invalid + } + private void pushConfigs(ConfigurationType type, File configPath) throws Exception { pushConfigs(type, configPath, Optional.empty()); } @@ -251,86 +300,118 @@ public class ConfigurationManagerIntegrationTest { } public interface RedirectCallback { - void call(String[] args) throws Exception; } - private String redirectSystemOut(final String[] args, RedirectCallback callback) - throws Exception { + private String redirectSystemOut(final String[] args, RedirectCallback callback) throws Exception { PrintStream os = System.out; - try (OutputStream baos = new ByteArrayOutputStream(); - PrintStream ps = new PrintStream(baos)) { + try (OutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos)) { System.setOut(ps); callback.call(args); System.out.flush(); System.setOut(os); return baos.toString(); + } finally { System.setOut(os); } } /** - *{ - "parserClassName": "org.apache.metron.parsers.GrokParser", - "sensorTopic": "squid", - "parserConfig": { - "grokPath": "/patterns/squid", - "patternLabel": "SQUID_DELIMITED", - "timestampField": "timestamp" - }, - "fieldTransformations" : [ - { - "transformation" : "STELLAR" - ,"output" : [ "full_hostname", "domain_without_subdomains" ] - ,"config" : { - "full_hostname" : "URL_TO_HOST(url)" - ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" - } - } - ] - } + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "squid", + * "parserConfig": { + * "grokPath": "/patterns/squid", + * "patternLabel": "SQUID_DELIMITED", + * "timestampField": "timestamp" + * }, + * "fieldTransformations" : [ + * { + * "transformation" : "STELLAR", + * "output" : [ "full_hostname", "domain_without_subdomains" ], + * "config" : { + * "full_hostname" : "URL_TO_HOST(url)", + * "domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" + * } + * } + * ] + * } */ @Multiline private static String squidParserConfig; @Test - public void writes_single_parser_config_to_zookeeper() throws Exception { + public void testPushParser() throws Exception { + + // create a parser config File configFile = new File(parsersDir, "myparser.json"); TestUtils.write(configFile, squidParserConfig); + + // push the parser config pushConfigs(PARSER, configDir, Optional.of("myparser")); + + // validate byte[] expected = JSONUtils.INSTANCE.toJSONPretty(squidParserConfig); byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(PARSER, Optional.of("myparser")), 1)); Assert.assertThat(actual, equalTo(expected)); } /** + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "invalidFieldForParserConfig": "22" + * } + */ + @Multiline + private static String badParserConfig; + + @Test(expected = RuntimeException.class) + public void testPushParserWithBadConfig() throws Exception { + + // create a parser config + File configFile = new File(parsersDir, "badparser.json"); + TestUtils.write(configFile, badParserConfig); + + // push the parser config + pushConfigs(PARSER, configDir, Optional.of("badparser")); + + // exception expected as the parser config is invalid + } + + /** * { - "enrichment" : { - "fieldMap": { - "geo": ["ip_dst_addr", "ip_src_addr"], - "host": ["host"] - } - }, - "threatIntel": { - "fieldMap": { - "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] - }, - "fieldToTypeMap": { - "ip_src_addr" : ["malicious_ip"], - "ip_dst_addr" : ["malicious_ip"] - } - } - } + * "enrichment" : { + * "fieldMap": { + * "geo": ["ip_dst_addr", "ip_src_addr"], + * "host": ["host"] + * } + * }, + * "threatIntel": { + * "fieldMap": { + * "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] + * }, + * "fieldToTypeMap": { + * "ip_src_addr" : ["malicious_ip"], + * "ip_dst_addr" : ["malicious_ip"] + * } + * } + * } */ @Multiline private static String someEnrichmentConfig; @Test - public void writes_single_enrichment_config_to_zookeeper() throws Exception { + public void testPushEnrichment() throws Exception { + + // create enrichment config File configFile = new File(enrichmentsDir, "myenrichment.json"); TestUtils.write(configFile, someEnrichmentConfig); + + // push enrichment config pushConfigs(ENRICHMENT, configDir, Optional.of("myenrichment")); + + // validate byte[] expected = JSONUtils.INSTANCE.toJSONPretty(someEnrichmentConfig); byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(ENRICHMENT, Optional.of("myenrichment")), 1)); Assert.assertThat(actual, equalTo(expected)); @@ -338,37 +419,153 @@ public class ConfigurationManagerIntegrationTest { /** * { - "hdfs" : { - "index": "myindex", - "batchSize": 5, - "enabled" : true - }, - "elasticsearch" : { - "index": "myindex", - "batchSize": 5, - "enabled" : true - }, - "solr" : { - "index": "myindex", - "batchSize": 5, - "enabled" : true - } - } + * "enrichment" : { + * "fieldMap": { + * "geo": ["ip_dst_addr", "ip_src_addr"], + * "host": ["host"] + * } + * }, + * "invalidField": { + * + * } + * } + */ + @Multiline + private static String badEnrichmentConfig; + + @Test(expected = RuntimeException.class) + public void testPushEnrichmentWithBadConfig() throws Exception { + + // create enrichment config + File configFile = new File(enrichmentsDir, "badenrichment.json"); + TestUtils.write(configFile, badEnrichmentConfig); + + // push enrichment config + pushConfigs(ENRICHMENT, configDir, Optional.of("badenrichment")); + + // exception expected as the enrichment config is invalid + } + + /** + * { + * "hdfs" : { + * "index": "myindex", + * "batchSize": 5, + * "enabled" : true + * }, + * "elasticsearch" : { + * "index": "myindex", + * "batchSize": 5, + * "enabled" : true + * }, + * "solr" : { + * "index": "myindex", + * "batchSize": 5, + * "enabled" : true + * } + * } */ @Multiline private static String someIndexingConfig; @Test - public void writes_single_indexing_config_to_zookeeper() throws Exception { + public void testPushIndexing() throws Exception { + + // write the indexing config File configFile = new File(indexingDir, "myindex.json"); TestUtils.write(configFile, someIndexingConfig); + + // push the index config pushConfigs(INDEXING, configDir, Optional.of("myindex")); + + // validate byte[] expected = JSONUtils.INSTANCE.toJSONPretty(someIndexingConfig); byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(INDEXING, Optional.of("myindex")), 1)); Assert.assertThat(actual, equalTo(expected)); } /** + * { + * "hdfs" + * } + */ + @Multiline + private static String badIndexingConfig; + + @Test(expected = RuntimeException.class) + public void testPushIndexingWithBadConfig() throws Exception { + + // write the indexing config + File configFile = new File(indexingDir, "myindex.json"); + TestUtils.write(configFile, badIndexingConfig); + + // push the index config + pushConfigs(INDEXING, configDir, Optional.of("myindex")); + + // exception expected as the indexing config is invalid + } + + /** + * { + * "profiles": [ + * { + * "profile": "hello-world", + * "onlyif": "exists(ip_src_addr)", + * "foreach": "ip_src_addr", + * "init": { "count": "0" }, + * "update": { "count": "count + 1" }, + * "result": "count" + * } + * ] + * } + */ + @Multiline + private static String someProfilerConfig; + + @Test + public void testPushProfiler() throws Exception { + + // create the profiler config + File configFile = new File(configDir, "profiler.json"); + TestUtils.write(configFile, someProfilerConfig); + + // push the profiler config + Optional<String> configName = Optional.empty(); + pushConfigs(PROFILER, configDir, configName); + + // validate + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(someProfilerConfig); + byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(PROFILER, configName), 1)); + Assert.assertThat(actual, equalTo(expected)); + } + + /** + * { + * "profiles": [ + * { + * "profile": "invalid; missing foreach, result, etc" + * } + * ] + * } + */ + @Multiline + private static String badProfilerConfig; + + @Test(expected = RuntimeException.class) + public void testPushProfilerWithBadConfig() throws Exception { + + // write the indexing config + File configFile = new File(configDir, "profiler.json"); + TestUtils.write(configFile, badProfilerConfig); + + // push the index config + Optional<String> configName = Optional.empty(); + pushConfigs(PROFILER, configDir, configName); + + // exception expected as the profiler config is invalid + } + + /** * [ { "op": "replace", "path": "/a", "value": [ "new1", "new2" ] } ] */ @Multiline @@ -381,13 +578,21 @@ public class ConfigurationManagerIntegrationTest { private static String expectedSomeConfig; @Test - public void patches_global_config_from_file() throws Exception { + public void testPatchGlobalFromFile() throws Exception { + + // create a patch file File patchFile = new File(tmpDir, "global-config-patch.json"); TestUtils.write(patchFile, somePatchConfig); + + // create the global config File configFile = new File(configDir, "global.json"); - TestUtils.write(configFile, someConfig); + TestUtils.write(configFile, globalConfig); pushConfigs(GLOBAL, configDir, Optional.of("global")); + + // patch the global config patchConfigs(GLOBAL, Optional.of(patchFile), Optional.of("global"), Optional.empty(), Optional.empty(), Optional.empty()); + + // validate the patch byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedSomeConfig); byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(GLOBAL, Optional.of("global")), 1)); Assert.assertThat(actual, equalTo(expected)); @@ -417,60 +622,108 @@ public class ConfigurationManagerIntegrationTest { } /** - * [ { "op": "replace", "path": "/parserConfig/timestampField", "value": "heyjoe" } ] + * [ + * { + * "op": "replace", + * "path": "/parserConfig/timestampField", + * "value": "heyjoe" + * } + * ] */ @Multiline public static String someParserPatch; /** - *{ - "parserClassName": "org.apache.metron.parsers.GrokParser", - "sensorTopic": "squid", - "parserConfig": { - "grokPath": "/patterns/squid", - "patternLabel": "SQUID_DELIMITED", - "timestampField": "heyjoe" - }, - "fieldTransformations" : [ - { - "transformation" : "STELLAR" - ,"output" : [ "full_hostname", "domain_without_subdomains" ] - ,"config" : { - "full_hostname" : "URL_TO_HOST(url)" - ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" - } - } - ] - } + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "squid", + * "parserConfig": { + * "grokPath": "/patterns/squid", + * "patternLabel": "SQUID_DELIMITED", + * "timestampField": "heyjoe" + * }, + * "fieldTransformations" : [ + * { + * "transformation" : "STELLAR", + * "output" : [ "full_hostname", "domain_without_subdomains" ], + * "config" : { + * "full_hostname" : "URL_TO_HOST(url)", + * "domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" + * } + * } + * ] + * } */ @Multiline public static String expectedPatchedParser; @Test - public void patches_parser_config_from_file() throws Exception { + public void testPatchParserFromFile() throws Exception { + + // create a patch file File patchFile = new File(tmpDir, "parser-patch.json"); TestUtils.write(patchFile, someParserPatch); + + // create a parser configuration File configFile = new File(parsersDir, "myparser.json"); TestUtils.write(configFile, squidParserConfig); pushConfigs(PARSER, configDir, Optional.of("myparser")); + + // patch the configuration patchConfigs(PARSER, Optional.of(patchFile), Optional.of("myparser"), Optional.empty(), Optional.empty(), Optional.empty()); + + // validate the patch byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedPatchedParser); byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(PARSER, Optional.of("myparser")), 1)); Assert.assertThat(actual, equalTo(expected)); } @Test - public void patches_parser_config_from_key_value() throws Exception { + public void testPatchParserFromKeyValue() throws Exception { + + // push the parser config File configFile = new File(parsersDir, "myparser.json"); TestUtils.write(configFile, squidParserConfig); pushConfigs(PARSER, configDir, Optional.of("myparser")); + + // patch the parser configuration patchConfigs(PARSER, Optional.empty(), Optional.of("myparser"), Optional.of(ADD), Optional.of("/parserConfig/timestampField"), Optional.of("\"\"heyjoe\"\"")); + + // validate the patch byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedPatchedParser); byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(PARSER, Optional.of("myparser")), 1)); Assert.assertThat(actual, equalTo(expected)); } /** + * [ + * { + * "op": "add", + * "path": "/invalidFieldForParserConfig", + * "value": "22" + * } + * ] + */ + @Multiline + public static String badParserPatch; + + @Test(expected = RuntimeException.class) + public void testPatchParserWithBadConfig() throws Exception { + + // create a patch file that when applied makes the parser config invalid + File patchFile = new File(tmpDir, "parser-patch.json"); + TestUtils.write(patchFile, badParserPatch); + + // create a parser configuration + File configFile = new File(parsersDir, "myparser.json"); + TestUtils.write(configFile, squidParserConfig); + pushConfigs(PARSER, configDir, Optional.of("myparser")); + + // patch the configuration + patchConfigs(PARSER, Optional.of(patchFile), Optional.of("myparser"), Optional.empty(), Optional.empty(), Optional.empty()); + } + + /** * { * "a": "b", * "foo": { @@ -484,16 +737,53 @@ public class ConfigurationManagerIntegrationTest { private static String expectedComplexConfig; @Test - public void patches_global_config_from_complex_key_value() throws Exception { + public void testPatchGlobalFromComplexKeyValue() throws Exception { + + // write a global configuration File configFile = new File(configDir, "global.json"); - TestUtils.write(configFile, someConfig); + TestUtils.write(configFile, globalConfig); pushConfigs(GLOBAL, configDir, Optional.of("global")); + + // patch the global configuration patchConfigs(GLOBAL, Optional.empty(), Optional.of("global"), Optional.of(ADD), Optional.of("/foo"), Optional.of("{ \"bar\" : { \"baz\" : [ \"bazval1\", \"bazval2\" ] } }")); + + // validate the patch byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedComplexConfig); byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(GLOBAL, Optional.of("global")), 1)); Assert.assertThat(actual, equalTo(expected)); } + /** + * [ + * { + * "op": "add", + * "path": "/invalidFieldForProfilerConfig", + * "value": "22" + * } + * ] + */ + @Multiline + public static String badProfilerPatch; + + @Test(expected = RuntimeException.class) + public void testPatchProfilerWithBadConfig() throws Exception { + + // create a patch file that when applied makes the profiler config invalid + File patchFile = new File(tmpDir, "patch.json"); + TestUtils.write(patchFile, badProfilerPatch); + + // create the profiler config + File configFile = new File(configDir, "profiler.json"); + TestUtils.write(configFile, someProfilerConfig); + + // push the profiler config + pushConfigs(PROFILER, configDir, Optional.empty()); + + // patch the profiler config + patchConfigs(PROFILER, Optional.of(patchFile), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + } + + @After public void tearDown() throws IOException { client.close(); http://git-wip-us.apache.org/repos/asf/metron/blob/b43b3cc3/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java index 145d7ce..8f8cb54 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java @@ -88,32 +88,10 @@ public class ConfigurationsUtilsTest { } /** - * { - * "foo" : "bar" - * } + * { "foo": "bar" } */ @Multiline - private static String someConfig; - - @Test - public void modifies_global_configuration() throws Exception { - ConfigurationType type = ConfigurationType.GLOBAL; - ConfigurationsUtils - .writeConfigToZookeeper(type, JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl); - byte[] actual = ConfigurationsUtils.readConfigBytesFromZookeeper(type, zookeeperUrl); - assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(someConfig))); - } - - @Test - public void modifies_single_parser_configuration() throws Exception { - ConfigurationType type = ConfigurationType.PARSER; - String parserName = "a-happy-metron-parser"; - ConfigurationsUtils.writeConfigToZookeeper(type, Optional.of(parserName), - JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl); - byte[] actual = ConfigurationsUtils - .readConfigBytesFromZookeeper(type, Optional.of(parserName), zookeeperUrl); - assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(someConfig))); - } + private static String someGlobalConfig; /** * [ @@ -125,7 +103,7 @@ public class ConfigurationsUtilsTest { * ] */ @Multiline - private static String patchSomeConfig; + private static String patchGlobalConfig; /** * { @@ -133,7 +111,63 @@ public class ConfigurationsUtilsTest { * } */ @Multiline - private static String modifiedSomeConfig; + private static String modifiedGlobalConfig; + + /** + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "squid" + * } + */ + @Multiline + private static String someParserConfig; + + /** + * [ + * { + * "op": "replace", + * "path": "/sensorTopic", + * "value": "new-squid-topic" + * } + * ] + */ + @Multiline + private static String patchParserConfig; + + /** + * { + * "parserClassName": "org.apache.metron.parsers.GrokParser", + * "sensorTopic": "new-squid-topic" + * } + */ + @Multiline + private static String modifiedParserConfig; + + @Test + public void modifiedGlobalConfiguration() throws Exception { + + // write global configuration + ConfigurationType type = ConfigurationType.GLOBAL; + ConfigurationsUtils.writeConfigToZookeeper(type, JSONUtils.INSTANCE.toJSONPretty(someParserConfig), zookeeperUrl); + + // validate the modified global configuration + byte[] actual = ConfigurationsUtils.readConfigBytesFromZookeeper(type, zookeeperUrl); + assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(someParserConfig))); + } + + @Test + public void modifiesSingleParserConfiguration() throws Exception { + + // write parser configuration + ConfigurationType type = ConfigurationType.PARSER; + String parserName = "a-happy-metron-parser"; + byte[] config = JSONUtils.INSTANCE.toJSONPretty(someParserConfig); + ConfigurationsUtils.writeConfigToZookeeper(type, Optional.of(parserName), config, zookeeperUrl); + + // validate the modified parser configuration + byte[] actual = ConfigurationsUtils.readConfigBytesFromZookeeper(type, Optional.of(parserName), zookeeperUrl); + assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(someParserConfig))); + } /** * Note: the current configuration structure mixes abstractions based on the configuration type @@ -142,23 +176,40 @@ public class ConfigurationsUtilsTest { * are stored. The semantics are similar but slightly different. */ @Test - public void patches_global_configuration_via_patch_json() throws Exception { - ConfigurationType type = ConfigurationType.GLOBAL; - String parserName = "patched-metron-global-config"; - ConfigurationsUtils.writeConfigToZookeeper(type, JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl); - ConfigurationsUtils.applyConfigPatchToZookeeper(type, JSONUtils.INSTANCE.toJSONPretty(patchSomeConfig), zookeeperUrl); + public void patchesGlobalConfigurationViaPatchJSON() throws Exception { + + // setup zookeeper with a configuration + final ConfigurationType type = ConfigurationType.GLOBAL; + byte[] config = JSONUtils.INSTANCE.toJSONPretty(someGlobalConfig); + ConfigurationsUtils.writeConfigToZookeeper(type, config, zookeeperUrl); + + // patch the configuration + byte[] patch = JSONUtils.INSTANCE.toJSONPretty(patchGlobalConfig); + ConfigurationsUtils.applyConfigPatchToZookeeper(type, patch, zookeeperUrl); + + // validate the patched configuration byte[] actual = ConfigurationsUtils.readConfigBytesFromZookeeper(type, zookeeperUrl); - assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(modifiedSomeConfig))); + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(modifiedGlobalConfig); + assertThat(actual, equalTo(expected)); } @Test - public void patches_parser_configuration_via_patch_json() throws Exception { - ConfigurationType type = ConfigurationType.PARSER; - String parserName = "patched-metron-parser"; - ConfigurationsUtils.writeConfigToZookeeper(type, Optional.of(parserName), JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl); - ConfigurationsUtils.applyConfigPatchToZookeeper(type, Optional.of(parserName), JSONUtils.INSTANCE.toJSONPretty(patchSomeConfig), zookeeperUrl); + public void patchesParserConfigurationViaPatchJSON() throws Exception { + + // setup zookeeper with a configuration + final ConfigurationType type = ConfigurationType.PARSER; + final String parserName = "patched-metron-parser"; + byte[] config = JSONUtils.INSTANCE.toJSONPretty(someParserConfig); + ConfigurationsUtils.writeConfigToZookeeper(type, Optional.of(parserName), config, zookeeperUrl); + + // patch the configuration + byte[] patch = JSONUtils.INSTANCE.toJSONPretty(patchParserConfig); + ConfigurationsUtils.applyConfigPatchToZookeeper(type, Optional.of(parserName), patch, zookeeperUrl); + + // validate the patched configuration byte[] actual = ConfigurationsUtils.readConfigBytesFromZookeeper(type, Optional.of(parserName), zookeeperUrl); - assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(modifiedSomeConfig))); + byte[] expected = JSONUtils.INSTANCE.toJSONPretty(modifiedParserConfig); + assertThat(actual, equalTo(expected)); } @After
