http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java index 36863e3..a9446ce 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java @@ -24,6 +24,7 @@ import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.common.configuration.extensions.ParserExtensionConfig; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.metron.common.utils.JSONUtils; @@ -70,6 +71,17 @@ public class ConfigurationsUtils { writeToZookeeper(GLOBAL.getZookeeperRoot(), globalConfig, client); } + public static void writeGlobalBundlePropertiesToZookeeper(byte[] bundleProperties, String zookeeperUrl) throws Exception{ + try(CuratorFramework client = getClient(zookeeperUrl)){ + client.start(); + writeGlobalBundlePropertiesToZookeeper(bundleProperties,client); + } + } + + public static void writeGlobalBundlePropertiesToZookeeper(byte[] bundleProperties, CuratorFramework client) throws Exception{ + writeToZookeeper(Constants.ZOOKEEPER_ROOT + "/bundle.properties", bundleProperties, client); + } + public static void writeProfilerConfigToZookeeper(byte[] config, CuratorFramework client) throws Exception { PROFILER.deserialize(new String(config)); writeToZookeeper(PROFILER.getZookeeperRoot(), config, client); @@ -92,10 +104,31 @@ public class ConfigurationsUtils { writeToZookeeper(PARSER.getZookeeperRoot() + "/" + sensorType, configData, client); } + public static void writeParserExtensionConfigToZookeeper(String extensionID, byte[] configData, CuratorFramework client) throws Exception { + ParserExtensionConfig c = (ParserExtensionConfig) PARSER_EXTENSION.deserialize(new String(configData)); + writeToZookeeper(PARSER_EXTENSION.getZookeeperRoot() + "/" + extensionID, configData, client); + } + + public static void writeParserExtensionConfigToZookeeper(String extensionID, ParserExtensionConfig parserExtensionConfig, String zookeeperUrl) throws Exception { + writeParserExtensionConfigToZookeeper(extensionID, JSONUtils.INSTANCE.toJSON(parserExtensionConfig), zookeeperUrl); + } + + public static void writeParserExtensionConfigToZookeeper(String extensionID, byte[] configData, String zookeeperUrl) throws Exception { + try(CuratorFramework client = getClient(zookeeperUrl)) { + client.start(); + writeParserExtensionConfigToZookeeper(extensionID, configData, client); + } + } + + public static void deleteParsesrExtensionConfig(String extensionID, CuratorFramework client)throws Exception{ + deleteFromZookeeper(PARSER_EXTENSION.getZookeeperRoot() + "/" + extensionID, client); + } + public static void writeSensorIndexingConfigToZookeeper(String sensorType, Map<String, Object> sensorIndexingConfig, String zookeeperUrl) throws Exception { writeSensorIndexingConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSON(sensorIndexingConfig), zookeeperUrl); } + public static void writeSensorIndexingConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); @@ -143,6 +176,10 @@ public class ConfigurationsUtils { } } + public static void deleteFromZookeeper(String path, CuratorFramework client) throws Exception{ + client.delete().forPath(path); + } + public static void updateConfigsFromZookeeper(Configurations configurations, CuratorFramework client) throws Exception { configurations.updateGlobalConfig(readGlobalConfigBytesFromZookeeper(client)); } @@ -179,6 +216,11 @@ public class ConfigurationsUtils { return JSONUtils.INSTANCE.load(new ByteArrayInputStream(readFromZookeeper(PARSER.getZookeeperRoot() + "/" + sensorType, client)), SensorParserConfig.class); } + public static ParserExtensionConfig readParserExtensionConfigFromZookeeper(String extensionID, CuratorFramework client) throws Exception { + return JSONUtils.INSTANCE.load(new ByteArrayInputStream(readFromZookeeper(PARSER_EXTENSION.getZookeeperRoot() + "/" + extensionID, client)), ParserExtensionConfig.class); + } + + public static byte[] readGlobalConfigBytesFromZookeeper(CuratorFramework client) throws Exception { return readFromZookeeper(GLOBAL.getZookeeperRoot(), client); } @@ -223,7 +265,7 @@ public class ConfigurationsUtils { } public static void uploadConfigsToZookeeper(String rootFilePath, CuratorFramework client) throws Exception { - uploadConfigsToZookeeper(rootFilePath, rootFilePath, rootFilePath, rootFilePath, rootFilePath, client); + uploadConfigsToZookeeper(rootFilePath, rootFilePath, rootFilePath, rootFilePath, rootFilePath,rootFilePath, client); } public static void uploadConfigsToZookeeper(String globalConfigPath, @@ -232,6 +274,16 @@ public class ConfigurationsUtils { String indexingConfigPath, String profilerConfigPath, CuratorFramework client) throws Exception { + uploadConfigsToZookeeper(globalConfigPath,parsersConfigPath,enrichmentsConfigPath,indexingConfigPath,profilerConfigPath,null,client); + } + + public static void uploadConfigsToZookeeper(String globalConfigPath, + String parsersConfigPath, + String enrichmentsConfigPath, + String indexingConfigPath, + String profilerConfigPath, + String parserExtensionPath, + CuratorFramework client) throws Exception { // global if (globalConfigPath != null) { @@ -240,6 +292,18 @@ public class ConfigurationsUtils { setupStellarStatically(client, Optional.of(new String(globalConfig))); ConfigurationsUtils.writeGlobalConfigToZookeeper(readGlobalConfigFromFile(globalConfigPath), client); } + final byte[] globalBundleProperties = readBundlePropertiesFromFile(globalConfigPath); + if (globalBundleProperties.length > 0){ + ConfigurationsUtils.writeGlobalBundlePropertiesToZookeeper(globalBundleProperties, client); + } + } + + // parser configs + if(parserExtensionPath != null) { + Map<String, byte[]> parserExtensionConfigs = readParserExtensionConfigsFromFile(parserExtensionPath); + for (String parserExtension : parserExtensionConfigs.keySet()) { + ConfigurationsUtils.writeParserExtensionConfigToZookeeper(parserExtension, parserExtensionConfigs.get(parserExtension), client); + } } // parsers @@ -318,6 +382,15 @@ public class ConfigurationsUtils { return globalConfig; } + public static byte[] readBundlePropertiesFromFile(String rootPath) throws IOException { + byte[] bundleProperties = new byte[0]; + File configPath = new File(rootPath,"bundle.properties"); + if(configPath.exists()){ + bundleProperties = Files.readAllBytes(configPath.toPath()); + } + return bundleProperties; + } + public static Map<String, byte[]> readSensorParserConfigsFromFile(String rootPath) throws IOException { return readSensorConfigsFromFile(rootPath, PARSER); } @@ -345,6 +418,20 @@ public class ConfigurationsUtils { return config; } + public static Map<String, byte[]> readParserExtensionConfigsFromFile(String rootPath) throws IOException { + Map<String, byte[]> parserExtensionConfigs = new HashMap<>(); + File configPath = new File(rootPath, PARSER_EXTENSION.getDirectory()); + if (configPath.exists()) { + File[] children = configPath.listFiles(); + if (children != null) { + for (File file : children) { + parserExtensionConfigs.put(FilenameUtils.removeExtension(file.getName()), Files.readAllBytes(file.toPath())); + } + } + } + return parserExtensionConfigs; + } + public static Map<String, byte[]> readSensorConfigsFromFile(String rootPath, ConfigurationType configType) throws IOException { Map<String, byte[]> sensorConfigs = new HashMap<>(); File configPath = new File(rootPath, configType.getDirectory());
http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/extensions/ParserExtensionConfig.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/extensions/ParserExtensionConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/extensions/ParserExtensionConfig.java new file mode 100644 index 0000000..9e8fb19 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/extensions/ParserExtensionConfig.java @@ -0,0 +1,192 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.configuration.extensions; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.common.utils.JSONUtils; + +import java.io.IOException; +import java.io.Serializable; +import java.util.*; + +public class ParserExtensionConfig implements Serializable{ + private String extensionAssemblyName; + private String extensionBundleName; + private String extensionsBundleID; + private String extensionsBundleVersion; + private Set<String> parserExtensionParserNames; + private Map<String,SensorParserConfig> defaultParserConfigs; + private Map<String,SensorEnrichmentConfig> defaultEnrichementConfigs; + private Map<String,Map<String,Object>> defaultIndexingConfigs; + private Map<String,Map<String,Object>> defaultElasticSearchTemplates; + + public String getExtensionAssemblyName() { + return extensionAssemblyName; + } + + public void setExtensionAssemblyName(String extensionAssemblyName) { + this.extensionAssemblyName = extensionAssemblyName; + } + + public String getExtensionBundleName() { + return extensionBundleName; + } + + public void setExtensionBundleName(String extensionBundleName) { + this.extensionBundleName = extensionBundleName; + } + + public String getExtensionsBundleID() { + return extensionsBundleID; + } + + public void setExtensionsBundleID(String extensionsBundleID) { + this.extensionsBundleID = extensionsBundleID; + } + + public String getExtensionsBundleVersion() { + return extensionsBundleVersion; + } + + public void setExtensionsBundleVersion(String extensionsBundleVersion) { + this.extensionsBundleVersion = extensionsBundleVersion; + } + + public Set<String> getParserExtensionParserNames() { + if(this.parserExtensionParserNames != null) { + return ImmutableSet.copyOf(this.parserExtensionParserNames); + } + return ImmutableSet.of(); + } + + public void setParserExtensionParserNames(Set<String> parserExtensionParserNames) { + this.parserExtensionParserNames = new HashSet(parserExtensionParserNames); + } + + public Map<String,SensorParserConfig> getDefaultParserConfigs() { + if(this.defaultParserConfigs != null){ + return ImmutableMap.copyOf(this.defaultParserConfigs); + } + return ImmutableMap.of(); + } + + public void setDefaultParserConfigs(Map<String,SensorParserConfig> defaultParserConfigs) { + this.defaultParserConfigs = new HashMap<>(defaultParserConfigs); + } + + public Map<String,SensorEnrichmentConfig> getDefaultEnrichementConfigs() { + if(this.defaultEnrichementConfigs != null){ + return ImmutableMap.copyOf(this.defaultEnrichementConfigs); + } + return ImmutableMap.of(); + } + + public void setDefaultEnrichementConfigs(Map<String,SensorEnrichmentConfig> defaultEnrichementConfigs) { + this.defaultEnrichementConfigs = new HashMap<>(defaultEnrichementConfigs); + } + + public Map<String,Map<String, Object>> getDefaultIndexingConfigs() { + if(this.defaultIndexingConfigs != null){ + return ImmutableMap.copyOf(this.defaultIndexingConfigs); + } + return ImmutableMap.of(); + } + + public void setDefaultIndexingConfigs(Map<String,Map<String, Object>> defaultIndexingConfigs) { + this.defaultIndexingConfigs = new HashMap<>(defaultIndexingConfigs); + } + + public void setParserExtensionParserName(Collection<String> parserExtensionParserNames) { + this.parserExtensionParserNames = new HashSet(); + this.parserExtensionParserNames.addAll(parserExtensionParserNames); + } + + public Map<String, Map<String, Object>> getDefaultElasticSearchTemplates() { + if(this.defaultElasticSearchTemplates != null){ + return ImmutableMap.copyOf(this.defaultElasticSearchTemplates); + } + return ImmutableMap.of(); + } + + public void setDefaultElasticSearchTemplates(Map<String, Map<String, Object>> defaultElasticSearchTemplates) { + this.defaultElasticSearchTemplates = new HashMap<>(defaultElasticSearchTemplates); + } + + public static ParserExtensionConfig fromBytes(byte[] config) throws IOException { + ParserExtensionConfig ret = JSONUtils.INSTANCE.load(new String(config), ParserExtensionConfig.class); + return ret; + } + + public String toJSON() throws JsonProcessingException { + return JSONUtils.INSTANCE.toJSON(this, true); + } + + @Override + public String toString() { + return "SensorParserConfig{" + + "extensionAssemblyName='" + extensionAssemblyName + '\'' + + ", extensionBundleName='" + extensionBundleName + '\'' + + ", extensionsBundleID='" + extensionsBundleID + '\'' + + ", parserExtensionParserNames='" + String.join(",",this.parserExtensionParserNames) + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ParserExtensionConfig that = (ParserExtensionConfig) o; + + if (getExtensionAssemblyName() != null ? !getExtensionAssemblyName().equals(that.getExtensionAssemblyName()) : that.getExtensionAssemblyName() != null) + return false; + if (getExtensionBundleName() != null ? !getExtensionBundleName().equals(that.getExtensionBundleName()) : that.getExtensionBundleName() != null) + return false; + if (getExtensionsBundleID() != null ? !getExtensionsBundleID().equals(that.getExtensionsBundleID()) : that.getExtensionsBundleID() != null) + return false; + if (getExtensionsBundleVersion() != null ? !getExtensionsBundleVersion().equals(that.getExtensionsBundleVersion()) : that.getExtensionsBundleVersion() != null) + return false; + if (getDefaultParserConfigs() != null ? !getDefaultParserConfigs().equals(that.getDefaultParserConfigs()) : that.getDefaultParserConfigs() != null) + return false; + if (getDefaultEnrichementConfigs() != null ? !getDefaultEnrichementConfigs().equals(that.getDefaultEnrichementConfigs()) : that.getDefaultEnrichementConfigs() != null) + return false; + if (getDefaultIndexingConfigs() != null ? !getDefaultIndexingConfigs().equals(that.getDefaultIndexingConfigs()) : that.getDefaultIndexingConfigs() != null) + return false; + if (getDefaultElasticSearchTemplates() != null ? !getDefaultElasticSearchTemplates().equals(that.getDefaultElasticSearchTemplates()) : that.getDefaultElasticSearchTemplates() != null) + return false; + return getParserExtensionParserNames() != null ? getParserExtensionParserNames().equals(that.getParserExtensionParserNames()) : that.getParserExtensionParserNames() == null; + } + + @Override + public int hashCode() { + int result = getExtensionAssemblyName() != null ? getExtensionAssemblyName().hashCode() : 0; + result = 31 * result + (getExtensionBundleName() != null ? getExtensionBundleName().hashCode() : 0); + result = 31 * result + (getExtensionsBundleID() != null ? getExtensionsBundleID().hashCode() : 0); + result = 31 * result + (getExtensionsBundleVersion() != null ? getExtensionsBundleVersion().hashCode() : 0); + result = 31 * result + (getDefaultParserConfigs() != null ? getDefaultParserConfigs().hashCode() : 0); + result = 31 * result + (getDefaultEnrichementConfigs() != null ? getDefaultEnrichementConfigs().hashCode() : 0); + result = 31 * result + (getDefaultIndexingConfigs() != null ? getDefaultIndexingConfigs().hashCode() : 0); + result = 31 * result + (getDefaultElasticSearchTemplates() != null ? getDefaultElasticSearchTemplates().hashCode() : 0); + result = 31 * result + (getParserExtensionParserNames() != null ? getParserExtensionParserNames().hashCode() : 0); + return result; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ResourceLoader.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ResourceLoader.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ResourceLoader.java new file mode 100644 index 0000000..de2f039 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ResourceLoader.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.metron.common.utils; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; + +/** + * <p> The ResourceLoader loads resources from paths. If configured, it will handle 'rooting' the + * paths to correctly reflect the current configuration. This means that a configured path + * '/foo/bar' can be correctly resolved to /some/hdfs/root/foo/bar or ./foo/bar through + * configuration of the current root.</p> + * + * <p> The ResourceLoader addresses issues that arise when the root path is a configuration option, + * but there are coded paths in configurations</p> + * + * If no configuration is provided, the defaults will be used. If the global configuration does + * not have a "metron.apps.hdfs.dir" setting then the default will be used. </p> + * <p>This class implements {@link AutoCloseable} and should + * ideally be used with a try with resources pattern</p> + */ +public class ResourceLoader implements AutoCloseable { + + /** + * Builds a new ResourceLoader. + */ + public static class Builder { + + private Configuration fileSystemConfiguration; + private Map<String, Object> configuration; + + /** + * Provide a {@link Configuration}. + * + * @param fileSystemConfiguration Hadoop Configuration + * @return The Builder + */ + public Builder withFileSystemConfiguration(Configuration fileSystemConfiguration) { + this.fileSystemConfiguration = fileSystemConfiguration; + return this; + } + + /** + * Provide a configuration Map This map should have an inner map at key "globalConfig", and that + * map should have a key "metron.apps.hdfs.dir". + * + * @param configuration the Map + * @return the Builder + */ + public Builder withConfiguration(Map<String, Object> configuration) { + this.configuration = configuration; + return this; + } + + /** + * Builds a new {@link ResourceLoader}. + * + * @return an instance of ResourceLoader + */ + public ResourceLoader build() { + return new ResourceLoader(configuration, fileSystemConfiguration); + } + } + + + private static final String DEFAULT_ROOT_DIR = StringUtils.EMPTY; + private String root = DEFAULT_ROOT_DIR; + private FileSystem fileSystem; + private Configuration fileSystemConfiguration; + + private ResourceLoader(){} + + @SuppressWarnings("unchecked") + private ResourceLoader(Map<String, Object> configuration, Configuration fileSystemConfiguration) { + if (configuration != null) { + if (configuration.containsKey("metron.apps.hdfs.dir")) { + root = (String) configuration.get("metron.apps.hdfs.dir"); + } + } + if (fileSystemConfiguration != null) { + this.fileSystemConfiguration = fileSystemConfiguration; + } else { + this.fileSystemConfiguration = new Configuration(); + } + } + + /** + * getResources returns resource streams, mapped to their names. If the passed resourceLocation is + * a directory, then all resources located in that directory and subdirectories will be loaded and + * returned. If the passed resourceLocation is not a directory, the that location will be loaded + * and returned + * + * @param resourceLocation The location from which to attempt to load resources + * @return Map of resource names to InputStreams + * @throws IOException if the resource doesn't exist, or if there is a configured root that does + * not exist + */ + public Map<String, InputStream> getResources(String resourceLocation) throws IOException { + Map<String, InputStream> streamMap = new HashMap<>(); + if (fileSystem == null) { + fileSystem = FileSystem.get(fileSystemConfiguration); + } + Path path = new Path(resourceLocation); + if (!StringUtils.isEmpty(root) && !resourceLocation.startsWith(root)) { + // mergePaths does not handle merging and separators well + // as in merging '/root' and 'child/foo' will result in + // a path of '/rootchild/foo' + // this is almost certainly not what you would expect + // we will fix this up here + String thisRoot = root; + if(!thisRoot.endsWith("/") && !resourceLocation.startsWith("/")) { + thisRoot = thisRoot + "/"; + } + Path rootPath = new Path(thisRoot); + if (!fileSystem.exists(rootPath)) { + throw new FileNotFoundException(thisRoot); + } + path = Path.mergePaths(rootPath, path); + } + if (fileSystem.exists(path)) { + if (fileSystem.isDirectory(path)) { + RemoteIterator<LocatedFileStatus> fileStatusRemoteIterator = fileSystem + .listFiles(path, true); + while (fileStatusRemoteIterator.hasNext()) { + LocatedFileStatus locatedFileStatus = fileStatusRemoteIterator.next(); + if (!locatedFileStatus.isDirectory()) { + streamMap + .put(locatedFileStatus.getPath().getName(), + fileSystem.open(locatedFileStatus.getPath())); + } + } + } else { + streamMap.put(resourceLocation, fileSystem.open(path)); + } + } else { + InputStream classLoaderStream = getClass().getResourceAsStream(resourceLocation); + if (classLoaderStream == null) { + throw new FileNotFoundException(resourceLocation); + } + streamMap.put(resourceLocation, classLoaderStream); + } + return streamMap; + } + + @Override + public void close() throws Exception { + if (fileSystem != null) { + fileSystem.close(); + fileSystem = null; + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java index 0c72183..68dbe2a 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java @@ -40,6 +40,7 @@ public class ConfigurationsUtilsTest { private byte[] expectedGlobalConfig; private Map<String, byte[]> expectedSensorParserConfigMap; private Map<String, byte[]> expectedSensorEnrichmentConfigMap; + private static final String TEST_SENSOR_TYPE = "asa"; @Before public void setup() throws Exception { @@ -48,8 +49,8 @@ public class ConfigurationsUtilsTest { client = ConfigurationsUtils.getClient(zookeeperUrl); client.start(); expectedGlobalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH); - expectedSensorParserConfigMap = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH); - expectedSensorEnrichmentConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); + expectedSensorParserConfigMap = ConfigurationsUtils.readSensorParserConfigsFromFile(String.format(TestConstants.A_PARSER_CONFIGS_PATH_FMT, TEST_SENSOR_TYPE,TEST_SENSOR_TYPE)); + expectedSensorEnrichmentConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(String.format(TestConstants.A_PARSER_CONFIGS_PATH_FMT,TEST_SENSOR_TYPE,TEST_SENSOR_TYPE)); } @Test @@ -60,16 +61,15 @@ public class ConfigurationsUtilsTest { Assert.assertTrue(Arrays.equals(expectedGlobalConfig, actualGlobalConfigBytes)); Assert.assertTrue(expectedSensorParserConfigMap.size() > 0); - String testSensorType = "yaf"; - byte[] expectedSensorParserConfigBytes = expectedSensorParserConfigMap.get(testSensorType); - ConfigurationsUtils.writeSensorParserConfigToZookeeper(testSensorType, expectedSensorParserConfigBytes, zookeeperUrl); - byte[] actualSensorParserConfigBytes = ConfigurationsUtils.readSensorParserConfigBytesFromZookeeper(testSensorType, client); + byte[] expectedSensorParserConfigBytes = expectedSensorParserConfigMap.get(TEST_SENSOR_TYPE); + ConfigurationsUtils.writeSensorParserConfigToZookeeper(TEST_SENSOR_TYPE, expectedSensorParserConfigBytes, zookeeperUrl); + byte[] actualSensorParserConfigBytes = ConfigurationsUtils.readSensorParserConfigBytesFromZookeeper(TEST_SENSOR_TYPE, client); Assert.assertTrue(Arrays.equals(expectedSensorParserConfigBytes, actualSensorParserConfigBytes)); Assert.assertTrue(expectedSensorEnrichmentConfigMap.size() > 0); - byte[] expectedSensorEnrichmentConfigBytes = expectedSensorEnrichmentConfigMap.get(testSensorType); - ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(testSensorType, expectedSensorEnrichmentConfigBytes, zookeeperUrl); - byte[] actualSensorEnrichmentConfigBytes = ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(testSensorType, client); + byte[] expectedSensorEnrichmentConfigBytes = expectedSensorEnrichmentConfigMap.get(TEST_SENSOR_TYPE); + ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(TEST_SENSOR_TYPE, expectedSensorEnrichmentConfigBytes, zookeeperUrl); + byte[] actualSensorEnrichmentConfigBytes = ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(TEST_SENSOR_TYPE, client); Assert.assertTrue(Arrays.equals(expectedSensorEnrichmentConfigBytes, actualSensorEnrichmentConfigBytes)); String name = "testConfig"; http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java index 5a77b4f..b2638e5 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java @@ -33,11 +33,12 @@ import java.util.Map; public class SensorEnrichmentConfigTest { + private static final String TEST_SENSOR_TYPE = "yaf"; @Test public void test() throws IOException { EqualsVerifier.forClass(SensorEnrichmentConfig.class).suppress(Warning.NONFINAL_FIELDS).usingGetClass().verify(); - Map<String, byte[]> testSensorConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); - byte[] sensorConfigBytes = testSensorConfigMap.get("yaf"); + Map<String, byte[]> testSensorConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(String.format(TestConstants.A_PARSER_CONFIGS_PATH_FMT, TEST_SENSOR_TYPE,TEST_SENSOR_TYPE)); + byte[] sensorConfigBytes = testSensorConfigMap.get(TEST_SENSOR_TYPE); SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig.fromBytes(sensorConfigBytes); Assert.assertNotNull(sensorEnrichmentConfig); Assert.assertTrue(sensorEnrichmentConfig.toString() != null && sensorEnrichmentConfig.toString().length() > 0); @@ -45,7 +46,7 @@ public class SensorEnrichmentConfigTest { @Test public void testSerDe() throws IOException { - for(File enrichmentConfig : new File(new File(TestConstants.ENRICHMENTS_CONFIGS_PATH), "enrichments").listFiles()) { + for(File enrichmentConfig : new File(new File(String.format(TestConstants.A_PARSER_CONFIGS_PATH_FMT, TEST_SENSOR_TYPE,TEST_SENSOR_TYPE)), "enrichments").listFiles()) { SensorEnrichmentConfig config = null; try (BufferedReader br = new BufferedReader(new FileReader(enrichmentConfig))) { String parserStr = IOUtils.toString(br); http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/extensions/ParserExtensionConfigTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/extensions/ParserExtensionConfigTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/extensions/ParserExtensionConfigTest.java new file mode 100644 index 0000000..6e101d5 --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/extensions/ParserExtensionConfigTest.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.configuration.extensions; + +import org.apache.commons.io.IOUtils; +import org.apache.metron.TestConstants; +import org.junit.Assert; +import org.junit.Test; + +import java.io.*; + +public class ParserExtensionConfigTest { + @Test + public void testSerDe() throws IOException { + File parserConfig = new File(TestConstants.SAMPLE_EXTENSIONS_PARSER_CONFIG_PATH, "metron-test-parsers.json"); + ParserExtensionConfig config = null; + try (BufferedReader br = new BufferedReader(new FileReader(parserConfig))) { + String parserStr = IOUtils.toString(br); + config = ParserExtensionConfig.fromBytes(parserStr.getBytes()); + } + ParserExtensionConfig config2 = ParserExtensionConfig.fromBytes(config.toJSON().getBytes()); + Assert.assertEquals(config2, config); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/ResourceLoaderTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/ResourceLoaderTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/ResourceLoaderTest.java new file mode 100644 index 0000000..8e609e5 --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/ResourceLoaderTest.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.metron.common.utils; + +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ResourceLoaderTest { + + @BeforeClass + public static void before() throws Exception { + File f = new File("target/test"); + if (!f.exists()) { + f.createNewFile(); + } + + f = new File("target/resdir"); + if (!f.exists()) { + f.mkdir(); + } + + f = new File("target/resdir/test1"); + if (!f.exists()) { + f.createNewFile(); + } + } + + @Test(expected = FileNotFoundException.class) + public void testNonFileThrowsException() throws Exception { + try (ResourceLoader loader = new ResourceLoader.Builder().build()) { + loader.getResources("/usr/nope"); + } + } + + @Test + public void getResourcesShouldReturnResourceWithNoRootSet() throws Exception { + try (ResourceLoader loader = new ResourceLoader.Builder().build()) { + Map<String, InputStream> resources = loader.getResources("target/test"); + Assert.assertNotNull(resources.get("target/test")); + } + } + + @Test + public void getResourcesShouldReturnResourceWithRootSet() throws Exception { + Map<String, Object> config = ImmutableMap.of("metron.apps.hdfs.dir", "./target/resdir/"); + try (ResourceLoader loader = new ResourceLoader.Builder().withConfiguration(config).build()) { + Map<String, InputStream> resources = loader.getResources("/test1"); + Assert.assertNotNull(resources.get("/test1")); + } + } + + @Test + public void getResourcesShouldReturnResourceWithNoRootSetAndExistingFSConfig() throws Exception { + try (ResourceLoader loader = new ResourceLoader.Builder() + .withFileSystemConfiguration(new Configuration()).build()) { + Map<String, InputStream> resources = loader.getResources("target/test"); + Assert.assertNotNull(resources.get("target/test")); + } + } + + @Test + public void getResourcesShouldReturnResourceWithRootSetAndExistingFSConfig() throws Exception { + Map<String, Object> config = ImmutableMap.of("metron.apps.hdfs.dir", "./target/resdir/"); + try (ResourceLoader loader = new ResourceLoader.Builder().withConfiguration(config) + .withFileSystemConfiguration(new Configuration()).build()) { + Map<String, InputStream> resources = loader.getResources("/test1"); + Assert.assertNotNull(resources.get("/test1")); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/asa.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/asa.json b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/asa.json deleted file mode 100644 index fa07468..0000000 --- a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/asa.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "enrichment" : { - "fieldMap": { - "geo": ["ip_dst_addr", "ip_src_addr"] - } - } -} - http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/bro.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/bro.json b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/bro.json deleted file mode 100644 index 57d0365..0000000 --- a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/bro.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "enrichment" : { - "fieldMap": { - "geo": ["ip_dst_addr", "ip_src_addr"], - "host": ["host"] - } - }, - "threatIntel": { - "fieldMap": { - "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] - }, - "fieldToTypeMap": { - "ip_src_addr" : ["malicious_ip"], - "ip_dst_addr" : ["malicious_ip"] - } - } -} - http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/snort.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/snort.json b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/snort.json deleted file mode 100644 index 8cd8197..0000000 --- a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/snort.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "enrichment" : { - "fieldMap": - { - "geo": ["ip_dst_addr", "ip_src_addr"], - "host": ["host"] - } - }, - "threatIntel" : { - "fieldMap": - { - "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] - }, - "fieldToTypeMap": - { - "ip_src_addr" : ["malicious_ip"], - "ip_dst_addr" : ["malicious_ip"] - }, - "triageConfig" : { - "riskLevelRules" : [ - { - "rule" : "not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24'))", - "score" : 10 - } - ], - "aggregator" : "MAX" - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/websphere.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/websphere.json b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/websphere.json deleted file mode 100644 index 149957b..0000000 --- a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/websphere.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "enrichment": { - "fieldMap": { - "geo": [ - "ip_src_addr" - ], - "host": [ - "ip_src_addr" - ] - }, - "fieldToTypeMap": { - "ip_src_addr": [ - "playful_classification" - ] - } - } -} - http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/yaf.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/yaf.json b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/yaf.json deleted file mode 100644 index d02cb85..0000000 --- a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/yaf.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "enrichment" : { - "fieldMap": - { - "geo": ["ip_dst_addr", "ip_src_addr"], - "host": ["host"] - } - }, - "threatIntel": { - "fieldMap": - { - "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] - }, - "fieldToTypeMap": - { - "ip_src_addr" : ["malicious_ip"], - "ip_dst_addr" : ["malicious_ip"] - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java index e4625ac..7877953 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java @@ -25,6 +25,7 @@ import org.apache.metron.integration.UnableToStartException; import org.apache.metron.integration.components.ZKServerComponent; import org.apache.zookeeper.KeeperException; +import javax.swing.text.html.Option; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -45,6 +46,7 @@ public class ConfigUploadComponent implements InMemoryComponent { private String profilerConfigPath; private Optional<Consumer<ConfigUploadComponent>> postStartCallback = Optional.empty(); private Optional<String> globalConfig = Optional.empty(); + private Optional<byte[]> bundleProperties = Optional.empty(); private Map<String, SensorParserConfig> parserSensorConfigs = new HashMap<>(); public ConfigUploadComponent withConnectionString(String connectionString) { @@ -90,6 +92,11 @@ public class ConfigUploadComponent implements InMemoryComponent { return this; } + public ConfigUploadComponent withBundleProperties(byte[] propBytes){ + this.bundleProperties = Optional.ofNullable(propBytes); + return this; + } + public ConfigUploadComponent withPostStartCallback(Consumer<ConfigUploadComponent> f) { this.postStartCallback = Optional.ofNullable(f); return this; @@ -156,6 +163,10 @@ public class ConfigUploadComponent implements InMemoryComponent { if(globalConfig.isPresent()) { writeGlobalConfigToZookeeper(globalConfig.get().getBytes(), zookeeperUrl); } + if(bundleProperties.isPresent()){ + writeGlobalBundlePropertiesToZookeeper(bundleProperties.get(),zookeeperUrl); + } + if(postStartCallback.isPresent()) { postStartCallback.get().accept(this); } http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/README.md b/metron-platform/metron-extensions/README.md new file mode 100644 index 0000000..85e80a0 --- /dev/null +++ b/metron-platform/metron-extensions/README.md @@ -0,0 +1,46 @@ +# Metron Extensions + +[Metron Extensions](extension_terms.md#metron-extensions) allow the Metron system to be extended with new capabilities and support for new types of data. +Extensions may be new telemetry parsers or libraries of Stellar functions (Coming Soon). + +Providing an extension mechanism, which includes maven archetype support, installation and loading is important not only for allowing new capabilities to be added to Metron, +but for allowing those capabilities to be developed and maintained outside the Metron code tree, without the need or rather the requiement to fork the project in order to integrate changes. + +The may be multiple types of [Metron Extensions](extension_terms.md#metron-extensions), and each extension type will define it's own specific structure +for packaging, and have it's own deployment and installation details. + +The important idea in [extension packaging](extension_packaging.md) is that a parser extension should be able to be deployed as a single +self contained unit. + +## Exposing functionality + +[Metron Extensions](extension_terms.md#metron-extensions) expose functionality through the [ClassIndex Library](https://github.com/atteo/classindex). +The default dependency on ClassIndex produced by the Metron Extension archetypes ensures that any classes in an +extension bundle that derive from an interface or base class in the system are exposed automatically at compile time +without the need to manual expose provided service interfaces ( such as you would have to do with ServiceLoader implementations). + +As an example: Parsers. Metron parsers implement the MessageParser interface. + +The MessageParser is defined as such: + +``` +@IndexSubclasses +public interface MessageParser<T> extends Configurable + +``` +Therefore any class in a parser extension that implements this interface will therefore be exposed to the extension host/consumer. +Also, there may be more than one extension type exposed with in a given extension. This means, for example, that a parser extension may include +more than one parser classes, each with it's own configurations but sharing a bundle. See [Parser Extension Packaging](metron-parser-extensions/parser_extension_packaging.md). + + +## Metron System Extensions + +### Parsers + +The parsers that are developed and deployed with Metron are developed as [Metron Extensions](extension_terms.md#metron-extensions) as well. +See [Adding a new Parser to Metron System](metron-parser-extensions/adding_system_parsers.md) for a guide to adding a new parser to be built and deployed as part of Metron itself. + + + +[Extension Terms](extension_terms.md) + http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/extension_deployment.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/extension_deployment.md b/metron-platform/metron-extensions/extension_deployment.md new file mode 100644 index 0000000..9900d1a --- /dev/null +++ b/metron-platform/metron-extensions/extension_deployment.md @@ -0,0 +1,20 @@ +# Metron Extension Deployment + +Metron Extensions have varied packaging, as it is defined by the extension type and it's requirements. +What is common across extension types is the deployment of the [Metron Bundles](../../bundles-lib) containing +the extension library (if present). + +These Bundles are deployed to HDFS under /apps/metron/extension_lib. +The /apps/metron/extension_working directory is used at runtime by the bundle system. + +> NOTE: Bundles may also be deployed locally on the cluster under /usr/metron/VERSION/ + +```bash + drwxrwxr-x - metron hadoop 0 2017-06-27 16:15 /apps/metron/extension_lib + drwxrwxr-x - metron hadoop 0 2017-06-27 16:16 /apps/metron/extension_working +``` + + +See specific extension deployment information + +- [Parser Extension Deployment](metron-parser-extensions/parser_extension_deployment.md) http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/extension_packaging.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/extension_packaging.md b/metron-platform/metron-extensions/extension_packaging.md new file mode 100644 index 0000000..ac56ff3 --- /dev/null +++ b/metron-platform/metron-extensions/extension_packaging.md @@ -0,0 +1,16 @@ +# Metron Extension Packaging + +Metron Extensions have varied packaging, as it is defined by the extension type and it's requirements. + +## The Package + +The package itself is a tar.gz file created at build time. The configuration for this packaging +in the project as produced by the specific extension archetype, and within the +XXXX-assembly module, specifically in the src/main/assembly/assembly.xml file. + + + + +See specific extension packaging information + +- [Parser Extension Packaging](metron-parser-extensions/parser_extension_packaging.md) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/extension_terms.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/extension_terms.md b/metron-platform/metron-extensions/extension_terms.md new file mode 100644 index 0000000..3589bca --- /dev/null +++ b/metron-platform/metron-extensions/extension_terms.md @@ -0,0 +1,25 @@ +# Metron Extension Terms + + +### [Metron Extension](../metron-extensions) +> Additional functionality for the Metron system +> This functionality can be defined as code libraries and dependencies and configuration, or possibly configuration alone +> There are different types of extensions, each with it's own specific composition +> Extensions are package as an [Extension Package](#extension-package) + +### [Extension Package](extension_packaging.md) +> The compressed file that contains the configurations and code libraries and dependencies of the extension +> The Extension Library and dependencies are themselves packaged as a [Metron Bundle](../../bundles-lib) +> Extension Archetypes produce projects with an assembly module that will create an extension package +> Extension packages can be installed/uninstalled into Metron through the REST-API (NOTE: METRON-942) + +### [Bundle](../../bundles-lib) + +> Bundles are the runtime components of an extension. +> +> A bundle is jar package that includes the extension jar file, and all of it's dependency jars ( that are not provided by the system). +> It also includes some metadata about the extension. +> +> Bundles are deployed into the system in HDFS, and are unpacked and loaded into applications that use extensions. +> Each bundle is loaded with it's own classloader, which will allow for isolation of dependencies between extensions (for things not loaded by +> by the host application itself). http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/README.md b/metron-platform/metron-extensions/metron-parser-extensions/README.md new file mode 100644 index 0000000..60beba0 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/README.md @@ -0,0 +1,9 @@ +# Metron Parser Extensions + +## Introduction + +Parsers are pluggable components which are used to transform raw data +(textual or raw bytes) into JSON messages suitable for downstream +enrichment and indexing. + +Parsers are package and deployed as Metron Extensions http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/adding_system_parsers.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/adding_system_parsers.md b/metron-platform/metron-extensions/metron-parser-extensions/adding_system_parsers.md new file mode 100644 index 0000000..99de8f4 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/adding_system_parsers.md @@ -0,0 +1,345 @@ +# Developing a new Metron Parser Extension (Metron) + +This guide discusses how to add a new Metron Parser Extension to the Metron +codebase. + +## Setup +Make sure you have a good, working branch that builds and passes testing +This includes building and installing the bundles-maven-plugin +and the metron-parser-extension-archetype. + +## Create a parser extension +```commandline +cd metron-platform/metron-extensions/metron-parser-extensions +run mvn -U archetype:generate -DarchetypeCatalog=local +``` +- select Apache Maven Parser Extension Archetype for Metron +- fill out the archetype variables: +``` commandline + groupId : org.apache.metron + artifactId: metron-parser-foo-extension + version: $METRON_VERSION + package: org.apache.metron.parsers + metronVersion: $METRON_VERSION + parserClassName: name of parser Java classes, such as BasicFoo + parserName: the parser name as it appears in the system (yaf, asa, bro) foo +``` +So you should see the following output: + +``` commandline +Choose archetype: +1: local -> org.apache.metron:metron-parser-extension-archetype (Apache Parser Extension Archetype for Metron) +Choose a number or apply filter (format: [groupId:]artifactId, case sensitive contains): : 1 +Define value for property 'groupId': org.apache.metron +Define value for property 'artifactId': metron-parser-foo-extension +Define value for property 'version' 1.0-SNAPSHOT: : 0.4.0 +Define value for property 'package' org.apache.metron: : org.apache.metron.parsers +Define value for property 'metronVersion': 0.4.0 +Define value for property 'parserClassName' (should match expression '^[A-Z].*$'): BasicFoo +Define value for property 'parserName' (should match expression '^[a-z]+[A-Z,a-z]+$'): foo +Confirm properties configuration: +groupId: org.apache.metron +artifactId: metron-parser-foo-extension +version: 0.4.0 +package: org.apache.metron.parsers +metronVersion: 0.4.0 +parserClassName: BasicFoo +parserName: foo +``` + +At this point, you should have a new extension project in the directory named metron-parser-foo-extension with the following files: + +``` commandline +. +âââ metron-parser-foo +â  âââ README.md +â  âââ pom.xml +â  âââ src +â  âââ main +â  â  âââ config +â  â  â  âââ elasticsearch +â  â  â  â  âââ foo_index.template +â  â  â  âââ zookeeper +â  â  â  âââ enrichments +â  â  â  â  âââ foo.json +â  â  â  âââ indexing +â  â  â  â  âââ foo.json +â  â  â  âââ parsers +â  â  â  âââ foo.json +â  â  âââ java +â  â  â  âââ org +â  â  â  âââ apache +â  â  â  âââ metron +â  â  â  âââ parsers +â  â  â  âââ foo +â  â  â  âââ BasicFooParser.java +â  â  âââ resources +â  â  âââ META-INF +â  â  â  âââ LICENSE +â  â  â  âââ NOTICE +â  â  âââ patterns +â  â  âââ common +â  âââ test +â  âââ java +â  â  âââ org +â  â  âââ apache +â  â  âââ metron +â  â  âââ parsers +â  â  âââ foo +â  â  â  âââ BasicFooParserTest.java +â  â  âââ integration +â  â  âââ BasicFooIntegrationTest.java +â  âââ resources +â  âââ config +â  â  âââ zookeeper +â  â  âââ bundle.properties +â  â  âââ global.json +â  âââ data +â  â  âââ parsed +â  â  â  âââ test.parsed +â  â  âââ raw +â  â  âââ test.raw +â  âââ log4j.properties +âââ metron-parser-foo-assembly +â  âââ metron-parser-foo-assembly.iml +â  âââ pom.xml +â  âââ src +â  âââ main +â  âââ assembly +â  âââ assembly.xml +âââ metron-parser-foo-bundle +â  âââ metron-parser-foo-bundle.iml +â  âââ pom.xml +âââ pom.xml +âââ tree.txt + +37 directories, 26 files + +``` + +Verify that the metron-parser-extensions/pom.xml references the new module, or add it as such: + +```xml +<modules> + <module>metron-parser-asa-extension</module> + <module>metron-parser-bro-extension</module> + <module>metron-parser-cef-extension</module> + <module>metron-parser-fireeye-extension</module> + <module>metron-parser-ise-extension</module> + <module>metron-parser-lancope-extension</module> + <module>metron-parser-logstash-extension</module> + <module>metron-parser-paloalto-extension</module> + <module>metron-parser-snort-extension</module> + <module>metron-parser-sourcefire-extension</module> + <module>metron-parser-websphere-extension</module> + <module>metron-parser-squid-extension</module> + <module>metron-parser-yaf-extension</module> + <module>metron-parser-foo-extension</module> + <module>metron-parser-bundle-tests</module> + </modules> +``` + + Build metron and run tests to ensure you are starting with a working set + The archetype produces a very simple parser with unit and integration tests + + ``` commandline + [INFO] + [INFO] --- jacoco-maven-plugin:0.7.9:report (report) @ metron-parser-foo --- + [INFO] Loading execution data file /Users/you/src/apache/forks/metron/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-foo-extension/metron-parser-foo/target/jacoco.exec + [INFO] Analyzed bundle 'metron-parser-foo' with 1 classes + [INFO] + [INFO] --- maven-jar-plugin:3.0.2:jar (default-jar) @ metron-parser-foo --- + [INFO] + [INFO] --- maven-jar-plugin:3.0.2:test-jar (default) @ metron-parser-foo --- + [INFO] + [INFO] --- maven-surefire-plugin:2.18:test (integration-tests) @ metron-parser-foo --- + [INFO] Surefire report directory: /Users/you/src/apache/forks/metron/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-foo-extension/metron-parser-foo/target/surefire-reports + + ------------------------------------------------------- + T E S T S + ------------------------------------------------------- + Running org.apache.metron.parsers.integration.BasicFooIntegrationTest + Running Pathed Sample Data Validation on sensorType foo + Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 43.021 sec - in org.apache.metron.parsers.integration.BasicFooIntegrationTest + + Results : + + + + + Tests run: 1, Failures: 0, Errors: 0, Skipped: 0 + + + [INFO] metron-parser-foo-extension ........................ SUCCESS [ 0.002 s] + [INFO] metron-parser-foo .................................. SUCCESS [ 44.879 s] + [INFO] metron-parser-foo-bundle ........................... SUCCESS [ 0.011 s] + [INFO] metron-parser-foo-assembly ......................... SUCCESS [ 0.030 s] + +``` +--------- + +## Build your parser + + - Edit the configurations for indexing, enrichment, parser + - You must remove the config/elasticsearch directory created by the archetype! + - add grok patterns if you want + - remove the sample parser and write your own + - etc + - If your parser is only configuration ( like yaf ), still write the tests etc, + but get rid of main java code, and edit the metron-parser-foo-assembly/src/main/assembly/assembly.xml as we have done with yaf: + +```xml + + <!-- YAF IS CONFIGURATION ONLY AT THIS TIME + <fileSet> + <directory>${project.basedir}/../metron-parser-yaf-bundle/target</directory> + <includes> + <include>metron-parser-yaf-bundle-${project.version}.bundle</include> + </includes> + <outputDirectory>/lib</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + </fileSet> + --> +``` + + + +## Add your extension to the install + + Once you are satisfied with your parser, it is time to add it to the install + + - First, you have to make sure that it is copied from it's build directory to the correct directory for the repo build. + + Open and edit metron-deployment/packaging/docker/rpm-docker/pom.xml + add a resource entry under the <!-- extensions --> area as such: + +```xml + + <resource> + <directory>${metron_dir}/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-foo-extension/metron-parser-foo-assembly/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> + +``` + +## Add the extension to the rpm spec + + Open metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec + + - Add a source entry such as + ```commandline + Source24: metron-parser-foo-assembly-%{full_version}-archive.tar.gz + ``` + + - Under %install add a mkdir entry such as + ```commandline + mkdir -p %{buildroot}%{metron_extensions_etc_parsers}/foo + ``` + - Under copy source files and untar make an entry such as + ```commandline + tar -xzf %{SOURCE24} -C %{buildroot}%{metron_extensions_etc_parsers}/foo + ``` + - If your parser has code and is not just configuration, under move the bundles from config to extensions lib make an entry as such + ```commandline + mv %{buildroot}%{metron_extensions_etc_parsers}/foo/lib/*.bundle %{buildroot}%{metron_extensions_lib}/ + ``` + - Add a package entry for the parser. Examine the other entries for parser extensions and make sure your entries match + +```commandline + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +%package parser-extension-foo +Summary: Metron Foo Parser Extension Files +Group: Applications/Internet +Provides: parser-extension-foo = %{version} + +%description parser-extension-foo +This package installs the Metron FOO Parser Extension files + +%files parser-extension-foo +%defattr(-,root,root,755) +%dir %{metron_root} +%dir %{metron_home} +%dir %{metron_extensions_etc} +%dir %{metron_extensions_etc_parsers} +%dir %{metron_extensions_etc_parsers}/foo +%dir %{metron_extensions_etc_parsers}/foo/config +%dir %{metron_extensions_etc_parsers}/foo/config/zookeeper +%dir %{metron_extensions_etc_parsers}/foo/config/zookeeper/parsers +%dir %{metron_extensions_etc_parsers}/foo/config/zookeeper/enrichments +%dir %{metron_extensions_etc_parsers}/foo/config/zookeeper/indexing +%dir %{metron_extensions_etc_parsers}/foo/patterns +%dir %{metron_extensions_lib} +%{metron_extensions_etc_parsers}/foo/config/zookeeper/parsers/foo.json +%{metron_extensions_etc_parsers}/foo/config/zookeeper/enrichments/foo.json +%{metron_extensions_etc_parsers}/foo/config/zookeeper/indexing/foo.json +%{metron_extensions_etc_parsers}/foo/patterns/common +%attr(0644,root,root) %{metron_extensions_lib}/metron-parser-foo-bundle-%{full_version}.bundle + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``` + +- Add to the changelog in the spec + + +## Build the rpms + +```commandline +cd metron-deployment/ +mvn package -P build-rpms +``` + +You should see your new rpm: + +```commandline + +â[]-(~/src/apache/forks/metron/metron-deployment/packaging/docker/rpm-docker/RPMS/noarch)-[]- +â> ls +metron-common-0.4.0-201706071954.noarch.rpm metron-parser-extension-fireeye-0.4.0-201706071954.noarch.rpm metron-parser-extension-websphere-0.4.0-201706071954.noarch.rpm +metron-config-0.4.0-201706071954.noarch.rpm metron-parser-extension-foo-0.4.0-201706071954.noarch.rpm metron-parser-extension-yaf-0.4.0-201706071954.noarch.rpm +metron-data-management-0.4.0-201706071954.noarch.rpm metron-parser-extension-ise-0.4.0-201706071954.noarch.rpm metron-parsers-0.4.0-201706071954.noarch.rpm +metron-elasticsearch-0.4.0-201706071954.noarch.rpm metron-parser-extension-lancope-0.4.0-201706071954.noarch.rpm metron-pcap-0.4.0-201706071954.noarch.rpm +metron-enrichment-0.4.0-201706071954.noarch.rpm metron-parser-extension-logstash-0.4.0-201706071954.noarch.rpm metron-profiler-0.4.0-201706071954.noarch.rpm +metron-indexing-0.4.0-201706071954.noarch.rpm metron-parser-extension-paloalto-0.4.0-201706071954.noarch.rpm metron-rest-0.4.0-201706071954.noarch.rpm +metron-parser-extension-asa-0.4.0-201706071954.noarch.rpm metron-parser-extension-snort-0.4.0-201706071954.noarch.rpm metron-solr-0.4.0-201706071954.noarch.rpm +metron-parser-extension-bro-0.4.0-201706071954.noarch.rpm metron-parser-extension-sourcefire-0.4.0-201706071954.noarch.rpm +metron-parser-extension-cef-0.4.0-201706071954.noarch.rpm metron-parser-extension-squid-0.4.0-201706071954.noarch.rpm + +``` + +## Integrate with the ambari mpack + - Add to the all_parsers list + +Edit metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +and add your parser to the list, as such: + +```commandline +all_parsers = "asa,bro,cef,fireeye,ise,lancope,logstash,paloalto,foo,snort,sourcefire,squid,websphere,yaf" +``` + + - Add your parser rpm to the metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml +```commandline +<package> + <name>metron-parser-extension-foo</name> +</package> +``` + +## Run Vagrant full_dev_platform + +```commandline +cd metron-deployment/vagrant/full_dev_platform +vagrant up +``` +- Verify the parser is installed + - open http://node1:4200/sensors ( default user and password are user and password) + - verify that foo is in the list + +You should be able to stop a running sensor and start foo, verify in the storm ui that it is loaded and running + + + + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-assembly/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-assembly/pom.xml new file mode 100644 index 0000000..b0b07c0 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-assembly/pom.xml @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-asa-extension</artifactId> + <version>0.4.1</version> + </parent> + + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-asa-assembly</artifactId> + <version>0.4.1</version> + <packaging>pom</packaging> + <name>metron-parser-asa-assembly</name> + + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptor>src/main/assembly/assembly.xml</descriptor> + </configuration> + <executions> + <execution> + <id>make-assembly</id> <!-- this is used for inheritance merges --> + <phase>package</phase> <!-- bind to the packaging phase --> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-assembly/src/main/assembly/assembly.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-assembly/src/main/assembly/assembly.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-assembly/src/main/assembly/assembly.xml new file mode 100644 index 0000000..986d3ce --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-assembly/src/main/assembly/assembly.xml @@ -0,0 +1,65 @@ +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<assembly> + <id>archive</id> + <formats> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${project.basedir}/../metron-parser-asa/src/main/config</directory> + <outputDirectory>/config</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + <excludes> + <exclude>**/*.formatted</exclude> + <exclude>**/*.filtered</exclude> + </excludes> + <fileMode>0644</fileMode> + <lineEnding>unix</lineEnding> + <filtered>true</filtered> + </fileSet> + <fileSet> + <directory>${project.basedir}/../metron-parser-asa/src/main/resources/patterns</directory> + <outputDirectory>/patterns</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + <excludes> + <exclude>**/*.formatted</exclude> + <exclude>**/*.filtered</exclude> + </excludes> + <fileMode>0644</fileMode> + <lineEnding>unix</lineEnding> + </fileSet> + <fileSet> + <directory>${project.basedir}/../metron-parser-asa/src/main/scripts</directory> + <outputDirectory>/bin</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + <excludes> + <exclude>**/*.formatted</exclude> + <exclude>**/*.filtered</exclude> + </excludes> + <fileMode>0755</fileMode> + <lineEnding>unix</lineEnding> + <filtered>true</filtered> + </fileSet> + <fileSet> + <directory>${project.basedir}/../metron-parser-asa-bundle/target</directory> + <includes> + <include>metron-parser-asa-bundle-${project.version}.bundle</include> + </includes> + <outputDirectory>/lib</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + </fileSet> + </fileSets> +</assembly> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-bundle/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-bundle/pom.xml new file mode 100644 index 0000000..bfff635 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa-bundle/pom.xml @@ -0,0 +1,42 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-asa-extension</artifactId> + <version>0.4.1</version> + </parent> + + <artifactId>metron-parser-asa-bundle</artifactId> + <version>0.4.1</version> + <name>metron-parser-asa-bundle</name> + <packaging>bundle</packaging> + <properties> + <maven.javadoc.skip>true</maven.javadoc.skip> + <source.skip>false</source.skip> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-asa</artifactId> + <version>0.4.1</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/README.md b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/README.md new file mode 100644 index 0000000..f03e5cb --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/README.md @@ -0,0 +1,3 @@ +# ASA Parser + +## Introduction http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/pom.xml new file mode 100644 index 0000000..4f8a356 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/pom.xml @@ -0,0 +1,86 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-asa-extension</artifactId> + <version>0.4.1</version> + </parent> + <artifactId>metron-parser-asa</artifactId> + <version>0.4.1</version> + <name>metron-parser-asa</name> + <packaging>jar</packaging> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parsers</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.atteo.classindex</groupId> + <artifactId>classindex</artifactId> + <version>${global_classindex_version}</version> + <scope>provided</scope> + </dependency> + <!-- testing --> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-extensions-testing</artifactId> + <version>${project.parent.version}</version> + <type>pom</type> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>${global_jar_version}</version> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + <resources> + <resource> + <directory>src/main/resources</directory> + </resource> + <resource> + <directory>src/main/patterns</directory> + </resource> + <resource> + <directory>src/test/resources</directory> + </resource> + </resources> + </build> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/enrichments/asa.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/enrichments/asa.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/enrichments/asa.json new file mode 100644 index 0000000..fa07468 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/enrichments/asa.json @@ -0,0 +1,8 @@ +{ + "enrichment" : { + "fieldMap": { + "geo": ["ip_dst_addr", "ip_src_addr"] + } + } +} + http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/indexing/asa.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/indexing/asa.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/indexing/asa.json new file mode 100644 index 0000000..153ccff --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/indexing/asa.json @@ -0,0 +1,18 @@ +{ + "hdfs" : { + "index": "asa", + "batchSize": 5, + "enabled" : true + }, + "elasticsearch" : { + "index": "asa", + "batchSize": 5, + "enabled" : true + }, + "solr" : { + "index": "asa", + "batchSize": 5, + "enabled" : true + } +} + http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/parsers/asa.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/parsers/asa.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/parsers/asa.json new file mode 100644 index 0000000..489a047 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/config/zookeeper/parsers/asa.json @@ -0,0 +1,7 @@ +{ + "parserClassName": "org.apache.metron.parsers.asa.BasicAsaParser", + "sensorTopic": "asa", + "parserConfig": { + "deviceTimeZone": "UTC" + } +}
