http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabaseTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabaseTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabaseTest.java index 238f8e0..913c661 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabaseTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabaseTest.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.metron.common.dsl.Context; +import org.apache.metron.stellar.dsl.Context; import org.apache.metron.test.utils.UnitTestHelper; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser;
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java index 724221d..cd1006c 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java @@ -17,16 +17,15 @@ */ package org.apache.metron.enrichment.adapters.stellar; -import com.google.common.collect.ImmutableList; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.StellarEnrichmentTest; import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.MapVariableResolver; -import org.apache.metron.common.dsl.VariableResolver; -import org.apache.metron.common.stellar.StellarProcessor; import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.MapVariableResolver; +import org.apache.metron.stellar.dsl.VariableResolver; import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java index 2149cb9..a7b2d7d 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java @@ -24,7 +24,6 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; -import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; @@ -32,8 +31,6 @@ import org.apache.metron.TestConstants; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase; -import org.apache.metron.enrichment.bolt.ErrorEnrichmentBolt; -import org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt; import org.apache.metron.enrichment.converter.EnrichmentHelper; import org.apache.metron.enrichment.converter.EnrichmentKey; import org.apache.metron.enrichment.converter.EnrichmentValue; @@ -44,7 +41,6 @@ import org.apache.metron.enrichment.stellar.SimpleHBaseEnrichmentFunctions; import org.apache.metron.hbase.TableProvider; import org.apache.metron.integration.BaseIntegrationTest; import org.apache.metron.integration.ComponentRunner; -import org.apache.metron.integration.Processor; import org.apache.metron.integration.ProcessorResult; import org.apache.metron.integration.components.FluxTopologyComponent; import org.apache.metron.integration.components.KafkaComponent; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/DocumentationGenerator.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/DocumentationGenerator.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/DocumentationGenerator.java index ed371d0..c9178c0 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/DocumentationGenerator.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/DocumentationGenerator.java @@ -18,8 +18,8 @@ package org.apache.metron.enrichment.stellar; import com.google.common.collect.Lists; -import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver; -import org.apache.metron.common.dsl.StellarFunctionInfo; +import org.apache.metron.stellar.dsl.functions.resolver.SingletonFunctionResolver; +import org.apache.metron.stellar.dsl.StellarFunctionInfo; import java.util.Collections; import java.util.List; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctionsTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctionsTest.java index e8f8f71..8619b9f 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctionsTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoEnrichmentFunctionsTest.java @@ -20,9 +20,9 @@ package org.apache.metron.enrichment.stellar; import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.stellar.StellarProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase; import org.apache.metron.test.utils.UnitTestHelper; import org.json.simple.JSONObject; @@ -141,7 +141,7 @@ public class GeoEnrichmentFunctionsTest { Assert.assertEquals("Remote IP should return country result based on DB", expectedSubsetMessage, result); } - @Test(expected=org.apache.metron.common.dsl.ParseException.class) + @Test(expected=org.apache.metron.stellar.dsl.ParseException.class) @SuppressWarnings("unchecked") public void testGetTooManyParams() throws Exception { String stellar = "GEO_GET('216.160.83.56', ['country', 'city', 'dmaCode', 'location_point'], 'garbage')"; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java index 9c8df16..b7668f3 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java @@ -21,13 +21,12 @@ package org.apache.metron.enrichment.stellar; import com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.stellar.StellarProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.enrichment.converter.EnrichmentHelper; import org.apache.metron.enrichment.converter.EnrichmentKey; import org.apache.metron.enrichment.converter.EnrichmentValue; -import org.apache.metron.enrichment.lookup.EnrichmentLookup; import org.apache.metron.enrichment.lookup.LookupKV; import org.apache.metron.hbase.TableProvider; import org.apache.metron.test.mock.MockHTable; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java index b3a0868..63f720b 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/threatintel/triage/ThreatTriageTest.java @@ -23,8 +23,8 @@ import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.RuleScore; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatScore; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.StellarFunctions; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.metron.common.utils.JSONUtils; import org.junit.Assert; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-integration-test/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml index 54f4a6b..69416f4 100644 --- a/metron-platform/metron-integration-test/pom.xml +++ b/metron-platform/metron-integration-test/pom.xml @@ -217,6 +217,11 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-common</artifactId> <version>${global_hadoop_version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/StellarClasspathFunctionResolver.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/StellarClasspathFunctionResolver.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/StellarClasspathFunctionResolver.java new file mode 100644 index 0000000..fb4ec75 --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/StellarClasspathFunctionResolver.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.integration; + +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.metron.integration.components.MRComponent; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Properties; + +import static org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver.Config.STELLAR_VFS_PATHS; + +public class StellarClasspathFunctionResolver { + static MRComponent component; + static Configuration configuration; + @BeforeClass + public static void setup() { + component = new MRComponent().withBasePath("target"); + component.start(); + configuration = component.getConfiguration(); + + try { + FileSystem fs = FileSystem.newInstance(configuration); + fs.mkdirs(new Path("/classpath-resources")); + fs.copyFromLocalFile(new Path("src/test/classpath-resources/custom-1.0-SNAPSHOT.jar"), new Path("/classpath-resources")); + } catch (IOException e) { + throw new RuntimeException("Unable to start cluster", e); + } + } + + @AfterClass + public static void teardown() { + component.stop(); + } + + + public static ClasspathFunctionResolver create(Properties config) { + ClasspathFunctionResolver resolver = new ClasspathFunctionResolver(); + + Context context = new Context.Builder() + .with(Context.Capabilities.STELLAR_CONFIG, () -> config) + .build(); + resolver.initialize(context); + + return resolver; + } + + @Test + public void test() throws Exception { + Properties config = new Properties(); + config.put(STELLAR_VFS_PATHS.param(), configuration.get("fs.defaultFS") + "/classpath-resources/.*.jar"); + ClasspathFunctionResolver resolver = create(config); + HashSet<String> functions = new HashSet<>(Lists.newArrayList(resolver.getFunctions())); + Assert.assertTrue(functions.contains("NOW")); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml index 4184668..d32dfff 100644 --- a/metron-platform/metron-management/pom.xml +++ b/metron-platform/metron-management/pom.xml @@ -51,6 +51,11 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> <groupId>com.jakewharton.fliptables</groupId> <artifactId>fliptables</artifactId> <version>1.0.2</version> @@ -126,13 +131,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.metron</groupId> - <artifactId>metron-common</artifactId> - <version>${project.parent.version}</version> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> <version>${global_hadoop_version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java index ecebe8b..cb78db0 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java @@ -18,12 +18,9 @@ package org.apache.metron.management; import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; -import org.apache.commons.configuration.ConfigurationUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.TreeCache; import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.curator.framework.recipes.cache.TreeCacheListener; @@ -34,11 +31,11 @@ import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.configuration.IndexingConfigurations; import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.common.utils.JSONUtils; import java.util.*; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/main/java/org/apache/metron/management/EnrichmentConfigFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/EnrichmentConfigFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/EnrichmentConfigFunctions.java index 29f950e..b65f118 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/EnrichmentConfigFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/EnrichmentConfigFunctions.java @@ -20,21 +20,17 @@ package org.apache.metron.management; import com.fasterxml.jackson.core.JsonProcessingException; import com.jakewharton.fliptables.FlipTable; import org.apache.log4j.Logger; -import org.apache.metron.common.configuration.FieldTransformer; -import org.apache.metron.common.configuration.IndexingConfigurations; import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; import org.apache.metron.common.utils.JSONUtils; import java.util.*; import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; -import static org.apache.metron.common.configuration.ConfigurationType.INDEXING; public class EnrichmentConfigFunctions { http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java index 4d64803..4effae8 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/FileSystemFunctions.java @@ -17,24 +17,21 @@ */ package org.apache.metron.management; -import com.google.common.base.Splitter; -import com.google.common.collect.Iterables; import com.jakewharton.fliptables.FlipTable; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.log4j.Logger; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.stellar.common.utils.ConversionUtils; import java.io.IOException; import java.net.URI; import java.text.DateFormat; import java.util.*; -import java.util.function.Function; public class FileSystemFunctions { private static final Logger LOG = Logger.getLogger(FileSystemFunctions.class); http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/main/java/org/apache/metron/management/GrokFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/GrokFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/GrokFunctions.java index d97832c..757219c 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/GrokFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/GrokFunctions.java @@ -22,10 +22,10 @@ import oi.thekraken.grok.api.Grok; import oi.thekraken.grok.api.Match; import oi.thekraken.grok.api.exception.GrokException; import org.apache.log4j.Logger; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; import java.io.InputStream; import java.io.InputStreamReader; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/main/java/org/apache/metron/management/IndexingConfigFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/IndexingConfigFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/IndexingConfigFunctions.java index 2c5bc27..4a90171 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/IndexingConfigFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/IndexingConfigFunctions.java @@ -20,11 +20,11 @@ package org.apache.metron.management; import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.log4j.Logger; import org.apache.metron.common.configuration.IndexingConfigurations; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.common.utils.JSONUtils; import java.util.HashMap; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java index c5428d2..1b0af51 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java @@ -18,25 +18,21 @@ package org.apache.metron.management; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.stellar.common.utils.ConversionUtils; import java.util.*; import java.util.concurrent.ExecutionException; -import static org.apache.metron.common.dsl.Context.Capabilities.GLOBAL_CONFIG; +import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; /** * Kafka functions available in Stellar. http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/main/java/org/apache/metron/management/ParserConfigFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ParserConfigFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ParserConfigFunctions.java index d85516e..9150798 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ParserConfigFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ParserConfigFunctions.java @@ -18,19 +18,16 @@ package org.apache.metron.management; import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.base.Splitter; -import com.google.common.collect.Iterables; import com.jakewharton.fliptables.FlipTable; import org.apache.log4j.Logger; -import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.common.configuration.SensorParserConfig; -import org.apache.metron.common.dsl.*; -import org.apache.metron.common.field.transformation.FieldTransformation; import org.apache.metron.common.field.transformation.FieldTransformations; -import org.apache.metron.common.stellar.shell.StellarExecutor; import org.apache.metron.common.utils.JSONUtils; -import org.jboss.aesh.console.Console; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; import java.util.*; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java index 5ab506a..a191526 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java @@ -19,23 +19,18 @@ package org.apache.metron.management; import com.jakewharton.fliptables.FlipTable; import org.apache.commons.io.IOUtils; -import org.apache.commons.io.input.NullInputStream; import org.apache.commons.lang3.text.WordUtils; import org.apache.log4j.Logger; -import org.apache.metron.common.dsl.*; -import org.apache.metron.common.stellar.shell.PausableInput; -import org.apache.metron.common.stellar.shell.StellarExecutor; -import org.apache.metron.common.utils.ConversionUtils; -import org.jboss.aesh.console.AeshProcess; +import org.apache.metron.stellar.common.shell.PausableInput; +import org.apache.metron.stellar.common.shell.StellarExecutor; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.*; import org.jboss.aesh.console.Console; import java.io.*; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.nio.charset.Charset; import java.util.*; -import static org.apache.metron.common.stellar.shell.StellarExecutor.CONSOLE; +import static org.apache.metron.stellar.common.shell.StellarExecutor.CONSOLE; public class ShellFunctions { private static final Logger LOG = Logger.getLogger(ShellFunctions.class); http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java index a16dea7..eb330a8 100644 --- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java +++ b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ThreatTriageFunctions.java @@ -18,23 +18,19 @@ package org.apache.metron.management; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; import com.jakewharton.fliptables.FlipTable; import org.apache.log4j.Logger; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.RiskLevelRule; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig; import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.common.utils.JSONUtils; -import java.io.IOException; import java.util.*; import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java index 972eed7..ade5943 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java @@ -25,8 +25,8 @@ import org.apache.curator.test.TestingServer; import org.apache.log4j.Level; import org.apache.metron.common.cli.ConfigurationManager; import org.apache.metron.common.configuration.ConfigurationsUtils; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.test.utils.UnitTestHelper; import org.json.simple.parser.JSONParser; import org.json.simple.JSONObject; @@ -40,7 +40,7 @@ import java.util.HashMap; import static org.apache.metron.TestConstants.PARSER_CONFIGS_PATH; import static org.apache.metron.TestConstants.SAMPLE_CONFIG_PATH; import static org.apache.metron.management.utils.FileUtils.slurp; -import static org.apache.metron.common.utils.StellarProcessorUtils.run; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; public class ConfigurationFunctionsTest { private static TestingServer testZkServer; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java index 7d8e20a..d7dd8f5 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java @@ -21,12 +21,11 @@ import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.stellar.StellarProcessor; -import org.apache.metron.common.stellar.shell.StellarExecutor; import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.shell.StellarExecutor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -154,8 +153,6 @@ public class EnrichmentConfigFunctionsTest { Assert.assertEquals(variables.get("upper").getExpression(), get(stellarFunctions,"upper")); } - - @Test public void testAddHasExisting() { String newConfig = (String) run( http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java index e0bad79..92bf79a 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java @@ -20,7 +20,7 @@ package org.apache.metron.management; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.metron.common.dsl.Context; +import org.apache.metron.stellar.dsl.Context; import org.junit.*; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/test/java/org/apache/metron/management/GrokFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/GrokFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/GrokFunctionsTest.java index 6158978..52c5e5f 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/GrokFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/GrokFunctionsTest.java @@ -19,12 +19,12 @@ package org.apache.metron.management; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.apache.metron.common.dsl.Context; +import org.apache.metron.stellar.dsl.Context; import org.junit.Assert; import org.junit.Test; import java.util.HashMap; -import static org.apache.metron.common.utils.StellarProcessorUtils.run; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; public class GrokFunctionsTest { private String grokExpr = "%{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?"; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java index 6fec2ec..f43a79c 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/IndexingConfigFunctionsTest.java @@ -19,11 +19,10 @@ package org.apache.metron.management; import com.google.common.collect.ImmutableMap; import org.apache.metron.common.configuration.IndexingConfigurations; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.stellar.StellarProcessor; -import org.apache.metron.common.stellar.shell.StellarExecutor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.shell.StellarExecutor; import org.junit.Assert; import org.junit.Before; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/test/java/org/apache/metron/management/KafkaFunctionsIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/KafkaFunctionsIntegrationTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/KafkaFunctionsIntegrationTest.java index 97cfb65..e639c63 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/KafkaFunctionsIntegrationTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/KafkaFunctionsIntegrationTest.java @@ -18,10 +18,10 @@ package org.apache.metron.management; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.common.dsl.functions.resolver.SimpleFunctionResolver; -import org.apache.metron.common.stellar.StellarProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.integration.BaseIntegrationTest; import org.apache.metron.integration.ComponentRunner; import org.apache.metron.integration.components.KafkaComponent; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java index f425140..0587000 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java @@ -21,8 +21,8 @@ import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.common.configuration.SensorParserConfig; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.stellar.shell.StellarExecutor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.common.shell.StellarExecutor; import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Before; @@ -34,7 +34,7 @@ import java.util.Map; import static org.apache.metron.TestConstants.PARSER_CONFIGS_PATH; import static org.apache.metron.management.utils.FileUtils.slurp; import static org.apache.metron.common.configuration.ConfigurationType.PARSER; -import static org.apache.metron.common.utils.StellarProcessorUtils.run; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; public class ParserConfigFunctionsTest { http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java index cf40de8..0f334ee 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java @@ -19,17 +19,15 @@ package org.apache.metron.management; import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.stellar.shell.StellarExecutor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.common.shell.StellarExecutor; import org.junit.Assert; import org.junit.Test; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; -import static org.apache.metron.common.utils.StellarProcessorUtils.run; +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; public class ShellFunctionsTest { http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java index 68be5d3..b5593b0 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ThreatTriageFunctionsTest.java @@ -21,11 +21,10 @@ import com.google.common.collect.ImmutableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; import org.apache.metron.common.configuration.enrichment.threatintel.RiskLevelRule; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.stellar.StellarProcessor; -import org.apache.metron.common.stellar.shell.StellarExecutor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.common.shell.StellarExecutor; import org.junit.Assert; import org.junit.Before; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 56506a7..6fe2ff0 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -23,13 +23,12 @@ import org.apache.metron.common.bolt.ConfiguredParserBolt; import org.apache.metron.common.configuration.FieldTransformer; import org.apache.metron.common.configuration.FieldValidator; import org.apache.metron.common.configuration.SensorParserConfig; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.StellarFunctions; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.metron.common.error.MetronError; import org.apache.metron.common.message.MessageGetStrategy; import org.apache.metron.common.message.MessageGetters; import org.apache.metron.common.utils.ErrorUtils; -import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase; import org.apache.metron.parsers.filters.Filters; import org.apache.metron.parsers.interfaces.MessageFilter; import org.apache.metron.parsers.interfaces.MessageParser; @@ -60,7 +59,7 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable { //default filter is noop, so pass everything through. private MessageFilter<JSONObject> filter; private WriterHandler writer; - private org.apache.metron.common.dsl.Context stellarContext; + private Context stellarContext; private transient MessageGetStrategy messageGetStrategy; public ParserBolt( String zookeeperUrl , String sensorType http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java index 52d45c9..9f7cd51 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java @@ -20,7 +20,7 @@ package org.apache.metron.parsers.csv; import com.google.common.collect.ImmutableList; import org.apache.metron.common.csv.CSVConverter; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.slf4j.Logger; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java index 9d51519..9cdafa3 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java @@ -17,12 +17,10 @@ */ package org.apache.metron.parsers.filters; -import org.apache.commons.configuration.Configuration; -import org.apache.metron.common.dsl.Context; +import org.apache.metron.stellar.dsl.Context; import org.apache.metron.parsers.interfaces.MessageFilter; import org.json.simple.JSONObject; -import java.io.Serializable; import java.util.HashSet; import java.util.List; import java.util.Map; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java index a564e62..15a035a 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java @@ -18,9 +18,12 @@ package org.apache.metron.parsers.filters; -import org.apache.metron.common.dsl.*; -import org.apache.metron.common.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.common.stellar.StellarPredicateProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.MapVariableResolver; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +import org.apache.metron.stellar.common.StellarPredicateProcessor; import org.apache.metron.parsers.interfaces.MessageFilter; import org.json.simple.JSONObject; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java index abcb57a..b7b91c0 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java @@ -17,10 +17,7 @@ */ package org.apache.metron.parsers.interfaces; -import org.apache.metron.common.dsl.Context; - -import java.io.Serializable; -import java.util.Map; +import org.apache.metron.stellar.dsl.Context; public interface MessageFilter<T> extends Configurable{ http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java index c68e101..8ff4f93 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java @@ -22,8 +22,6 @@ import org.apache.metron.storm.kafka.flux.SpoutConfiguration; import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.StormSubmitter; -import org.apache.storm.hbase.security.AutoHBase; -import org.apache.storm.hdfs.common.security.AutoHDFS; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.utils.Utils; import com.fasterxml.jackson.core.type.TypeReference; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java index 885fc3c..8441409 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java @@ -19,7 +19,7 @@ package org.apache.metron.filters; import com.google.common.collect.ImmutableMap; -import org.apache.metron.common.dsl.Context; +import org.apache.metron.stellar.dsl.Context; import org.apache.metron.parsers.filters.Filters; import org.apache.metron.parsers.interfaces.MessageFilter; import org.json.simple.JSONObject; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java index 835f17e..49f111d 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java @@ -22,23 +22,20 @@ import org.apache.metron.common.configuration.*; import org.apache.metron.common.error.MetronError; import org.apache.metron.test.error.MetronErrorJSONMatcher; -import org.apache.metron.test.utils.UnitTestHelper; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.tuple.Tuple; import com.google.common.collect.ImmutableList; import org.apache.metron.common.configuration.writer.ParserWriterConfiguration; import org.apache.metron.common.configuration.writer.WriterConfiguration; -import org.apache.metron.common.dsl.Context; +import org.apache.metron.stellar.dsl.Context; import org.apache.metron.common.writer.BulkMessageWriter; import org.adrianwalker.multilinestring.Multiline; import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.SensorParserConfig; -import org.apache.metron.common.utils.ErrorUtils; import org.apache.metron.common.writer.BulkWriterResponse; import org.apache.metron.parsers.BasicParser; -import org.apache.metron.parsers.csv.CSVParser; import org.apache.metron.test.bolt.BaseBoltTest; import org.apache.metron.parsers.interfaces.MessageFilter; import org.apache.metron.parsers.interfaces.MessageParser; @@ -49,7 +46,6 @@ import org.junit.Assert; import org.junit.Test; import org.mockito.Mock; -import java.io.File; import java.io.IOException; import java.util.*; import java.util.stream.Collectors; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java index 7476bcf..d4d1c02 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java @@ -24,7 +24,7 @@ import org.adrianwalker.multilinestring.Multiline; import org.apache.hadoop.hbase.util.Bytes; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.SensorParserConfig; -import org.apache.metron.common.dsl.Context; +import org.apache.metron.stellar.dsl.Context; import org.apache.metron.common.field.validation.FieldValidation; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.enrichment.integration.components.ConfigUploadComponent; @@ -37,7 +37,6 @@ import org.apache.metron.parsers.csv.CSVParser; import org.apache.metron.parsers.integration.components.ParserTopologyComponent; import org.apache.metron.test.utils.UnitTestHelper; import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; import org.junit.Assert; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-pcap/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml index 7b9c9e5..020db07 100644 --- a/metron-platform/metron-pcap/pom.xml +++ b/metron-platform/metron-pcap/pom.xml @@ -140,13 +140,6 @@ <version>${project.parent.version}</version> </dependency> <dependency> - <groupId>org.apache.metron</groupId> - <artifactId>metron-common</artifactId> - <version>${project.parent.version}</version> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>${global_hadoop_version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java index e5a15e7..b32cdbc 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java @@ -18,10 +18,8 @@ package org.apache.metron.pcap.filter; -import org.apache.metron.common.Constants; -import org.apache.metron.common.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.VariableResolver; -import java.util.EnumMap; import java.util.HashMap; import java.util.Map; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/fixed/FixedPcapFilter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/fixed/FixedPcapFilter.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/fixed/FixedPcapFilter.java index 001b500..1954f1a 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/fixed/FixedPcapFilter.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/fixed/FixedPcapFilter.java @@ -21,18 +21,16 @@ package org.apache.metron.pcap.filter.fixed; import com.google.common.base.Joiner; import org.apache.hadoop.conf.Configuration; import org.apache.metron.common.Constants; -import org.apache.metron.common.dsl.MapVariableResolver; -import org.apache.metron.common.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.MapVariableResolver; +import org.apache.metron.stellar.dsl.VariableResolver; import org.apache.metron.pcap.PacketInfo; import org.apache.metron.pcap.PcapHelper; import org.apache.metron.pcap.filter.PcapFilter; import org.apache.metron.pcap.filter.PcapFilterConfigurator; import org.apache.metron.pcap.filter.PcapFilters; -import org.apache.metron.pcap.filter.PcapFieldResolver; import org.apache.metron.pcap.pattern.ByteArrayMatchingUtil; import javax.xml.bind.DatatypeConverter; -import java.util.EnumMap; import java.util.Map; import java.util.concurrent.ExecutionException; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/query/QueryPcapFilter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/query/QueryPcapFilter.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/query/QueryPcapFilter.java index d6fa203..552a5ae 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/query/QueryPcapFilter.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/query/QueryPcapFilter.java @@ -19,20 +19,17 @@ package org.apache.metron.pcap.filter.query; import org.apache.hadoop.conf.Configuration; -import org.apache.metron.common.Constants; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.MapVariableResolver; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.stellar.StellarPredicateProcessor; -import org.apache.metron.common.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.MapVariableResolver; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.common.StellarPredicateProcessor; +import org.apache.metron.stellar.dsl.VariableResolver; import org.apache.metron.pcap.PacketInfo; import org.apache.metron.pcap.PcapHelper; -import org.apache.metron.pcap.filter.PcapFieldResolver; import org.apache.metron.pcap.filter.PcapFilter; import org.apache.metron.pcap.filter.PcapFilterConfigurator; import org.apache.metron.pcap.filter.PcapFilters; -import java.util.EnumMap; import java.util.Map; public class QueryPcapFilter implements PcapFilter { http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/pattern/ByteArrayMatcherFunction.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/pattern/ByteArrayMatcherFunction.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/pattern/ByteArrayMatcherFunction.java index a4a74f3..1e80cc8 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/pattern/ByteArrayMatcherFunction.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/pattern/ByteArrayMatcherFunction.java @@ -17,10 +17,10 @@ */ package org.apache.metron.pcap.pattern; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; import javax.xml.bind.DatatypeConverter; import java.util.List; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-pcap/src/test/java/org/apache/metron/pcap/pattern/ByteArrayMatchingUtilTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/test/java/org/apache/metron/pcap/pattern/ByteArrayMatchingUtilTest.java b/metron-platform/metron-pcap/src/test/java/org/apache/metron/pcap/pattern/ByteArrayMatchingUtilTest.java index 37b3f3f..0f1d0ec 100644 --- a/metron-platform/metron-pcap/src/test/java/org/apache/metron/pcap/pattern/ByteArrayMatchingUtilTest.java +++ b/metron-platform/metron-pcap/src/test/java/org/apache/metron/pcap/pattern/ByteArrayMatchingUtilTest.java @@ -17,9 +17,7 @@ */ package org.apache.metron.pcap.pattern; -import com.google.common.collect.ImmutableMap; -import org.apache.metron.common.utils.StellarProcessorUtils; -import org.apache.metron.pcap.pattern.ByteArrayMatchingUtil; +import org.apache.metron.stellar.common.utils.StellarProcessorUtils; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -28,7 +26,6 @@ import org.junit.runners.Parameterized; import javax.xml.bind.DatatypeConverter; import java.util.*; import java.util.concurrent.ExecutionException; -import java.util.function.Predicate; @RunWith(Parameterized.class) public class ByteArrayMatchingUtilTest { http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-storm-kafka/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-storm-kafka/pom.xml b/metron-platform/metron-storm-kafka/pom.xml index 5c28b34..b14c2a9 100644 --- a/metron-platform/metron-storm-kafka/pom.xml +++ b/metron-platform/metron-storm-kafka/pom.xml @@ -94,6 +94,11 @@ <artifactId>metron-common</artifactId> <version>${project.parent.version}</version> </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SpoutConfiguration.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SpoutConfiguration.java b/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SpoutConfiguration.java index 2a4586d..847de8b 100644 --- a/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SpoutConfiguration.java +++ b/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SpoutConfiguration.java @@ -18,8 +18,7 @@ package org.apache.metron.storm.kafka.flux; import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.storm.kafka.spout.KafkaSpoutConfig; import java.util.ArrayList; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java index dbdd93d..0e73a3b 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java @@ -23,7 +23,7 @@ import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.writer.BulkMessageWriter; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.common.writer.BulkWriterResponse; import org.json.simple.JSONObject; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java index c5d1e4f..1ba9a6b 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java @@ -18,13 +18,11 @@ package org.apache.metron.writer.hdfs; import org.apache.metron.common.configuration.IndexingConfigurations; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.MapVariableResolver; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.dsl.VariableResolver; -import org.apache.metron.common.stellar.StellarCompiler; -import org.apache.metron.common.stellar.StellarProcessor; -import org.apache.metron.common.utils.SerDeUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.MapVariableResolver; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.dsl.VariableResolver; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.storm.task.TopologyContext; import org.apache.storm.tuple.Tuple; import org.apache.metron.common.configuration.writer.WriterConfiguration; @@ -40,7 +38,6 @@ import org.json.simple.JSONObject; import java.io.*; import java.util.*; -import java.util.function.Function; public class HdfsWriter implements BulkMessageWriter<JSONObject>, Serializable { List<RotationAction> rotationActions = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java index 1884f5d..7ce9b9b 100644 --- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java +++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java @@ -24,7 +24,7 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.writer.MessageWriter; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.common.utils.KafkaUtils; import org.apache.metron.common.utils.StringUtils; import org.apache.metron.writer.AbstractWriter; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/pom.xml ---------------------------------------------------------------------- diff --git a/metron-stellar/pom.xml b/metron-stellar/pom.xml new file mode 100644 index 0000000..a352bd6 --- /dev/null +++ b/metron-stellar/pom.xml @@ -0,0 +1,88 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.metron</groupId> + <artifactId>metron-stellar</artifactId> + <packaging>pom</packaging> + <name>metron-stellar</name> + <parent> + <artifactId>Metron</artifactId> + <groupId>org.apache.metron</groupId> + <version>0.4.0</version> + </parent> + <description>DSL for stream analytics</description> + <url>https://metron.apache.org/</url> + <scm> + <connection>scm:git:https://git-wip-us.apache.org/repos/asf/metron.git</connection> + <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/metron.git</developerConnection> + <tag>HEAD</tag> + <url>https://git-wip-us.apache.org/repos/asf/metron</url> + </scm> + + <licenses> + <license> + <name>The Apache Software License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + <distribution>repo</distribution> + </license> + </licenses> + <modules> + <module>stellar-common</module> + </modules> + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>${global_slf4j_version}</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <version>${global_slf4j_version}</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${global_junit_version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>1.6.6</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito</artifactId> + <version>1.6.6</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.adrianwalker</groupId> + <artifactId>multiline-string</artifactId> + <version>0.1.2</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-all</artifactId> + <version>1.3</version> + <scope>test</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-stellar/stellar-common/3rdPartyStellar.md ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/3rdPartyStellar.md b/metron-stellar/stellar-common/3rdPartyStellar.md new file mode 100644 index 0000000..b87c68f --- /dev/null +++ b/metron-stellar/stellar-common/3rdPartyStellar.md @@ -0,0 +1,160 @@ +# Custom Stellar Functions + +Metron is fundamentally a programmable, extensible system +and Stellar is the extension language. We have some great Stellar functions +available out of the box and we'll be adding more over time, but they may +not quite scratch quite your particular itch. + +Of course, we'd love to have your contribution inside of Metron if you think it +general purpose enough, but not every function is general-purpose or it may rely +on libraries those licenses aren't acceptable for an Apache project. In that case, then you will +be wondering how to add your custom function to a running instance of Metron. + +## Building Your Own Function + +Let's say that I need a function that returns the current time in milliseconds +since the epoch. I notice that there's nothing like that currently in Metron, +so I embark on the adventure of adding it for my cluster. + +I will presume that you have an installed Metron into your local maven repo via `mvn install` . In the future, when we publish to a maven repo, +you will not need this. I will depend on 0.4.0 for the +purpose of this demonstration + +### Hack, Hack, Hack + +I like to use Maven, so we'll use that for this demonstration, but you can use whatever +build system that you like. Here's my favorite way to build a project with groupId `com.mycompany.stellar` +and artifactId of `tempus` +`mvn archetype:create -DgroupId=com.mycompany.stellar -DartifactId=tempus -DarchetypeArtifactId=maven-archetype-quickstart` + +First, we should depend on `metron-common` and we can do that by adjusting the `pom.xml` just created: +``` +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>com.mycompany.stellar</groupId> + <artifactId>tempus</artifactId> + <version>1.0-SNAPSHOT</version> + <packaging>jar</packaging> + + <name>Stellar Time Functions</name> + <url>http://mycompany.com</url> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-common</artifactId> + <version>0.4.0</version> + <!-- NOTE: We will want to depend on the deployed common on the classpath. --> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>3.8.1</version> + <scope>test</scope> + </dependency> + </dependencies> +</project> +``` + +Let's add our implementation in `src/main/java/com/mycompany/stellar/TimeFunctions.java` with the following content: +``` +package com.notmetron.stellar; + +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; + +import java.util.List; + +public class TimeFunction { + @Stellar( name="NOW", + description = "Right now!", + params = {}, + returns="Timestamp" + ) + public static class Now implements StellarFunction { + + public Object apply(List<Object> list, Context context) throws ParseException { + return System.currentTimeMillis(); + } + + public void initialize(Context context) { } + + public boolean isInitialized() { + return true; + } + } +} +``` + +Now we can build the project via `mvn package` which will create a `target/tempus-1.0-SNAPSHOT.jar` file. + +## Install the Function + +Now that we have a jar with our custom function, we must make Metron aware of it. + +### Deploy the Jar + +First you need to place the jar in HDFS, if we have it on an access node, one way to do that is: +* `hadoop fs -put tempus-1.0-SNAPSHOT.jar /apps/metron/stellar` +This presumes that: +* you've standardized on `/apps/metron/stellar` as the location for custom jars +* you are running the command from an access node with the `hadoop` command installed +* you are running from a user that has write access to `/apps/metron/stellar` + +### Set Global Config + +You may not need this if your Metron administrator already has this setup. + +With that dispensed with, we need to ensure that Metron knows to look at that location. +We need to ensure that the `stellar.function.paths` property in the `global.json` is in place that makes Metron aware +to look for Stellar functions in `/apps/metron/stellar` on HDFS. + +This property looks like, the following for a vagrant install +``` +{ + "es.clustername": "metron", + "es.ip": "node1", + "es.port": "9300", + "es.date.format": "yyyy.MM.dd.HH", + "stellar.function.paths" : "hdfs://node1:8020/apps/metron/stellar/.*.jar", +} +``` + +The `stellar.function.paths` property takes a comma separated list of URIs or URIs with regex expressions at the end. +Also, note path is prefaced by the HDFS default name, which, if you do not know, can be found by executing, +`hdfs getconf -confKey fs.default.name`, such as +``` +[root@node1 ~]# hdfs getconf -confKey fs.default.name +hdfs://node1:8020 +``` +### Use the Function + +Now that we have deployed the function, if we want to use it, +any running topologies that use Stellar will need to be restarted. + +Beyond that, let's take a look at it in the REPL: +``` +Stellar, Go! +Please note that functions are loading lazily in the background and will be unavailable until loaded fully. +{es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH, stellar.function.paths=hdfs://node1:8020/apps/metron/stellar/.*.jar, profiler.client.period.duration=1, profiler.client.period.duration.units=MINUTES} +[Stellar]>>> # Get the help for NOW +[Stellar]>>> ?NOW +Functions loaded, you may refer to functions now... +NOW +Description: Right now! + +Returns: Timestamp +[Stellar]>>> # Try to run the NOW function, which we added: +[Stellar]>>> NOW() +1488400515655 +[Stellar]>>> # Looks like I got a timestamp, success! +```
