[48/50] [abbrv] metron git commit: METRON-1539: Specialized RENAME field transformer closes apache/incubator-metron#1002
METRON-1539: Specialized RENAME field transformer closes apache/incubator-metron#1002 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2b4f0b84 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2b4f0b84 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2b4f0b84 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 2b4f0b84062d65f9400421d66ec3b7d6d093bebf Parents: 1c5435c Author: cstellaAuthored: Wed Apr 25 11:49:56 2018 -0400 Committer: cstella Committed: Wed Apr 25 11:49:56 2018 -0400 -- .../common/configuration/FieldTransformer.java | 4 +- .../transformation/FieldTransformations.java| 1 + .../transformation/RenameTransformation.java| 55 +++ .../transformation/FieldTransformationTest.java | 17 +--- .../RenameTransformationTest.java | 99 metron-platform/metron-parsers/README.md| 25 - 6 files changed, 183 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java -- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java index df80691..43ce9d8 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java @@ -94,7 +94,9 @@ public class FieldTransformer implements Serializable { if (output == null || output.isEmpty()) { if (input == null || input.isEmpty()) { - throw new IllegalStateException("You must specify an input field if you want to leave the output fields empty"); + //both are empty, so let's set them both to null + output = null; + input = null; } else { output = input; } http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java -- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java index a905123..95ff390 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java @@ -25,6 +25,7 @@ public enum FieldTransformations { ,REMOVE(new RemoveTransformation()) ,STELLAR(new StellarTransformation()) ,SELECT(new SelectTransformation()) + ,RENAME(new RenameTransformation()) ; FieldTransformation mapping; FieldTransformations(FieldTransformation mapping) { http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java -- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java new file mode 100644 index 000..f8b9374 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package
[47/50] [abbrv] metron git commit: METRON-1520: Add caching for stellar field transformations closes apache/incubator-metron#990
METRON-1520: Add caching for stellar field transformations closes apache/incubator-metron#990 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1c5435cc Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1c5435cc Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1c5435cc Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 1c5435ccbe96c03e59c6a18d681da43561769dba Parents: 37e3fd3 Author: cstellaAuthored: Wed Apr 25 11:48:44 2018 -0400 Committer: cstella Committed: Wed Apr 25 11:48:44 2018 -0400 -- metron-platform/Performance-tuning-guide.md | 13 ++ .../configuration/SensorParserConfig.java | 15 ++ .../transformation/StellarTransformation.java | 3 +- .../StellarTransformationTest.java | 30 metron-platform/metron-parsers/README.md| 13 ++ .../apache/metron/parsers/bolt/ParserBolt.java | 15 +- .../stellar/common/CachingStellarProcessor.java | 144 +++ .../org/apache/metron/stellar/dsl/Context.java | 43 +- .../common/CachingStellarProcessorTest.java | 104 ++ 9 files changed, 371 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/1c5435cc/metron-platform/Performance-tuning-guide.md -- diff --git a/metron-platform/Performance-tuning-guide.md b/metron-platform/Performance-tuning-guide.md index e2d1ae2..c2d19d6 100644 --- a/metron-platform/Performance-tuning-guide.md +++ b/metron-platform/Performance-tuning-guide.md @@ -60,6 +60,19 @@ parallelism will leave you with idle consumers since Kafka limits the max number important because Kafka has certain ordering guarantees for message delivery per partition that would not be possible if more than one consumer in a given consumer group were able to read from that partition. +## Sensor Topology Tuning Suggestions + +If you are using stellar field transformations in your sensors, by default, stellar expressions +are not cached. Sensors that use stellar field transformations by see a performance +boost by turning on caching via setting the `cacheConfig` +[property](metron-parsers#parser_configuration). +This is beneficial if your transformations: + +* Are complex (e.g. `ENRICHMENT_GET` calls or other high latency calls) +* All Yield the same results for the same inputs ( caching is either off or applied to all transformations) + * If any of your transformations are non-deterministic, caching should not be used as it will result in the likelihood of incorrect results being returned. + + ## Component Tuning Levers ### High Level Overview http://git-wip-us.apache.org/repos/asf/metron/blob/1c5435cc/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java -- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java index 2d0ccd8..d347481 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java @@ -45,11 +45,26 @@ public class SensorParserConfig implements Serializable { private Integer parserNumTasks = 1; private Integer errorWriterParallelism = 1; private Integer errorWriterNumTasks = 1; + private Map cacheConfig = new HashMap<>(); private Map spoutConfig = new HashMap<>(); private String securityProtocol = null; private Map stormConfig = new HashMap<>(); /** + * Cache config for stellar field transformations. + * * stellar.cache.maxSize - The maximum number of elements in the cache. + * * stellar.cache.maxTimeRetain - The maximum amount of time an element is kept in the cache (in minutes). + * @return + */ + public Map getCacheConfig() { +return cacheConfig; + } + + public void setCacheConfig(Map cacheConfig) { +this.cacheConfig = cacheConfig; + } + + /** * Return the number of workers for the topology. This property will be used for the parser unless overridden on the CLI. * @return */ http://git-wip-us.apache.org/repos/asf/metron/blob/1c5435cc/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java -- diff --git
[40/50] [abbrv] metron git commit: METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984
METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/82212ba8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/82212ba8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/82212ba8 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 82212ba818a0ef3b92505e8d144487c69a8d4a44 Parents: 3fcbf8b Author: nickwallenAuthored: Tue Apr 17 09:43:16 2018 -0400 Committer: nickallen Committed: Tue Apr 17 09:43:16 2018 -0400 -- .../packaging/ambari/metron-mpack/pom.xml | 7 +- .../configuration/metron-enrichment-env.xml | 130 +++- .../package/scripts/enrichment_commands.py | 20 +- .../package/scripts/enrichment_master.py| 12 +- .../package/scripts/params/params_linux.py | 19 +- .../enrichment-splitjoin.properties.j2 | 63 ++ .../templates/enrichment-unified.properties.j2 | 60 ++ .../METRON/CURRENT/themes/metron_theme.json | 151 - .../docker/rpm-docker/SPECS/metron.spec | 5 +- .../main/config/enrichment-splitjoin.properties | 63 ++ .../config/enrichment-splitjoin.properties.j2 | 63 ++ .../main/config/enrichment-unified.properties | 69 +++ .../config/enrichment-unified.properties.j2 | 60 ++ .../src/main/config/enrichment.properties | 64 -- .../src/main/config/enrichment.properties.j2| 63 -- .../main/flux/enrichment/remote-splitjoin.yaml | 590 +++ .../main/flux/enrichment/remote-unified.yaml| 71 ++- .../src/main/flux/enrichment/remote.yaml| 590 --- .../main/scripts/start_enrichment_topology.sh | 16 +- .../integration/EnrichmentIntegrationTest.java | 61 +- .../UnifiedEnrichmentIntegrationTest.java | 71 +++ 21 files changed, 1433 insertions(+), 815 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/pom.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml index 491e8dd..1843eb7 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml @@ -110,7 +110,8 @@ ${basedir}/../../../../metron-platform/metron-enrichment/src/main/config - enrichment.properties.j2 + enrichment-splitjoin.properties.j2 + enrichment-unified.properties.j2 false @@ -171,8 +172,10 @@ ${basedir}/src/main/resources/common-services/METRON/CURRENT/package/templates -enrichment.properties.j2 + enrichment-unified.properties.j2 + enrichment-splitjoin.properties.j2 elasticsearch.properties.j2 +hdfs.properties.j2 http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml index 9737660..81b135c 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml @@ -18,6 +18,10 @@ limitations under the License. --> + + geoip_url http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz @@ -33,6 +37,10 @@ content + + enrichment_kafka_start Enrichment Topology Spout Offset @@ -81,6 +89,10 @@ indexing Threat Intel Error Topic + + enrichment_hbase_table enrichment @@
[45/50] [abbrv] metron git commit: METRON-1511 Unable to Serialize Profiler Configuration (nickwallen) closes apache/metron#982
METRON-1511 Unable to Serialize Profiler Configuration (nickwallen) closes apache/metron#982 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b5bf9a98 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b5bf9a98 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b5bf9a98 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: b5bf9a98725f866a7fee6470a8e763d17cc69ffd Parents: a41611b Author: nickwallenAuthored: Mon Apr 23 09:36:06 2018 -0400 Committer: nickallen Committed: Mon Apr 23 09:36:06 2018 -0400 -- .../configuration/profiler/ProfileConfig.java | 57 -- .../profiler/ProfileResultExpressions.java | 4 +- .../profiler/ProfileTriageExpressions.java | 8 ++ .../configuration/profiler/ProfilerConfig.java | 81 -- .../profiler/ProfileConfigTest.java | 102 ++--- .../profiler/ProfilerConfigTest.java| 109 +-- 6 files changed, 310 insertions(+), 51 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/b5bf9a98/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java -- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java index f5b46e6..f2272c3 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileConfig.java @@ -18,12 +18,15 @@ package org.apache.metron.common.configuration.profiler; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.metron.common.utils.JSONUtils; +import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -264,15 +267,47 @@ public class ProfileConfig implements Serializable { @Override public String toString() { -return "ProfileConfig{" + -"profile='" + profile + '\'' + -", foreach='" + foreach + '\'' + -", onlyif='" + onlyif + '\'' + -", init=" + init + -", update=" + update + -", groupBy=" + groupBy + -", result=" + result + -", expires=" + expires + -'}'; +return new ToStringBuilder(this) +.append("profile", profile) +.append("foreach", foreach) +.append("onlyif", onlyif) +.append("init", init) +.append("update", update) +.append("groupBy", groupBy) +.append("result", result) +.append("expires", expires) +.toString(); + } + + /** + * Deserialize a {@link ProfileConfig}. + * + * @param bytes Raw bytes containing a UTF-8 JSON String. + * @return The Profile definition. + * @throws IOException + */ + public static ProfileConfig fromBytes(byte[] bytes) throws IOException { +return JSONUtils.INSTANCE.load(new String(bytes), ProfileConfig.class); + } + + /** + * Deserialize a {@link ProfileConfig}. + * + * @param json A String containing JSON. + * @return The Profile definition. + * @throws IOException + */ + public static ProfileConfig fromJSON(String json) throws IOException { +return JSONUtils.INSTANCE.load(json, ProfileConfig.class); + } + + /** + * Serialize the profile definition to a JSON string. + * + * @return The Profiler configuration serialized as a JSON string. + * @throws JsonProcessingException + */ + public String toJSON() throws JsonProcessingException { +return JSONUtils.INSTANCE.toJSON(this, true); } } http://git-wip-us.apache.org/repos/asf/metron/blob/b5bf9a98/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java -- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java index 82af223..5bcec72 100644 ---
[29/50] [abbrv] metron git commit: METRON-1518 Build Failure When Using Profile HDP-2.5.0.0 (nickwallen) closes apache/metron#986
METRON-1518 Build Failure When Using Profile HDP-2.5.0.0 (nickwallen) closes apache/metron#986 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ed50d48b Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ed50d48b Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ed50d48b Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: ed50d48bb47cf3f301884f6e18fe4efc8c1b91f1 Parents: a8b555d Author: nickwallenAuthored: Tue Apr 10 17:16:20 2018 -0400 Committer: nickallen Committed: Tue Apr 10 17:16:20 2018 -0400 -- .../profiler/bolt/ProfileBuilderBolt.java | 51 +--- 1 file changed, 11 insertions(+), 40 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/ed50d48b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java -- diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java index fb3d2d0..ca02b58 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java @@ -42,13 +42,11 @@ import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.zookeeper.SimpleEventListener; import org.apache.metron.zookeeper.ZKCache; -import org.apache.storm.StormTimer; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseWindowedBolt; import org.apache.storm.tuple.Tuple; -import org.apache.storm.utils.Utils; import org.apache.storm.windowing.TupleWindow; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -59,9 +57,9 @@ import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.function.Supplier; import static java.lang.String.format; import static org.apache.metron.profiler.bolt.ProfileSplitterBolt.ENTITY_TUPLE_FIELD; @@ -155,8 +153,8 @@ public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable { private FlushSignal activeFlushSignal; /** - * A timer that flushes expired profiles on a regular interval. The expired profiles - * are flushed on a separate thread. + * An executor that flushes expired profiles at a regular interval on a separate + * thread. * * Flushing expired profiles ensures that any profiles that stop receiving messages * for an extended period of time will continue to be flushed. @@ -164,7 +162,7 @@ public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable { * This introduces concurrency issues as the bolt is no longer single threaded. Due * to this, all access to the {@code MessageDistributor} needs to be protected. */ - private StormTimer expiredFlushTimer; + private transient ScheduledExecutorService flushExpiredExecutor; public ProfileBuilderBolt() { this.emitters = new ArrayList<>(); @@ -202,7 +200,7 @@ public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable { this.configurations = new ProfilerConfigurations(); this.activeFlushSignal = new FixedFrequencyFlushSignal(periodDurationMillis); setupZookeeper(); -startExpiredFlushTimer(); +startFlushingExpiredProfiles(); } @Override @@ -210,7 +208,7 @@ public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable { try { zookeeperCache.close(); zookeeperClient.close(); - expiredFlushTimer.close(); + flushExpiredExecutor.shutdown(); } catch(Throwable e) { LOG.error("Exception when cleaning up", e); @@ -421,39 +419,12 @@ public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable { } /** - * Converts milliseconds to seconds and handles an ugly cast. - * - * @param millis Duration in milliseconds. - * @return Duration in seconds. - */ - private int toSeconds(long millis) { -return (int) TimeUnit.MILLISECONDS.toSeconds(millis); - } - - /** - * Creates a timer that regularly flushes expired profiles on a separate thread. - */ - private void startExpiredFlushTimer() { - -
[35/50] [abbrv] metron git commit: METRON-1347: Indexing Topology should fail tuples without a source.type (cstella via mmiklavc) closes apache/metron#863
METRON-1347: Indexing Topology should fail tuples without a source.type (cstella via mmiklavc) closes apache/metron#863 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bfe90ef1 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bfe90ef1 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bfe90ef1 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: bfe90ef1e579be53a14d9fd0e4dc19fc6a81baf0 Parents: 53124d9 Author: cstellaAuthored: Fri Apr 13 11:17:00 2018 -0600 Committer: Michael Miklavcic Committed: Fri Apr 13 11:17:00 2018 -0600 -- .../bolt/BulkMessageWriterBoltTest.java | 25 ++ metron-platform/metron-indexing/README.md | 6 +++ .../writer/bolt/BulkMessageWriterBolt.java | 51 ++-- 3 files changed, 68 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/bfe90ef1/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java -- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java index 308638e..dedf5e6 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java @@ -118,6 +118,31 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest { private MessageGetStrategy messageGetStrategy; @Test + public void testSensorTypeMissing() throws Exception { +BulkMessageWriterBolt bulkMessageWriterBolt = new BulkMessageWriterBolt("zookeeperUrl") + .withBulkMessageWriter(bulkMessageWriter).withMessageGetter(MessageGetters.JSON_FROM_FIELD.name()) +.withMessageGetterField("message"); +bulkMessageWriterBolt.setCuratorFramework(client); +bulkMessageWriterBolt.setZKCache(cache); + bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(sensorType, +new FileInputStream(sampleSensorIndexingConfigPath)); + +bulkMessageWriterBolt.declareOutputFields(declarer); +verify(declarer, times(1)).declareStream(eq("error"), argThat( +new FieldsMatcher("message"))); +Map stormConf = new HashMap(); +bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector); +BulkWriterComponent component = mock(BulkWriterComponent.class); +bulkMessageWriterBolt.setWriterComponent(component); +verify(bulkMessageWriter, times(1)).init(eq(stormConf),any(TopologyContext.class), any(WriterConfiguration.class)); +JSONObject message = (JSONObject) new JSONParser().parse(sampleMessageString); +message.remove("source.type"); +when(tuple.getValueByField("message")).thenReturn(message); +bulkMessageWriterBolt.execute(tuple); +verify(component, times(1)).error(eq("null"), any(), any(), any()); + } + + @Test public void testFlushOnBatchSize() throws Exception { BulkMessageWriterBolt bulkMessageWriterBolt = new BulkMessageWriterBolt("zookeeperUrl") .withBulkMessageWriter(bulkMessageWriter).withMessageGetter(MessageGetters.JSON_FROM_FIELD.name()) http://git-wip-us.apache.org/repos/asf/metron/blob/bfe90ef1/metron-platform/metron-indexing/README.md -- diff --git a/metron-platform/metron-indexing/README.md b/metron-platform/metron-indexing/README.md index d351d7c..f4a4501 100644 --- a/metron-platform/metron-indexing/README.md +++ b/metron-platform/metron-indexing/README.md @@ -32,6 +32,12 @@ Indices are written in batch and the batch size and batch timeout are specified [Sensor Indexing Configuration](#sensor-indexing-configuration) via the `batchSize` and `batchTimeout` parameters. These configs are variable by sensor type. +## Minimal Assumptions for Message Structure + +At minimum, a message should have a `source.type` field. +Without this field, the message tuple will be failed and not written +with an appropriate error indicated in the Storm UI and logs. + ## Indexing Architecture ![Architecture](indexing_arch.png) http://git-wip-us.apache.org/repos/asf/metron/blob/bfe90ef1/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java -- diff --git
[26/50] [abbrv] metron git commit: METRON-1505 Intermittent Profiler Integration Test Failure (nickwallen) closes apache/metron#977
METRON-1505 Intermittent Profiler Integration Test Failure (nickwallen) closes apache/metron#977 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/46bc63db Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/46bc63db Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/46bc63db Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 46bc63dbcfe9f0ddabfd4821958962a2dac9378e Parents: ab4f8e6 Author: nickwallenAuthored: Sat Apr 7 11:28:01 2018 -0400 Committer: nickallen Committed: Sat Apr 7 11:28:01 2018 -0400 -- .../profiler/DefaultMessageDistributor.java | 54 +++- .../src/main/flux/profiler/remote.yaml | 2 + .../profiler/bolt/ProfileBuilderBolt.java | 149 +++--- .../profiler/bolt/ProfileSplitterBolt.java | 1 - .../config/zookeeper/percentiles/profiler.json | 12 - .../processing-time-test/profiler.json | 11 + .../zookeeper/readme-example-1/profiler.json| 17 -- .../zookeeper/readme-example-2/profiler.json| 18 -- .../zookeeper/readme-example-3/profiler.json| 11 - .../zookeeper/readme-example-4/profiler.json| 11 - .../profiler/bolt/ProfileBuilderBoltTest.java | 130 +++-- .../integration/ProfilerIntegrationTest.java| 274 +-- .../configuration/profiler/ProfileConfig.java | 49 ++-- .../ZKConfigurationsCacheIntegrationTest.java | 4 +- .../org/apache/metron/hbase/bolt/HBaseBolt.java | 22 +- 15 files changed, 319 insertions(+), 446 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/46bc63db/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java -- diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java index ea5126f..70f4228 100644 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java +++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java @@ -25,6 +25,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.RemovalListener; import com.google.common.cache.RemovalNotification; +import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.stellar.dsl.Context; import org.json.simple.JSONObject; @@ -33,7 +34,6 @@ import org.slf4j.LoggerFactory; import java.lang.invoke.MethodHandles; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -74,7 +74,7 @@ public class DefaultMessageDistributor implements MessageDistributor { * messages. Once it has not received messages for a period of time, it is * moved to the expired cache. */ - private transient Cache activeCache; + private transient Cache activeCache; /** * A cache of expired profiles. @@ -85,7 +85,7 @@ public class DefaultMessageDistributor implements MessageDistributor { * can flush the state of the expired profile. If the client does not flush * the expired profiles, this state will be lost forever. */ - private transient Cache expiredCache; + private transient Cache expiredCache; /** * Create a new message distributor. @@ -222,7 +222,7 @@ public class DefaultMessageDistributor implements MessageDistributor { * @param cache The cache to flush. * @return The measurements captured when flushing the profiles. */ - private List flushCache(Cache cache) { + private List flushCache(Cache cache) { List measurements = new ArrayList<>(); for(ProfileBuilder profileBuilder: cache.asMap().values()) { @@ -262,11 +262,19 @@ public class DefaultMessageDistributor implements MessageDistributor { /** * Builds the key that is used to lookup the {@link ProfileBuilder} within the cache. * + * The cache key is built using the hash codes of the profile and entity name. If the profile + * definition is ever changed, the same cache entry will not be reused. This ensures that no + * state can be carried over from the old definition into the new, which might result in an + * invalid profile measurement. + * * @param profile The profile definition.
[42/50] [abbrv] metron git commit: METRON-1502 Upgrade Doxia plugin to 1.8 (justinleet) closes apache/metron#974
METRON-1502 Upgrade Doxia plugin to 1.8 (justinleet) closes apache/metron#974 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/daf543b1 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/daf543b1 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/daf543b1 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: daf543b10423c2cfc2b740e39647ef4eb3863872 Parents: 08252f5 Author: justinleetAuthored: Tue Apr 17 13:32:23 2018 -0400 Committer: leet Committed: Tue Apr 17 13:32:23 2018 -0400 -- metron-deployment/amazon-ec2/README.md | 88 +- metron-deployment/packaging/ambari/README.md| 168 +-- .../packaging/packer-build/README.md| 2 +- metron-interface/metron-rest/README.md | 56 +++ metron-platform/metron-enrichment/README.md | 2 +- metron-sensors/pycapa/README.md | 84 +- site-book/pom.xml | 4 +- .../src-resources/templates/site.xml.template | 6 +- 8 files changed, 206 insertions(+), 204 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/daf543b1/metron-deployment/amazon-ec2/README.md -- diff --git a/metron-deployment/amazon-ec2/README.md b/metron-deployment/amazon-ec2/README.md index 73a3d70..b2efc9e 100644 --- a/metron-deployment/amazon-ec2/README.md +++ b/metron-deployment/amazon-ec2/README.md @@ -46,39 +46,39 @@ Any platform that supports these tools is suitable, but the following instructio 1. Install Homebrew by running the following command in a terminal. Refer to the [Homebrew](http://brew.sh/) home page for the latest installation instructions. - ``` - /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - ``` +``` +/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +``` 2. With Homebrew installed, run the following command in a terminal to install all of the required tools and dependencies. - ``` - brew update - brew tap caskroom/versions - brew cask install java8 vagrant virtualbox - brew install maven git node - ``` +``` +brew update +brew tap caskroom/versions +brew cask install java8 vagrant virtualbox +brew install maven git node +``` 3. Install Ansible by following the instructions [here](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip). 4. Ensure that a public SSH key is located at `~/.ssh/id_rsa.pub`. - ``` - $ cat ~/.ssh/id_rsa.pub - ssh-rsa B3NzaC1yc2EDAQABAAABAQChv5GJxPjR39UJV7VY17ivbLVlxFrH7UHwh1Jsjem4d1eYiAtde5N2y65/HRNxWbhYli9ED8k0/MRP92ejewucEbrPNq5mytPqdC4IvZ98Ln2GbqTDwvlP3T7xa/wYFOpFsOmXXql8216wSrnrS4f3XK7ze34S6/VmY+lsBYnr3dzyj8sG/mexpJgFS/w83mWJV0e/ryf4Hd7P6DZ5fO+nmTXfKNK22ga4ctcnbZ+toYcPL+ODCh8598XCKVo97XjwF5OxN3vl1p1HHguo3cHB4H1OIaqX5mUt59gFIZcAXUME89PO6NUiZDd3RTstpf125nQVkQAHu2fvW96/f037 nick@localhost - ``` +``` +$ cat ~/.ssh/id_rsa.pub +ssh-rsa B3NzaC1yc2EDAQABAAABAQChv5GJxPjR39UJV7VY17ivbLVlxFrH7UHwh1Jsjem4d1eYiAtde5N2y65/HRNxWbhYli9ED8k0/MRP92ejewucEbrPNq5mytPqdC4IvZ98Ln2GbqTDwvlP3T7xa/wYFOpFsOmXXql8216wSrnrS4f3XK7ze34S6/VmY+lsBYnr3dzyj8sG/mexpJgFS/w83mWJV0e/ryf4Hd7P6DZ5fO+nmTXfKNK22ga4ctcnbZ+toYcPL+ODCh8598XCKVo97XjwF5OxN3vl1p1HHguo3cHB4H1OIaqX5mUt59gFIZcAXUME89PO6NUiZDd3RTstpf125nQVkQAHu2fvW96/f037 nick@localhost +``` - If this file does not exist, run the following command at a terminal and accept all defaults. Only the public key, not the private key, will be uploaded to Amazon and configured on each host to enable SSH connectivity. While it is possible to create and use an alternative key those details will not be covered. +If this file does not exist, run the following command at a terminal and accept all defaults. Only the public key, not the private key, will be uploaded to Amazon and configured on each host to enable SSH connectivity. While it is possible to create and use an alternative key those details will not be covered. - ``` - ssh-keygen -t rsa - ``` +``` +ssh-keygen -t rsa +``` 5. Ensure the JAVA_HOME environment variable is set - ``` - export JAVA_HOME="/Library/Java/JavaVirtualMachines/jdk1.8.0_162.jdk/Contents/Home" - ``` +``` +export JAVA_HOME="/Library/Java/JavaVirtualMachines/jdk1.8.0_162.jdk/Contents/Home" +``` Notice: You must replace the path with the installed JDK version path @@ -103,20 +103,20 @@ Having successfully created your Amazon Web Services account, hopefully you will 1. Use the Amazon access key by exporting its values via the shell's
[11/50] [abbrv] metron git commit: METRON-590 Enable Use of Event Time in Profiler (nickwallen) closes apache/metron#965
http://git-wip-us.apache.org/repos/asf/metron/blob/3083b471/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignalTest.java -- diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignalTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignalTest.java new file mode 100644 index 000..b8949c5 --- /dev/null +++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignalTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.metron.profiler.bolt; + +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests the {@code FixedFrequencyFlushSignal} class. + */ +public class FixedFrequencyFlushSignalTest { + + @Test + public void testSignalFlush() { + +FixedFrequencyFlushSignal signal = new FixedFrequencyFlushSignal(1000); + +// not time to flush yet +assertFalse(signal.isTimeToFlush()); + +// advance time +signal.update(5000); + +// not time to flush yet +assertFalse(signal.isTimeToFlush()); + +// advance time +signal.update(7000); + +// time to flush +assertTrue(signal.isTimeToFlush()); + } + + @Test + public void testOutOfOrderTimestamps() { +FixedFrequencyFlushSignal signal = new FixedFrequencyFlushSignal(1000); + +// advance time, out-of-order +signal.update(5000); +signal.update(1000); +signal.update(7000); +signal.update(3000); + +// need to flush @ 5000 + 1000 = 6000. if anything > 6000 (even out-of-order), then it should signal a flush +assertTrue(signal.isTimeToFlush()); + } + + @Test(expected = IllegalArgumentException.class) + public void testNegativeFrequency() { +new FixedFrequencyFlushSignal(-1000); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/3083b471/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/KafkaDestinationHandlerTest.java -- diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/KafkaDestinationHandlerTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/KafkaDestinationHandlerTest.java deleted file mode 100644 index c3f2584..000 --- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/KafkaDestinationHandlerTest.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.metron.profiler.bolt; - -import com.google.common.collect.ImmutableMap; -import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.common.configuration.profiler.ProfileConfig; -import org.apache.metron.common.utils.JSONUtils; -import org.apache.metron.profiler.ProfileMeasurement; -import org.apache.metron.statistics.OnlineStatisticsProvider; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.tuple.Values; -import org.json.simple.JSONObject; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; - -import
[21/50] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json deleted file mode 100644 index 037f1c6..000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json +++ /dev/null @@ -1,88 +0,0 @@ -{ "create" : { "_id": "all-metron-index", "_type": "index-pattern" } } -{"title":"*_index_*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\" :true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\", \"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false, \"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scri
[25/50] [abbrv] metron git commit: METRON-1449 Set Zookeeper URL for Stellar Running in Zeppelin Notebook (nickwallen) closes apache/metron#931
METRON-1449 Set Zookeeper URL for Stellar Running in Zeppelin Notebook (nickwallen) closes apache/metron#931 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ab4f8e65 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ab4f8e65 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ab4f8e65 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: ab4f8e65e13aaea3a10491290a7411b3b6dc955e Parents: 0ab39a3 Author: nickwallenAuthored: Fri Apr 6 16:40:01 2018 -0400 Committer: nickallen Committed: Fri Apr 6 16:40:01 2018 -0400 -- metron-platform/metron-management/pom.xml | 1 - .../shell/DefaultStellarShellExecutor.java | 32 +++--- .../shell/DefaultStellarShellExecutorTest.java | 11 ++ metron-stellar/stellar-zeppelin/README.md | 80 -- metron-stellar/stellar-zeppelin/pom.xml | 12 +++ .../stellar/zeppelin/StellarInterpreter.java| 95 ++--- .../zeppelin/StellarInterpreterProperty.java| 79 ++ .../StellarInterpreterPropertyTest.java | 62 +++ .../zeppelin/StellarInterpreterTest.java| 60 ++- .../integration/ConfigUploadComponent.java | 82 +++ .../StellarInterpreterIntegrationTest.java | 104 +++ 11 files changed, 501 insertions(+), 117 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-platform/metron-management/pom.xml -- diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml index c185662..962fd46 100644 --- a/metron-platform/metron-management/pom.xml +++ b/metron-platform/metron-management/pom.xml @@ -183,7 +183,6 @@ - http://git-wip-us.apache.org/repos/asf/metron/blob/ab4f8e65/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java -- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java index 3f2c495..781a0cf 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java @@ -52,6 +52,7 @@ import java.io.ByteArrayInputStream; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -271,10 +272,7 @@ public class DefaultStellarShellExecutor implements StellarShellExecutor { globals = (Map ) capability.get(); } else { - // if it does not exist, create it. this creates the global config for the current stellar executor - // session only. this does not change the global config maintained externally in zookeeper - globals = new HashMap<>(); - getContext().addCapability(GLOBAL_CONFIG, () -> globals); + throw new IllegalStateException("'GLOBAL_CONFIG' is missing"); } return globals; @@ -344,21 +342,25 @@ public class DefaultStellarShellExecutor implements StellarShellExecutor { * @param zkClient An optional Zookeeper client. */ private Context createContext(Properties properties, Optional zkClient) throws Exception { +Context.Builder contextBuilder = new Context.Builder(); +Map globals; +if (zkClient.isPresent()) { -Context.Builder contextBuilder = new Context.Builder() -.with(SHELL_VARIABLES, () -> variables) -.with(STELLAR_CONFIG, () -> properties); + // fetch globals from zookeeper + globals = fetchGlobalConfig(zkClient.get()); + contextBuilder.with(ZOOKEEPER_CLIENT, () -> zkClient.get()); -// load global configuration from zookeeper -if (zkClient.isPresent()) { - Map global = fetchGlobalConfig(zkClient.get()); - contextBuilder - .with(GLOBAL_CONFIG, () -> global) - .with(ZOOKEEPER_CLIENT, () -> zkClient.get()) - .with(STELLAR_CONFIG, () -> getStellarConfig(global, properties)); +} else { + + // use empty globals to allow a user to '%define' their own + globals = new HashMap<>(); } -return contextBuilder.build(); +return contextBuilder +.with(SHELL_VARIABLES, () -> variables) +
[18/50] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py new file mode 100755 index 000..f0903ac --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py @@ -0,0 +1,95 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from elasticsearch import Elasticsearch +from elasticsearch.helpers import bulk +import cPickle as pickle +import argparse, sys, os.path +import errno +import os + + +class DashboardIndex(object): + +def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs): +""" +:arg host: hostname of the node (default: localhost) +:arg port: port to use (integer, default: 9200) +:arg url_prefix: optional url prefix for elasticsearch +:arg timeout: default timeout in seconds (float, default: 10) +""" +self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}]) + +def get(self): +""" +Get .kibana index from Elasticsearch +""" +dotkibana = self.es.search(index='.kibana', size = 100) +return dotkibana['hits']['hits'] + +def load(self,filespec): +""" +Save Index data on local filesystem +:args filespec: path/filename for saved file +""" +data=[] +with open(filespec,'rb') as fp: +data = pickle.load(fp) +return data + +def save(self,filename,data): +""" +Save Index data on local filesystem +:args filespec: path/filename for saved file +""" +with open(filename,'wb') as fp: +pickle.dump(data,fp) + +def put(self,data): +""" +Bulk write data to Elasticsearch +:args data: data to be written (note: index name is specified in data) +""" +bulk(self.es,data) + +def main(self,args): + +if args.save: +print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec)) +self.save(filename=args.filespec,data=di.get()) +else: +""" +Loads Kibana Dashboard definition from disk and replaces .kibana on index +:args filespec: path/filename for saved file +""" +if not os.path.isfile(args.filespec): +raise IOError( +errno.ENOENT, os.strerror(errno.ENOENT), args.filespec) +self.es.indices.delete(index='.kibana', ignore=[400, 404]) +self.put(data=di.load(filespec=args.filespec)) + +if __name__ == '__main__': + +parser = argparse.ArgumentParser() +parser.add_argument("hostname", help="ES Hostname or IP", type=str) +parser.add_argument("port", help="ES Port", type=int) +parser.add_argument("filespec", help="file to be pushed from or saved to", type=str) +parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true") +args = parser.parse_args() +di = DashboardIndex(host=args.hostname,port=args.port) +di.main(args) http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
[33/50] [abbrv] metron git commit: METRON-1521: JSONMapParser is no longer serializable closes apache/incubator-metron#991
METRON-1521: JSONMapParser is no longer serializable closes apache/incubator-metron#991 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/eb5b2d42 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/eb5b2d42 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/eb5b2d42 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: eb5b2d42b83901c05c15e00b9c3c5bce5dd72ab9 Parents: ea6992f Author: cstellaAuthored: Thu Apr 12 09:33:12 2018 -0400 Committer: cstella Committed: Thu Apr 12 09:33:12 2018 -0400 -- .../java/org/apache/metron/parsers/json/JSONMapParser.java | 4 ++-- .../org/apache/metron/parsers/integration/ParserDriver.java| 6 +- 2 files changed, 7 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/eb5b2d42/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java -- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java index bddf35d..f5d67f9 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java @@ -89,8 +89,7 @@ public class JSONMapParser extends BasicParser { public static final String JSONP_QUERY = "jsonpQuery"; private MapStrategy mapStrategy = MapStrategy.DROP; - private TypeRef >> typeRef = new TypeRef
>>() { - }; + private transient TypeRef
>> typeRef = null; private String jsonpQuery = null; @@ -99,6 +98,7 @@ public class JSONMapParser extends BasicParser { String strategyStr = (String) config.getOrDefault(MAP_STRATEGY_CONFIG, MapStrategy.DROP.name()); mapStrategy = MapStrategy.valueOf(strategyStr); if (config.containsKey(JSONP_QUERY)) { + typeRef = new TypeRef
>>() { }; jsonpQuery = (String) config.get(JSONP_QUERY); Configuration.setDefaults(new Configuration.Defaults() { http://git-wip-us.apache.org/repos/asf/metron/blob/eb5b2d42/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java -- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java index deb0217..b03ea80 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java @@ -18,6 +18,7 @@ package org.apache.metron.parsers.integration; import com.google.common.collect.ImmutableList; +import org.apache.commons.lang.SerializationUtils; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.configuration.FieldValidator; import org.apache.metron.common.configuration.ParserConfigurations; @@ -42,6 +43,7 @@ import org.mockito.Matchers; import java.io.Closeable; import java.io.IOException; +import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -54,7 +56,7 @@ import static org.mockito.Mockito.when; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ParserDriver { +public class ParserDriver implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(ParserBolt.class); public static class CollectingWriter implements MessageWriter{ List
output; @@ -151,6 +153,8 @@ public class ParserDriver { public ProcessorResult > run(List
in) { ShimParserBolt bolt = new ShimParserBolt(new ArrayList<>()); +byte[] b = SerializationUtils.serialize(bolt); +ShimParserBolt b2 = (ShimParserBolt) SerializationUtils.deserialize(b); OutputCollector collector = mock(OutputCollector.class); bolt.prepare(null, null, collector); for(byte[] record : in) {
[28/50] [abbrv] metron git commit: METRON-1465:Support for Elasticsearch X-pack (wardbekker via mmiklavc) closes apache/metron#946
METRON-1465:Support for Elasticsearch X-pack (wardbekker via mmiklavc) closes apache/metron#946 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a8b555dc Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a8b555dc Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a8b555dc Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: a8b555dcc9f548d7b91789a46d9435b4d8b17581 Parents: 3ba9ae2 Author: wardbekkerAuthored: Mon Apr 9 13:14:13 2018 -0600 Committer: Michael Miklavcic Committed: Mon Apr 9 13:14:13 2018 -0600 -- metron-deployment/Kerberos-manual-setup.md | 209 +++ .../roles/metron-builder/tasks/build-debs.yml | 2 +- .../roles/metron-builder/tasks/build-rpms.yml | 2 +- .../METRON/CURRENT/configuration/metron-env.xml | 2 - .../metron-rest/src/main/scripts/metron-rest.sh | 9 + .../src/main/config/zookeeper/global.json | 5 +- .../apache/metron/common/utils/HDFSUtils.java | 59 ++ .../metron/common/utils/ReflectionUtils.java| 66 +- .../elasticsearch/dao/ElasticsearchDao.java | 33 ++- .../elasticsearch/utils/ElasticsearchUtils.java | 107 -- .../writer/ElasticsearchWriter.java | 8 +- .../scripts/start_elasticsearch_topology.sh | 8 +- .../writer/ElasticsearchWriterTest.java | 19 +- .../stellar/common/utils/ConversionUtils.java | 19 +- 14 files changed, 486 insertions(+), 62 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/a8b555dc/metron-deployment/Kerberos-manual-setup.md -- diff --git a/metron-deployment/Kerberos-manual-setup.md b/metron-deployment/Kerberos-manual-setup.md index 47a63d8..456703a 100644 --- a/metron-deployment/Kerberos-manual-setup.md +++ b/metron-deployment/Kerberos-manual-setup.md @@ -30,6 +30,7 @@ This document provides instructions for kerberizing Metron's Vagrant-based devel * [Start Metron](#start-metron) * [Push Data](#push-data) * [More Information](#more-information) +* [Elasticseach X-Pack](#x-pack) Setup - @@ -533,3 +534,211 @@ In order to correct this, you should: ### References * [https://github.com/apache/storm/blob/master/SECURITY.md](https://github.com/apache/storm/blob/master/SECURITY.md) + +X-Pack +-- + +First, stop the random_access_indexing topology through the Storm UI or from the CLI, e.g. + +``` +storm kill random_access_indexing +``` + +Here are instructions for enabling X-Pack with Elasticsearch and Kibana: https://www.elastic.co/guide/en/x-pack/5.6/installing-xpack.html + +You need to be sure to add the appropriate username and password for Elasticsearch and Kibana to enable external connections from Metron components. e.g. the following will create a user "transport_client_user" with password "changeme" and "superuser" credentials. + +``` +sudo /usr/share/elasticsearch/bin/x-pack/users useradd transport_client_user -p changeme -r superuser +``` + +Once you've picked a password to connect to ES, you need to upload a 1-line file to HDFS with that password in it. Metron will use this file to securely read the password in order to connect to ES securely. + +Here is an example using "changeme" as the password + +``` +echo changeme > /tmp/xpack-password +sudo -u hdfs hdfs dfs -mkdir /apps/metron/elasticsearch/ +sudo -u hdfs hdfs dfs -put /tmp/xpack-password /apps/metron/elasticsearch/ +sudo -u hdfs hdfs dfs -chown metron:metron /apps/metron/elasticsearch/xpack-password +``` + +New settings have been added to configure the Elasticsearch client. By default the client will run as the normal ES prebuilt transport client. If you enable X-Pack you should set the es.client.class as shown below. + +Add the es settings to global.json + +``` +/usr/metron/0.4.3/config/zookeeper/global.json -> + + "es.client.settings" : { + "es.client.class" : "org.elasticsearch.xpack.client.PreBuiltXPackTransportClient", + "es.xpack.username" : "transport_client_user", + "es.xpack.password.file" : "/apps/metron/elasticsearch/xpack-password" + } +``` + +Submit the update to Zookeeper + +``` +$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i METRON_HOME/config/zookeeper/ -z $ZOOKEEPER +``` + +The last step before restarting the topology is to create a custom X-Pack shaded and relocated jar. This is up to you because of licensing restrictions, but here is a sample Maven pom file that should help. + +``` + + +http://maven.apache.org/POM/4.0.0; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> +4.0.0 +org.elasticsearch +
[37/50] [abbrv] metron git commit: METRON-1522 Fix the typo errors at profile debugger readme (MohanDV via nickwallen) closes apache/metron#992
METRON-1522 Fix the typo errors at profile debugger readme (MohanDV via nickwallen) closes apache/metron#992 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1d3e7fcd Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1d3e7fcd Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1d3e7fcd Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 1d3e7fcd7db64e7cb6d986802240fcbae174ae91 Parents: f8b7c58 Author: MohanDVAuthored: Fri Apr 13 16:50:14 2018 -0400 Committer: nickallen Committed: Fri Apr 13 16:50:14 2018 -0400 -- metron-analytics/metron-profiler/README.md | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/1d3e7fcd/metron-analytics/metron-profiler/README.md -- diff --git a/metron-analytics/metron-profiler/README.md b/metron-analytics/metron-profiler/README.md index 218ec66..01918aa 100644 --- a/metron-analytics/metron-profiler/README.md +++ b/metron-analytics/metron-profiler/README.md @@ -152,8 +152,8 @@ Creating and refining profiles is an iterative process. Iterating against a liv * If a message is needed by two profiles, then two routes have been followed. ``` - [Stellar]>>> p := PROFILER_INIT(conf) - [Stellar]>>> p + [Stellar]>>> profiler := PROFILER_INIT(conf) + [Stellar]>>> profiler Profiler{1 profile(s), 0 messages(s), 0 route(s)} ``` @@ -172,11 +172,11 @@ Creating and refining profiles is an iterative process. Iterating against a liv 1. Apply the message to your Profiler, as many times as you like. ``` - [Stellar]>>> PROFILER_APPLY(msg, p) + [Stellar]>>> PROFILER_APPLY(msg, profiler) Profiler{1 profile(s), 1 messages(s), 1 route(s)} ``` ``` - [Stellar]>>> PROFILER_APPLY(msg, p) + [Stellar]>>> PROFILER_APPLY(msg, profiler) Profiler{1 profile(s), 2 messages(s), 2 route(s)} ``` @@ -205,7 +205,7 @@ Creating and refining profiles is an iterative process. Iterating against a liv ``` Apply those 10 messages to your profile(s). ``` - [Stellar]>>> PROFILER_APPLY(msgs, p) + [Stellar]>>> PROFILER_APPLY(msgs, profiler) Profiler{1 profile(s), 10 messages(s), 10 route(s)} ```
[32/50] [abbrv] metron git commit: METRON-1516 Support for Ansible 2.5.0 (ottobackwards) closes apache/metron#989
METRON-1516 Support for Ansible 2.5.0 (ottobackwards) closes apache/metron#989 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ea6992fd Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ea6992fd Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ea6992fd Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: ea6992fd943c552c41565c7a320804cb58b733bd Parents: 62d1a1b Author: ottobackwardsAuthored: Thu Apr 12 06:15:38 2018 -0400 Committer: otto Committed: Thu Apr 12 06:15:38 2018 -0400 -- metron-deployment/amazon-ec2/README.md | 2 +- metron-deployment/amazon-ec2/playbook.yml | 4 ++-- metron-deployment/ansible/playbooks/metron_full_install.yml | 4 ++-- metron-deployment/development/centos6/README.md | 2 +- metron-deployment/development/ubuntu14/README.md| 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/ea6992fd/metron-deployment/amazon-ec2/README.md -- diff --git a/metron-deployment/amazon-ec2/README.md b/metron-deployment/amazon-ec2/README.md index bc259ec..73a3d70 100644 --- a/metron-deployment/amazon-ec2/README.md +++ b/metron-deployment/amazon-ec2/README.md @@ -38,7 +38,7 @@ Getting Started The host used to deploy Apache Metron will need the following software tools installed. The following versions are known to work as of the time of this writing, but by no means are these the only working versions. - - Ansible 2.0.0.2 or 2.2.2.0 + - Ansible 2.0.0.2, 2.2.2.0, or 2.5.0 - Python 2.7.11 - Maven 3.3.9 http://git-wip-us.apache.org/repos/asf/metron/blob/ea6992fd/metron-deployment/amazon-ec2/playbook.yml -- diff --git a/metron-deployment/amazon-ec2/playbook.yml b/metron-deployment/amazon-ec2/playbook.yml index de64490..470a181 100644 --- a/metron-deployment/amazon-ec2/playbook.yml +++ b/metron-deployment/amazon-ec2/playbook.yml @@ -23,8 +23,8 @@ - conf/defaults.yml pre_tasks: - name: Verify Ansible Version - fail: msg="Metron Requires Ansible 2.0.0.2 or 2.2.2.0, current version is {{ ansible_version }}" - when: "ansible_version.full | version_compare('2.2.2.0', '!=') and ansible_version.full | version_compare('2.0.0.2', '!=')" + fail: msg="Metron Requires Ansible 2.0.0.2, 2.2.2.0 or 2.5.0, current version is {{ ansible_version }}" + when: "ansible_version.full | version_compare('2.2.2.0', '!=') and ansible_version.full | version_compare('2.0.0.2', '!=') and ansible_version.full | version_compare('2.5.0', '!=')" tasks: - include: tasks/create-keypair.yml - include: tasks/create-vpc.yml http://git-wip-us.apache.org/repos/asf/metron/blob/ea6992fd/metron-deployment/ansible/playbooks/metron_full_install.yml -- diff --git a/metron-deployment/ansible/playbooks/metron_full_install.yml b/metron-deployment/ansible/playbooks/metron_full_install.yml index b517671..099d810 100644 --- a/metron-deployment/ansible/playbooks/metron_full_install.yml +++ b/metron-deployment/ansible/playbooks/metron_full_install.yml @@ -18,8 +18,8 @@ - hosts: all pre_tasks: - name: Verify Ansible Version - fail: msg="Metron Requires Ansible 2.0.0.2 or 2.2.2.0, current version is {{ ansible_version }}" - when: "ansible_version.full | version_compare('2.2.2.0', '!=') and ansible_version.full | version_compare('2.0.0.2', '!=')" + fail: msg="Metron Requires Ansible 2.0.0.2, 2.2.2.0, or 2.5.0, current version is {{ ansible_version }}" + when: "ansible_version.full | version_compare('2.2.2.0', '!=') and ansible_version.full | version_compare('2.0.0.2', '!=') and ansible_version.full | version_compare('2.5.0', '!=')" - include: metron_build.yml tags: http://git-wip-us.apache.org/repos/asf/metron/blob/ea6992fd/metron-deployment/development/centos6/README.md -- diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md index bd8553c..ec85be3 100644 --- a/metron-deployment/development/centos6/README.md +++ b/metron-deployment/development/centos6/README.md @@ -29,7 +29,7 @@ Getting Started The computer used to deploy Apache Metron will need to have the following components installed. - - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0) + - [Ansible](https://github.com/ansible/ansible) (2.0.0.2, 2.2.2.0, or 2.5.0) - [Docker](https://www.docker.com/community-edition) -
[19/50] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json new file mode 100644 index 000..037f1c6 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboard-bulkload.json @@ -0,0 +1,88 @@ +{ "create" : { "_id": "all-metron-index", "_type": "index-pattern" } } +{"title":"*_index_*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\" :true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\", \"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false, \"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scri
[14/50] [abbrv] metron git commit: METRON-1491: The indexing topology restart logic is wrong (cstella via mmiklavc) closes apache/metron#964
METRON-1491: The indexing topology restart logic is wrong (cstella via mmiklavc) closes apache/metron#964 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/5ed9631a Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/5ed9631a Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/5ed9631a Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 5ed9631a2936ec60d0ea6557ca4396cffdadc688 Parents: 3083b47 Author: cstellaAuthored: Tue Mar 20 16:08:02 2018 -0600 Committer: Michael Miklavcic Committed: Tue Mar 20 16:08:02 2018 -0600 -- .../package/scripts/indexing_commands.py| 43 1 file changed, 35 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/5ed9631a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index 4c862f0..fd78119 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -181,7 +181,7 @@ class IndexingCommands: def start_batch_indexing_topology(self, env): Logger.info('Starting ' + self.__batch_indexing_topology) -if not self.is_topology_active(env): +if not self.is_batch_topology_active(env): if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, @@ -200,7 +200,7 @@ class IndexingCommands: def start_random_access_indexing_topology(self, env): Logger.info('Starting ' + self.__random_access_indexing_topology) -if not self.is_topology_active(env): +if not self.is_random_access_topology_active(env): if self.__params.security_enabled: metron_security.kinit(self.__params.kinit_path_local, self.__params.metron_keytab_path, @@ -263,21 +263,48 @@ class IndexingCommands: def restart_indexing_topology(self, env): Logger.info('Restarting the indexing topologies') -self.stop_indexing_topology(env) +self.restart_batch_indexing_topology(env) +self.restart_random_access_indexing_topology(env) + +def restart_batch_indexing_topology(self, env): +Logger.info('Restarting the batch indexing topology') +self.stop_batch_indexing_topology(env) + +# Wait for old topology to be cleaned up by Storm, before starting again. +retries = 0 +topology_active = self.is_batch_topology_active(env) +while topology_active and retries < 3: +Logger.info('Existing batch topology still active. Will wait and retry') +time.sleep(10) +retries += 1 +topology_active = self.is_batch_topology_active(env) + +if not topology_active: +Logger.info('Waiting for storm kill to complete') +time.sleep(30) +self.start_batch_indexing_topology(env) +Logger.info('Done restarting the batch indexing topology') +else: +Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') + +def restart_random_access_indexing_topology(self, env): +Logger.info('Restarting the random access indexing topology') +self.stop_random_access_indexing_topology(env) # Wait for old topology to be cleaned up by Storm, before starting again. retries = 0 -topology_active = self.is_topology_active(env) -while self.is_topology_active(env) and retries < 3: -Logger.info('Existing topology still active. Will wait and retry') +topology_active = self.is_random_access_topology_active(env) +while topology_active and retries < 3: +Logger.info('Existing random access topology still active. Will wait and retry') time.sleep(10) retries += 1 +topology_active = self.is_random_access_topology_active(env) if not topology_active: Logger.info('Waiting for storm kill to complete') time.sleep(30) -
[24/50] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0ab39a32 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0ab39a32 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0ab39a32 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 0ab39a32b61104812a6a9e69e999e7053e3e7e90 Parents: 0d847cf Author: mmiklavcAuthored: Thu Apr 5 09:07:48 2018 -0600 Committer: Michael Miklavcic Committed: Thu Apr 5 09:07:48 2018 -0600 -- .../roles/ambari_master/defaults/main.yml | 2 + .../ambari_master/tasks/elasticsearch_mpack.yml | 26 ++ .../ansible/roles/ambari_master/tasks/main.yml | 3 +- .../roles/ambari_master/tasks/metron_mpack.yml | 26 ++ .../ansible/roles/ambari_master/tasks/mpack.yml | 26 -- .../roles/load_web_templates/tasks/main.yml | 2 +- .../manual-install/Manual_Install_CentOS6.md| 4 +- metron-deployment/packaging/ambari/README.md| 25 +- .../ambari/elasticsearch-mpack/README.md| 62 + .../ambari/elasticsearch-mpack/pom.xml | 95 +++ .../src/main/assemblies/elasticsearch-mpack.xml | 43 +++ .../ELASTICSEARCH/5.6.2/metainfo.xml| 29 ++ .../ELASTICSEARCH/5.6.2/repos/repoinfo.xml | 45 .../addon-services/KIBANA/5.6.2/metainfo.xml| 30 +++ .../KIBANA/5.6.2/quicklinks/quicklinks.json | 27 ++ .../KIBANA/5.6.2/repos/repoinfo.xml | 60 + .../5.6.2/configuration/elastic-env.xml | 86 ++ .../5.6.2/configuration/elastic-jvm-options.xml | 144 ++ .../5.6.2/configuration/elastic-site.xml| 198 ++ .../5.6.2/configuration/elastic-sysconfig.xml | 97 +++ .../5.6.2/configuration/elastic-systemd.xml | 30 +++ .../ELASTICSEARCH/5.6.2/metainfo.xml| 97 +++ .../5.6.2/package/scripts/elastic_commands.py | 266 +++ .../5.6.2/package/scripts/elastic_master.py | 72 + .../5.6.2/package/scripts/elastic_slave.py | 71 + .../5.6.2/package/scripts/params.py | 108 .../5.6.2/package/scripts/properties_config.py | 34 +++ .../5.6.2/package/scripts/service_check.py | 114 .../5.6.2/package/scripts/status_params.py | 27 ++ .../templates/elasticsearch.master.yaml.j2 | 77 ++ .../templates/elasticsearch.slave.yaml.j2 | 78 ++ .../templates/elasticsearch_limits.conf.j2 | 20 ++ .../5.6.2/quicklinks/quicklinks.json| 43 +++ .../ELASTICSEARCH/5.6.2/role_command_order.json | 8 + .../KIBANA/5.6.2/configuration/kibana-env.xml | 72 + .../KIBANA/5.6.2/configuration/kibana-site.xml | 113 .../common-services/KIBANA/5.6.2/metainfo.xml | 84 ++ .../KIBANA/5.6.2/package/scripts/common.py | 56 .../5.6.2/package/scripts/kibana_master.py | 81 ++ .../KIBANA/5.6.2/package/scripts/params.py | 50 .../KIBANA/5.6.2/quicklinks/quicklinks.json | 28 ++ .../src/main/resources/mpack.json | 76 ++ .../packaging/ambari/metron-mpack/README.md | 20 +- .../src/main/assemblies/metron-mpack.xml| 14 - .../ELASTICSEARCH/5.6.2/metainfo.xml| 29 -- .../ELASTICSEARCH/5.6.2/repos/repoinfo.xml | 45 .../addon-services/KIBANA/5.6.2/metainfo.xml| 30 --- .../KIBANA/5.6.2/quicklinks/quicklinks.json | 27 -- .../KIBANA/5.6.2/repos/repoinfo.xml | 60 - .../5.6.2/configuration/elastic-env.xml | 86 -- .../5.6.2/configuration/elastic-jvm-options.xml | 144 -- .../5.6.2/configuration/elastic-site.xml| 198 -- .../5.6.2/configuration/elastic-sysconfig.xml | 97 --- .../5.6.2/configuration/elastic-systemd.xml | 30 --- .../ELASTICSEARCH/5.6.2/metainfo.xml| 97 --- .../5.6.2/package/scripts/elastic_commands.py | 266 --- .../5.6.2/package/scripts/elastic_master.py | 72 - .../5.6.2/package/scripts/elastic_slave.py | 71 - .../5.6.2/package/scripts/params.py | 108 .../5.6.2/package/scripts/properties_config.py | 34 --- .../5.6.2/package/scripts/service_check.py | 114 .../5.6.2/package/scripts/status_params.py | 27 -- .../templates/elasticsearch.master.yaml.j2 | 77 -- .../templates/elasticsearch.slave.yaml.j2 | 78 -- .../templates/elasticsearch_limits.conf.j2 | 20 -- .../5.6.2/quicklinks/quicklinks.json| 43 --- .../ELASTICSEARCH/5.6.2/role_command_order.json | 8 - .../KIBANA/5.6.2/configuration/kibana-env.xml | 72 - .../KIBANA/5.6.2/configuration/kibana-site.xml | 113
[02/50] [abbrv] metron git commit: METRON-1488: user_settings hbase table does not have acls set up for kerberos closes apache/metron#962
METRON-1488: user_settings hbase table does not have acls set up for kerberos closes apache/metron#962 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/26c5d306 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/26c5d306 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/26c5d306 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 26c5d3065bd7a635699083e8131d4a7629bf3b4d Parents: 37662d3 Author: cstellaAuthored: Thu Mar 15 12:06:56 2018 -0400 Committer: cstella Committed: Thu Mar 15 12:06:56 2018 -0400 -- .../common-services/METRON/CURRENT/package/scripts/rest_master.py | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/metron/blob/26c5d306/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py index 36a68c6..2f2d3f9 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py @@ -51,6 +51,8 @@ class RestMaster(Script): commands.init_kafka_topics() if not commands.is_hbase_configured(): commands.create_hbase_tables() +if params.security_enabled and not commands.is_hbase_acl_configured(): +commands.set_hbase_acls() if params.security_enabled and not commands.is_kafka_acl_configured(): commands.init_kafka_acls() commands.set_kafka_acl_configured()
[08/50] [abbrv] metron git commit: METRON-1483: Create a tool to monitor performance of the topologies closes apache/incubator-metron#958
http://git-wip-us.apache.org/repos/asf/metron/blob/46ad9d93/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java -- diff --git a/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java new file mode 100644 index 000..5d5c240 --- /dev/null +++ b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.performance.sampler; + +import java.util.Random; + +public class UnbiasedSampler implements Sampler { + + @Override + public int sample(Random rng, int limit) { +return rng.nextInt(limit); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/46ad9d93/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java -- diff --git a/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java new file mode 100644 index 000..c13f236 --- /dev/null +++ b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.performance.util; + +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public enum KafkaUtil { + INSTANCE; + + public List getTopicPartition(KafkaConsumerconsumer, String topic) { + +List partitions = consumer.partitionsFor(topic); +List ret = new ArrayList<>(partitions.size()); +for(PartitionInfo par : partitions) { + ret.add(new TopicPartition(topic, par.partition())); +} +return ret; + } + + public Map getKafkaOffsetMap(KafkaConsumer consumer, String topic ) { +Map ret = new HashMap<>(); +if(!consumer.subscription().contains(topic)) { + consumer.subscribe(Collections.singletonList(topic)); +} +consumer.poll(0); +List partitions = getTopicPartition(consumer, topic); +consumer.seekToEnd(partitions); +for(TopicPartition par : partitions) { + ret.put(par.partition(), consumer.position(par)-1); +} +return ret; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/46ad9d93/metron-contrib/metron-performance/src/main/scripts/load_tool.sh -- diff --git a/metron-contrib/metron-performance/src/main/scripts/load_tool.sh b/metron-contrib/metron-performance/src/main/scripts/load_tool.sh new file mode 100755 index 000..1b56a6e --- /dev/null +++ b/metron-contrib/metron-performance/src/main/scripts/load_tool.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license
[44/50] [abbrv] metron git commit: METRON-1528: Fix missing file in metron.spec (mmiklavc via mmiklavc) closes apache/metron#996
METRON-1528: Fix missing file in metron.spec (mmiklavc via mmiklavc) closes apache/metron#996 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a41611b1 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a41611b1 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a41611b1 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: a41611b1ae2bb17fa9333ef6f965749652e95538 Parents: e094914 Author: mmiklavcAuthored: Wed Apr 18 07:35:01 2018 -0600 Committer: Michael Miklavcic Committed: Wed Apr 18 07:35:01 2018 -0600 -- metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/metron/blob/a41611b1/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec -- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 1f40105..15469d9 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -123,6 +123,7 @@ This package installs the Metron common files %{metron_home} %dir %{metron_home}/lib %{metron_home}/bin/zk_load_configs.sh %{metron_home}/bin/stellar +%{metron_home}/bin/cluster_info.py %{metron_home}/config/zookeeper/global.json %attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}.jar
[06/50] [abbrv] metron git commit: METRON-1493 Unhelpful Error Message When Assignment Expressions Fail (nickwallen) closes apache/metron#966
METRON-1493 Unhelpful Error Message When Assignment Expressions Fail (nickwallen) closes apache/metron#966 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/52dd9fb8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/52dd9fb8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/52dd9fb8 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 52dd9fb852bef5998dac83109ac6e122860be489 Parents: 9c5d9d7 Author: nickwallenAuthored: Fri Mar 16 10:16:07 2018 -0400 Committer: nickallen Committed: Fri Mar 16 10:16:07 2018 -0400 -- .../common/shell/specials/AssignmentCommand.java | 2 +- .../common/shell/specials/AssignmentCommandTest.java | 14 ++ 2 files changed, 15 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/52dd9fb8/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java -- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java index e253b3b..664e01e 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java @@ -79,7 +79,7 @@ public class AssignmentCommand implements SpecialCommand { return result; } else { - return error("Assignment expression failed"); + return result; } } } http://git-wip-us.apache.org/repos/asf/metron/blob/52dd9fb8/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java -- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java index 899effb..1b5c9d5 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java @@ -149,6 +149,20 @@ public class AssignmentCommandTest { assertFalse(executor.getState().containsKey("x")); } + /** + * If an assignment expression fails, the error message should explain + * why the expression fails. + */ + @Test + public void testErrorMessageWhenAssignmentFails() { +StellarResult result = command.execute("x := 0/0", executor); + +// validate the result +assertTrue(result.isError()); +assertTrue(result.getException().isPresent()); +assertEquals(ArithmeticException.class, result.getException().get().getClass()); + } + @Test public void testAssignNull() { StellarResult result = command.execute("x := NULL", executor);
[49/50] [abbrv] metron git commit: Merge remote-tracking branch 'origin/master' into feature/METRON-1416-upgrade-solr
http://git-wip-us.apache.org/repos/asf/metron/blob/d0a4e4c0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java -- diff --cc metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java index 762a272,000..9f2414a mode 100644,00..100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java @@@ -1,478 -1,0 +1,478 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.solr.dao; + +import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.search.Group; +import org.apache.metron.indexing.dao.search.GroupOrder; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; +import org.apache.metron.indexing.dao.search.GroupResult; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.search.SortField; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.solr.matcher.ModifiableSolrParamsMatcher; +import org.apache.metron.solr.matcher.SolrQueryMatcher; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.FieldStatsInfo; +import org.apache.solr.client.solrj.response.PivotField; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.IsCollectionContaining.hasItems; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mockStatic; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({CollectionAdminRequest.class}) +public class SolrSearchDaoTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private SolrClient client; + private AccessConfig accessConfig; + private SolrSearchDao solrSearchDao; + + @SuppressWarnings("unchecked") + @Before + public void setUp() throws Exception { +client = mock(SolrClient.class); +accessConfig = mock(AccessConfig.class); +solrSearchDao = new SolrSearchDao(client, accessConfig); +mockStatic(CollectionAdminRequest.class); + when(CollectionAdminRequest.listCollections(client)).thenReturn(Arrays.asList("bro", "snort")); + } + + @Test + public void searchShouldProperlyReturnSearchResponse() throws Exception
[36/50] [abbrv] metron git commit: METRON-1519 Indexing Error Topic Property Not Displayed in MPack (nickwallen) closes apache/metron#987
METRON-1519 Indexing Error Topic Property Not Displayed in MPack (nickwallen) closes apache/metron#987 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f8b7c585 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f8b7c585 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f8b7c585 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: f8b7c5852310a088bd15ca5d21ba4f98b51521be Parents: bfe90ef Author: nickwallenAuthored: Fri Apr 13 14:47:17 2018 -0400 Committer: nickallen Committed: Fri Apr 13 14:47:17 2018 -0400 -- .../common-services/METRON/CURRENT/themes/metron_theme.json| 6 -- 1 file changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/f8b7c585/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 234b551..364b3ef 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -514,6 +514,10 @@ "subsection-name": "subsection-indexing-kafka" }, { + "config": "metron-indexing-env/indexing_error_topic", + "subsection-name": "subsection-indexing-kafka" +}, +{ "config": "metron-indexing-env/update_hbase_table", "subsection-name": "subsection-indexing-update" }, @@ -553,7 +557,6 @@ "config": "metron-indexing-env/batch_indexing_topology_max_spout_pending", "subsection-name": "subsection-indexing-hdfs" }, - { "config": "metron-indexing-env/ra_indexing_kafka_spout_parallelism", "subsection-name": "subsection-indexing-storm" @@ -562,7 +565,6 @@ "config": "metron-indexing-env/batch_indexing_kafka_spout_parallelism", "subsection-name": "subsection-indexing-hdfs" }, - { "config": "metron-indexing-env/ra_indexing_writer_parallelism", "subsection-name": "subsection-indexing-storm"
[30/50] [abbrv] metron git commit: METRON-1510 Update Metron website to include info about github update subscription (anandsubbu) closes apache/metron#981
METRON-1510 Update Metron website to include info about github update subscription (anandsubbu) closes apache/metron#981 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/438893b7 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/438893b7 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/438893b7 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 438893b78c852e34d7edb965840500a78503d299 Parents: ed50d48 Author: anandsubbuAuthored: Wed Apr 11 11:03:37 2018 +0530 Committer: anandsubbu Committed: Wed Apr 11 11:03:37 2018 +0530 -- site/community/index.md | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/438893b7/site/community/index.md -- diff --git a/site/community/index.md b/site/community/index.md index 7c09b14..b222ba3 100644 --- a/site/community/index.md +++ b/site/community/index.md @@ -121,11 +121,15 @@ title: Apache Metron Community General & Public Discussion [mailto:user-subscr...@metron.apache.org;>Subscribe] [mailto:user-unsubscr...@metron.apache.org;>Unsubscribe] -[http://mail-archives.apache.org/mod_mbox/metron-user/;>Archives] +[https://lists.apache.org/list.html?u...@metron.apache.org;>Archives] Code & Documentation Change [mailto:dev-subscr...@metron.apache.org;>Subscribe] [mailto:dev-unsubscr...@metron.apache.org;>Unsubscribe] -[http://mail-archives.apache.org/mod_mbox/metron-dev/;>Archives] +[https://lists.apache.org/list.html?d...@metron.apache.org;>Archives] + Issues & Github updates +[mailto:issues-subscr...@metron.apache.org;>Subscribe] +[mailto:issues-unsubscr...@metron.apache.org;>Unsubscribe] +[https://lists.apache.org/list.html?iss...@metron.apache.org;>Archives]
[10/50] [abbrv] metron git commit: METRON-590 Enable Use of Event Time in Profiler (nickwallen) closes apache/metron#965
http://git-wip-us.apache.org/repos/asf/metron/blob/3083b471/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index ceb9e4e..ccce022 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -287,6 +287,8 @@ profiler_input_topic = config['configurations']['metron-enrichment-env']['enrich profiler_kafka_start = config['configurations']['metron-profiler-env']['profiler_kafka_start'] profiler_period_duration = config['configurations']['metron-profiler-env']['profiler_period_duration'] profiler_period_units = config['configurations']['metron-profiler-env']['profiler_period_units'] +profiler_window_duration = config['configurations']['metron-profiler-env']['profiler_window_duration'] +profiler_window_units = config['configurations']['metron-profiler-env']['profiler_window_units'] profiler_ttl = config['configurations']['metron-profiler-env']['profiler_ttl'] profiler_ttl_units = config['configurations']['metron-profiler-env']['profiler_ttl_units'] profiler_hbase_batch = config['configurations']['metron-profiler-env']['profiler_hbase_batch'] @@ -302,6 +304,11 @@ profiler_hbase_acl_configured_flag_file = status_params.profiler_hbase_acl_confi if not len(profiler_topology_worker_childopts) == 0: profiler_topology_worker_childopts += ' ' profiler_topology_worker_childopts += config['configurations']['metron-profiler-env']['profiler_topology_worker_childopts'] +profiler_max_routes_per_bolt=config['configurations']['metron-profiler-env']['profiler_max_routes_per_bolt'] +profiler_window_lag=config['configurations']['metron-profiler-env']['profiler_window_lag'] +profiler_window_lag_units=config['configurations']['metron-profiler-env']['profiler_window_lag_units'] +profiler_topology_message_timeout_secs=config['configurations']['metron-profiler-env']['profiler_topology_message_timeout_secs'] +profiler_topology_max_spout_pending=config['configurations']['metron-profiler-env']['profiler_topology_max_spout_pending'] # Indexing ra_indexing_kafka_start = config['configurations']['metron-indexing-env']['ra_indexing_kafka_start'] http://git-wip-us.apache.org/repos/asf/metron/blob/3083b471/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 index 06fd209..fabdaa7 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 @@ -22,6 +22,10 @@ topology.worker.childopts={{profiler_topology_worker_childopts}} topology.auto-credentials={{topology_auto_credentials}} +profiler.workers={{profiler_topology_workers}} +profiler.executors={{profiler_acker_executors}} +topology.message.timeout.secs={{profiler_topology_message_timeout_secs}} +topology.max.spout.pending={{profiler_topology_max_spout_pending}} # Profiler # @@ -29,10 +33,16 @@ profiler.input.topic={{enrichment_output_topic}} profiler.output.topic={{enrichment_input_topic}} profiler.period.duration={{profiler_period_duration}} profiler.period.duration.units={{profiler_period_units}} -profiler.workers={{profiler_topology_workers}} -profiler.executors={{profiler_acker_executors}} +profiler.window.duration={{profiler_window_duration}} +profiler.window.duration.units={{profiler_window_units}} profiler.ttl={{profiler_ttl}} profiler.ttl.units={{profiler_ttl_units}} +profiler.window.lag={{profiler_window_lag}} +profiler.window.lag.units={{profiler_window_lag_units}} +profiler.max.routes.per.bolt={{profiler_max_routes_per_bolt}} + +# HBase # + profiler.hbase.salt.divisor=1000 profiler.hbase.table={{profiler_hbase_table}} profiler.hbase.column.family={{profiler_hbase_cf}} @@ -43,6 +53,5 @@ profiler.hbase.flush.interval.seconds={{profiler_hbase_flush_interval}}
[05/50] [abbrv] metron git commit: METRON-1397 Support for JSON Path and complex documents in JSONMapParser closes apache/incubator-metron#914
METRON-1397 Support for JSON Path and complex documents in JSONMapParser closes apache/incubator-metron#914 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9c5d9d76 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9c5d9d76 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9c5d9d76 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 9c5d9d76644fc07bae36644906f52e0422f33d0e Parents: 85d1247 Author: ottobackwardsAuthored: Thu Mar 15 14:17:31 2018 -0400 Committer: cstella Committed: Thu Mar 15 14:17:31 2018 -0400 -- dependencies_with_url.csv | 3 + .../docker/rpm-docker/SPECS/metron.spec | 1 + .../jsonMapQuery/parsed/jsonMapExampleParsed| 2 + .../data/jsonMapQuery/raw/jsonMapExampleOutput | 1 + metron-platform/metron-parsers/README.md| 12 ++ metron-platform/metron-parsers/pom.xml | 5 + .../config/zookeeper/parsers/jsonMapQuery.json | 5 + .../metron/parsers/json/JSONMapParser.java | 145 + .../JSONMapQueryIntegrationTest.java| 36 .../validation/SampleDataValidation.java| 2 +- .../parsers/json/JSONMapParserQueryTest.java| 201 +++ .../metron/test/utils/ValidationUtils.java | 46 - 12 files changed, 406 insertions(+), 53 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/dependencies_with_url.csv -- diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index e2b947b..1e73eb1 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -22,6 +22,9 @@ com.flipkart.zjsonpatch:zjsonpatch:jar:0.3.4:compile,Apache v2, https://github.c com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD license,http://code.google.com/p/protobuf com.google.protobuf:protobuf-java:jar:2.6.1:compile,New BSD license,http://code.google.com/p/protobuf com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/ +com.jayway.jsonpath:json-path:jar:2.3.0:compile,Apache v2,https://github.com/json-path/JsonPath +net.minidev:accessors-smart:jar:1.2:compile,Apache v2,https://github.com/netplex/json-smart-v2 +net.minidev:json-smart:jar:2.3:compile,Apache v2,https://github.com/netplex/json-smart-v2 com.maxmind.db:maxmind-db:jar:1.2.1:compile,CC-BY-SA 3.0,https://github.com/maxmind/MaxMind-DB com.maxmind.geoip2:geoip2:jar:2.8.0:compile,Apache v2,https://github.com/maxmind/GeoIP2-java com.sun.xml.bind:jaxb-impl:jar:2.2.3-1:compile,CDDL,http://jaxb.java.net/ http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec -- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 265d595..cc01d7c 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -147,6 +147,7 @@ This package installs the Metron Parser files %{metron_home}/bin/start_parser_topology.sh %{metron_home}/config/zookeeper/parsers/bro.json %{metron_home}/config/zookeeper/parsers/jsonMap.json +%{metron_home}/config/zookeeper/parsers/jsonMapQuery.json %{metron_home}/config/zookeeper/parsers/snort.json %{metron_home}/config/zookeeper/parsers/squid.json %{metron_home}/config/zookeeper/parsers/websphere.json http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed -- diff --git a/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed new file mode 100644 index 000..e614bda --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed @@ -0,0 +1,2 @@ +{ "string" : "bar", "number" : 2, "ignored" : [ "blah" ], "original_string":"{ \"string\" : \"bar\", \"number\" : 2, \"ignored\" : [ \"blah\" ] }","timestamp":1, "source.type":"jsonMapQuery","guid":"this-is-random-uuid-will-be-36-chars" } +{ "number" : 7 , "original_string" : "{ \"number\" : 7 }", "source.type":"jsonMapQuery","timestamp":1,"guid":"this-is-random-uuid-will-be-36-chars"} http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/raw/jsonMapExampleOutput