[31/52] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py -- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py new file mode 100755 index 000..24f2306 --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script + +def yamlify_variables(var) : + if isinstance(var, type(True)): +return str(var).lower() + else: +return var + +# server configurations +config = Script.get_config() + +masters_also_are_datanodes = config['configurations']['elastic-site']['masters_also_are_datanodes'] +elastic_home = config['configurations']['elastic-sysconfig']['elastic_home'] +data_dir = config['configurations']['elastic-sysconfig']['data_dir'] +work_dir = config['configurations']['elastic-sysconfig']['work_dir'] +conf_dir = config['configurations']['elastic-sysconfig']['conf_dir'] +heap_size = config['configurations']['elastic-sysconfig']['heap_size'] +max_open_files = config['configurations']['elastic-sysconfig']['max_open_files'] +max_map_count = config['configurations']['elastic-sysconfig']['max_map_count'] + +elastic_user = config['configurations']['elastic-env']['elastic_user'] +elastic_group = config['configurations']['elastic-env']['elastic_group'] +log_dir = config['configurations']['elastic-env']['elastic_log_dir'] +pid_dir = config['configurations']['elastic-env']['elastic_pid_dir'] + +hostname = config['hostname'] +java64_home = config['hostLevelParams']['java_home'] +elastic_env_sh_template = config['configurations']['elastic-env']['content'] +sysconfig_template = config['configurations']['elastic-sysconfig']['content'] + +cluster_name = config['configurations']['elastic-site']['cluster_name'] +zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts'] + +path_data = config['configurations']['elastic-site']['path_data'] +http_cors_enabled = config['configurations']['elastic-site']['http_cors_enabled'] +http_port = config['configurations']['elastic-site']['http_port'] +transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port'] + +recover_after_time = config['configurations']['elastic-site']['recover_after_time'] +gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes'] +expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes'] +index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count'] +index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size'] +index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval'] +indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type'] +index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards'] +index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas'] +indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size'] +bootstrap_memory_lock = yamlify_variables(config['configurations']['elastic-site']['bootstrap_memory_lock']) +threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size'] +cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries'] +cluster_routing_allocation_disk_watermark_low =
[43/52] [abbrv] metron git commit: METRON-1347: Indexing Topology should fail tuples without a source.type (cstella via mmiklavc) closes apache/metron#863
METRON-1347: Indexing Topology should fail tuples without a source.type (cstella via mmiklavc) closes apache/metron#863 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bfe90ef1 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bfe90ef1 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bfe90ef1 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: bfe90ef1e579be53a14d9fd0e4dc19fc6a81baf0 Parents: 53124d9 Author: cstellaAuthored: Fri Apr 13 11:17:00 2018 -0600 Committer: Michael Miklavcic Committed: Fri Apr 13 11:17:00 2018 -0600 -- .../bolt/BulkMessageWriterBoltTest.java | 25 ++ metron-platform/metron-indexing/README.md | 6 +++ .../writer/bolt/BulkMessageWriterBolt.java | 51 ++-- 3 files changed, 68 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/bfe90ef1/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java -- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java index 308638e..dedf5e6 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java @@ -118,6 +118,31 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest { private MessageGetStrategy messageGetStrategy; @Test + public void testSensorTypeMissing() throws Exception { +BulkMessageWriterBolt bulkMessageWriterBolt = new BulkMessageWriterBolt("zookeeperUrl") + .withBulkMessageWriter(bulkMessageWriter).withMessageGetter(MessageGetters.JSON_FROM_FIELD.name()) +.withMessageGetterField("message"); +bulkMessageWriterBolt.setCuratorFramework(client); +bulkMessageWriterBolt.setZKCache(cache); + bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(sensorType, +new FileInputStream(sampleSensorIndexingConfigPath)); + +bulkMessageWriterBolt.declareOutputFields(declarer); +verify(declarer, times(1)).declareStream(eq("error"), argThat( +new FieldsMatcher("message"))); +Map stormConf = new HashMap(); +bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector); +BulkWriterComponent component = mock(BulkWriterComponent.class); +bulkMessageWriterBolt.setWriterComponent(component); +verify(bulkMessageWriter, times(1)).init(eq(stormConf),any(TopologyContext.class), any(WriterConfiguration.class)); +JSONObject message = (JSONObject) new JSONParser().parse(sampleMessageString); +message.remove("source.type"); +when(tuple.getValueByField("message")).thenReturn(message); +bulkMessageWriterBolt.execute(tuple); +verify(component, times(1)).error(eq("null"), any(), any(), any()); + } + + @Test public void testFlushOnBatchSize() throws Exception { BulkMessageWriterBolt bulkMessageWriterBolt = new BulkMessageWriterBolt("zookeeperUrl") .withBulkMessageWriter(bulkMessageWriter).withMessageGetter(MessageGetters.JSON_FROM_FIELD.name()) http://git-wip-us.apache.org/repos/asf/metron/blob/bfe90ef1/metron-platform/metron-indexing/README.md -- diff --git a/metron-platform/metron-indexing/README.md b/metron-platform/metron-indexing/README.md index d351d7c..f4a4501 100644 --- a/metron-platform/metron-indexing/README.md +++ b/metron-platform/metron-indexing/README.md @@ -32,6 +32,12 @@ Indices are written in batch and the batch size and batch timeout are specified [Sensor Indexing Configuration](#sensor-indexing-configuration) via the `batchSize` and `batchTimeout` parameters. These configs are variable by sensor type. +## Minimal Assumptions for Message Structure + +At minimum, a message should have a `source.type` field. +Without this field, the message tuple will be failed and not written +with an appropriate error indicated in the Storm UI and logs. + ## Indexing Architecture ![Architecture](indexing_arch.png) http://git-wip-us.apache.org/repos/asf/metron/blob/bfe90ef1/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java -- diff --git
[35/52] [abbrv] metron git commit: METRON-1504: Enriching missing values does not match the semantics between the new enrichment topology and old closes apache/incubator-metron#976
METRON-1504: Enriching missing values does not match the semantics between the new enrichment topology and old closes apache/incubator-metron#976 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3ba9ae25 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3ba9ae25 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3ba9ae25 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 3ba9ae25126bc393e3b9307f4ffe63ac66a909f4 Parents: 46bc63d Author: cstellaAuthored: Mon Apr 9 11:47:51 2018 -0400 Committer: cstella Committed: Mon Apr 9 11:47:51 2018 -0400 -- .../enrichment/parallel/ParallelEnricher.java | 3 + .../integration/EnrichmentIntegrationTest.java | 12 +++- .../parallel/ParallelEnricherTest.java | 59 +++- 3 files changed, 70 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java -- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java index 6ddb892..b10c148 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java @@ -170,6 +170,9 @@ public class ParallelEnricher { for(Object o : m.keySet()) { String field = (String) o; Object value = m.get(o); + if(value == null) { +continue; + } CacheKey cacheKey = new CacheKey(field, value, config); String prefix = adapter.getOutputPrefix(cacheKey); Supplier supplier = () -> { http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java -- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java index 267ca62..3c55c95 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java @@ -95,7 +95,15 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest { private static List getInputMessages(String path){ try{ - return TestUtils.readSampleData(path); + List ret = TestUtils.readSampleData(path); + { +//we want one of the fields without a destination IP to ensure that enrichments can function +Map sansDestinationIp = JSONUtils.INSTANCE.load(new String(ret.get(ret.size() -1)) + , JSONUtils.MAP_SUPPLIER); +sansDestinationIp.remove(Constants.Fields.DST_ADDR.getName()); +ret.add(JSONUtils.INSTANCE.toJSONPretty(sansDestinationIp)); + } + return ret; }catch(IOException ioe){ return null; } @@ -262,8 +270,6 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest { //ensure we always have a source ip and destination ip Assert.assertNotNull(jsonDoc.get(SRC_IP)); -Assert.assertNotNull(jsonDoc.get(DST_IP)); - Assert.assertNotNull(jsonDoc.get("ALL_CAPS")); Assert.assertNotNull(jsonDoc.get("map.blah")); Assert.assertNull(jsonDoc.get("map")); http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java -- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java index 4a4573b..d4fcdf4 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java @@ -79,7 +79,39
[36/52] [abbrv] metron git commit: METRON-1465:Support for Elasticsearch X-pack (wardbekker via mmiklavc) closes apache/metron#946
METRON-1465:Support for Elasticsearch X-pack (wardbekker via mmiklavc) closes apache/metron#946 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a8b555dc Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a8b555dc Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a8b555dc Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: a8b555dcc9f548d7b91789a46d9435b4d8b17581 Parents: 3ba9ae2 Author: wardbekkerAuthored: Mon Apr 9 13:14:13 2018 -0600 Committer: Michael Miklavcic Committed: Mon Apr 9 13:14:13 2018 -0600 -- metron-deployment/Kerberos-manual-setup.md | 209 +++ .../roles/metron-builder/tasks/build-debs.yml | 2 +- .../roles/metron-builder/tasks/build-rpms.yml | 2 +- .../METRON/CURRENT/configuration/metron-env.xml | 2 - .../metron-rest/src/main/scripts/metron-rest.sh | 9 + .../src/main/config/zookeeper/global.json | 5 +- .../apache/metron/common/utils/HDFSUtils.java | 59 ++ .../metron/common/utils/ReflectionUtils.java| 66 +- .../elasticsearch/dao/ElasticsearchDao.java | 33 ++- .../elasticsearch/utils/ElasticsearchUtils.java | 107 -- .../writer/ElasticsearchWriter.java | 8 +- .../scripts/start_elasticsearch_topology.sh | 8 +- .../writer/ElasticsearchWriterTest.java | 19 +- .../stellar/common/utils/ConversionUtils.java | 19 +- 14 files changed, 486 insertions(+), 62 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/a8b555dc/metron-deployment/Kerberos-manual-setup.md -- diff --git a/metron-deployment/Kerberos-manual-setup.md b/metron-deployment/Kerberos-manual-setup.md index 47a63d8..456703a 100644 --- a/metron-deployment/Kerberos-manual-setup.md +++ b/metron-deployment/Kerberos-manual-setup.md @@ -30,6 +30,7 @@ This document provides instructions for kerberizing Metron's Vagrant-based devel * [Start Metron](#start-metron) * [Push Data](#push-data) * [More Information](#more-information) +* [Elasticseach X-Pack](#x-pack) Setup - @@ -533,3 +534,211 @@ In order to correct this, you should: ### References * [https://github.com/apache/storm/blob/master/SECURITY.md](https://github.com/apache/storm/blob/master/SECURITY.md) + +X-Pack +-- + +First, stop the random_access_indexing topology through the Storm UI or from the CLI, e.g. + +``` +storm kill random_access_indexing +``` + +Here are instructions for enabling X-Pack with Elasticsearch and Kibana: https://www.elastic.co/guide/en/x-pack/5.6/installing-xpack.html + +You need to be sure to add the appropriate username and password for Elasticsearch and Kibana to enable external connections from Metron components. e.g. the following will create a user "transport_client_user" with password "changeme" and "superuser" credentials. + +``` +sudo /usr/share/elasticsearch/bin/x-pack/users useradd transport_client_user -p changeme -r superuser +``` + +Once you've picked a password to connect to ES, you need to upload a 1-line file to HDFS with that password in it. Metron will use this file to securely read the password in order to connect to ES securely. + +Here is an example using "changeme" as the password + +``` +echo changeme > /tmp/xpack-password +sudo -u hdfs hdfs dfs -mkdir /apps/metron/elasticsearch/ +sudo -u hdfs hdfs dfs -put /tmp/xpack-password /apps/metron/elasticsearch/ +sudo -u hdfs hdfs dfs -chown metron:metron /apps/metron/elasticsearch/xpack-password +``` + +New settings have been added to configure the Elasticsearch client. By default the client will run as the normal ES prebuilt transport client. If you enable X-Pack you should set the es.client.class as shown below. + +Add the es settings to global.json + +``` +/usr/metron/0.4.3/config/zookeeper/global.json -> + + "es.client.settings" : { + "es.client.class" : "org.elasticsearch.xpack.client.PreBuiltXPackTransportClient", + "es.xpack.username" : "transport_client_user", + "es.xpack.password.file" : "/apps/metron/elasticsearch/xpack-password" + } +``` + +Submit the update to Zookeeper + +``` +$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i METRON_HOME/config/zookeeper/ -z $ZOOKEEPER +``` + +The last step before restarting the topology is to create a custom X-Pack shaded and relocated jar. This is up to you because of licensing restrictions, but here is a sample Maven pom file that should help. + +``` + + +http://maven.apache.org/POM/4.0.0; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> +4.0.0 +org.elasticsearch +
[23/52] [abbrv] metron git commit: METRON-1500 Enhance 'prepare-commit' to Support Feature Branches (nickwallen) closes apache/metron#971
METRON-1500 Enhance 'prepare-commit' to Support Feature Branches (nickwallen) closes apache/metron#971 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9e95d4b6 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9e95d4b6 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9e95d4b6 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 9e95d4b61410c8033f0a4ea51e831566d5d933d3 Parents: 5ed9631 Author: nickwallenAuthored: Fri Mar 23 12:23:34 2018 -0400 Committer: nickallen Committed: Fri Mar 23 12:23:34 2018 -0400 -- dev-utilities/committer-utils/prepare-commit | 27 ++- 1 file changed, 22 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/9e95d4b6/dev-utilities/committer-utils/prepare-commit -- diff --git a/dev-utilities/committer-utils/prepare-commit b/dev-utilities/committer-utils/prepare-commit index 67116cb..ae8d7ab 100755 --- a/dev-utilities/committer-utils/prepare-commit +++ b/dev-utilities/committer-utils/prepare-commit @@ -19,9 +19,9 @@ # not likely to change METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron.git; BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git; -BASE_BRANCH=master CONFIG_FILE=~/.metron-prepare-commit GITHUB_REMOTE="origin" +BASE_BRANCH=master # does a config file already exist? if [ -f $CONFIG_FILE ]; then @@ -114,6 +114,11 @@ if [ ! -d "$WORK" ]; then read -p " origin repo [$ORIGIN]: " INPUT [ -n "$INPUT" ] && ORIGIN=$INPUT + # what branch did the PR get submitted against? could be a feature branch + BASE_BRANCH=`curl -s https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/$PR | python -c 'import sys, json; print json.load(sys.stdin)["base"]["ref"]'` + read -p " base branch to merge into [$BASE_BRANCH]: " INPUT + [ -n "$INPUT" ] && BASE_BRANCH=$INPUT + # clone the repository and fetch updates mkdir -p $WORK git clone $ORIGIN $WORK @@ -125,11 +130,23 @@ if [ ! -d "$WORK" ]; then # fetch any changes from upstream git remote add upstream $UPSTREAM - git fetch upstream $BASE_BRANCH + if git fetch upstream "$BASE_BRANCH"; then + +if [ $BASE_BRANCH = "master" ]; then + # merge any changes from upstream + git checkout $BASE_BRANCH + git merge upstream/$BASE_BRANCH - # merge any changes from upstream - git checkout $BASE_BRANCH - git merge upstream/$BASE_BRANCH +else + # create a local branch from the remote feature branch + git checkout -B $BASE_BRANCH upstream/$BASE_BRANCH + +fi + + else +# unable to fetch the base branch +exit $? + fi else
[52/52] [abbrv] metron git commit: METRON-1528: Fix missing file in metron.spec (mmiklavc via mmiklavc) closes apache/metron#996
METRON-1528: Fix missing file in metron.spec (mmiklavc via mmiklavc) closes apache/metron#996 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a41611b1 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a41611b1 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a41611b1 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: a41611b1ae2bb17fa9333ef6f965749652e95538 Parents: e094914 Author: mmiklavcAuthored: Wed Apr 18 07:35:01 2018 -0600 Committer: Michael Miklavcic Committed: Wed Apr 18 07:35:01 2018 -0600 -- metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/metron/blob/a41611b1/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec -- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 1f40105..15469d9 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -123,6 +123,7 @@ This package installs the Metron common files %{metron_home} %dir %{metron_home}/lib %{metron_home}/bin/zk_load_configs.sh %{metron_home}/bin/stellar +%{metron_home}/bin/cluster_info.py %{metron_home}/config/zookeeper/global.json %attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}.jar
[47/52] [abbrv] metron git commit: METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984
http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml -- diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml deleted file mode 100644 index fd7ceff..000 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml +++ /dev/null @@ -1,590 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: "enrichment" -config: -topology.workers: ${enrichment.workers} -topology.acker.executors: ${enrichment.acker.executors} -topology.worker.childopts: ${topology.worker.childopts} -topology.auto-credentials: ${topology.auto-credentials} -topology.max.spout.pending: ${topology.max.spout.pending} - -components: - -# Enrichment -- id: "stellarEnrichmentAdapter" -className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter" -configMethods: -- name: "ofType" -args: -- "ENRICHMENT" - -# Any kafka props for the producer go here. -- id: "kafkaWriterProps" -className: "java.util.HashMap" -configMethods: - - name: "put" - args: - - "security.protocol" - - "${kafka.security.protocol}" - -- id: "stellarEnrichment" -className: "org.apache.metron.enrichment.configuration.Enrichment" -constructorArgs: -- "stellar" -- ref: "stellarEnrichmentAdapter" - -- id: "geoEnrichmentAdapter" -className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter" -- id: "geoEnrichment" -className: "org.apache.metron.enrichment.configuration.Enrichment" -constructorArgs: -- "geo" -- ref: "geoEnrichmentAdapter" -- id: "hostEnrichmentAdapter" -className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter" -constructorArgs: -- '${enrichment.host.known_hosts}' -- id: "hostEnrichment" -className: "org.apache.metron.enrichment.configuration.Enrichment" -constructorArgs: -- "host" -- ref: "hostEnrichmentAdapter" - -- id: "simpleHBaseEnrichmentConfig" -className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseConfig" -configMethods: -- name: "withProviderImpl" -args: -- "${hbase.provider.impl}" -- name: "withHBaseTable" -args: -- "${enrichment.simple.hbase.table}" -- name: "withHBaseCF" -args: -- "${enrichment.simple.hbase.cf}" -- id: "simpleHBaseEnrichmentAdapter" -className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseAdapter" -configMethods: - -name: "withConfig" -args: -- ref: "simpleHBaseEnrichmentConfig" -- id: "simpleHBaseEnrichment" -className: "org.apache.metron.enrichment.configuration.Enrichment" -constructorArgs: - - "hbaseEnrichment" - - ref: "simpleHBaseEnrichmentAdapter" -- id: "enrichments" -className: "java.util.ArrayList" -configMethods: -- name: "add" -args: -- ref: "geoEnrichment" -- name: "add" -args: -- ref: "hostEnrichment" -- name: "add" -args: -- ref: "simpleHBaseEnrichment" -- name: "add" -args: -- ref: "stellarEnrichment" - -#enrichment error -- id: "enrichmentErrorKafkaWriter" -className: "org.apache.metron.writer.kafka.KafkaWriter" -configMethods: -- name: "withTopic" -args: -- "${enrichment.error.topic}" -- name: "withZkQuorum" -args: -- "${kafka.zk}" -
[46/52] [abbrv] metron git commit: METRON-1515: Errors loading stellar functions currently bomb the entire topology, they should be recoverable closes apache/incubator-metron#985
METRON-1515: Errors loading stellar functions currently bomb the entire topology, they should be recoverable closes apache/incubator-metron#985 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3fcbf8b4 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3fcbf8b4 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3fcbf8b4 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 3fcbf8b4e4e38f9c50842b8af857092b091c7c40 Parents: 1d3e7fc Author: cstellaAuthored: Mon Apr 16 15:12:11 2018 -0400 Committer: cstella Committed: Mon Apr 16 15:12:11 2018 -0400 -- .../resolver/ClasspathFunctionResolver.java | 45 +++- .../resolver/ClasspathFunctionResolverTest.java | 30 + 2 files changed, 65 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/3fcbf8b4/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java -- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java index 85aa015..b17233a 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/ClasspathFunctionResolver.java @@ -34,6 +34,7 @@ import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.dsl.StellarFunction; +import org.atteo.classindex.ClassFilter; import org.atteo.classindex.ClassIndex; import org.reflections.util.FilterBuilder; @@ -219,6 +220,17 @@ public class ClasspathFunctionResolver extends BaseFunctionResolver { } } + protected Iterable getStellarClasses(ClassLoader cl) { +return ClassIndex.getAnnotated(Stellar.class, cl); + } + + protected boolean includeClass(Class c, FilterBuilder filterBuilder) + { +boolean isAssignable = StellarFunction.class.isAssignableFrom(c); +boolean isFiltered = filterBuilder.apply(c.getCanonicalName()); +return isAssignable && isFiltered; + } + /** * Returns a set of classes that should undergo further interrogation for resolution * (aka discovery) of Stellar functions. @@ -254,16 +266,29 @@ public class ClasspathFunctionResolver extends BaseFunctionResolver { Set classes = new HashSet<>(); Set ret = new HashSet<>(); for(ClassLoader cl : cls) { - for(Class c : ClassIndex.getAnnotated(Stellar.class, cl)) { -LOG.debug("{}: Found class: {}", cl.getClass().getCanonicalName(), c.getCanonicalName()); -boolean isAssignable = StellarFunction.class.isAssignableFrom(c); -boolean isFiltered = filterBuilder.apply(c.getCanonicalName()); -if( isAssignable && isFiltered ) { - String className = c.getName(); - if(!classes.contains(className)) { -LOG.debug("{}: Added class: {}", cl.getClass().getCanonicalName(), className); -ret.add((Class) c); -classes.add(className); + for(Class c : getStellarClasses(cl)) { +try { + LOG.debug("{}: Found class: {}", cl.getClass().getCanonicalName(), c.getCanonicalName()); + if (includeClass(c, filterBuilder)) { +String className = c.getName(); +if (!classes.contains(className)) { + LOG.debug("{}: Added class: {}", cl.getClass().getCanonicalName(), className); + ret.add((Class) c); + classes.add(className); +} + } +} +catch(Error le) { + //we have had some error loading a stellar function. This could mean that + //the classpath is unstable (e.g. old copies of jars are on the classpath). + try { +LOG.error("Skipping class " + c.getName() + ": " + le.getMessage() ++ ", please check that there are not old versions of stellar functions on the classpath.", le); + } + catch(Error ie) { +//it's possible that getName() will throw an exception if the class is VERY malformed. +LOG.error("Skipping class: " + le.getMessage() ++ ", please check that there are not old versions of stellar functions on the classpath.", le); } } }
[34/52] [abbrv] metron git commit: METRON-1505 Intermittent Profiler Integration Test Failure (nickwallen) closes apache/metron#977
METRON-1505 Intermittent Profiler Integration Test Failure (nickwallen) closes apache/metron#977 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/46bc63db Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/46bc63db Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/46bc63db Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 46bc63dbcfe9f0ddabfd4821958962a2dac9378e Parents: ab4f8e6 Author: nickwallenAuthored: Sat Apr 7 11:28:01 2018 -0400 Committer: nickallen Committed: Sat Apr 7 11:28:01 2018 -0400 -- .../profiler/DefaultMessageDistributor.java | 54 +++- .../src/main/flux/profiler/remote.yaml | 2 + .../profiler/bolt/ProfileBuilderBolt.java | 149 +++--- .../profiler/bolt/ProfileSplitterBolt.java | 1 - .../config/zookeeper/percentiles/profiler.json | 12 - .../processing-time-test/profiler.json | 11 + .../zookeeper/readme-example-1/profiler.json| 17 -- .../zookeeper/readme-example-2/profiler.json| 18 -- .../zookeeper/readme-example-3/profiler.json| 11 - .../zookeeper/readme-example-4/profiler.json| 11 - .../profiler/bolt/ProfileBuilderBoltTest.java | 130 +++-- .../integration/ProfilerIntegrationTest.java| 274 +-- .../configuration/profiler/ProfileConfig.java | 49 ++-- .../ZKConfigurationsCacheIntegrationTest.java | 4 +- .../org/apache/metron/hbase/bolt/HBaseBolt.java | 22 +- 15 files changed, 319 insertions(+), 446 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/46bc63db/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java -- diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java index ea5126f..70f4228 100644 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java +++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java @@ -25,6 +25,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.RemovalListener; import com.google.common.cache.RemovalNotification; +import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.stellar.dsl.Context; import org.json.simple.JSONObject; @@ -33,7 +34,6 @@ import org.slf4j.LoggerFactory; import java.lang.invoke.MethodHandles; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -74,7 +74,7 @@ public class DefaultMessageDistributor implements MessageDistributor { * messages. Once it has not received messages for a period of time, it is * moved to the expired cache. */ - private transient Cache activeCache; + private transient Cache activeCache; /** * A cache of expired profiles. @@ -85,7 +85,7 @@ public class DefaultMessageDistributor implements MessageDistributor { * can flush the state of the expired profile. If the client does not flush * the expired profiles, this state will be lost forever. */ - private transient Cache expiredCache; + private transient Cache expiredCache; /** * Create a new message distributor. @@ -222,7 +222,7 @@ public class DefaultMessageDistributor implements MessageDistributor { * @param cache The cache to flush. * @return The measurements captured when flushing the profiles. */ - private List flushCache(Cache cache) { + private List flushCache(Cache cache) { List measurements = new ArrayList<>(); for(ProfileBuilder profileBuilder: cache.asMap().values()) { @@ -262,11 +262,19 @@ public class DefaultMessageDistributor implements MessageDistributor { /** * Builds the key that is used to lookup the {@link ProfileBuilder} within the cache. * + * The cache key is built using the hash codes of the profile and entity name. If the profile + * definition is ever changed, the same cache entry will not be reused. This ensures that no + * state can be carried over from the old definition into the new, which might result in an + * invalid profile measurement. + * * @param profile The profile
[29/52] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json deleted file mode 100644 index 037f1c6..000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json +++ /dev/null @@ -1,88 +0,0 @@ -{ "create" : { "_id": "all-metron-index", "_type": "index-pattern" } } -{"title":"*_index_*","timeFieldName":"timestamp","notExpandable":true,"fields":"[{\"name\":\"AA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RA\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"RD\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TC\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"TTLs\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Z\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\" :true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:geoadapter:end:ts\",\"type\":\"date\", \"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:hostfromjsonlistadapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:begin:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"adapter:threatinteladapter:end:ts\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"addl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzer\",\"type\":\"string\",\"count\":0,\"scripted\":false, \"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"analyzers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"answers\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"app\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"arg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"assigned_ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_attempts\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_success\",\"type\":\"boolean\",\"count\":0,\"scri
[51/52] [abbrv] metron git commit: METRON-1445: Update performance tuning guide with more explicit parameter instructions (mmiklavc via mmiklavc) closes apache/metron#988
METRON-1445: Update performance tuning guide with more explicit parameter instructions (mmiklavc via mmiklavc) closes apache/metron#988 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e0949142 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e0949142 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e0949142 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: e0949142dd682a84e59fea09066de8024f106f13 Parents: daf543b Author: mmiklavcAuthored: Tue Apr 17 12:31:37 2018 -0600 Committer: Michael Miklavcic Committed: Tue Apr 17 12:31:37 2018 -0600 -- metron-platform/Performance-tuning-guide.md | 244 +++- metron-platform/metron-common/README.md | 32 ++ .../src/main/scripts/cluster_info.py| 389 +++ 3 files changed, 659 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/e0949142/metron-platform/Performance-tuning-guide.md -- diff --git a/metron-platform/Performance-tuning-guide.md b/metron-platform/Performance-tuning-guide.md index 7d79ace..e2d1ae2 100644 --- a/metron-platform/Performance-tuning-guide.md +++ b/metron-platform/Performance-tuning-guide.md @@ -17,6 +17,14 @@ limitations under the License. --> # Metron Performance Tuning Guide +- [Overview](#overview) +- [General Tuning Suggestions](#general-tuning-suggestions) +- [Component Tuning Levers](#component-tuning-levers) +- [Use Case Specific Tuning Suggestions](#use-case-specific-tuning-suggestions) +- [Debugging](#debugging) +- [Issues](#issues) +- [Reference](#reference) + ## Overview This document provides guidance from our experiences tuning the Apache Metron Storm topologies for maximum performance. You'll find @@ -31,20 +39,33 @@ pipe, and the majority of these options assist in tweaking the various pipe widt ## General Tuning Suggestions +### Storm Executors vs. Tasks + Note that there is currently no method for specifying the number of tasks from the number of executors in Flux topologies (enrichment, indexing). By default, the number of tasks will equal the number of executors. Logically, setting the number of tasks equal to the number of executors is sensible. Storm enforces num executors <= num tasks. The reason you might set the number of tasks higher than the number of executors is for future performance tuning and rebalancing without the need to bring down your topologies. The number of tasks is fixed at topology startup time whereas the number of executors can be increased up to a maximum value equal to the number of tasks. -When configuring Storm Kafka spouts, we found that the default values for poll.timeout.ms, offset.commit.period.ms, and max.uncommitted.offsets worked well in nearly all cases. -As a general rule, it was optimal to set spout parallelism equal to the number of partitions used in your Kafka topic. Any greater +### Kafka Spout Configuration + +When configuring Storm Kafka spouts, we found that the default values for + +- `poll.timeout.ms` +- `offset.commit.period.ms` +- `max.uncommitted.offsets ` + +worked well in nearly all cases. As a general rule, it was optimal to set spout parallelism equal to the number of partitions used in your Kafka topic. Any greater parallelism will leave you with idle consumers since Kafka limits the max number of consumers to the number of partitions. This is important because Kafka has certain ordering guarantees for message delivery per partition that would not be possible if more than one consumer in a given consumer group were able to read from that partition. ## Component Tuning Levers +### High Level Overview + +There are a number of levers that can be set while tuning a Metron cluster. The main services to interact with for performance tuning are: Kafka, Storm, HDFS, and indexing (Elasticsearch or Solr). For each service, here is a high level breakdown of the major knobs and levers that can be modified while tuning your cluster. + - Kafka - Number partitions - Storm @@ -70,12 +91,15 @@ for more details. ### Storm Tuning + Overview + There are quite a few options you will be confronted with when tuning your Storm topologies and this is largely trial and error. As a general rule of thumb, we recommend starting with the defaults and smaller numbers in terms of parallelism while iteratively working up until the desired performance is achieved. You will find the offset lag tool indispensable while verifying your settings. We won't go into a full discussion about Storm's architecture - see references section for more info - but there are some general rules of
[16/52] [abbrv] metron git commit: METRON-1483: Create a tool to monitor performance of the topologies closes apache/incubator-metron#958
http://git-wip-us.apache.org/repos/asf/metron/blob/46ad9d93/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java -- diff --git a/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java new file mode 100644 index 000..5d5c240 --- /dev/null +++ b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/sampler/UnbiasedSampler.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.performance.sampler; + +import java.util.Random; + +public class UnbiasedSampler implements Sampler { + + @Override + public int sample(Random rng, int limit) { +return rng.nextInt(limit); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/46ad9d93/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java -- diff --git a/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java new file mode 100644 index 000..c13f236 --- /dev/null +++ b/metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/util/KafkaUtil.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.performance.util; + +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public enum KafkaUtil { + INSTANCE; + + public List getTopicPartition(KafkaConsumerconsumer, String topic) { + +List partitions = consumer.partitionsFor(topic); +List ret = new ArrayList<>(partitions.size()); +for(PartitionInfo par : partitions) { + ret.add(new TopicPartition(topic, par.partition())); +} +return ret; + } + + public Map getKafkaOffsetMap(KafkaConsumer consumer, String topic ) { +Map ret = new HashMap<>(); +if(!consumer.subscription().contains(topic)) { + consumer.subscribe(Collections.singletonList(topic)); +} +consumer.poll(0); +List partitions = getTopicPartition(consumer, topic); +consumer.seekToEnd(partitions); +for(TopicPartition par : partitions) { + ret.put(par.partition(), consumer.position(par)-1); +} +return ret; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/46ad9d93/metron-contrib/metron-performance/src/main/scripts/load_tool.sh -- diff --git a/metron-contrib/metron-performance/src/main/scripts/load_tool.sh b/metron-contrib/metron-performance/src/main/scripts/load_tool.sh new file mode 100755 index 000..1b56a6e --- /dev/null +++ b/metron-contrib/metron-performance/src/main/scripts/load_tool.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license
[39/52] [abbrv] metron git commit: METRON-1494 Profiler Emits Messages to Kafka When Not Needed (nickwallen) closes apache/metron#967
METRON-1494 Profiler Emits Messages to Kafka When Not Needed (nickwallen) closes apache/metron#967 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/62d1a1bf Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/62d1a1bf Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/62d1a1bf Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 62d1a1bf7e8b9b3ee2f260c358719ea5080c9045 Parents: 438893b Author: nickwallenAuthored: Wed Apr 11 17:57:09 2018 -0400 Committer: nickallen Committed: Wed Apr 11 17:57:09 2018 -0400 -- .../metron/profiler/DefaultProfileBuilder.java | 5 + .../bolt/FixedFrequencyFlushSignal.java | 13 +- .../metron/profiler/bolt/HBaseEmitter.java | 12 +- .../metron/profiler/bolt/KafkaEmitter.java | 78 +-- .../profiler/bolt/ProfileSplitterBolt.java | 5 + .../metron/profiler/bolt/HBaseEmitterTest.java | 120 +++ .../metron/profiler/bolt/KafkaEmitterTest.java | 201 +-- 7 files changed, 358 insertions(+), 76 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java -- diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java index 4b564c9..66034ac 100644 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java +++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java @@ -124,8 +124,13 @@ public class DefaultProfileBuilder implements ProfileBuilder, Serializable { */ @Override public void apply(JSONObject message, long timestamp) { +LOG.debug("Applying message to profile; profile={}, entity={}, timestamp={}", +profileName, entity, timestamp); + try { if (!isInitialized()) { +LOG.debug("Initializing profile; profile={}, entity={}, timestamp={}", +profileName, entity, timestamp); // execute each 'init' expression assign(definition.getInit(), message, "init"); http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java -- diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java index b9f57dd..8c0a0b1 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java @@ -94,7 +94,8 @@ public class FixedFrequencyFlushSignal implements FlushSignal { // set the next time to flush flushTime = currentTime + flushFrequency; - LOG.debug("Setting flush time; flushTime={}, currentTime={}, flushFreq={}", + LOG.debug("Setting flush time; '{}' ms until flush; flushTime={}, currentTime={}, flushFreq={}", + timeToNextFlush(), flushTime, currentTime, flushFrequency); @@ -112,7 +113,7 @@ public class FixedFrequencyFlushSignal implements FlushSignal { boolean flush = currentTime > flushTime; LOG.debug("Flush={}, '{}' ms until flush; currentTime={}, flushTime={}", flush, -flush ? 0 : (flushTime-currentTime), +timeToNextFlush(), currentTime, flushTime); @@ -123,4 +124,12 @@ public class FixedFrequencyFlushSignal implements FlushSignal { public long currentTimeMillis() { return currentTime; } + + /** + * Returns the number of milliseconds to the next flush. + * @return The time left until the next flush. + */ + private long timeToNextFlush() { +return Math.max(0, flushTime - currentTime); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java -- diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java
[14/52] [abbrv] metron git commit: METRON-1493 Unhelpful Error Message When Assignment Expressions Fail (nickwallen) closes apache/metron#966
METRON-1493 Unhelpful Error Message When Assignment Expressions Fail (nickwallen) closes apache/metron#966 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/52dd9fb8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/52dd9fb8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/52dd9fb8 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 52dd9fb852bef5998dac83109ac6e122860be489 Parents: 9c5d9d7 Author: nickwallenAuthored: Fri Mar 16 10:16:07 2018 -0400 Committer: nickallen Committed: Fri Mar 16 10:16:07 2018 -0400 -- .../common/shell/specials/AssignmentCommand.java | 2 +- .../common/shell/specials/AssignmentCommandTest.java | 14 ++ 2 files changed, 15 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/52dd9fb8/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java -- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java index e253b3b..664e01e 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommand.java @@ -79,7 +79,7 @@ public class AssignmentCommand implements SpecialCommand { return result; } else { - return error("Assignment expression failed"); + return result; } } } http://git-wip-us.apache.org/repos/asf/metron/blob/52dd9fb8/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java -- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java index 899effb..1b5c9d5 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/specials/AssignmentCommandTest.java @@ -149,6 +149,20 @@ public class AssignmentCommandTest { assertFalse(executor.getState().containsKey("x")); } + /** + * If an assignment expression fails, the error message should explain + * why the expression fails. + */ + @Test + public void testErrorMessageWhenAssignmentFails() { +StellarResult result = command.execute("x := 0/0", executor); + +// validate the result +assertTrue(result.isError()); +assertTrue(result.getException().isPresent()); +assertEquals(ArithmeticException.class, result.getException().get().getClass()); + } + @Test public void testAssignNull() { StellarResult result = command.execute("x := NULL", executor);
[26/52] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py new file mode 100755 index 000..f0903ac --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py @@ -0,0 +1,95 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from elasticsearch import Elasticsearch +from elasticsearch.helpers import bulk +import cPickle as pickle +import argparse, sys, os.path +import errno +import os + + +class DashboardIndex(object): + +def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs): +""" +:arg host: hostname of the node (default: localhost) +:arg port: port to use (integer, default: 9200) +:arg url_prefix: optional url prefix for elasticsearch +:arg timeout: default timeout in seconds (float, default: 10) +""" +self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}]) + +def get(self): +""" +Get .kibana index from Elasticsearch +""" +dotkibana = self.es.search(index='.kibana', size = 100) +return dotkibana['hits']['hits'] + +def load(self,filespec): +""" +Save Index data on local filesystem +:args filespec: path/filename for saved file +""" +data=[] +with open(filespec,'rb') as fp: +data = pickle.load(fp) +return data + +def save(self,filename,data): +""" +Save Index data on local filesystem +:args filespec: path/filename for saved file +""" +with open(filename,'wb') as fp: +pickle.dump(data,fp) + +def put(self,data): +""" +Bulk write data to Elasticsearch +:args data: data to be written (note: index name is specified in data) +""" +bulk(self.es,data) + +def main(self,args): + +if args.save: +print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec)) +self.save(filename=args.filespec,data=di.get()) +else: +""" +Loads Kibana Dashboard definition from disk and replaces .kibana on index +:args filespec: path/filename for saved file +""" +if not os.path.isfile(args.filespec): +raise IOError( +errno.ENOENT, os.strerror(errno.ENOENT), args.filespec) +self.es.indices.delete(index='.kibana', ignore=[400, 404]) +self.put(data=di.load(filespec=args.filespec)) + +if __name__ == '__main__': + +parser = argparse.ArgumentParser() +parser.add_argument("hostname", help="ES Hostname or IP", type=str) +parser.add_argument("port", help="ES Port", type=int) +parser.add_argument("filespec", help="file to be pushed from or saved to", type=str) +parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true") +args = parser.parse_args() +di = DashboardIndex(host=args.hostname,port=args.port) +di.main(args) http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
[25/52] [abbrv] metron git commit: METRON-1501 Parser messages that fail to validate are dropped silently (cestella via justinleet) closes apache/metron#972
METRON-1501 Parser messages that fail to validate are dropped silently (cestella via justinleet) closes apache/metron#972 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0d847cf5 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0d847cf5 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0d847cf5 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 0d847cf5f91dc0d3b3b6838eb4b4de4aa2cf2fec Parents: 19b237d Author: cestellaAuthored: Tue Apr 3 10:29:19 2018 -0400 Committer: leet Committed: Tue Apr 3 10:29:19 2018 -0400 -- metron-platform/metron-parsers/README.md| 29 +++- .../apache/metron/parsers/bolt/ParserBolt.java | 17 .../metron/parsers/bolt/ParserBoltTest.java | 16 --- 3 files changed, 51 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/0d847cf5/metron-platform/metron-parsers/README.md -- diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md index 3d9fdfe..6b9d62e 100644 --- a/metron-platform/metron-parsers/README.md +++ b/metron-platform/metron-parsers/README.md @@ -45,7 +45,34 @@ There are two general types types of parsers: * `ERROR` : Throw an error when a multidimensional map is encountered * `jsonpQuery` : A [JSON Path](#json_path) query string. If present, the result of the JSON Path query should be a list of messages. This is useful if you have a JSON document which contains a list or array of messages embedded in it, and you do not have another means of splitting the message. * A field called `timestamp` is expected to exist and, if it does not, then current time is inserted. - + +## Parser Error Routing + +Currently, we have a few mechanisms for either deferring processing of +messages or marking messages as invalid. + +### Invalidation Errors + +There are two reasons a message will be marked as invalid: +* Fail [global validation](../metron-common#validation-framework) +* Fail the parser's validate function (generally that means to not have a `timestamp` field or a `original_string` field. + +Those messages which are marked as invalid are sent to the error queue +with an indication that they are invalid in the error message. + +### Parser Errors + +Errors, which are defined as unexpected exceptions happening during the +parse, are sent along to the error queue with a message indicating that +there was an error in parse along with a stacktrace. This is to +distinguish from the invalid messages. + +## Filtered + +One can also filter a message by specifying a `filterClassName` in the +parser config. Filtered messages are just dropped rather than passed +through. + ## Parser Architecture ![Architecture](parser_arch.png) http://git-wip-us.apache.org/repos/asf/metron/blob/0d847cf5/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java -- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 6fc4ed7..e996f14 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -193,23 +193,28 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable { message.put(Constants.GUID, UUID.randomUUID().toString()); } - if (parser.validate(message) && (filter == null || filter.emitTuple(message, stellarContext))) { -numWritten++; -List failedValidators = getFailedValidators(message, fieldValidations); -if(failedValidators.size() > 0) { + if (filter == null || filter.emitTuple(message, stellarContext)) { +boolean isInvalid = !parser.validate(message); +List failedValidators = null; +if(!isInvalid) { + failedValidators = getFailedValidators(message, fieldValidations); + isInvalid = !failedValidators.isEmpty(); +} +if( isInvalid) { MetronError error = new MetronError() .withErrorType(Constants.ErrorType.PARSER_INVALID) .withSensorType(getSensorType()) .addRawMessage(message); - Set errorFields = failedValidators.stream() + Set errorFields = failedValidators == null?null:failedValidators.stream()
[49/52] [abbrv] metron git commit: METRON-1527: Remove dead test file sitting in source folder (mmiklavc via mmiklavc) closes apache/metron#994
METRON-1527: Remove dead test file sitting in source folder (mmiklavc via mmiklavc) closes apache/metron#994 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08252f59 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08252f59 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08252f59 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 08252f59b2df1f4ef149b899c944f6a8a0aa049e Parents: 82212ba Author: mmiklavcAuthored: Tue Apr 17 10:55:08 2018 -0600 Committer: Michael Miklavcic Committed: Tue Apr 17 10:55:08 2018 -0600 -- .../org/apache/metron/common/writer/test.json | 31 1 file changed, 31 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/08252f59/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/test.json -- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/test.json b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/test.json deleted file mode 100644 index 023cd63..000 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/test.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "index": "bro", - "batchSize": 1, - "enrichment" : { -"fieldMap": -{ - "geo": ["ip_dst_addr", "ip_src_addr"], - "host": ["host"] -} - }, - "threatIntel" : { -"fieldMap": -{ - "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] -}, -"fieldToTypeMap": -{ - "ip_src_addr" : ["malicious_ip"], - "ip_dst_addr" : ["malicious_ip"] -}, -"triageConfig" : { - "riskLevelRules" : [ -{ - "rule" : "ip_src_addr == '31.24.30.31'", - "score" : 10 -} - ], - "aggregator" : "MAX" -} - } -}
[32/52] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0ab39a32 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0ab39a32 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0ab39a32 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 0ab39a32b61104812a6a9e69e999e7053e3e7e90 Parents: 0d847cf Author: mmiklavcAuthored: Thu Apr 5 09:07:48 2018 -0600 Committer: Michael Miklavcic Committed: Thu Apr 5 09:07:48 2018 -0600 -- .../roles/ambari_master/defaults/main.yml | 2 + .../ambari_master/tasks/elasticsearch_mpack.yml | 26 ++ .../ansible/roles/ambari_master/tasks/main.yml | 3 +- .../roles/ambari_master/tasks/metron_mpack.yml | 26 ++ .../ansible/roles/ambari_master/tasks/mpack.yml | 26 -- .../roles/load_web_templates/tasks/main.yml | 2 +- .../manual-install/Manual_Install_CentOS6.md| 4 +- metron-deployment/packaging/ambari/README.md| 25 +- .../ambari/elasticsearch-mpack/README.md| 62 + .../ambari/elasticsearch-mpack/pom.xml | 95 +++ .../src/main/assemblies/elasticsearch-mpack.xml | 43 +++ .../ELASTICSEARCH/5.6.2/metainfo.xml| 29 ++ .../ELASTICSEARCH/5.6.2/repos/repoinfo.xml | 45 .../addon-services/KIBANA/5.6.2/metainfo.xml| 30 +++ .../KIBANA/5.6.2/quicklinks/quicklinks.json | 27 ++ .../KIBANA/5.6.2/repos/repoinfo.xml | 60 + .../5.6.2/configuration/elastic-env.xml | 86 ++ .../5.6.2/configuration/elastic-jvm-options.xml | 144 ++ .../5.6.2/configuration/elastic-site.xml| 198 ++ .../5.6.2/configuration/elastic-sysconfig.xml | 97 +++ .../5.6.2/configuration/elastic-systemd.xml | 30 +++ .../ELASTICSEARCH/5.6.2/metainfo.xml| 97 +++ .../5.6.2/package/scripts/elastic_commands.py | 266 +++ .../5.6.2/package/scripts/elastic_master.py | 72 + .../5.6.2/package/scripts/elastic_slave.py | 71 + .../5.6.2/package/scripts/params.py | 108 .../5.6.2/package/scripts/properties_config.py | 34 +++ .../5.6.2/package/scripts/service_check.py | 114 .../5.6.2/package/scripts/status_params.py | 27 ++ .../templates/elasticsearch.master.yaml.j2 | 77 ++ .../templates/elasticsearch.slave.yaml.j2 | 78 ++ .../templates/elasticsearch_limits.conf.j2 | 20 ++ .../5.6.2/quicklinks/quicklinks.json| 43 +++ .../ELASTICSEARCH/5.6.2/role_command_order.json | 8 + .../KIBANA/5.6.2/configuration/kibana-env.xml | 72 + .../KIBANA/5.6.2/configuration/kibana-site.xml | 113 .../common-services/KIBANA/5.6.2/metainfo.xml | 84 ++ .../KIBANA/5.6.2/package/scripts/common.py | 56 .../5.6.2/package/scripts/kibana_master.py | 81 ++ .../KIBANA/5.6.2/package/scripts/params.py | 50 .../KIBANA/5.6.2/quicklinks/quicklinks.json | 28 ++ .../src/main/resources/mpack.json | 76 ++ .../packaging/ambari/metron-mpack/README.md | 20 +- .../src/main/assemblies/metron-mpack.xml| 14 - .../ELASTICSEARCH/5.6.2/metainfo.xml| 29 -- .../ELASTICSEARCH/5.6.2/repos/repoinfo.xml | 45 .../addon-services/KIBANA/5.6.2/metainfo.xml| 30 --- .../KIBANA/5.6.2/quicklinks/quicklinks.json | 27 -- .../KIBANA/5.6.2/repos/repoinfo.xml | 60 - .../5.6.2/configuration/elastic-env.xml | 86 -- .../5.6.2/configuration/elastic-jvm-options.xml | 144 -- .../5.6.2/configuration/elastic-site.xml| 198 -- .../5.6.2/configuration/elastic-sysconfig.xml | 97 --- .../5.6.2/configuration/elastic-systemd.xml | 30 --- .../ELASTICSEARCH/5.6.2/metainfo.xml| 97 --- .../5.6.2/package/scripts/elastic_commands.py | 266 --- .../5.6.2/package/scripts/elastic_master.py | 72 - .../5.6.2/package/scripts/elastic_slave.py | 71 - .../5.6.2/package/scripts/params.py | 108 .../5.6.2/package/scripts/properties_config.py | 34 --- .../5.6.2/package/scripts/service_check.py | 114 .../5.6.2/package/scripts/status_params.py | 27 -- .../templates/elasticsearch.master.yaml.j2 | 77 -- .../templates/elasticsearch.slave.yaml.j2 | 78 -- .../templates/elasticsearch_limits.conf.j2 | 20 -- .../5.6.2/quicklinks/quicklinks.json| 43 --- .../ELASTICSEARCH/5.6.2/role_command_order.json | 8 - .../KIBANA/5.6.2/configuration/kibana-env.xml | 72 - .../KIBANA/5.6.2/configuration/kibana-site.xml | 113
[48/52] [abbrv] metron git commit: METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984
METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/82212ba8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/82212ba8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/82212ba8 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 82212ba818a0ef3b92505e8d144487c69a8d4a44 Parents: 3fcbf8b Author: nickwallenAuthored: Tue Apr 17 09:43:16 2018 -0400 Committer: nickallen Committed: Tue Apr 17 09:43:16 2018 -0400 -- .../packaging/ambari/metron-mpack/pom.xml | 7 +- .../configuration/metron-enrichment-env.xml | 130 +++- .../package/scripts/enrichment_commands.py | 20 +- .../package/scripts/enrichment_master.py| 12 +- .../package/scripts/params/params_linux.py | 19 +- .../enrichment-splitjoin.properties.j2 | 63 ++ .../templates/enrichment-unified.properties.j2 | 60 ++ .../METRON/CURRENT/themes/metron_theme.json | 151 - .../docker/rpm-docker/SPECS/metron.spec | 5 +- .../main/config/enrichment-splitjoin.properties | 63 ++ .../config/enrichment-splitjoin.properties.j2 | 63 ++ .../main/config/enrichment-unified.properties | 69 +++ .../config/enrichment-unified.properties.j2 | 60 ++ .../src/main/config/enrichment.properties | 64 -- .../src/main/config/enrichment.properties.j2| 63 -- .../main/flux/enrichment/remote-splitjoin.yaml | 590 +++ .../main/flux/enrichment/remote-unified.yaml| 71 ++- .../src/main/flux/enrichment/remote.yaml| 590 --- .../main/scripts/start_enrichment_topology.sh | 16 +- .../integration/EnrichmentIntegrationTest.java | 61 +- .../UnifiedEnrichmentIntegrationTest.java | 71 +++ 21 files changed, 1433 insertions(+), 815 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/pom.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml index 491e8dd..1843eb7 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml @@ -110,7 +110,8 @@ ${basedir}/../../../../metron-platform/metron-enrichment/src/main/config - enrichment.properties.j2 + enrichment-splitjoin.properties.j2 + enrichment-unified.properties.j2 false @@ -171,8 +172,10 @@ ${basedir}/src/main/resources/common-services/METRON/CURRENT/package/templates -enrichment.properties.j2 + enrichment-unified.properties.j2 + enrichment-splitjoin.properties.j2 elasticsearch.properties.j2 +hdfs.properties.j2 http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml index 9737660..81b135c 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml @@ -18,6 +18,10 @@ limitations under the License. --> + + geoip_url http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz @@ -33,6 +37,10 @@ content + + enrichment_kafka_start Enrichment Topology Spout Offset @@ -81,6 +89,10 @@ indexing Threat Intel Error Topic + + enrichment_hbase_table
[44/52] [abbrv] metron git commit: METRON-1519 Indexing Error Topic Property Not Displayed in MPack (nickwallen) closes apache/metron#987
METRON-1519 Indexing Error Topic Property Not Displayed in MPack (nickwallen) closes apache/metron#987 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f8b7c585 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f8b7c585 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f8b7c585 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: f8b7c5852310a088bd15ca5d21ba4f98b51521be Parents: bfe90ef Author: nickwallenAuthored: Fri Apr 13 14:47:17 2018 -0400 Committer: nickallen Committed: Fri Apr 13 14:47:17 2018 -0400 -- .../common-services/METRON/CURRENT/themes/metron_theme.json| 6 -- 1 file changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/f8b7c585/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 234b551..364b3ef 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -514,6 +514,10 @@ "subsection-name": "subsection-indexing-kafka" }, { + "config": "metron-indexing-env/indexing_error_topic", + "subsection-name": "subsection-indexing-kafka" +}, +{ "config": "metron-indexing-env/update_hbase_table", "subsection-name": "subsection-indexing-update" }, @@ -553,7 +557,6 @@ "config": "metron-indexing-env/batch_indexing_topology_max_spout_pending", "subsection-name": "subsection-indexing-hdfs" }, - { "config": "metron-indexing-env/ra_indexing_kafka_spout_parallelism", "subsection-name": "subsection-indexing-storm" @@ -562,7 +565,6 @@ "config": "metron-indexing-env/batch_indexing_kafka_spout_parallelism", "subsection-name": "subsection-indexing-hdfs" }, - { "config": "metron-indexing-env/ra_indexing_writer_parallelism", "subsection-name": "subsection-indexing-storm"
[41/52] [abbrv] metron git commit: METRON-1521: JSONMapParser is no longer serializable closes apache/incubator-metron#991
METRON-1521: JSONMapParser is no longer serializable closes apache/incubator-metron#991 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/eb5b2d42 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/eb5b2d42 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/eb5b2d42 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: eb5b2d42b83901c05c15e00b9c3c5bce5dd72ab9 Parents: ea6992f Author: cstellaAuthored: Thu Apr 12 09:33:12 2018 -0400 Committer: cstella Committed: Thu Apr 12 09:33:12 2018 -0400 -- .../java/org/apache/metron/parsers/json/JSONMapParser.java | 4 ++-- .../org/apache/metron/parsers/integration/ParserDriver.java| 6 +- 2 files changed, 7 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/eb5b2d42/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java -- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java index bddf35d..f5d67f9 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java @@ -89,8 +89,7 @@ public class JSONMapParser extends BasicParser { public static final String JSONP_QUERY = "jsonpQuery"; private MapStrategy mapStrategy = MapStrategy.DROP; - private TypeRef >> typeRef = new TypeRef
>>() { - }; + private transient TypeRef
>> typeRef = null; private String jsonpQuery = null; @@ -99,6 +98,7 @@ public class JSONMapParser extends BasicParser { String strategyStr = (String) config.getOrDefault(MAP_STRATEGY_CONFIG, MapStrategy.DROP.name()); mapStrategy = MapStrategy.valueOf(strategyStr); if (config.containsKey(JSONP_QUERY)) { + typeRef = new TypeRef
>>() { }; jsonpQuery = (String) config.get(JSONP_QUERY); Configuration.setDefaults(new Configuration.Defaults() { http://git-wip-us.apache.org/repos/asf/metron/blob/eb5b2d42/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java -- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java index deb0217..b03ea80 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java @@ -18,6 +18,7 @@ package org.apache.metron.parsers.integration; import com.google.common.collect.ImmutableList; +import org.apache.commons.lang.SerializationUtils; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.configuration.FieldValidator; import org.apache.metron.common.configuration.ParserConfigurations; @@ -42,6 +43,7 @@ import org.mockito.Matchers; import java.io.Closeable; import java.io.IOException; +import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -54,7 +56,7 @@ import static org.mockito.Mockito.when; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ParserDriver { +public class ParserDriver implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(ParserBolt.class); public static class CollectingWriter implements MessageWriter{ List
output; @@ -151,6 +153,8 @@ public class ParserDriver { public ProcessorResult > run(List
in) { ShimParserBolt bolt = new ShimParserBolt(new ArrayList<>()); +byte[] b = SerializationUtils.serialize(bolt); +ShimParserBolt b2 = (ShimParserBolt) SerializationUtils.deserialize(b); OutputCollector collector = mock(OutputCollector.class); bolt.prepare(null, null, collector); for(byte[] record : in) {
[20/52] [abbrv] metron git commit: METRON-590 Enable Use of Event Time in Profiler (nickwallen) closes apache/metron#965
http://git-wip-us.apache.org/repos/asf/metron/blob/3083b471/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/StandAloneProfilerTest.java -- diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/StandAloneProfilerTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/StandAloneProfilerTest.java new file mode 100644 index 000..2269c86 --- /dev/null +++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/StandAloneProfilerTest.java @@ -0,0 +1,255 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.metron.profiler; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.configuration.profiler.ProfilerConfig; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.stellar.dsl.Context; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests the StandAloneProfiler class. + */ +public class StandAloneProfilerTest { + + /** + * { + * "profiles": [ + * ] + * } + */ + @Multiline + private String noProfiles; + + /** + * { + * "profiles": [ + * { + *"profile": "profile1", + *"foreach": "'global'", + *"init": { "count": 0 }, + *"update": { "count": "count + 1" }, + *"result": "count" + * } + * ] + * } + */ + @Multiline + private String oneProfile; + + /** + * { + * "profiles": [ + * { + *"profile": "profile1", + *"foreach": "'global1'", + *"result": "'result'" + * }, + * { + *"profile": "profile2", + *"foreach": "'global2'", + *"result": "'result'" + * } + * ] + * } + */ + @Multiline + private String twoProfiles; + + /** + * { + * "ip_src_addr": "10.0.0.1", + * "ip_dst_addr": "10.0.0.20", + * "protocol": "HTTP", + * "timestamp": 2, + * } + */ + @Multiline + private String messageJson; + + private JSONObject message; + + private long periodDurationMillis = TimeUnit.MINUTES.toMillis(15); + + private Context context = Context.EMPTY_CONTEXT(); + + @Before + public void setup() throws Exception { + +// parse the input message +JSONParser parser = new JSONParser(); +message = (JSONObject) parser.parse(messageJson); + } + + @Test + public void testWithOneProfile() throws Exception { + +StandAloneProfiler profiler = createProfiler(oneProfile); +profiler.apply(message); +profiler.apply(message); +profiler.apply(message); + +List measurements = profiler.flush(); +assertEquals(1, measurements.size()); + +// expect 1 measurement for the 1 profile that has been defined +ProfileMeasurement m = measurements.get(0); +assertEquals("profile1", m.getProfileName()); +assertEquals(3, m.getProfileValue()); + } + + + @Test + public void testWithTwoProfiles() throws Exception { + +StandAloneProfiler profiler = createProfiler(twoProfiles); +profiler.apply(message); +profiler.apply(message); +profiler.apply(message); + +List measurements = profiler.flush(); +assertEquals(2, measurements.size()); + +// expect 2 measurements, 1 for each profile +List expected = Arrays.asList(new String[] { "profile1", "profile2" }); +{ + ProfileMeasurement m = measurements.get(0); + assertTrue(expected.contains(m.getProfileName())); + assertEquals("result", m.getProfileValue()); +} +{ + ProfileMeasurement m = measurements.get(1); + assertTrue(expected.contains(m.getProfileName())); + assertEquals("result", m.getProfileValue()); +} + } + + /** + * The message count and route count will
[05/52] [abbrv] metron git commit: METRON-1463: Adjust the groupings and shuffles in enrichment to be more efficient closes apache/metron#944
METRON-1463: Adjust the groupings and shuffles in enrichment to be more efficient closes apache/metron#944 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9fb0d06e Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9fb0d06e Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9fb0d06e Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 9fb0d06e644597032da58c5df020a841b93d19c0 Parents: 1d95b83 Author: cstellaAuthored: Wed Mar 7 09:21:52 2018 -0500 Committer: cstella Committed: Wed Mar 7 09:21:52 2018 -0500 -- .../src/main/flux/enrichment/remote.yaml| 42 +--- .../enrichment/bolt/EnrichmentJoinBolt.java | 4 +- .../enrichment/bolt/EnrichmentSplitterBolt.java | 4 +- .../enrichment/bolt/GenericEnrichmentBolt.java | 2 +- .../enrichment/bolt/ThreatIntelJoinBolt.java| 4 +- .../bolt/ThreatIntelSplitterBolt.java | 4 +- 6 files changed, 28 insertions(+), 32 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml -- diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml index 7ee1b2e..fd7ceff 100644 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml @@ -411,7 +411,7 @@ streams: from: "kafkaSpout" to: "enrichmentSplitBolt" grouping: -type: SHUFFLE +type: LOCAL_OR_SHUFFLE #enrichment - name: "enrichmentSplit -> host" @@ -420,14 +420,15 @@ streams: grouping: streamId: "host" type: FIELDS -args: ["key"] +args: ["message"] + - name: "enrichmentSplit -> geo" from: "enrichmentSplitBolt" to: "geoEnrichmentBolt" grouping: streamId: "geo" type: FIELDS -args: ["key"] +args: ["message"] - name: "enrichmentSplit -> stellar" from: "enrichmentSplitBolt" @@ -435,7 +436,7 @@ streams: grouping: streamId: "stellar" type: FIELDS -args: ["key"] +args: ["message"] - name: "enrichmentSplit -> simpleHBaseEnrichmentBolt" @@ -444,7 +445,7 @@ streams: grouping: streamId: "hbaseEnrichment" type: FIELDS -args: ["key"] +args: ["message"] - name: "splitter -> join" from: "enrichmentSplitBolt" @@ -453,6 +454,7 @@ streams: streamId: "message" type: FIELDS args: ["key"] + - name: "geo -> join" from: "geoEnrichmentBolt" to: "enrichmentJoinBolt" @@ -460,6 +462,7 @@ streams: streamId: "geo" type: FIELDS args: ["key"] + - name: "stellar -> join" from: "stellarEnrichmentBolt" to: "enrichmentJoinBolt" @@ -468,8 +471,6 @@ streams: type: FIELDS args: ["key"] - - - name: "simpleHBaseEnrichmentBolt -> join" from: "simpleHBaseEnrichmentBolt" to: "enrichmentJoinBolt" @@ -477,6 +478,7 @@ streams: streamId: "hbaseEnrichment" type: FIELDS args: ["key"] + - name: "host -> join" from: "hostEnrichmentBolt" to: "enrichmentJoinBolt" @@ -491,32 +493,28 @@ streams: to: "enrichmentErrorOutputBolt" grouping: streamId: "error" -type: FIELDS -args: ["message"] +type: LOCAL_OR_SHUFFLE - name: "stellarEnrichmentBolt -> enrichmentErrorOutputBolt" from: "stellarEnrichmentBolt" to: "enrichmentErrorOutputBolt" grouping: streamId: "error" -type: FIELDS -args: ["message"] +type: LOCAL_OR_SHUFFLE - name: "hostEnrichmentBolt -> enrichmentErrorOutputBolt" from: "hostEnrichmentBolt" to: "enrichmentErrorOutputBolt" grouping: streamId: "error" -type: FIELDS -args: ["message"] +type: LOCAL_OR_SHUFFLE - name: "simpleHBaseEnrichmentBolt -> enrichmentErrorOutputBolt" from: "simpleHBaseEnrichmentBolt" to: "enrichmentErrorOutputBolt" grouping: streamId: "error" -type: FIELDS -args: ["message"] +type: LOCAL_OR_SHUFFLE #threat intel - name: "enrichmentJoin -> threatSplit"
[06/52] [abbrv] metron git commit: METRON-1467: Replace guava caches in places where the keyspace might be large closes apache/metron#947
METRON-1467: Replace guava caches in places where the keyspace might be large closes apache/metron#947 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/abb152b8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/abb152b8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/abb152b8 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: abb152b83631001ad067849dfaefd2d4e3b0cdb4 Parents: 9fb0d06 Author: cstellaAuthored: Wed Mar 7 11:20:56 2018 -0500 Committer: cstella Committed: Wed Mar 7 11:20:56 2018 -0500 -- metron-interface/metron-rest/pom.xml| 5 +++ metron-platform/metron-enrichment/pom.xml | 2 +- .../enrichment/bolt/GenericEnrichmentBolt.java | 19 --- .../apache/metron/enrichment/bolt/JoinBolt.java | 34 +--- .../bolt/GenericEnrichmentBoltTest.java | 2 +- .../metron/enrichment/bolt/JoinBoltTest.java| 7 ++-- metron-stellar/stellar-common/pom.xml | 5 +++ .../stellar/common/BaseStellarProcessor.java| 31 +++--- .../stellar/dsl/functions/DateFunctions.java| 8 ++--- pom.xml | 1 + 10 files changed, 56 insertions(+), 58 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/abb152b8/metron-interface/metron-rest/pom.xml -- diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml index 44bad97..dcdea2b 100644 --- a/metron-interface/metron-rest/pom.xml +++ b/metron-interface/metron-rest/pom.xml @@ -38,6 +38,11 @@ 2.6.4 + + com.github.ben-manes.caffeine + caffeine + ${global_caffeine_version} + org.springframework.kafka spring-kafka http://git-wip-us.apache.org/repos/asf/metron/blob/abb152b8/metron-platform/metron-enrichment/pom.xml -- diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml index bcfb41b..1dffd8b 100644 --- a/metron-platform/metron-enrichment/pom.xml +++ b/metron-platform/metron-enrichment/pom.xml @@ -70,7 +70,7 @@ com.github.ben-manes.caffeine caffeine - 2.6.2 + ${global_caffeine_version} http://git-wip-us.apache.org/repos/asf/metron/blob/abb152b8/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java -- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java index 7d67d2d..0677453 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java @@ -18,13 +18,13 @@ package org.apache.metron.enrichment.bolt; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import java.util.HashSet; import java.util.Map; import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.StringUtils; + +import com.github.benmanes.caffeine.cache.CacheLoader; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import org.apache.metron.common.Constants; import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt; import org.apache.metron.common.configuration.ConfigurationType; @@ -146,13 +146,8 @@ public class GenericEnrichmentBolt extends ConfiguredEnrichmentBolt { throw new IllegalStateException("MAX_TIME_RETAIN_MINUTES must be specified"); if (this.adapter == null) throw new IllegalStateException("Adapter must be specified"); -loader = new CacheLoader () { - @Override - public JSONObject load(CacheKey key) throws Exception { -return adapter.enrich(key); - } -}; -cache = CacheBuilder.newBuilder().maximumSize(maxCacheSize) +loader = key -> adapter.enrich(key); +cache = Caffeine.newBuilder().maximumSize(maxCacheSize) .expireAfterWrite(maxTimeRetain, TimeUnit.MINUTES) .build(loader); boolean success = adapter.initializeAdapter(getConfigurations().getGlobalConfig()); @@ -228,7 +223,7 @@ public class GenericEnrichmentBolt extends ConfiguredEnrichmentBolt {
[37/52] [abbrv] metron git commit: METRON-1518 Build Failure When Using Profile HDP-2.5.0.0 (nickwallen) closes apache/metron#986
METRON-1518 Build Failure When Using Profile HDP-2.5.0.0 (nickwallen) closes apache/metron#986 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ed50d48b Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ed50d48b Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ed50d48b Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: ed50d48bb47cf3f301884f6e18fe4efc8c1b91f1 Parents: a8b555d Author: nickwallenAuthored: Tue Apr 10 17:16:20 2018 -0400 Committer: nickallen Committed: Tue Apr 10 17:16:20 2018 -0400 -- .../profiler/bolt/ProfileBuilderBolt.java | 51 +--- 1 file changed, 11 insertions(+), 40 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/ed50d48b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java -- diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java index fb3d2d0..ca02b58 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java @@ -42,13 +42,11 @@ import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.zookeeper.SimpleEventListener; import org.apache.metron.zookeeper.ZKCache; -import org.apache.storm.StormTimer; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseWindowedBolt; import org.apache.storm.tuple.Tuple; -import org.apache.storm.utils.Utils; import org.apache.storm.windowing.TupleWindow; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -59,9 +57,9 @@ import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.function.Supplier; import static java.lang.String.format; import static org.apache.metron.profiler.bolt.ProfileSplitterBolt.ENTITY_TUPLE_FIELD; @@ -155,8 +153,8 @@ public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable { private FlushSignal activeFlushSignal; /** - * A timer that flushes expired profiles on a regular interval. The expired profiles - * are flushed on a separate thread. + * An executor that flushes expired profiles at a regular interval on a separate + * thread. * * Flushing expired profiles ensures that any profiles that stop receiving messages * for an extended period of time will continue to be flushed. @@ -164,7 +162,7 @@ public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable { * This introduces concurrency issues as the bolt is no longer single threaded. Due * to this, all access to the {@code MessageDistributor} needs to be protected. */ - private StormTimer expiredFlushTimer; + private transient ScheduledExecutorService flushExpiredExecutor; public ProfileBuilderBolt() { this.emitters = new ArrayList<>(); @@ -202,7 +200,7 @@ public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable { this.configurations = new ProfilerConfigurations(); this.activeFlushSignal = new FixedFrequencyFlushSignal(periodDurationMillis); setupZookeeper(); -startExpiredFlushTimer(); +startFlushingExpiredProfiles(); } @Override @@ -210,7 +208,7 @@ public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable { try { zookeeperCache.close(); zookeeperClient.close(); - expiredFlushTimer.close(); + flushExpiredExecutor.shutdown(); } catch(Throwable e) { LOG.error("Exception when cleaning up", e); @@ -421,39 +419,12 @@ public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable { } /** - * Converts milliseconds to seconds and handles an ugly cast. - * - * @param millis Duration in milliseconds. - * @return Duration in seconds. - */ - private int toSeconds(long millis) { -return (int) TimeUnit.MILLISECONDS.toSeconds(millis); - } - - /** - * Creates a timer that regularly flushes expired profiles on a separate thread. - */ - private void
[38/52] [abbrv] metron git commit: METRON-1510 Update Metron website to include info about github update subscription (anandsubbu) closes apache/metron#981
METRON-1510 Update Metron website to include info about github update subscription (anandsubbu) closes apache/metron#981 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/438893b7 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/438893b7 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/438893b7 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 438893b78c852e34d7edb965840500a78503d299 Parents: ed50d48 Author: anandsubbuAuthored: Wed Apr 11 11:03:37 2018 +0530 Committer: anandsubbu Committed: Wed Apr 11 11:03:37 2018 +0530 -- site/community/index.md | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/438893b7/site/community/index.md -- diff --git a/site/community/index.md b/site/community/index.md index 7c09b14..b222ba3 100644 --- a/site/community/index.md +++ b/site/community/index.md @@ -121,11 +121,15 @@ title: Apache Metron Community General & Public Discussion [mailto:user-subscr...@metron.apache.org;>Subscribe] [mailto:user-unsubscr...@metron.apache.org;>Unsubscribe] -[http://mail-archives.apache.org/mod_mbox/metron-user/;>Archives] +[https://lists.apache.org/list.html?u...@metron.apache.org;>Archives] Code & Documentation Change [mailto:dev-subscr...@metron.apache.org;>Subscribe] [mailto:dev-unsubscr...@metron.apache.org;>Unsubscribe] -[http://mail-archives.apache.org/mod_mbox/metron-dev/;>Archives] +[https://lists.apache.org/list.html?d...@metron.apache.org;>Archives] + Issues & Github updates +[mailto:issues-subscr...@metron.apache.org;>Subscribe] +[mailto:issues-unsubscr...@metron.apache.org;>Unsubscribe] +[https://lists.apache.org/list.html?iss...@metron.apache.org;>Archives]
[02/52] [abbrv] metron git commit: METRON-1460: Create a complementary non-split-join enrichment topology closes apache/metron#940
http://git-wip-us.apache.org/repos/asf/metron/blob/1d95b831/metron-platform/metron-enrichment/unified_enrichment_arch.svg -- diff --git a/metron-platform/metron-enrichment/unified_enrichment_arch.svg b/metron-platform/metron-enrichment/unified_enrichment_arch.svg new file mode 100644 index 000..e42d394 --- /dev/null +++ b/metron-platform/metron-enrichment/unified_enrichment_arch.svg @@ -0,0 +1,14 @@ + +http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd;> +http://www.w3.org/2000/svg; xmlns:xlink="http://www.w3.org/1999/xlink; width="611px" height="671px" version="1.1" content="mxfile userAgent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36 version=8.3.6 editor=www.draw.io type=devicediagram id=831675e5-f95b-138c-0e1a-018afd21d13b name=Page-17b3Z0qvIkiX8NHn5tzEKuGSeQWKGO+Z5nnn6hm9nnqqsk1XVbXXqtP1l2pa5t4QQBOER7ss9llb8BtPtwU/hUKh9kja/QUBy/AYzv0EQDoH338+B89cBBEd/HcinMvl1CPyXA2Z5pb8fBH4/upZJOv/pxKXvm6Uc/nww7rsujZc/HQunqd//fFrWN3++6xDm6d8dMOOw+fujbpksxR+P9fqX40Ja5sUfdwZfxK9PojCu86lfu9/v9xsEZz9/fn3chn9c6/cHnYsw6fd/dQhmf4Ppqe+XX6/ag06bp2v/6LZf3+P+nU//1u4p7Zb/ky/8bpYtbNb0jxb/tGs5/+iLYmmb+xX4G0z9rbXA/SYJ5yJNfn8TNmXe3a/j+77pdB/Y0mkp7w4lf/+gLZPkueJzjeG5cnvkz9D5X7/MBf3697nsM2SA//XCntfPeIKe63f9Ehe/3+une3/u/NOmZerrv1np7kAqK5uG7pt++mk/jKZEnMDP8b5bfh9qP9f89c2/OjMpp3tUlf3T8LlfnytT v3fU/Vjp8e92Nvg3E94zI+3bdJnO+5Tfv0Cg/+v3/v59WoAI8uv9/i+D7I8pUPzr8QX8fjD8fVznf7v2v9j2fvG7ef/a1K+/MzXbTWVctE/r/63R/7BS2f7MlL+ZUwmjtHn3c/l770T9svTtfULzfED9bfT/0atd36V/MRqWfngGzTz8mrxZeTz2pH7uRv5xFPjjyP06CZfwN5j89Rbihi7/DaJLh9KNHZD5vCfvP5ppF6yd3684/P6LOmjSv/+lr1f3G0Rd98ugARr24xgImYIJjRqRAbfpmiWyDDXwEm1neJ8I+9e8SDErxmQODJIkeexK26ReD7WmfXSWiWJQzt/GJe84+TpdqQtDEmdWiOoUS6NdOMTzYqgR8T27fOOZ9UbTfn3kr6OzyqMy4C093cXpCSyAp/t+dfaaZALEiKR0Fh1zieNaYGZ93y6Dwx6jPhZ+/qNw/P43bp2Gvm7Dch1rle9dqNmr1HahFy5aPmKKqTpOQCYHV656gC2aPhW0g+9vHyeuvgXulIbCdTJlndK1+txmv0c2Fwkf1E7lWL/fhukmLJeFZyF7f/J5R/PTKbeFGs9wqLdLtbLs4QbdE7U6TMkl6acuHBSdshUvhfrTMON1DdcuC1qngfe4pCb7KiQays3HdIxiHUEpBwHcaqwPHHErQuR9Erk9T+vJJ3ikpiCbOBY6DRZSFToRpZ1mzGmc7pTnkulJEYd4qwYkqubD6v2tBLC1dzudxnZfyDHbzpk+2kYZI5/okq8PHoOStMHnMYAxz11Oz9sjUuD5NOkdXzrur3FVgJtL8cog0LnfuTglTI8JanjWKYtDNDZUYBw8S0m4h+EHgkyIninMfuySld59pnl6wkrrWHG2i7nMz4TM+4sndGmWVXxngC22EeDTh+iu3lemtNf1AYmnOVoBXdq6qsL9GlogQV7uOUjZHgOMoNhqQI+93PsAtvCiG1zpB8gVqCde1X3Mo6V7VFBowwYto1Xku4TWD b1Y/CC54UwHOktWcn21pYL26j3PuZSBu+3daQbm9rc9KMlkhbHMaly35ACsJ2OKFka9KgK/Nlxh3gRTKyhLWIXgNkmVhLa79/fX6K3ljh1DGQTpipm/j3AApJvxjKDQ5+6f5deQJbN3iWN371Bq4M8YBrg6rKonR58kxAJvOP0ZE3yd3Gc09uv1/swbaQA8TxSAJAjHkjWFo0V4goufj1l2BW8DysY844VCaABj5ftVsAWEXpPnjLi6V0Tt1DogVHArnxU+kvCDIxTje71P7AbFn+gI3KtNyIdnMqXRBEMDjrB0ksqCdym+TSLnLLsg3Jd1Epqu4Whb3MW8UlKahHZvgckdRWymMV0J/+lPoaWHZ4qIWr6HXiNFmPDxDxOLF9J4uiVTCTg7IP/EmDJUAntMgKS/H4DbDjgfnzk20f7tyblXH6q820Uod6zbVSDaXB80weB29syhLsD4umnMVUrJynjOf0usrqrGpYG5XgIC1rf8WxiQ8BlQW3AC+P6WZ0uFa5wtpV7rI0KRadcEkHheLmfWt7xH5dszcHg6k/5pisQc0HJXVXFENRblazVXfJ5Ouu+eRR3WjQCS0LCmGSGuwJVDewr64tS36IPvtd3H+zTANnF1mj9948Qm4Joju7qR77SFpHloEdLPhOL2GoTfHitWhnhOJEvJsggYDpzWsptDpmwOCLkMsm5nt6enzM8bdYwcjkS9MAZQTyBmczAd8UXE4RVuJ3mv29d+D9K2Ofx4TZXope66td8eTleAushSpuRTY2IjoAva3X+G4jPdKeyA91+uVWEJhjxCDNhV/7RuB0J01Vwt4K9Ph4DACaXfUquPfcDeLToQg+BSldZgrkS6r0dkJsXqgu0LIqm/2lkJQjboda7dxWcKdI+tztttnFE3x1zQZGObX4VLx4lYAKiwPc6g0FSpt5OIuEzw6cgPGJSHWWf+roqYEwf0ceBRxq/3/Tj5cQbv5rlucAxcSxj2SfXsO6CJMX LfgoTr1ek5Hlu7uP/pydGJku1zBweroGZ00d8yxD5+KyGM4KcveKqgWj4Mi/72muDbYRHpjSV5dCJIKjAe+CmtTJoXdRuibJ/GLrdeyeU2mDVb2vg209gG3HaOF3e9u8elshdM8Z04ChCP2xxfMQt75URLuJ3woWz/FYTEExToVCo6dsxtWwl8oZfbT0MrTaQAUpKqUO6GraFfTesBxCfh+tnqYtuBisdfePqV2eg99bks4525C+NK7jfUiiAxm1VUTPTAe1ObUNU11aCvzwDACuOuceKV9Mk7pW6dYH2j2rsLWpdhXiehbEcgFldUuTm5Hn1HzD+hpDu3smqPgwxJHctWhZsS5f1+hx9D74PhMBjbhq9D3xisTfmnOXqJrvWDIkbqczEwjnze8oV8xrCwE+meChRX0fvA1hr3pl3qGX9Ihby6fJCC+KqrQvDR28u6TgBh7+lN3DN1gF5iwIe6XTUYUTmCThtAz8y0jNa8TweY+liQOsasK1I8SOHaHsAEz+/opNTuNIImZTwBoipEBaTHdc8PRX+9ZSSczbBTnOlis4rK/JiZTvc85tdsUQUCacxSadgP/Jjn/JOg9nsrsnkZp9cF84VHf/Kn9y7+QTAc/qonxkVbikBNBFGp+vmQP5UdG5yMZdnG9evhGtGGd0lKR2ZegGz0feMpyVL3wDxx6gYyn5ruFNXeMB3q7+g7gVPDy3R8fGqzC2tDu0SuuYOwfsViv2Sh0155LDXnBXY9jYA7ChRAOq6zNjZghFy1S1noCM7A+34G9Zk1gfDMxpDhVDYmr6vVvOltq9l6eyjL5g4V8bwHSjhejIiWkAe1lcrNfJ64sI0xAwbIYLR9Zr9kuznvmMJVD/T2rPid39flyB25yvTKwa6VwDZO5c8QCBt8+rzDy4F/9/2WLuDrJa3FsU9rb/FaQTIKjiKIZtR2FxOVZ/6MuVZ4r3nXcB/QgswA3XgDRiLquKdkF+xa8DgT1cIqCRZz0Qg
[07/52] [abbrv] metron git commit: METRON-1471: Migrate shuffle connections to local or shuffle closes apache/metron#949
METRON-1471: Migrate shuffle connections to local or shuffle closes apache/metron#949 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8bc7732e Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8bc7732e Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8bc7732e Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 8bc7732e4c904085e4f78469abded84635971632 Parents: abb152b Author: cstellaAuthored: Thu Mar 8 17:17:57 2018 -0500 Committer: cstella Committed: Thu Mar 8 17:17:57 2018 -0500 -- .../metron-profiler/src/main/flux/profiler/remote.yaml | 6 +++--- .../metron-indexing/src/main/flux/indexing/batch/remote.yaml | 4 ++-- .../src/main/flux/indexing/random_access/remote.yaml | 4 ++-- .../apache/metron/parsers/topology/ParserTopologyBuilder.java | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml -- diff --git a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml index 752640a..9ec5ba4 100644 --- a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml +++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml @@ -168,7 +168,7 @@ streams: from: "kafkaSpout" to: "splitterBolt" grouping: -type: SHUFFLE +type: LOCAL_OR_SHUFFLE - name: "splitter -> builder" from: "splitterBolt" @@ -182,11 +182,11 @@ streams: to: "hbaseBolt" grouping: streamId: "hbase" -type: SHUFFLE +type: LOCAL_OR_SHUFFLE - name: "builder -> kafka" from: "builderBolt" to: "kafkaBolt" grouping: streamId: "kafka" -type: SHUFFLE +type: LOCAL_OR_SHUFFLE http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml -- diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml index 85e3baa..3087dd9 100644 --- a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml +++ b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml @@ -158,7 +158,7 @@ streams: from: "kafkaSpout" to: "hdfsIndexingBolt" grouping: -type: SHUFFLE +type: LOCAL_OR_SHUFFLE - name: "hdfsBolt -> errorIndexingBolt" @@ -166,4 +166,4 @@ streams: to: "indexingErrorBolt" grouping: streamId: "error" -type: SHUFFLE +type: LOCAL_OR_SHUFFLE http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml -- diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml index cadc1ec..429ba45 100644 --- a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml +++ b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml @@ -130,11 +130,11 @@ streams: from: "kafkaSpout" to: "indexingBolt" grouping: -type: SHUFFLE +type: LOCAL_OR_SHUFFLE - name: "indexingBolt -> errorIndexingBolt" from: "indexingBolt" to: "indexingErrorBolt" grouping: streamId: "error" -type: SHUFFLE +type: LOCAL_OR_SHUFFLE http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java -- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java index 1039e56..2865dd6 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java @@ -126,14 +126,14 @@ public class
[03/52] [abbrv] metron git commit: METRON-1460: Create a complementary non-split-join enrichment topology closes apache/metron#940
http://git-wip-us.apache.org/repos/asf/metron/blob/1d95b831/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java -- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java new file mode 100644 index 000..5f82b1c --- /dev/null +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/WorkerPoolStrategies.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.enrichment.parallel; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Function; + +/** + * The strategy to use to construct the thread pool. + */ +public enum WorkerPoolStrategies { + /** + * Fixed thread pool + */ + FIXED(numThreads -> Executors.newFixedThreadPool(numThreads)), + /** + * Work stealing thread pool. + */ + WORK_STEALING(numThreads -> Executors.newWorkStealingPool(numThreads)) + ; + Functioncreator; + WorkerPoolStrategies(Function creator) { +this.creator = creator; + } + + public ExecutorService create(int numThreads) { +return creator.apply(numThreads); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/1d95b831/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java -- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java index ab3d462..63d39c5 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java @@ -21,6 +21,7 @@ import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; @@ -28,6 +29,7 @@ import org.apache.metron.enrichment.lookup.EnrichmentLookup; import org.apache.metron.enrichment.lookup.handler.KeyWithContext; import org.apache.metron.hbase.TableProvider; import org.apache.metron.enrichment.converter.EnrichmentKey; +import org.json.simple.JSONObject; import sun.management.Sensor; import javax.annotation.Nullable; @@ -118,4 +120,18 @@ public class EnrichmentUtils { } } + public static JSONObject adjustKeys(JSONObject enrichedMessage, JSONObject enrichedField, String field, String prefix) { +if ( !enrichedField.isEmpty()) { + for (Object enrichedKey : enrichedField.keySet()) { +if(!StringUtils.isEmpty(prefix)) { + enrichedMessage.put(field + "." + enrichedKey, enrichedField.get(enrichedKey)); +} +else { + enrichedMessage.put(enrichedKey, enrichedField.get(enrichedKey)); +} + } +} +return enrichedMessage; + } + } http://git-wip-us.apache.org/repos/asf/metron/blob/1d95b831/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java -- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java index 7898ccd..870d709 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java +++
[28/52] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py deleted file mode 100755 index f0903ac..000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from elasticsearch import Elasticsearch -from elasticsearch.helpers import bulk -import cPickle as pickle -import argparse, sys, os.path -import errno -import os - - -class DashboardIndex(object): - -def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs): -""" -:arg host: hostname of the node (default: localhost) -:arg port: port to use (integer, default: 9200) -:arg url_prefix: optional url prefix for elasticsearch -:arg timeout: default timeout in seconds (float, default: 10) -""" -self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}]) - -def get(self): -""" -Get .kibana index from Elasticsearch -""" -dotkibana = self.es.search(index='.kibana', size = 100) -return dotkibana['hits']['hits'] - -def load(self,filespec): -""" -Save Index data on local filesystem -:args filespec: path/filename for saved file -""" -data=[] -with open(filespec,'rb') as fp: -data = pickle.load(fp) -return data - -def save(self,filename,data): -""" -Save Index data on local filesystem -:args filespec: path/filename for saved file -""" -with open(filename,'wb') as fp: -pickle.dump(data,fp) - -def put(self,data): -""" -Bulk write data to Elasticsearch -:args data: data to be written (note: index name is specified in data) -""" -bulk(self.es,data) - -def main(self,args): - -if args.save: -print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec)) -self.save(filename=args.filespec,data=di.get()) -else: -""" -Loads Kibana Dashboard definition from disk and replaces .kibana on index -:args filespec: path/filename for saved file -""" -if not os.path.isfile(args.filespec): -raise IOError( -errno.ENOENT, os.strerror(errno.ENOENT), args.filespec) -self.es.indices.delete(index='.kibana', ignore=[400, 404]) -self.put(data=di.load(filespec=args.filespec)) - -if __name__ == '__main__': - -parser = argparse.ArgumentParser() -parser.add_argument("hostname", help="ES Hostname or IP", type=str) -parser.add_argument("port", help="ES Port", type=int) -parser.add_argument("filespec", help="file to be pushed from or saved to", type=str) -parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true") -args = parser.parse_args() -di = DashboardIndex(host=args.hostname,port=args.port) -di.main(args) http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
[15/52] [abbrv] metron git commit: METRON-1487 Define Performance Benchmarks for Enrichment Topology (nickwallen) closes apache/metron#961
METRON-1487 Define Performance Benchmarks for Enrichment Topology (nickwallen) closes apache/metron#961 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e3eeec38 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e3eeec38 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e3eeec38 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: e3eeec38a66e1f10c296c110d47bc8bc3e995629 Parents: 52dd9fb Author: nickwallenAuthored: Sat Mar 17 09:22:15 2018 -0400 Committer: nickallen Committed: Sat Mar 17 09:22:15 2018 -0400 -- metron-platform/Performance-tuning-guide.md | 2 + .../metron-enrichment/Performance.md| 514 +++ 2 files changed, 516 insertions(+) -- http://git-wip-us.apache.org/repos/asf/metron/blob/e3eeec38/metron-platform/Performance-tuning-guide.md -- diff --git a/metron-platform/Performance-tuning-guide.md b/metron-platform/Performance-tuning-guide.md index bc8801b..7d79ace 100644 --- a/metron-platform/Performance-tuning-guide.md +++ b/metron-platform/Performance-tuning-guide.md @@ -422,6 +422,7 @@ modifying the options outlined above, increasing the poll timeout, or both. ## Reference +* [Enrichment Performance](metron-enrichment/Performance.md) * http://storm.apache.org/releases/1.0.1/flux.html * https://stackoverflow.com/questions/17257448/what-is-the-task-in-storm-parallelism * http://storm.apache.org/releases/current/Understanding-the-parallelism-of-a-Storm-topology.html @@ -429,3 +430,4 @@ modifying the options outlined above, increasing the poll timeout, or both. * https://www.confluent.io/blog/how-to-choose-the-number-of-topicspartitions-in-a-kafka-cluster/ * https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.1/bk_storm-component-guide/content/storm-kafkaspout-perf.html + http://git-wip-us.apache.org/repos/asf/metron/blob/e3eeec38/metron-platform/metron-enrichment/Performance.md -- diff --git a/metron-platform/metron-enrichment/Performance.md b/metron-platform/metron-enrichment/Performance.md new file mode 100644 index 000..4016a0d --- /dev/null +++ b/metron-platform/metron-enrichment/Performance.md @@ -0,0 +1,514 @@ + + +# Enrichment Performance + +This guide defines a set of benchmarks used to measure the performance of the Enrichment topology. The guide also provides detailed steps on how to execute those benchmarks along with advice for tuning the Unified Enrichment topology. + +* [Benchmarks](#benchmarks) +* [Benchmark Execution](#benchmark-execution) +* [Performance Tuning](#performance-tuning) +* [Benchmark Results](#benchmark-results) + +## Benchmarks + +The following section describes a set of enrichments that will be used to benchmark the performance of the Enrichment topology. + +* [Geo IP Enrichment](#geo-ip-enrichment) +* [HBase Enrichment](#hbase-enrichment) +* [Stellar Enrichment](#stellar-enrichment) + +### Geo IP Enrichment + +This benchmark measures the performance of executing a Geo IP enrichment. Given a valid IP address the enrichment will append detailed location information for that IP. The location information is sourced from an external Geo IP data source like [Maxmind](https://github.com/maxmind/GeoIP2-java). + + Configuration + +Adding the following Stellar expression to the Enrichment topology configuration will define a Geo IP enrichment. +``` +geo := GEO_GET(ip_dst_addr) +``` + +After the enrichment process completes, the telemetry message will contain a set of fields with location information for the given IP address. +``` +{ + "ip_dst_addr":"151.101.129.140", + ... + "geo.city":"San Francisco", + "geo.country":"US", + "geo.dmaCode":"807", + "geo.latitude":"37.7697", + "geo.location_point":"37.7697,-122.3933", + "geo.locID":"5391959", + "geo.longitude":"-122.3933", + "geo.postalCode":"94107", + } +``` + +### HBase Enrichment + +This benchmark measures the performance of executing an enrichment that retrieves data from an external HBase table. This type of enrichment is useful for enriching telemetry from an Asset Database or other source of relatively static data. + + Configuration + +Adding the following Stellar expression to the Enrichment topology configuration will define an Hbase enrichment. This looks up the 'ip_dst_addr' within an HBase table 'top-1m' and returns a hostname. +``` +top1m := ENRICHMENT_GET('top-1m', ip_dst_addr, 'top-1m', 't') +``` + +After the telemetry has been enriched, it will contain the host and IP elements that were retrieved from the HBase table. +``` +{ + "ip_dst_addr":"151.101.2.166", + ... +
[12/52] [abbrv] metron git commit: METRON-1299 In MetronError tests, don't test for HostName if getHostName wouldn't work closes apache/incubator-metron#924
METRON-1299 In MetronError tests, don't test for HostName if getHostName wouldn't work closes apache/incubator-metron#924 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/85d12475 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/85d12475 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/85d12475 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 85d12475cf363f0b9d6bdd973ec9860550f73980 Parents: e69ce21 Author: ottobackwardsAuthored: Thu Mar 15 14:13:44 2018 -0400 Committer: cstella Committed: Thu Mar 15 14:13:44 2018 -0400 -- .../metron/common/error/MetronErrorTest.java | 18 +++--- 1 file changed, 15 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/85d12475/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java -- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java index 5e505a8..e7390de 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java @@ -18,15 +18,16 @@ package org.apache.metron.common.error; import com.google.common.collect.Sets; -import com.google.common.primitives.Bytes; +import org.apache.commons.lang3.StringUtils; import org.apache.metron.common.Constants; import org.json.simple.JSONObject; import org.junit.Before; import org.junit.Test; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Arrays; -import static org.apache.metron.common.Constants.ErrorFields.RAW_MESSAGE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -53,7 +54,18 @@ public class MetronErrorTest { assertEquals(Constants.ErrorType.PARSER_ERROR.getType(), errorJSON.get(Constants.ErrorFields.ERROR_TYPE.getName())); assertEquals("error", errorJSON.get(Constants.SENSOR_TYPE)); assertEquals("sensorType", errorJSON.get(Constants.ErrorFields.FAILED_SENSOR_TYPE.getName())); -assertTrue(((String) errorJSON.get(Constants.ErrorFields.HOSTNAME.getName())).length() > 0); + +String hostName = null; +try { + hostName = InetAddress.getLocalHost().getHostName(); +} catch (UnknownHostException uhe) { + // unable to get the hostname on this machine, don't test it +} + +if (!StringUtils.isEmpty(hostName)) { + assertTrue(((String) errorJSON.get(Constants.ErrorFields.HOSTNAME.getName())).length() > 0); + assertEquals(hostName, (String) errorJSON.get(Constants.ErrorFields.HOSTNAME.getName())); +} assertTrue(((long) errorJSON.get(Constants.ErrorFields.TIMESTAMP.getName())) > 0); }
metron git commit: METRON-1528: Fix missing file in metron.spec (mmiklavc via mmiklavc) closes apache/metron#996
Repository: metron Updated Branches: refs/heads/master e0949142d -> a41611b1a METRON-1528: Fix missing file in metron.spec (mmiklavc via mmiklavc) closes apache/metron#996 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a41611b1 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a41611b1 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a41611b1 Branch: refs/heads/master Commit: a41611b1ae2bb17fa9333ef6f965749652e95538 Parents: e094914 Author: mmiklavcAuthored: Wed Apr 18 07:35:01 2018 -0600 Committer: Michael Miklavcic Committed: Wed Apr 18 07:35:01 2018 -0600 -- metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/metron/blob/a41611b1/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec -- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 1f40105..15469d9 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -123,6 +123,7 @@ This package installs the Metron common files %{metron_home} %dir %{metron_home}/lib %{metron_home}/bin/zk_load_configs.sh %{metron_home}/bin/stellar +%{metron_home}/bin/cluster_info.py %{metron_home}/config/zookeeper/global.json %attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}.jar