METRON-1471: Migrate shuffle connections to local or shuffle closes apache/metron#949
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8bc7732e Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8bc7732e Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8bc7732e Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 8bc7732e4c904085e4f78469abded84635971632 Parents: abb152b Author: cstella <ceste...@gmail.com> Authored: Thu Mar 8 17:17:57 2018 -0500 Committer: cstella <ceste...@gmail.com> Committed: Thu Mar 8 17:17:57 2018 -0500 ---------------------------------------------------------------------- .../metron-profiler/src/main/flux/profiler/remote.yaml | 6 +++--- .../metron-indexing/src/main/flux/indexing/batch/remote.yaml | 4 ++-- .../src/main/flux/indexing/random_access/remote.yaml | 4 ++-- .../apache/metron/parsers/topology/ParserTopologyBuilder.java | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml index 752640a..9ec5ba4 100644 --- a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml +++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml @@ -168,7 +168,7 @@ streams: from: "kafkaSpout" to: "splitterBolt" grouping: - type: SHUFFLE + type: LOCAL_OR_SHUFFLE - name: "splitter -> builder" from: "splitterBolt" @@ -182,11 +182,11 @@ streams: to: "hbaseBolt" grouping: streamId: "hbase" - type: SHUFFLE + type: LOCAL_OR_SHUFFLE - name: "builder -> kafka" from: "builderBolt" to: "kafkaBolt" grouping: streamId: "kafka" - type: SHUFFLE + type: LOCAL_OR_SHUFFLE http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml index 85e3baa..3087dd9 100644 --- a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml +++ b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml @@ -158,7 +158,7 @@ streams: from: "kafkaSpout" to: "hdfsIndexingBolt" grouping: - type: SHUFFLE + type: LOCAL_OR_SHUFFLE - name: "hdfsBolt -> errorIndexingBolt" @@ -166,4 +166,4 @@ streams: to: "indexingErrorBolt" grouping: streamId: "error" - type: SHUFFLE + type: LOCAL_OR_SHUFFLE http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml index cadc1ec..429ba45 100644 --- a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml +++ b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml @@ -130,11 +130,11 @@ streams: from: "kafkaSpout" to: "indexingBolt" grouping: - type: SHUFFLE + type: LOCAL_OR_SHUFFLE - name: "indexingBolt -> errorIndexingBolt" from: "indexingBolt" to: "indexingErrorBolt" grouping: streamId: "error" - type: SHUFFLE + type: LOCAL_OR_SHUFFLE http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java index 1039e56..2865dd6 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java @@ -126,14 +126,14 @@ public class ParserTopologyBuilder { ParserBolt parserBolt = createParserBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig, outputTopic); builder.setBolt("parserBolt", parserBolt, parserParallelism) .setNumTasks(parserNumTasks) - .shuffleGrouping("kafkaSpout"); + .localOrShuffleGrouping("kafkaSpout"); // create the error bolt, if needed if (errorWriterNumTasks > 0) { WriterBolt errorBolt = createErrorBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig); builder.setBolt("errorMessageWriter", errorBolt, errorWriterParallelism) .setNumTasks(errorWriterNumTasks) - .shuffleGrouping("parserBolt", Constants.ERROR_STREAM); + .localOrShuffleGrouping("parserBolt", Constants.ERROR_STREAM); } return new ParserTopology(builder, stormConfigSupplier.get(parserConfig, Config.class));
