STORM-2416: move flux-examples and refactor storm-mqtt module
Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1b514cbd Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1b514cbd Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1b514cbd Branch: refs/heads/1.x-branch Commit: 1b514cbd0d029f7ff4bc32b1061350fede89beab Parents: d6c8298 Author: P. Taylor Goetz <[email protected]> Authored: Thu Mar 16 16:47:20 2017 -0400 Committer: P. Taylor Goetz <[email protected]> Committed: Thu Mar 16 16:47:20 2017 -0400 ---------------------------------------------------------------------- examples/flux-examples/README.md | 93 ++++++ examples/flux-examples/pom.xml | 152 +++++++++ .../flux/examples/StatefulWordCounter.java | 64 ++++ .../storm/flux/examples/TestPrintBolt.java | 39 +++ .../storm/flux/examples/TestWindowBolt.java | 47 +++ .../storm/flux/examples/WordCountClient.java | 74 ++++ .../apache/storm/flux/examples/WordCounter.java | 71 ++++ .../src/main/resources/hbase_bolt.properties | 18 + .../src/main/resources/hdfs_bolt.properties | 26 ++ .../src/main/resources/kafka_spout.yaml | 136 ++++++++ .../src/main/resources/multilang.yaml | 89 +++++ .../src/main/resources/simple_hbase.yaml | 92 +++++ .../src/main/resources/simple_hdfs.yaml | 105 ++++++ .../resources/simple_stateful_wordcount.yaml | 60 ++++ .../src/main/resources/simple_windowing.yaml | 69 ++++ .../src/main/resources/simple_wordcount.yaml | 68 ++++ external/flux/flux-examples/README.md | 93 ------ external/flux/flux-examples/pom.xml | 146 -------- .../flux/examples/StatefulWordCounter.java | 64 ---- .../storm/flux/examples/TestPrintBolt.java | 39 --- .../storm/flux/examples/TestWindowBolt.java | 47 --- .../storm/flux/examples/WordCountClient.java | 74 ---- .../apache/storm/flux/examples/WordCounter.java | 71 ---- .../src/main/resources/hbase_bolt.properties | 18 - .../src/main/resources/hdfs_bolt.properties | 26 -- .../src/main/resources/kafka_spout.yaml | 136 -------- .../src/main/resources/multilang.yaml | 89 ----- .../src/main/resources/simple_hbase.yaml | 92 ----- .../src/main/resources/simple_hdfs.yaml | 105 ------ .../resources/simple_stateful_wordcount.yaml | 60 ---- .../src/main/resources/simple_windowing.yaml | 69 ---- .../src/main/resources/simple_wordcount.yaml | 68 ---- external/flux/pom.xml | 1 - external/storm-mqtt/core/pom.xml | 125 ------- .../java/org/apache/storm/mqtt/MqttLogger.java | 36 -- .../java/org/apache/storm/mqtt/MqttMessage.java | 41 --- .../apache/storm/mqtt/MqttMessageMapper.java | 44 --- .../org/apache/storm/mqtt/MqttTupleMapper.java | 37 -- .../org/apache/storm/mqtt/bolt/MqttBolt.java | 105 ------ .../apache/storm/mqtt/common/MqttOptions.java | 334 ------------------- .../apache/storm/mqtt/common/MqttPublisher.java | 67 ---- .../org/apache/storm/mqtt/common/MqttUtils.java | 88 ----- .../org/apache/storm/mqtt/common/SslUtils.java | 64 ---- .../mqtt/mappers/ByteArrayMessageMapper.java | 34 -- .../storm/mqtt/mappers/StringMessageMapper.java | 37 -- .../apache/storm/mqtt/spout/AckableMessage.java | 71 ---- .../org/apache/storm/mqtt/spout/MqttSpout.java | 262 --------------- .../storm/mqtt/ssl/DefaultKeyStoreLoader.java | 97 ------ .../apache/storm/mqtt/ssl/KeyStoreLoader.java | 35 -- .../storm/mqtt/trident/MqttPublishFunction.java | 85 ----- .../storm/mqtt/StormMqttIntegrationTest.java | 152 --------- external/storm-mqtt/pom.xml | 143 ++++++-- .../java/org/apache/storm/mqtt/MqttLogger.java | 36 ++ .../java/org/apache/storm/mqtt/MqttMessage.java | 41 +++ .../apache/storm/mqtt/MqttMessageMapper.java | 44 +++ .../org/apache/storm/mqtt/MqttTupleMapper.java | 37 ++ .../org/apache/storm/mqtt/bolt/MqttBolt.java | 105 ++++++ .../apache/storm/mqtt/common/MqttOptions.java | 334 +++++++++++++++++++ .../apache/storm/mqtt/common/MqttPublisher.java | 67 ++++ .../org/apache/storm/mqtt/common/MqttUtils.java | 88 +++++ .../org/apache/storm/mqtt/common/SslUtils.java | 64 ++++ .../mqtt/mappers/ByteArrayMessageMapper.java | 34 ++ .../storm/mqtt/mappers/StringMessageMapper.java | 37 ++ .../apache/storm/mqtt/spout/AckableMessage.java | 71 ++++ .../org/apache/storm/mqtt/spout/MqttSpout.java | 262 +++++++++++++++ .../storm/mqtt/ssl/DefaultKeyStoreLoader.java | 97 ++++++ .../apache/storm/mqtt/ssl/KeyStoreLoader.java | 35 ++ .../storm/mqtt/trident/MqttPublishFunction.java | 85 +++++ .../storm/mqtt/StormMqttIntegrationTest.java | 152 +++++++++ pom.xml | 15 + storm-perf/pom.xml | 19 +- 71 files changed, 2915 insertions(+), 2966 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/README.md ---------------------------------------------------------------------- diff --git a/examples/flux-examples/README.md b/examples/flux-examples/README.md new file mode 100644 index 0000000..3d610b4 --- /dev/null +++ b/examples/flux-examples/README.md @@ -0,0 +1,93 @@ +# Flux Examples +A collection of examples illustrating various capabilities. + +## Building From Source and Running + +Checkout the projects source and perform a top level Maven build (i.e. from the `flux` directory): + +```bash +git clone https://github.com/apache/storm.git +cd storm +mvn install -DskipTests=true +``` + +This will create a shaded (i.e. "fat" or "uber") jar in the `external/flux/flux-examples/target` directory that can run/deployed with +the `storm` command: + +```bash +cd flux-examples +storm jar ./target/flux-examples-*-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_wordcount.yaml +``` + +The example YAML files are also packaged in the examples jar, so they can also be referenced with Flux's `--resource` +command line switch: + +```bash +storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local --resource /simple_wordcount.yaml +``` + +## Available Examples + +### [simple_wordcount.yaml](src/main/resources/simple_wordcount.yaml) + +This is a very basic wordcount example using Java spouts and bolts. It simply logs the running count of each word +received. + +### [multilang.yaml](src/main/resources/multilang.yaml) + +Another wordcount example that uses a spout written in JavaScript (node.js), a bolt written in Python, and two bolts +written in java. + +### [kafka_spout.yaml](src/main/resources/kafka_spout.yaml) + +This example illustrates how to configure Storm's `storm-kafka` spout using Flux YAML DSL `components`, `references`, +and `constructor arguments` constructs. + +### [simple_hdfs.yaml](src/main/resources/simple_hdfs.yaml) + +This example demonstrates using Flux to setup a storm-hdfs bolt to write to an HDFS cluster. It also demonstrates Flux's +variable substitution/filtering feature. + +To run the `simple_hdfs.yaml` example, copy the `hdfs_bolt.properties` file to a convenient location and change, at +least, the property `hdfs.url` to point to a HDFS cluster. Then you can run the example something like: + +```bash +storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hdfs.yaml --filter my_hdfs_bolt.properties +``` + +### [simple_hbase.yaml](src/main/resources/simple_hbase.yaml) + +This example illustrates how to use Flux to setup a storm-hbase bolt to write to HBase. + +To run the `simple_hbase.yaml` example, copy the `hbase_bolt.properties` file to a convenient location and change the properties + `hbase.rootdir` and `hbase.zookeeper.quorum`. Then you can run the example something like: + +```bash +storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hbase.yaml --filter my_hbase_bolt.properties +``` + +### [simple_windowing.yaml](src/main/resources/simple_windowing.yaml) + +This example illustrates how to use Flux to set up a storm topology that contains windowing operations. + +To run, + +```bash +storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_windowing.yaml +``` + +### [simple_stateful_wordcount.yaml](src/main/resources/simple_stateful_wordcount.yaml) + +Flux also supports stateful bolts which is illustrated with this example. It is basically an extension of the basic wordcount example. +The state is periodically saved (checkpointed) and restored when the topology is restarted. + +```bash +storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_stateful_wordcount.yaml +``` + +By default the state is stored in-memory only. As such you won't see a resumed state unless you configure to use Redis as the state backend. +Ensure that you have Redis running at `localhost:6379` and that `storm-redis-*.jar` is in the classpath. + +```bash +STORM_EXT_CLASSPATH=../../storm-redis/target storm jar ./target/flux-examples-*.jar -c topology.state.provider=org.apache.storm.redis.state.RedisKeyValueStateProvider org.apache.storm.flux.Flux --local ./src/main/resources/simple_stateful_wordcount.yaml +``` http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/pom.xml ---------------------------------------------------------------------- diff --git a/examples/flux-examples/pom.xml b/examples/flux-examples/pom.xml new file mode 100644 index 0000000..7fff5db --- /dev/null +++ b/examples/flux-examples/pom.xml @@ -0,0 +1,152 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <artifactId>storm</artifactId> + <groupId>org.apache.storm</groupId> + <version>1.1.0-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + + <artifactId>flux-examples</artifactId> + <packaging>jar</packaging> + + <name>flux-examples</name> + + <dependencies> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>flux-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>flux-wrappers</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-hdfs</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>commons-beanutils</groupId> + <artifactId>commons-beanutils-core</artifactId> + </exclusion> + <exclusion> + <groupId>commons-beanutils</groupId> + <artifactId>commons-beanutils</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>javax.servlet-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-hbase</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>servlet-api-2.5</artifactId> + </exclusion> + <exclusion> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-kafka</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>${storm.kafka.artifact.id}</artifactId> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <createDependencyReducedPom>true</createDependencyReducedPom> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.sf</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.dsa</exclude> + <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/*.rsa</exclude> + <exclude>META-INF/*.EC</exclude> + <exclude>META-INF/*.ec</exclude> + <exclude>META-INF/MSFTSIG.SF</exclude> + <exclude>META-INF/MSFTSIG.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <mainClass>org.apache.storm.flux.Flux</mainClass> + </transformer> + </transformers> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java new file mode 100644 index 0000000..5534888 --- /dev/null +++ b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.storm.flux.examples; + +import org.apache.storm.state.KeyValueState; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.base.BaseStatefulBolt; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; + +import java.util.Map; + +public class StatefulWordCounter extends BaseStatefulBolt<KeyValueState<String, Long>> { + + private KeyValueState<String, Long> wordCounts; + private OutputCollector collector; + + @SuppressWarnings("rawtypes") + @Override + public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { + this.collector = collector; + } + + @Override + public void initState(KeyValueState<String, Long> state) { + wordCounts = state; + } + + @Override + public void execute(Tuple tuple) { + String word = tuple.getString(0); + + Long count = wordCounts.get(word, 0L); + count++; + wordCounts.put(word, count); + + collector.emit(tuple, new Values(word, count)); + collector.ack(tuple); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("word", "count")); + } + +} http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java new file mode 100644 index 0000000..137e354 --- /dev/null +++ b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.storm.flux.examples; + +import org.apache.storm.topology.BasicOutputCollector; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseBasicBolt; +import org.apache.storm.tuple.Tuple; + +/** + * Prints the tuples to stdout + */ +public class TestPrintBolt extends BaseBasicBolt { + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + System.out.println(tuple); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer ofd) { + } + +} http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java new file mode 100644 index 0000000..8c904d9 --- /dev/null +++ b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.storm.flux.examples; + +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseWindowedBolt; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; +import org.apache.storm.windowing.TupleWindow; + +import java.util.Map; + +public class TestWindowBolt extends BaseWindowedBolt { + private OutputCollector collector; + + @Override + public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { + this.collector = collector; + } + + @Override + public void execute(TupleWindow inputWindow) { + collector.emit(new Values(inputWindow.get().size())); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("count")); + } +} http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java new file mode 100644 index 0000000..eb4fb7a --- /dev/null +++ b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.storm.flux.examples; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; + +import java.io.FileInputStream; +import java.util.Properties; + +/** + * Connects to the 'WordCount' HBase table and prints counts for each word. + * + * Assumes you have run (or are running) the YAML topology definition in + * <code>simple_hbase.yaml</code> + * + * You will also need to modify `src/main/resources/hbase-site.xml` + * to point to your HBase instance, and then repackage with `mvn package`. + * This is a known issue. + * + */ +public class WordCountClient { + + public static void main(String[] args) throws Exception { + Configuration config = HBaseConfiguration.create(); + if(args.length == 1){ + Properties props = new Properties(); + props.load(new FileInputStream(args[0])); + System.out.println("HBase configuration:"); + for(Object key : props.keySet()) { + System.out.println(key + "=" + props.get(key)); + config.set((String)key, props.getProperty((String)key)); + } + } else { + System.out.println("Usage: WordCountClient <hbase_config.properties>"); + System.exit(1); + } + + HTable table = new HTable(config, "WordCount"); + String[] words = new String[] {"nathan", "mike", "jackson", "golda", "bertels"}; + + for (String word : words) { + Get get = new Get(Bytes.toBytes(word)); + Result result = table.get(get); + + byte[] countBytes = result.getValue(Bytes.toBytes("cf"), Bytes.toBytes("count")); + byte[] wordBytes = result.getValue(Bytes.toBytes("cf"), Bytes.toBytes("word")); + + String wordStr = Bytes.toString(wordBytes); + long count = Bytes.toLong(countBytes); + System.out.println("Word: '" + wordStr + "', Count: " + count); + } + + } +} http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java new file mode 100644 index 0000000..7093105 --- /dev/null +++ b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.storm.flux.examples; + +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.BasicOutputCollector; +import org.apache.storm.topology.IBasicBolt; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseBasicBolt; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static org.apache.storm.utils.Utils.tuple; + +/** + * This bolt is used by the HBase example. It simply emits the first field + * found in the incoming tuple as "word", with a "count" of `1`. + * + * In this case, the downstream HBase bolt handles the counting, so a value + * of `1` will just increment the HBase counter by one. + */ +public class WordCounter extends BaseBasicBolt { + private static final Logger LOG = LoggerFactory.getLogger(WordCounter.class); + + + + @SuppressWarnings("rawtypes") + public void prepare(Map stormConf, TopologyContext context) { + } + + /* + * Just output the word value with a count of 1. + * The HBaseBolt will handle incrementing the counter. + */ + public void execute(Tuple input, BasicOutputCollector collector) { + collector.emit(tuple(input.getValues().get(0), 1)); + } + + public void cleanup() { + + } + + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("word", "count")); + } + + @Override + public Map<String, Object> getComponentConfiguration() { + return null; + } + +} http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/resources/hbase_bolt.properties ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/resources/hbase_bolt.properties b/examples/flux-examples/src/main/resources/hbase_bolt.properties new file mode 100644 index 0000000..f8ed50c --- /dev/null +++ b/examples/flux-examples/src/main/resources/hbase_bolt.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hbase.rootdir=hdfs://hadoop:54310/hbase +hbase.zookeeper.quorum=hadoop \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/resources/hdfs_bolt.properties ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/resources/hdfs_bolt.properties b/examples/flux-examples/src/main/resources/hdfs_bolt.properties new file mode 100644 index 0000000..7bcbe7a --- /dev/null +++ b/examples/flux-examples/src/main/resources/hdfs_bolt.properties @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# The HDFS url +hdfs.url=hdfs://hadoop:54310 + +# The HDFS directory where the bolt will write incoming data +hdfs.write.dir=/incoming + +# The HDFS directory where files will be moved once the bolt has +# finished writing to it. +hdfs.dest.dir=/complete \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/resources/kafka_spout.yaml ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/resources/kafka_spout.yaml b/examples/flux-examples/src/main/resources/kafka_spout.yaml new file mode 100644 index 0000000..7533ce4 --- /dev/null +++ b/examples/flux-examples/src/main/resources/kafka_spout.yaml @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Test ability to wire together shell spouts/bolts +--- + +# topology definition +# name to be used when submitting +name: "kafka-topology" + +# Components +# Components are analagous to Spring beans. They are meant to be used as constructor, +# property(setter), and builder arguments. +# +# for the time being, components must be declared in the order they are referenced +components: + - id: "stringScheme" + className: "org.apache.storm.kafka.StringScheme" + + - id: "stringMultiScheme" + className: "org.apache.storm.spout.SchemeAsMultiScheme" + constructorArgs: + - ref: "stringScheme" + + - id: "zkHosts" + className: "org.apache.storm.kafka.ZkHosts" + constructorArgs: + - "localhost:2181" + +# Alternative kafka config +# - id: "kafkaConfig" +# className: "org.apache.storm.kafka.KafkaConfig" +# constructorArgs: +# # brokerHosts +# - ref: "zkHosts" +# # topic +# - "myKafkaTopic" +# # clientId (optional) +# - "myKafkaClientId" + + - id: "spoutConfig" + className: "org.apache.storm.kafka.SpoutConfig" + constructorArgs: + # brokerHosts + - ref: "zkHosts" + # topic + - "myKafkaTopic" + # zkRoot + - "/kafkaSpout" + # id + - "myId" + properties: + - name: "ignoreZkOffsets" + value: true + - name: "scheme" + ref: "stringMultiScheme" + + + +# NOTE: We may want to consider some level of spring integration. For example, allowing component references +# to a spring `ApplicationContext`. + +# topology configuration +# this will be passed to the submitter as a map of config options +# +config: + topology.workers: 1 + # ... + +# spout definitions +spouts: + - id: "kafka-spout" + className: "org.apache.storm.kafka.KafkaSpout" + constructorArgs: + - ref: "spoutConfig" + +# bolt definitions +bolts: + - id: "splitsentence" + className: "org.apache.storm.flux.wrappers.bolts.FluxShellBolt" + constructorArgs: + # command line + - ["python", "splitsentence.py"] + # output fields + - ["word"] + parallelism: 1 + # ... + + - id: "log" + className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt" + parallelism: 1 + # ... + + - id: "count" + className: "org.apache.storm.testing.TestWordCounter" + parallelism: 1 + # ... + +#stream definitions +# stream definitions define connections between spouts and bolts. +# note that such connections can be cyclical +# custom stream groupings are also supported + +streams: + - name: "kafka --> split" # name isn't used (placeholder for logging, UI, etc.) + from: "kafka-spout" + to: "splitsentence" + grouping: + type: SHUFFLE + + - name: "split --> count" + from: "splitsentence" + to: "count" + grouping: + type: FIELDS + args: ["word"] + + - name: "count --> log" + from: "count" + to: "log" + grouping: + type: SHUFFLE http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/resources/multilang.yaml ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/resources/multilang.yaml b/examples/flux-examples/src/main/resources/multilang.yaml new file mode 100644 index 0000000..aaab5d3 --- /dev/null +++ b/examples/flux-examples/src/main/resources/multilang.yaml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test ability to wire together shell spouts/bolts +--- + +# topology definition +# name to be used when submitting +name: "shell-topology" + +# topology configuration +# this will be passed to the submitter as a map of config options +# +config: + topology.workers: 1 + # ... + +# spout definitions +spouts: + - id: "sentence-spout" + className: "org.apache.storm.flux.wrappers.spouts.FluxShellSpout" + # shell spout constructor takes 2 arguments: String[], String[] + constructorArgs: + # command line + - ["node", "randomsentence.js"] + # output fields + - ["word"] + parallelism: 1 + # ... + +# bolt definitions +bolts: + - id: "splitsentence" + className: "org.apache.storm.flux.wrappers.bolts.FluxShellBolt" + constructorArgs: + # command line + - ["python", "splitsentence.py"] + # output fields + - ["word"] + parallelism: 1 + # ... + + - id: "log" + className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt" + parallelism: 1 + # ... + + - id: "count" + className: "org.apache.storm.testing.TestWordCounter" + parallelism: 1 + # ... + +#stream definitions +# stream definitions define connections between spouts and bolts. +# note that such connections can be cyclical +# custom stream groupings are also supported + +streams: + - name: "spout --> split" # name isn't used (placeholder for logging, UI, etc.) + from: "sentence-spout" + to: "splitsentence" + grouping: + type: SHUFFLE + + - name: "split --> count" + from: "splitsentence" + to: "count" + grouping: + type: FIELDS + args: ["word"] + + - name: "count --> log" + from: "count" + to: "log" + grouping: + type: SHUFFLE http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/resources/simple_hbase.yaml ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/resources/simple_hbase.yaml b/examples/flux-examples/src/main/resources/simple_hbase.yaml new file mode 100644 index 0000000..93a2781 --- /dev/null +++ b/examples/flux-examples/src/main/resources/simple_hbase.yaml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# NOTE: To use this example, you will need to modify `src/main/resources/hbase-site.xml` +# to point to your HBase instance, and then repackage with `mvn package`. +# This is a known issue. + +# topology definition +# name to be used when submitting +name: "hbase-persistent-wordcount" + +# Components +components: + - id: "columnFields" + className: "org.apache.storm.tuple.Fields" + constructorArgs: + - ["word"] + + - id: "counterFields" + className: "org.apache.storm.tuple.Fields" + constructorArgs: + - ["count"] + + - id: "mapper" + className: "org.apache.storm.hbase.bolt.mapper.SimpleHBaseMapper" + configMethods: + - name: "withRowKeyField" + args: ["word"] + - name: "withColumnFields" + args: [ref: "columnFields"] + - name: "withCounterFields" + args: [ref: "counterFields"] + - name: "withColumnFamily" + args: ["cf"] + +# topology configuration +# this will be passed to the submitter as a map of config options +config: + topology.workers: 1 + hbase.conf: + hbase.rootdir: "${hbase.rootdir}" + hbase.zookeeper.quorum: "${hbase.zookeeper.quorum}" + +# spout definitions +spouts: + - id: "word-spout" + className: "org.apache.storm.testing.TestWordSpout" + parallelism: 1 + +# bolt definitions + +bolts: + - id: "count-bolt" + className: "org.apache.storm.flux.examples.WordCounter" + parallelism: 1 + + - id: "hbase-bolt" + className: "org.apache.storm.hbase.bolt.HBaseBolt" + constructorArgs: + - "WordCount" # HBase table name + - ref: "mapper" + configMethods: + - name: "withConfigKey" + args: ["hbase.conf"] + parallelism: 1 + +streams: + - name: "" # name isn't used (placeholder for logging, UI, etc.) + from: "word-spout" + to: "count-bolt" + grouping: + type: SHUFFLE + + - name: "" # name isn't used (placeholder for logging, UI, etc.) + from: "count-bolt" + to: "hbase-bolt" + grouping: + type: FIELDS + args: ["word"] http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/resources/simple_hdfs.yaml ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/resources/simple_hdfs.yaml b/examples/flux-examples/src/main/resources/simple_hdfs.yaml new file mode 100644 index 0000000..b8d4020 --- /dev/null +++ b/examples/flux-examples/src/main/resources/simple_hdfs.yaml @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test ability to wire together shell spouts/bolts +--- + +# topology definition +# name to be used when submitting +name: "hdfs-topology" + +# Components +# Components are analagous to Spring beans. They are meant to be used as constructor, +# property(setter), and builder arguments. +# +# for the time being, components must be declared in the order they are referenced +components: + - id: "syncPolicy" + className: "org.apache.storm.hdfs.bolt.sync.CountSyncPolicy" + constructorArgs: + - 1000 + - id: "rotationPolicy" + className: "org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy" + constructorArgs: + - 30 + - SECONDS + + - id: "fileNameFormat" + className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat" + configMethods: + - name: "withPath" + args: ["${hdfs.write.dir}"] + - name: "withExtension" + args: [".txt"] + + - id: "recordFormat" + className: "org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat" + configMethods: + - name: "withFieldDelimiter" + args: ["|"] + + - id: "rotationAction" + className: "org.apache.storm.hdfs.common.rotation.MoveFileAction" + configMethods: + - name: "toDestination" + args: ["${hdfs.dest.dir}"] + +# spout definitions +spouts: + - id: "spout-1" + className: "org.apache.storm.testing.TestWordSpout" + parallelism: 1 + # ... + +# bolt definitions + +bolts: + - id: "bolt-1" + className: "org.apache.storm.hdfs.bolt.HdfsBolt" + configMethods: + - name: "withConfigKey" + args: ["hdfs.config"] + - name: "withFsUrl" + args: ["${hdfs.url}"] + - name: "withFileNameFormat" + args: [ref: "fileNameFormat"] + - name: "withRecordFormat" + args: [ref: "recordFormat"] + - name: "withRotationPolicy" + args: [ref: "rotationPolicy"] + - name: "withSyncPolicy" + args: [ref: "syncPolicy"] + - name: "addRotationAction" + args: [ref: "rotationAction"] + parallelism: 1 + # ... + + - id: "bolt-2" + className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt" + parallelism: 1 + +streams: + - name: "" # name isn't used (placeholder for logging, UI, etc.) + from: "spout-1" + to: "bolt-1" + grouping: + type: SHUFFLE + + - name: "" # name isn't used (placeholder for logging, UI, etc.) + from: "spout-1" + to: "bolt-2" + grouping: + type: SHUFFLE http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/resources/simple_stateful_wordcount.yaml ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/resources/simple_stateful_wordcount.yaml b/examples/flux-examples/src/main/resources/simple_stateful_wordcount.yaml new file mode 100644 index 0000000..14b9b3a --- /dev/null +++ b/examples/flux-examples/src/main/resources/simple_stateful_wordcount.yaml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +# topology definition +# name to be used when submitting +name: "stateful-wordcount-topology" + +# topology configuration +# this will be passed to the submitter as a map of config options +# +config: + topology.workers: 1 + +# spout definitions +spouts: + - id: "spout-1" + className: "org.apache.storm.testing.TestWordSpout" + parallelism: 1 + +# bolt definitions +bolts: + - id: "bolt-1" + className: "org.apache.storm.flux.examples.StatefulWordCounter" + parallelism: 1 + + - id: "bolt-2" + className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt" + parallelism: 1 + +#stream definitions +# stream definitions define connections between spouts and bolts. +# note that such connections can be cyclical +streams: + - name: "spout-1 --> bolt-1" # name isn't used (placeholder for logging, UI, etc.) + from: "spout-1" + to: "bolt-1" + grouping: + type: FIELDS + args: ["word"] + + - name: "bolt-1 --> bolt2" + from: "bolt-1" + to: "bolt-2" + grouping: + type: SHUFFLE http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/resources/simple_windowing.yaml ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/resources/simple_windowing.yaml b/examples/flux-examples/src/main/resources/simple_windowing.yaml new file mode 100755 index 0000000..f2f74ff --- /dev/null +++ b/examples/flux-examples/src/main/resources/simple_windowing.yaml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +name: "sliding-window-topology" + +components: + - id: "windowLength" + className: "org.apache.storm.topology.base.BaseWindowedBolt$Count" + constructorArgs: + - 5 + - id: "slidingInterval" + className: "org.apache.storm.topology.base.BaseWindowedBolt$Count" + constructorArgs: + - 3 + +config: + topology.workers: 1 + +# spout definitions +spouts: + - id: "spout-1" + className: "org.apache.storm.testing.TestWordSpout" + parallelism: 1 + +# bolt definitions +bolts: + - id: "bolt-1" + className: "org.apache.storm.flux.examples.TestWindowBolt" + configMethods: + - name: "withWindow" + args: [ref: "windowLength", ref: "slidingInterval"] + parallelism: 1 + - id: "bolt-2" + className: "org.apache.storm.flux.examples.TestPrintBolt" + parallelism: 1 + + +#stream definitions +# stream definitions define connections between spouts and bolts. +# note that such connections can be cyclical +streams: + - name: "spout-1 --> bolt-1" # name isn't used (placeholder for logging, UI, etc.) +# id: "connection-1" + from: "spout-1" + to: "bolt-1" + grouping: + type: FIELDS + args: ["word"] + - name: "bolt-1 --> bolt-2" # name isn't used (placeholder for logging, UI, etc.) +# id: "connection-1" + from: "bolt-1" + to: "bolt-2" + grouping: + type: SHUFFLE http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/examples/flux-examples/src/main/resources/simple_wordcount.yaml ---------------------------------------------------------------------- diff --git a/examples/flux-examples/src/main/resources/simple_wordcount.yaml b/examples/flux-examples/src/main/resources/simple_wordcount.yaml new file mode 100644 index 0000000..6443a97 --- /dev/null +++ b/examples/flux-examples/src/main/resources/simple_wordcount.yaml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +# topology definition +# name to be used when submitting +name: "yaml-topology" + +# topology configuration +# this will be passed to the submitter as a map of config options +# +config: + topology.workers: 1 + +# spout definitions +spouts: + - id: "spout-1" + className: "org.apache.storm.testing.TestWordSpout" + parallelism: 1 + +# bolt definitions +bolts: + - id: "bolt-1" + className: "org.apache.storm.testing.TestWordCounter" + parallelism: 1 + + - id: "bolt-2" + className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt" + parallelism: 1 + +#stream definitions +# stream definitions define connections between spouts and bolts. +# note that such connections can be cyclical +streams: + - name: "spout-1 --> bolt-1" # name isn't used (placeholder for logging, UI, etc.) +# id: "connection-1" + from: "spout-1" + to: "bolt-1" + grouping: + type: FIELDS + args: ["word"] + + - name: "bolt-1 --> bolt2" + from: "bolt-1" + to: "bolt-2" + grouping: + type: SHUFFLE + + + + + + + http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/external/flux/flux-examples/README.md ---------------------------------------------------------------------- diff --git a/external/flux/flux-examples/README.md b/external/flux/flux-examples/README.md deleted file mode 100644 index 3d610b4..0000000 --- a/external/flux/flux-examples/README.md +++ /dev/null @@ -1,93 +0,0 @@ -# Flux Examples -A collection of examples illustrating various capabilities. - -## Building From Source and Running - -Checkout the projects source and perform a top level Maven build (i.e. from the `flux` directory): - -```bash -git clone https://github.com/apache/storm.git -cd storm -mvn install -DskipTests=true -``` - -This will create a shaded (i.e. "fat" or "uber") jar in the `external/flux/flux-examples/target` directory that can run/deployed with -the `storm` command: - -```bash -cd flux-examples -storm jar ./target/flux-examples-*-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_wordcount.yaml -``` - -The example YAML files are also packaged in the examples jar, so they can also be referenced with Flux's `--resource` -command line switch: - -```bash -storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local --resource /simple_wordcount.yaml -``` - -## Available Examples - -### [simple_wordcount.yaml](src/main/resources/simple_wordcount.yaml) - -This is a very basic wordcount example using Java spouts and bolts. It simply logs the running count of each word -received. - -### [multilang.yaml](src/main/resources/multilang.yaml) - -Another wordcount example that uses a spout written in JavaScript (node.js), a bolt written in Python, and two bolts -written in java. - -### [kafka_spout.yaml](src/main/resources/kafka_spout.yaml) - -This example illustrates how to configure Storm's `storm-kafka` spout using Flux YAML DSL `components`, `references`, -and `constructor arguments` constructs. - -### [simple_hdfs.yaml](src/main/resources/simple_hdfs.yaml) - -This example demonstrates using Flux to setup a storm-hdfs bolt to write to an HDFS cluster. It also demonstrates Flux's -variable substitution/filtering feature. - -To run the `simple_hdfs.yaml` example, copy the `hdfs_bolt.properties` file to a convenient location and change, at -least, the property `hdfs.url` to point to a HDFS cluster. Then you can run the example something like: - -```bash -storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hdfs.yaml --filter my_hdfs_bolt.properties -``` - -### [simple_hbase.yaml](src/main/resources/simple_hbase.yaml) - -This example illustrates how to use Flux to setup a storm-hbase bolt to write to HBase. - -To run the `simple_hbase.yaml` example, copy the `hbase_bolt.properties` file to a convenient location and change the properties - `hbase.rootdir` and `hbase.zookeeper.quorum`. Then you can run the example something like: - -```bash -storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hbase.yaml --filter my_hbase_bolt.properties -``` - -### [simple_windowing.yaml](src/main/resources/simple_windowing.yaml) - -This example illustrates how to use Flux to set up a storm topology that contains windowing operations. - -To run, - -```bash -storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_windowing.yaml -``` - -### [simple_stateful_wordcount.yaml](src/main/resources/simple_stateful_wordcount.yaml) - -Flux also supports stateful bolts which is illustrated with this example. It is basically an extension of the basic wordcount example. -The state is periodically saved (checkpointed) and restored when the topology is restarted. - -```bash -storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_stateful_wordcount.yaml -``` - -By default the state is stored in-memory only. As such you won't see a resumed state unless you configure to use Redis as the state backend. -Ensure that you have Redis running at `localhost:6379` and that `storm-redis-*.jar` is in the classpath. - -```bash -STORM_EXT_CLASSPATH=../../storm-redis/target storm jar ./target/flux-examples-*.jar -c topology.state.provider=org.apache.storm.redis.state.RedisKeyValueStateProvider org.apache.storm.flux.Flux --local ./src/main/resources/simple_stateful_wordcount.yaml -``` http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/external/flux/flux-examples/pom.xml ---------------------------------------------------------------------- diff --git a/external/flux/flux-examples/pom.xml b/external/flux/flux-examples/pom.xml deleted file mode 100644 index adfc605..0000000 --- a/external/flux/flux-examples/pom.xml +++ /dev/null @@ -1,146 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.storm</groupId> - <artifactId>flux</artifactId> - <version>1.1.0-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - - <artifactId>flux-examples</artifactId> - <packaging>jar</packaging> - - <name>flux-examples</name> - - <dependencies> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>flux-core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>flux-wrappers</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-hdfs</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>commons-beanutils</groupId> - <artifactId>commons-beanutils-core</artifactId> - </exclusion> - <exclusion> - <groupId>commons-beanutils</groupId> - <artifactId>commons-beanutils</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-api</artifactId> - </exclusion> - <exclusion> - <groupId>javax.servlet</groupId> - <artifactId>servlet-api</artifactId> - </exclusion> - <exclusion> - <groupId>javax.servlet</groupId> - <artifactId>javax.servlet-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-hbase</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>servlet-api-2.5</artifactId> - </exclusion> - <exclusion> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - </exclusion> - <exclusion> - <groupId>javax.servlet</groupId> - <artifactId>servlet-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-kafka</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>${storm.kafka.artifact.id}</artifactId> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <createDependencyReducedPom>true</createDependencyReducedPom> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.sf</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.dsa</exclude> - <exclude>META-INF/*.RSA</exclude> - <exclude>META-INF/*.rsa</exclude> - <exclude>META-INF/*.EC</exclude> - <exclude>META-INF/*.ec</exclude> - <exclude>META-INF/MSFTSIG.SF</exclude> - <exclude>META-INF/MSFTSIG.RSA</exclude> - </excludes> - </filter> - </filters> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>org.apache.storm.flux.Flux</mainClass> - </transformer> - </transformers> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java ---------------------------------------------------------------------- diff --git a/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java b/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java deleted file mode 100644 index 5534888..0000000 --- a/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.flux.examples; - -import org.apache.storm.state.KeyValueState; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.base.BaseStatefulBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -import java.util.Map; - -public class StatefulWordCounter extends BaseStatefulBolt<KeyValueState<String, Long>> { - - private KeyValueState<String, Long> wordCounts; - private OutputCollector collector; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - this.collector = collector; - } - - @Override - public void initState(KeyValueState<String, Long> state) { - wordCounts = state; - } - - @Override - public void execute(Tuple tuple) { - String word = tuple.getString(0); - - Long count = wordCounts.get(word, 0L); - count++; - wordCounts.put(word, count); - - collector.emit(tuple, new Values(word, count)); - collector.ack(tuple); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("word", "count")); - } - -} http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java ---------------------------------------------------------------------- diff --git a/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java b/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java deleted file mode 100644 index 137e354..0000000 --- a/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.flux.examples; - -import org.apache.storm.topology.BasicOutputCollector; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseBasicBolt; -import org.apache.storm.tuple.Tuple; - -/** - * Prints the tuples to stdout - */ -public class TestPrintBolt extends BaseBasicBolt { - - @Override - public void execute(Tuple tuple, BasicOutputCollector collector) { - System.out.println(tuple); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer ofd) { - } - -} http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java ---------------------------------------------------------------------- diff --git a/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java b/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java deleted file mode 100644 index 8c904d9..0000000 --- a/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.flux.examples; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseWindowedBolt; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Values; -import org.apache.storm.windowing.TupleWindow; - -import java.util.Map; - -public class TestWindowBolt extends BaseWindowedBolt { - private OutputCollector collector; - - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - this.collector = collector; - } - - @Override - public void execute(TupleWindow inputWindow) { - collector.emit(new Values(inputWindow.get().size())); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("count")); - } -} http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java ---------------------------------------------------------------------- diff --git a/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java b/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java deleted file mode 100644 index eb4fb7a..0000000 --- a/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.flux.examples; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; - -import java.io.FileInputStream; -import java.util.Properties; - -/** - * Connects to the 'WordCount' HBase table and prints counts for each word. - * - * Assumes you have run (or are running) the YAML topology definition in - * <code>simple_hbase.yaml</code> - * - * You will also need to modify `src/main/resources/hbase-site.xml` - * to point to your HBase instance, and then repackage with `mvn package`. - * This is a known issue. - * - */ -public class WordCountClient { - - public static void main(String[] args) throws Exception { - Configuration config = HBaseConfiguration.create(); - if(args.length == 1){ - Properties props = new Properties(); - props.load(new FileInputStream(args[0])); - System.out.println("HBase configuration:"); - for(Object key : props.keySet()) { - System.out.println(key + "=" + props.get(key)); - config.set((String)key, props.getProperty((String)key)); - } - } else { - System.out.println("Usage: WordCountClient <hbase_config.properties>"); - System.exit(1); - } - - HTable table = new HTable(config, "WordCount"); - String[] words = new String[] {"nathan", "mike", "jackson", "golda", "bertels"}; - - for (String word : words) { - Get get = new Get(Bytes.toBytes(word)); - Result result = table.get(get); - - byte[] countBytes = result.getValue(Bytes.toBytes("cf"), Bytes.toBytes("count")); - byte[] wordBytes = result.getValue(Bytes.toBytes("cf"), Bytes.toBytes("word")); - - String wordStr = Bytes.toString(wordBytes); - long count = Bytes.toLong(countBytes); - System.out.println("Word: '" + wordStr + "', Count: " + count); - } - - } -} http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java ---------------------------------------------------------------------- diff --git a/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java b/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java deleted file mode 100644 index 7093105..0000000 --- a/external/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.flux.examples; - -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.BasicOutputCollector; -import org.apache.storm.topology.IBasicBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseBasicBolt; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - -import static org.apache.storm.utils.Utils.tuple; - -/** - * This bolt is used by the HBase example. It simply emits the first field - * found in the incoming tuple as "word", with a "count" of `1`. - * - * In this case, the downstream HBase bolt handles the counting, so a value - * of `1` will just increment the HBase counter by one. - */ -public class WordCounter extends BaseBasicBolt { - private static final Logger LOG = LoggerFactory.getLogger(WordCounter.class); - - - - @SuppressWarnings("rawtypes") - public void prepare(Map stormConf, TopologyContext context) { - } - - /* - * Just output the word value with a count of 1. - * The HBaseBolt will handle incrementing the counter. - */ - public void execute(Tuple input, BasicOutputCollector collector) { - collector.emit(tuple(input.getValues().get(0), 1)); - } - - public void cleanup() { - - } - - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("word", "count")); - } - - @Override - public Map<String, Object> getComponentConfiguration() { - return null; - } - -} http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/external/flux/flux-examples/src/main/resources/hbase_bolt.properties ---------------------------------------------------------------------- diff --git a/external/flux/flux-examples/src/main/resources/hbase_bolt.properties b/external/flux/flux-examples/src/main/resources/hbase_bolt.properties deleted file mode 100644 index f8ed50c..0000000 --- a/external/flux/flux-examples/src/main/resources/hbase_bolt.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -hbase.rootdir=hdfs://hadoop:54310/hbase -hbase.zookeeper.quorum=hadoop \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/external/flux/flux-examples/src/main/resources/hdfs_bolt.properties ---------------------------------------------------------------------- diff --git a/external/flux/flux-examples/src/main/resources/hdfs_bolt.properties b/external/flux/flux-examples/src/main/resources/hdfs_bolt.properties deleted file mode 100644 index 7bcbe7a..0000000 --- a/external/flux/flux-examples/src/main/resources/hdfs_bolt.properties +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# The HDFS url -hdfs.url=hdfs://hadoop:54310 - -# The HDFS directory where the bolt will write incoming data -hdfs.write.dir=/incoming - -# The HDFS directory where files will be moved once the bolt has -# finished writing to it. -hdfs.dest.dir=/complete \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/external/flux/flux-examples/src/main/resources/kafka_spout.yaml ---------------------------------------------------------------------- diff --git a/external/flux/flux-examples/src/main/resources/kafka_spout.yaml b/external/flux/flux-examples/src/main/resources/kafka_spout.yaml deleted file mode 100644 index 7533ce4..0000000 --- a/external/flux/flux-examples/src/main/resources/kafka_spout.yaml +++ /dev/null @@ -1,136 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Test ability to wire together shell spouts/bolts ---- - -# topology definition -# name to be used when submitting -name: "kafka-topology" - -# Components -# Components are analagous to Spring beans. They are meant to be used as constructor, -# property(setter), and builder arguments. -# -# for the time being, components must be declared in the order they are referenced -components: - - id: "stringScheme" - className: "org.apache.storm.kafka.StringScheme" - - - id: "stringMultiScheme" - className: "org.apache.storm.spout.SchemeAsMultiScheme" - constructorArgs: - - ref: "stringScheme" - - - id: "zkHosts" - className: "org.apache.storm.kafka.ZkHosts" - constructorArgs: - - "localhost:2181" - -# Alternative kafka config -# - id: "kafkaConfig" -# className: "org.apache.storm.kafka.KafkaConfig" -# constructorArgs: -# # brokerHosts -# - ref: "zkHosts" -# # topic -# - "myKafkaTopic" -# # clientId (optional) -# - "myKafkaClientId" - - - id: "spoutConfig" - className: "org.apache.storm.kafka.SpoutConfig" - constructorArgs: - # brokerHosts - - ref: "zkHosts" - # topic - - "myKafkaTopic" - # zkRoot - - "/kafkaSpout" - # id - - "myId" - properties: - - name: "ignoreZkOffsets" - value: true - - name: "scheme" - ref: "stringMultiScheme" - - - -# NOTE: We may want to consider some level of spring integration. For example, allowing component references -# to a spring `ApplicationContext`. - -# topology configuration -# this will be passed to the submitter as a map of config options -# -config: - topology.workers: 1 - # ... - -# spout definitions -spouts: - - id: "kafka-spout" - className: "org.apache.storm.kafka.KafkaSpout" - constructorArgs: - - ref: "spoutConfig" - -# bolt definitions -bolts: - - id: "splitsentence" - className: "org.apache.storm.flux.wrappers.bolts.FluxShellBolt" - constructorArgs: - # command line - - ["python", "splitsentence.py"] - # output fields - - ["word"] - parallelism: 1 - # ... - - - id: "log" - className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt" - parallelism: 1 - # ... - - - id: "count" - className: "org.apache.storm.testing.TestWordCounter" - parallelism: 1 - # ... - -#stream definitions -# stream definitions define connections between spouts and bolts. -# note that such connections can be cyclical -# custom stream groupings are also supported - -streams: - - name: "kafka --> split" # name isn't used (placeholder for logging, UI, etc.) - from: "kafka-spout" - to: "splitsentence" - grouping: - type: SHUFFLE - - - name: "split --> count" - from: "splitsentence" - to: "count" - grouping: - type: FIELDS - args: ["word"] - - - name: "count --> log" - from: "count" - to: "log" - grouping: - type: SHUFFLE http://git-wip-us.apache.org/repos/asf/storm/blob/1b514cbd/external/flux/flux-examples/src/main/resources/multilang.yaml ---------------------------------------------------------------------- diff --git a/external/flux/flux-examples/src/main/resources/multilang.yaml b/external/flux/flux-examples/src/main/resources/multilang.yaml deleted file mode 100644 index aaab5d3..0000000 --- a/external/flux/flux-examples/src/main/resources/multilang.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Test ability to wire together shell spouts/bolts ---- - -# topology definition -# name to be used when submitting -name: "shell-topology" - -# topology configuration -# this will be passed to the submitter as a map of config options -# -config: - topology.workers: 1 - # ... - -# spout definitions -spouts: - - id: "sentence-spout" - className: "org.apache.storm.flux.wrappers.spouts.FluxShellSpout" - # shell spout constructor takes 2 arguments: String[], String[] - constructorArgs: - # command line - - ["node", "randomsentence.js"] - # output fields - - ["word"] - parallelism: 1 - # ... - -# bolt definitions -bolts: - - id: "splitsentence" - className: "org.apache.storm.flux.wrappers.bolts.FluxShellBolt" - constructorArgs: - # command line - - ["python", "splitsentence.py"] - # output fields - - ["word"] - parallelism: 1 - # ... - - - id: "log" - className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt" - parallelism: 1 - # ... - - - id: "count" - className: "org.apache.storm.testing.TestWordCounter" - parallelism: 1 - # ... - -#stream definitions -# stream definitions define connections between spouts and bolts. -# note that such connections can be cyclical -# custom stream groupings are also supported - -streams: - - name: "spout --> split" # name isn't used (placeholder for logging, UI, etc.) - from: "sentence-spout" - to: "splitsentence" - grouping: - type: SHUFFLE - - - name: "split --> count" - from: "splitsentence" - to: "count" - grouping: - type: FIELDS - args: ["word"] - - - name: "count --> log" - from: "count" - to: "log" - grouping: - type: SHUFFLE
