update hbase and hdfs examples
Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9f5f8227 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9f5f8227 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9f5f8227 Branch: refs/heads/0.10.x-branch Commit: 9f5f822720544e356682d3c007c3407f5c2e5058 Parents: 01702dc Author: P. Taylor Goetz <ptgo...@gmail.com> Authored: Wed Apr 8 12:50:59 2015 -0400 Committer: P. Taylor Goetz <ptgo...@gmail.com> Committed: Wed Apr 8 12:50:59 2015 -0400 ---------------------------------------------------------------------- .../storm/flux/examples/WordCountClient.java | 17 +++++++-- flux-examples/src/main/resources/hbase-site.xml | 36 -------------------- .../src/main/resources/hbase_bolt.properties | 2 ++ .../src/main/resources/hdfs_bolt.properties | 6 ++-- .../src/main/resources/simple_hbase.yaml | 3 +- .../src/main/resources/simple_hdfs.yaml | 6 ++-- 6 files changed, 24 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java ---------------------------------------------------------------------- diff --git a/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java b/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java index 55873d5..eb4fb7a 100644 --- a/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java +++ b/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java @@ -24,6 +24,9 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; +import java.io.FileInputStream; +import java.util.Properties; + /** * Connects to the 'WordCount' HBase table and prints counts for each word. * @@ -39,8 +42,17 @@ public class WordCountClient { public static void main(String[] args) throws Exception { Configuration config = HBaseConfiguration.create(); - if(args.length > 0){ - config.set("hbase.rootdir", args[0]); + if(args.length == 1){ + Properties props = new Properties(); + props.load(new FileInputStream(args[0])); + System.out.println("HBase configuration:"); + for(Object key : props.keySet()) { + System.out.println(key + "=" + props.get(key)); + config.set((String)key, props.getProperty((String)key)); + } + } else { + System.out.println("Usage: WordCountClient <hbase_config.properties>"); + System.exit(1); } HTable table = new HTable(config, "WordCount"); @@ -54,7 +66,6 @@ public class WordCountClient { byte[] wordBytes = result.getValue(Bytes.toBytes("cf"), Bytes.toBytes("word")); String wordStr = Bytes.toString(wordBytes); - System.out.println(wordStr); long count = Bytes.toLong(countBytes); System.out.println("Word: '" + wordStr + "', Count: " + count); } http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/hbase-site.xml ---------------------------------------------------------------------- diff --git a/flux-examples/src/main/resources/hbase-site.xml b/flux-examples/src/main/resources/hbase-site.xml deleted file mode 100644 index 06c3031..0000000 --- a/flux-examples/src/main/resources/hbase-site.xml +++ /dev/null @@ -1,36 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration> - <property> - <name>hbase.cluster.distributed</name> - <value>true</value> - </property> - <property> - <name>hbase.rootdir</name> - <value>hdfs://hadoop:54310/hbase</value> - </property> - <property> - <name>hbase.zookeeper.quorum</name> - <value>hadoop</value> - </property> -</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/hbase_bolt.properties ---------------------------------------------------------------------- diff --git a/flux-examples/src/main/resources/hbase_bolt.properties b/flux-examples/src/main/resources/hbase_bolt.properties new file mode 100644 index 0000000..9903b41 --- /dev/null +++ b/flux-examples/src/main/resources/hbase_bolt.properties @@ -0,0 +1,2 @@ +hbase.rootdir=hdfs://hadoop:54310/hbase +hbase.zookeeper.quorum=hadoop \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/hdfs_bolt.properties ---------------------------------------------------------------------- diff --git a/flux-examples/src/main/resources/hdfs_bolt.properties b/flux-examples/src/main/resources/hdfs_bolt.properties index 34a7a23..dd1307d 100644 --- a/flux-examples/src/main/resources/hdfs_bolt.properties +++ b/flux-examples/src/main/resources/hdfs_bolt.properties @@ -1,9 +1,9 @@ # The HDFS url -hdfs.url="hdfs://hadoop:54310" +hdfs.url=hdfs://hadoop:54310 # The HDFS directory where the bolt will write incoming data -hdfs.write.dir="/incoming" +hdfs.write.dir=/incoming # The HDFS directory where files will be moved once the bolt has # finished writing to it. -hdfs.dest.dir="/complete" \ No newline at end of file +hdfs.dest.dir=/complete \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/simple_hbase.yaml ---------------------------------------------------------------------- diff --git a/flux-examples/src/main/resources/simple_hbase.yaml b/flux-examples/src/main/resources/simple_hbase.yaml index 5eb70ed..62686d0 100644 --- a/flux-examples/src/main/resources/simple_hbase.yaml +++ b/flux-examples/src/main/resources/simple_hbase.yaml @@ -51,7 +51,8 @@ components: config: topology.workers: 1 hbase.conf: - hbase.rootdir: "hdfs://hadoop:54310/hbase" + hbase.rootdir: "${hbase.rootdir}" + hbase.zookeeper.quorum: "${hbase.zookeeper.quorum}" # spout definitions spouts: http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/simple_hdfs.yaml ---------------------------------------------------------------------- diff --git a/flux-examples/src/main/resources/simple_hdfs.yaml b/flux-examples/src/main/resources/simple_hdfs.yaml index ea7721d..9007869 100644 --- a/flux-examples/src/main/resources/simple_hdfs.yaml +++ b/flux-examples/src/main/resources/simple_hdfs.yaml @@ -41,7 +41,7 @@ components: className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat" configMethods: - name: "withPath" - args: [${hdfs.write.dir}] + args: ["${hdfs.write.dir}"] - name: "withExtension" args: [".txt"] @@ -55,7 +55,7 @@ components: className: "org.apache.storm.hdfs.common.rotation.MoveFileAction" configMethods: - name: "toDestination" - args: [${hdfs.dest.dir}] + args: ["${hdfs.dest.dir}"] # spout definitions spouts: @@ -73,7 +73,7 @@ bolts: - name: "withConfigKey" args: ["hdfs.config"] - name: "withFsUrl" - args: [${hdfs.url}] + args: ["${hdfs.url}"] - name: "withFileNameFormat" args: [ref: "fileNameFormat"] - name: "withRecordFormat"