[2/2] storm git commit: add missing license headers and clean up RAT report
add missing license headers and clean up RAT report Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2154048f Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2154048f Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2154048f Branch: refs/heads/master Commit: 2154048fd123a0960a4f5854ec4dd436bb775329 Parents: 285d943 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 17:38:14 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 17:38:14 2015 -0400 -- .travis.yml | 11 + LICENSE | 4 +- conf/jaas_kerberos.conf | 17 ++ dev-tools/test-ns.py| 15 ++ doap_Storm.rdf | 6 +- external/flux/LICENSE | 202 --- .../apache/storm/flux/test/SimpleTopology.java | 17 ++ .../storm/flux/test/SimpleTopologySource.java | 17 ++ .../test/SimpleTopologyWithConfigParam.java | 17 ++ .../org/apache/storm/flux/test/TestBolt.java| 17 ++ .../storm/flux/test/TridentTopologySource.java | 17 ++ .../existing-topology-method-override.yaml | 15 ++ .../existing-topology-reflection-config.yaml| 15 ++ .../configs/existing-topology-reflection.yaml | 15 ++ .../configs/existing-topology-trident.yaml | 15 ++ .../resources/configs/existing-topology.yaml| 15 ++ .../configs/invalid-existing-topology.yaml | 16 ++ .../src/test/resources/configs/test.properties | 16 ++ .../src/main/resources/config.properties| 15 ++ external/storm-hbase/LICENSE| 202 --- .../storm/hive/trident/HiveStateFactory.java| 17 ++ .../apache/storm/hive/trident/HiveUpdater.java | 17 ++ external/storm-jdbc/LICENSE | 202 --- .../storm/jdbc/common/ConnectionProvider.java | 17 ++ .../jdbc/common/HikariCPConnectionProvider.java | 17 ++ .../storm/jdbc/mapper/JdbcLookupMapper.java | 17 ++ .../jdbc/mapper/SimpleJdbcLookupMapper.java | 17 ++ external/storm-jdbc/src/test/sql/test.sql | 17 ++ .../ExponentialBackoffMsgRetryManagerTest.java | 17 ++ external/storm-redis/LICENSE| 202 --- .../redis/trident/WordCountLookupMapper.java| 17 ++ .../redis/trident/WordCountStoreMapper.java | 17 ++ pom.xml | 17 +- storm-core/src/clj/backtype/storm/converter.clj | 15 ++ .../src/dev/drpc-simple-acl-test-scenario.yaml | 17 ++ .../storm/messaging/ConnectionWithStatus.java | 17 ++ .../auth/authorizer/DRPCAuthorizerBase.java | 17 ++ .../authorizer/DRPCSimpleACLAuthorizer.java | 18 ++ .../authorizer/ImpersonationAuthorizer.java | 17 ++ .../auth/kerberos/jaas_kerberos_cluster.conf| 20 +- .../auth/kerberos/jaas_kerberos_launcher.conf | 19 ++ .../worker-launcher/.deps/worker-launcher.Po| 16 ++ .../auth/DefaultHttpCredentialsPlugin_test.clj | 15 ++ .../authorizer/DRPCSimpleACLAuthorizer_test.clj | 15 ++ .../storm/security/auth/drpc-auth-alice.jaas| 17 ++ .../storm/security/auth/drpc-auth-bob.jaas | 17 ++ .../storm/security/auth/drpc-auth-charlie.jaas | 17 ++ .../storm/security/auth/drpc-auth-server.jaas | 17 ++ 48 files changed, 692 insertions(+), 817 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/2154048f/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index 99da952..27484eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,14 @@ +# Licensed under the Apache License, Version 2.0 (the License); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. language: java jdk: - oraclejdk7 http://git-wip-us.apache.org/repos/asf/storm/blob/2154048f/LICENSE -- diff --git a/LICENSE b/LICENSE index f468287..8755d1b 100644 --- a/LICENSE +++ b/LICENSE @@ -477,8 +477,8 @@ THE SOFTWARE. For jquery dataTables bootstrap integration -(storm-core/src/ui/public/js/dataTables.bootstrap.min.js -storm-core/src/ui/public/css/dataTables.bootstrap.css) +(storm-core/src/ui/public/js/jquery.dataTables.1.10.4.min.js
[1/2] storm git commit: add missing license headers and clean up RAT report
Repository: storm Updated Branches: refs/heads/master 285d943b8 - 2154048fd http://git-wip-us.apache.org/repos/asf/storm/blob/2154048f/external/storm-redis/src/test/java/org/apache/storm/redis/trident/WordCountStoreMapper.java -- diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/trident/WordCountStoreMapper.java b/external/storm-redis/src/test/java/org/apache/storm/redis/trident/WordCountStoreMapper.java index 6521302..b930998 100644 --- a/external/storm-redis/src/test/java/org/apache/storm/redis/trident/WordCountStoreMapper.java +++ b/external/storm-redis/src/test/java/org/apache/storm/redis/trident/WordCountStoreMapper.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.storm.redis.trident; import backtype.storm.tuple.ITuple; http://git-wip-us.apache.org/repos/asf/storm/blob/2154048f/pom.xml -- diff --git a/pom.xml b/pom.xml index 74f539e..abae7fb 100644 --- a/pom.xml +++ b/pom.xml @@ -762,9 +762,17 @@ exclude**/src/py/**/exclude !-- the following are in the LICENSE file -- - exclude**/src/ui/public/js/jquery-1.6.2.min.js/exclude + exclude**/src/ui/public/js/jquery.dataTables.1.10.4.min.js/exclude + exclude**/src/ui/public/css/jquery.dataTables.1.10.4.min.css/exclude +exclude**/src/ui/public/images/*/exclude + exclude**/src/ui/public/js/bootstrap-3.3.1.min.js/exclude + exclude**/src/ui/public/css/bootstrap-3.3.1.min.css/exclude + exclude**/src/ui/public/js/dataTables.bootstrap.min.js/exclude + exclude**/src/ui/public/css/dataTables.bootstrap.css/exclude + exclude**/src/ui/public/js/jsonFormatter.min.js/exclude + exclude**/src/ui/public/css/jsonFormatter.min.css/exclude + exclude**/src/ui/public/js/jquery-1.11.1.min.js/exclude exclude**/src/ui/public/js/jquery.cookies.2.2.0.min.js/exclude - exclude**/src/ui/public/js/jquery.tablesorter.min.js/exclude exclude**/src/ui/public/js/moment.min.js/exclude exclude**/src/ui/public/js/jquery.blockUI.min.js/exclude exclude**/src/ui/public/js/url.min.js/exclude @@ -772,10 +780,13 @@ exclude**/src/ui/public/js/arbor-graphics.js/exclude exclude**/src/ui/public/js/arbor-tween.js/exclude exclude**/src/ui/public/js/jquery.mustache.js/exclude -exclude**/src/ui/public/js/purl.js/exclude !-- generated by shade plugin -- exclude**/dependency-reduced-pom.xml/exclude + +exclude**/docs/**/exclude +exclude**/.git/**/exclude +exclude**/derby.log/exclude /excludes /configuration /plugin http://git-wip-us.apache.org/repos/asf/storm/blob/2154048f/storm-core/src/clj/backtype/storm/converter.clj -- diff --git a/storm-core/src/clj/backtype/storm/converter.clj b/storm-core/src/clj/backtype/storm/converter.clj index 45e3033..7bfc14b 100644 --- a/storm-core/src/clj/backtype/storm/converter.clj +++ b/storm-core/src/clj/backtype/storm/converter.clj @@ -1,3 +1,18 @@ +;; Licensed to the Apache Software Foundation (ASF) under one +;; or more contributor license agreements. See the NOTICE file +;; distributed with this work for additional information +;; regarding copyright ownership. The ASF licenses this file +;; to you under the Apache License, Version 2.0 (the +;; License); you may not use this file except in compliance +;; with the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by
svn commit: r9209 - /release/storm/apache-storm-0.9.5/
Author: ptgoetz Date: Wed Jun 3 16:45:13 2015 New Revision: 9209 Log: release Apache Storm 0.9.5 Added: release/storm/apache-storm-0.9.5/ release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz (with props) release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.asc release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.md5 release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.sha release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.zip (with props) release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.zip.asc release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.zip.md5 release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.zip.sha release/storm/apache-storm-0.9.5/apache-storm-0.9.5.tar.gz (with props) release/storm/apache-storm-0.9.5/apache-storm-0.9.5.tar.gz.asc release/storm/apache-storm-0.9.5/apache-storm-0.9.5.tar.gz.md5 release/storm/apache-storm-0.9.5/apache-storm-0.9.5.tar.gz.sha release/storm/apache-storm-0.9.5/apache-storm-0.9.5.zip (with props) release/storm/apache-storm-0.9.5/apache-storm-0.9.5.zip.asc release/storm/apache-storm-0.9.5/apache-storm-0.9.5.zip.md5 release/storm/apache-storm-0.9.5/apache-storm-0.9.5.zip.sha Added: release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz == Binary file - no diff available. Propchange: release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz -- svn:mime-type = application/octet-stream Added: release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.asc == --- release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.asc (added) +++ release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.asc Wed Jun 3 16:45:13 2015 @@ -0,0 +1,12 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG/MacGPG2 v2.0.22 (Darwin) +Comment: GPGTools - https://gpgtools.org + +iQEcBAABCgAGBQJVaIJLAAoJEI3gOWLoC4/9t10H/1r99Cl3LnG+GRKOttRyv384 +RCkrufN969g1vAFUWdydHDsjQy7YxKxV7nqBu087qhbD2wF5KMfHGhua/aT4gBYR +sX+M7mUwhUiFgb+VUDQfIj8L/F2D2WHHH6IzFr5LMYjfBz3BijvYivbM9Mr/MISM +BtRqrofxy96QjLBB6N/0khpUsMiBnMIxEdsJAriGBeE8uusuU7dkkyJs4vzMM8/j +IRf9zcT0I3TGLS5qL0wEQTDawOK6V7mXWilGNvklN4YmhRIvyqATEFpFBKGa2J9j +rEaUoJDeZTEgV7ijWTf6/cNvr1aZdJlvuAfKqbn1X4MC7wxCi34x/1Zbp92uhuI= +=VOqK +-END PGP SIGNATURE- Added: release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.md5 == --- release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.md5 (added) +++ release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.md5 Wed Jun 3 16:45:13 2015 @@ -0,0 +1,6 @@ +apache-storm-0.9.5-src.tar.gz: 56 EA 68 + 88 C2 69 + C9 9B A2 + D8 44 1C + 23 06 D8 + C2 Added: release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.sha == --- release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.sha (added) +++ release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.tar.gz.sha Wed Jun 3 16:45:13 2015 @@ -0,0 +1,8 @@ +apache-storm-0.9.5-src.tar.gz: 4FCAC5E1 FFAD74CE + E375E7D3 15905BC6 + F9CF20B4 C11FAA9B + 0CD58EBF 2F16AA58 + 00B414EE CDEC9C3F + EE0492EA AE95756E + 8E292155 F9353AC2 + 094886E6 08CDC56D Added: release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.zip == Binary file - no diff available. Propchange: release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.zip -- svn:mime-type = application/octet-stream Added: release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.zip.asc == --- release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.zip.asc (added) +++ release/storm/apache-storm-0.9.5/apache-storm-0.9.5-src.zip.asc Wed Jun 3 16:45:13 2015 @@ -0,0 +1,12 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG/MacGPG2 v2.0.22 (Darwin) +Comment: GPGTools - https://gpgtools.org + +iQEcBAABCgAGBQJVaIJLAAoJEI3gOWLoC4/9HyQIAOtM+X0KBOK7FFzKcS5KtgrD +j7kMblVb0jljHxYdL200IhbIssppm361nUJbs6McPtsDLSxt0rx2tDQ53w4IivK3 +/fbISZ4Poo+aiWlss0qs73WKQl2LTEQoKvlp26TkaCg5pZtM+CDaWE1ARTuPCrmn +cGBOuLjYXsXdzkfePvGjpFygFpq6AJ4dl9f+lDZbafS0QJVyu0jzklpckKEAcGkj
[36/50] [abbrv] storm git commit: update EventHubClient library version to 0.9.1
update EventHubClient library version to 0.9.1 Signed-off-by: Shanyu Zhao shz...@microsoft.com Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/86f326a3 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/86f326a3 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/86f326a3 Branch: refs/heads/0.10.x-branch Commit: 86f326a3c0b40fc1b0eab65fcb17b438e433cfd6 Parents: 85aeb3d Author: Shanyu Zhao shz...@microsoft.com Authored: Fri May 29 14:05:28 2015 -0700 Committer: Shanyu Zhao shz...@microsoft.com Committed: Fri May 29 14:05:28 2015 -0700 -- external/storm-eventhubs/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/86f326a3/external/storm-eventhubs/pom.xml -- diff --git a/external/storm-eventhubs/pom.xml b/external/storm-eventhubs/pom.xml index 6d4a47b..5de412f 100755 --- a/external/storm-eventhubs/pom.xml +++ b/external/storm-eventhubs/pom.xml @@ -33,7 +33,7 @@ properties project.build.sourceEncodingUTF-8/project.build.sourceEncoding -eventhubs.client.version0.9/eventhubs.client.version +eventhubs.client.version0.9.1/eventhubs.client.version /properties build plugins
[37/50] [abbrv] storm git commit: STORM-842: Drop Support for Java 1.6
STORM-842: Drop Support for Java 1.6 Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fc736002 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fc736002 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fc736002 Branch: refs/heads/0.10.x-branch Commit: fc73600228c156a68327a342bf2c2da514620bbb Parents: ad98824 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Mon Jun 1 17:15:06 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Mon Jun 1 17:15:06 2015 -0400 -- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/fc736002/pom.xml -- diff --git a/pom.xml b/pom.xml index bfb9069..4729554 100644 --- a/pom.xml +++ b/pom.xml @@ -658,8 +658,8 @@ groupIdorg.apache.maven.plugins/groupId artifactIdmaven-compiler-plugin/artifactId configuration -source1.6/source -target1.6/target +source1.7/source +target1.7/target /configuration /plugin plugin
[43/50] [abbrv] storm git commit: add STORM-818 to changelog
add STORM-818 to changelog Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/23137753 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/23137753 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/23137753 Branch: refs/heads/0.10.x-branch Commit: 23137753250b4d8e8e15cfe661a9ba6cdc037273 Parents: 6978b58 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 13:08:46 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 13:08:46 2015 -0400 -- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/23137753/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index 6329872..02aa95d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 0.11.0 ## 0.10.0 + * STORM-818: storm-eventhubs configuration improvement and refactoring * STORM-842: Drop Support for Java 1.6 * STORM-835: Netty Client hold batch object until io operation complete * STORM-827: Allow AutoTGT to work with storm-hdfs too.
[38/50] [abbrv] storm git commit: Add initialization for ResilientEventHubReceiver
Add initialization for ResilientEventHubReceiver Otherwise, the recover logic will kick in for the first time. Signed-off-by: Shanyu Zhao shz...@microsoft.com Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9c2972ac Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9c2972ac Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9c2972ac Branch: refs/heads/0.10.x-branch Commit: 9c2972ac635ef5a7066b1abf19ffd5dc9e42718c Parents: 86f326a Author: Shanyu Zhao shz...@microsoft.com Authored: Mon Jun 1 18:23:11 2015 -0700 Committer: Shanyu Zhao shz...@microsoft.com Committed: Mon Jun 1 18:23:11 2015 -0700 -- .../java/org/apache/storm/eventhubs/spout/EventHubReceiverImpl.java | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/9c2972ac/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubReceiverImpl.java -- diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubReceiverImpl.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubReceiverImpl.java index 7454af4..0fcad99 100755 --- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubReceiverImpl.java +++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubReceiverImpl.java @@ -71,6 +71,7 @@ public class EventHubReceiverImpl implements IEventHubReceiver { long start = System.currentTimeMillis(); receiver = new ResilientEventHubReceiver(connectionString, entityName, partitionId, consumerGroupName, defaultCredits, filter); +receiver.initialize(); long end = System.currentTimeMillis(); logger.info(created eventhub receiver, time taken(ms): + (end-start));
[50/50] [abbrv] storm git commit: fix version in pom
fix version in pom Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b2642962 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b2642962 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b2642962 Branch: refs/heads/0.10.x-branch Commit: b26429621741a785658d1984d113b15d491052e3 Parents: c3cc4dc Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 22:02:44 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 22:02:44 2015 -0400 -- storm-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/b2642962/storm-core/pom.xml -- diff --git a/storm-core/pom.xml b/storm-core/pom.xml index 765a1dd..db54481 100644 --- a/storm-core/pom.xml +++ b/storm-core/pom.xml @@ -20,7 +20,7 @@ parent artifactIdstorm/artifactId groupIdorg.apache.storm/groupId -version0.11.0-SNAPSHOT/version +version0.10.0-SNAPSHOT/version relativePath../relativePath /parent groupIdorg.apache.storm/groupId
[41/50] [abbrv] storm git commit: update changelog for STORM-842
update changelog for STORM-842 Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a55bbbea Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a55bbbea Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a55bbbea Branch: refs/heads/0.10.x-branch Commit: a55bbbea85fa83c3e9ab664922580d464de565fe Parents: d285d94 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue Jun 2 17:16:53 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue Jun 2 17:16:53 2015 -0400 -- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/a55bbbea/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index f9f7999..6329872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 0.11.0 ## 0.10.0 + * STORM-842: Drop Support for Java 1.6 * STORM-835: Netty Client hold batch object until io operation complete * STORM-827: Allow AutoTGT to work with storm-hdfs too. * STORM-821: Adding connection provider interface to decouple jdbc connector from a single connection pooling implementation.
[13/50] [abbrv] storm git commit: improve error logging
improve error logging Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/01702dc5 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/01702dc5 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/01702dc5 Branch: refs/heads/0.10.x-branch Commit: 01702dc503e4f5fbcc499dcae48bb3ffad1229db Parents: ae305c7 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Apr 8 01:10:30 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Apr 8 01:10:30 2015 -0400 -- .../java/org/apache/storm/flux/FluxBuilder.java | 9 +- .../java/org/apache/storm/flux/TCKTest.java | 10 ++ .../src/test/resources/configs/bad_hbase.yaml | 98 3 files changed, 114 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/01702dc5/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java b/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java index 964c62e..57237b6 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java +++ b/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java @@ -286,7 +286,10 @@ public class FluxBuilder { LOG.debug(Found something seemingly compatible, attempting invocation...); obj = con.newInstance(getArgsWithListCoercian(cArgs, con.getParameterTypes())); } else { -throw new IllegalArgumentException(Couldn't find a suitable constructor.); +String msg = String.format(Couldn't find a suitable constructor for class '%s' with arguments '%s'., +clazz.getName(), +cArgs); +throw new IllegalArgumentException(msg); } } else { obj = clazz.newInstance(); @@ -419,9 +422,9 @@ public class FluxBuilder { Object[] methodArgs = getArgsWithListCoercian(args, method.getParameterTypes()); method.invoke(instance, methodArgs); } else { -LOG.warn(Unable to find method '{}' in class '{}' with arguments {}., +String msg = String.format(Unable to find configuration method '%s' in class '%s' with arguments %s., new Object[]{methodName, clazz.getName(), args}); -throw new IllegalArgumentException(Configuration method not found.); +throw new IllegalArgumentException(msg); } } } http://git-wip-us.apache.org/repos/asf/storm/blob/01702dc5/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java -- diff --git a/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java b/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java index 27abfbe..9456d1b 100644 --- a/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java +++ b/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java @@ -91,6 +91,16 @@ public class TCKTest { topology.validate(); } +@Test(expected = IllegalArgumentException.class) +public void testBadHbase() throws Exception { +TopologyDef topologyDef = FluxParser.parseResource(/configs/bad_hbase.yaml, false, true, null, false); +Config conf = FluxBuilder.buildConfig(topologyDef); +ExecutionContext context = new ExecutionContext(topologyDef, conf); +StormTopology topology = FluxBuilder.buildTopology(context); +assertNotNull(topology); +topology.validate(); +} + @Test public void testIncludes() throws Exception { TopologyDef topologyDef = FluxParser.parseResource(/configs/include_test.yaml, false, true, null, false); http://git-wip-us.apache.org/repos/asf/storm/blob/01702dc5/flux-core/src/test/resources/configs/bad_hbase.yaml -- diff --git a/flux-core/src/test/resources/configs/bad_hbase.yaml b/flux-core/src/test/resources/configs/bad_hbase.yaml new file mode 100644 index 000..5d91400 --- /dev/null +++ b/flux-core/src/test/resources/configs/bad_hbase.yaml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# License); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed
[45/50] [abbrv] storm git commit: add STORM-561 to changelog
add STORM-561 to changelog Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/285d943b Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/285d943b Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/285d943b Branch: refs/heads/0.10.x-branch Commit: 285d943b87f5037f0eaf3d45af64622f13017dea Parents: cb370a9 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 13:24:03 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 13:24:03 2015 -0400 -- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/285d943b/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index 02aa95d..aa390f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,11 @@ ## 0.11.0 ## 0.10.0 - * STORM-818: storm-eventhubs configuration improvement and refactoring * STORM-842: Drop Support for Java 1.6 * STORM-835: Netty Client hold batch object until io operation complete * STORM-827: Allow AutoTGT to work with storm-hdfs too. * STORM-821: Adding connection provider interface to decouple jdbc connector from a single connection pooling implementation. + * STORM-818: storm-eventhubs configuration improvement and refactoring * STORM-816: maven-gpg-plugin does not work with gpg 2.1 * STORM-811: remove old metastor_db before running tests again. * STORM-808: allow null to be parsed as null @@ -124,6 +124,7 @@ * STORM-567: Move Storm Documentation/Website from SVN to git * STORM-565: Fix NPE when topology.groups is null. * STORM-563: Kafka Spout doesn't pick up from the beginning of the queue unless forceFromStart specified. + * STORM-561: Add flux as an external module * STORM-557: High Quality Images for presentations * STORM-554: the type of first param topology should be ^StormTopology not ^TopologyContext * STORM-552: Add netty socket backlog config
[15/50] [abbrv] storm git commit: set default parallelism of spouts/bolts to 1
set default parallelism of spouts/bolts to 1 Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/abf2924e Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/abf2924e Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/abf2924e Branch: refs/heads/0.10.x-branch Commit: abf2924e8a89d9b9fc28037a4ce079c607f62939 Parents: 9f5f822 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Fri Apr 10 15:47:16 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Fri Apr 10 15:47:16 2015 -0400 -- .../src/main/java/org/apache/storm/flux/model/VertexDef.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/abf2924e/flux-core/src/main/java/org/apache/storm/flux/model/VertexDef.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/model/VertexDef.java b/flux-core/src/main/java/org/apache/storm/flux/model/VertexDef.java index f8bf607..e71bcc2 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/model/VertexDef.java +++ b/flux-core/src/main/java/org/apache/storm/flux/model/VertexDef.java @@ -23,7 +23,8 @@ package org.apache.storm.flux.model; */ public abstract class VertexDef extends BeanDef { -private int parallelism; +// default parallelism to 1 so if it's ommitted, the topology will still function. +private int parallelism = 1; public int getParallelism() { return parallelism;
[11/50] [abbrv] storm git commit: add basic docs for HBase example
add basic docs for HBase example Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/edc5744c Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/edc5744c Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/edc5744c Branch: refs/heads/0.10.x-branch Commit: edc5744c48b65c3d7298ec5a61affb6164fc4d96 Parents: 4a1db96 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue Apr 7 23:57:53 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue Apr 7 23:57:53 2015 -0400 -- flux-examples/README.md | 12 1 file changed, 12 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/edc5744c/flux-examples/README.md -- diff --git a/flux-examples/README.md b/flux-examples/README.md index 425ad98..b3798a6 100644 --- a/flux-examples/README.md +++ b/flux-examples/README.md @@ -54,3 +54,15 @@ least, the property `hdfs.url` to point to a HDFS cluster. Then you can run the storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hdfs.yaml --filter my_hdfs_bolt.properties ``` +### [simple_hbase.yaml](src/main/resources/simple_hbase.yaml) + +This example illustrates how to use Flux to setup a storm-hbase bolt to write to HBase. + +In order to use this example, you will need to edit the `src/main resrouces/hbase-site.xml` file to reflect your HBase +environment, and then rebuild the topology jar. + +You can do so by running the following Maven command in the `flux-examples` directory: + +```bash +mvn clean install +``` \ No newline at end of file
[08/50] [abbrv] storm git commit: update examples and docs for HDFS example
update examples and docs for HDFS example Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0c1e0aa8 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0c1e0aa8 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0c1e0aa8 Branch: refs/heads/0.10.x-branch Commit: 0c1e0aa81f39e473bcf3482448813d78065e1212 Parents: 3411bc7 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Mon Apr 6 23:48:38 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Mon Apr 6 23:48:38 2015 -0400 -- .../src/test/resources/configs/hdfs_test.yaml | 97 - flux-examples/README.md | 30 +- flux-examples/pom.xml | 6 ++ .../src/main/resources/hdfs_bolt.properties | 9 ++ flux-examples/src/main/resources/multilang.yaml | 89 flux-examples/src/main/resources/shell.yaml | 89 .../src/main/resources/simple_hdfs.yaml | 105 +++ 7 files changed, 238 insertions(+), 187 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/0c1e0aa8/flux-core/src/test/resources/configs/hdfs_test.yaml -- diff --git a/flux-core/src/test/resources/configs/hdfs_test.yaml b/flux-core/src/test/resources/configs/hdfs_test.yaml deleted file mode 100644 index c1d28d2..000 --- a/flux-core/src/test/resources/configs/hdfs_test.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# License); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an AS IS BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Test ability to wire together shell spouts/bolts - -# topology definition -# name to be used when submitting -name: hdfs-topology - -# Components -# Components are analagous to Spring beans. They are meant to be used as constructor, -# property(setter), and builder arguments. -# -# for the time being, components must be declared in the order they are referenced -components: - - id: syncPolicy -className: org.apache.storm.hdfs.bolt.sync.CountSyncPolicy -constructorArgs: - - 1000 - - id: rotationPolicy -className: org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy -constructorArgs: - - 5.0 - - MB - - - id: fileNameFormat -className: org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat -configMethods: - - name: withPath -args: [/tmp/foo/] - - name: withExtension -args: [.txt] - - - id: recordFormat -className: org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat -configMethods: - - name: withFieldDelimiter -args: [|] - - - id: rotationAction -className: org.apache.storm.hdfs.common.rotation.MoveFileAction -configMethods: - - name: toDestination -args: [/tmp/dest2] - -# spout definitions -spouts: - - id: spout-1 -className: backtype.storm.testing.TestWordSpout -parallelism: 1 -# ... - -# bolt definitions - -#HdfsBolt bolt = new HdfsBolt() -#.withConfigKey(hdfs.config) -#.withFsUrl(args[0]) -#.withFileNameFormat(fileNameFormat) -#.withRecordFormat(format) -#.withRotationPolicy(rotationPolicy) -#.withSyncPolicy(syncPolicy) -#.addRotationAction(new MoveFileAction().toDestination(/tmp/dest2/)); -bolts: - - id: bolt-1 -className: org.apache.storm.hdfs.bolt.HdfsBolt -configMethods: - - name: withConfigKey -args: [hdfs.config] - - name: withFsUrl -args: [hdfs://localhost:1234] - - name: withFileNameFormat -args: [ref: fileNameFormat] - - name: withRecordFormat -args: [ref: recordFormat] - - name: withRotationPolicy -args: [ref: rotationPolicy] - - name: withSyncPolicy -args: [ref: syncPolicy] - - name: addRotationAction -args: [ref: rotationAction] -parallelism: 1 -# ... - http://git-wip-us.apache.org/repos/asf/storm/blob/0c1e0aa8/flux-examples/README.md
[30/50] [abbrv] storm git commit: add back missing multi-lang scripts
add back missing multi-lang scripts Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9fad816f Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9fad816f Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9fad816f Branch: refs/heads/0.10.x-branch Commit: 9fad816fd5840d5ad7d2727db4c7dafe4f1355ba Parents: f7d2f7f Author: P. Taylor Goetz ptgo...@gmail.com Authored: Fri May 8 15:23:54 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Fri May 8 15:23:54 2015 -0400 -- .../main/resources/resources/randomsentence.js | 93 .../main/resources/resources/splitsentence.py | 24 + 2 files changed, 117 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/9fad816f/external/flux/flux-wrappers/src/main/resources/resources/randomsentence.js -- diff --git a/external/flux/flux-wrappers/src/main/resources/resources/randomsentence.js b/external/flux/flux-wrappers/src/main/resources/resources/randomsentence.js new file mode 100644 index 000..36fc5f5 --- /dev/null +++ b/external/flux/flux-wrappers/src/main/resources/resources/randomsentence.js @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Example for storm spout. Emits random sentences. + * The original class in java - storm.starter.spout.RandomSentenceSpout. + * + */ + +var storm = require('./storm'); +var Spout = storm.Spout; + + +var SENTENCES = [ +the cow jumped over the moon, +an apple a day keeps the doctor away, +four score and seven years ago, +snow white and the seven dwarfs, +i am at two with nature] + +function RandomSentenceSpout(sentences) { +Spout.call(this); +this.runningTupleId = 0; +this.sentences = sentences; +this.pending = {}; +}; + +RandomSentenceSpout.prototype = Object.create(Spout.prototype); +RandomSentenceSpout.prototype.constructor = RandomSentenceSpout; + +RandomSentenceSpout.prototype.getRandomSentence = function() { +return this.sentences[getRandomInt(0, this.sentences.length - 1)]; +} + +RandomSentenceSpout.prototype.nextTuple = function(done) { +var self = this; +var sentence = this.getRandomSentence(); +var tup = [sentence]; +var id = this.createNextTupleId(); +this.pending[id] = tup; +//This timeout can be removed if TOPOLOGY_SLEEP_SPOUT_WAIT_STRATEGY_TIME_MS is configured to 100 +setTimeout(function() { +self.emit({tuple: tup, id: id}, function(taskIds) { +self.log(tup + ' sent to task ids - ' + taskIds); +}); +done(); +},100); +} + +RandomSentenceSpout.prototype.createNextTupleId = function() { +var id = this.runningTupleId; +this.runningTupleId++; +return id; +} + +RandomSentenceSpout.prototype.ack = function(id, done) { +this.log('Received ack for - ' + id); +delete this.pending[id]; +done(); +} + +RandomSentenceSpout.prototype.fail = function(id, done) { +var self = this; +this.log('Received fail for - ' + id + '. Retrying.'); +this.emit({tuple: this.pending[id], id:id}, function(taskIds) { +self.log(self.pending[id] + ' sent to task ids - ' + taskIds); +}); +done(); +} + +/** + * Returns a random integer between min (inclusive) and max (inclusive) + */ +function getRandomInt(min, max) { +return Math.floor(Math.random() * (max - min + 1)) + min; +} + +new RandomSentenceSpout(SENTENCES).run(); http://git-wip-us.apache.org/repos/asf/storm/blob/9fad816f/external/flux/flux-wrappers/src/main/resources/resources/splitsentence.py -- diff --git a/external/flux/flux-wrappers/src/main/resources/resources/splitsentence.py b/external/flux/flux-wrappers/src/main/resources/resources/splitsentence.py new file mode 100644 index 000..300105f --- /dev/null +++ b/external/flux/flux-wrappers/src/main/resources/resources/splitsentence.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software
[49/50] [abbrv] storm git commit: Merge branch 'master' into 0.10.x-branch
Merge branch 'master' into 0.10.x-branch Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c3cc4dc8 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c3cc4dc8 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c3cc4dc8 Branch: refs/heads/0.10.x-branch Commit: c3cc4dc893d3a3886e1c28745960df5268509d22 Parents: ea81f96 2154048 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 20:39:38 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 20:39:38 2015 -0400 -- .travis.yml | 11 + LICENSE | 4 +- conf/jaas_kerberos.conf | 17 ++ dev-tools/test-ns.py| 15 ++ doap_Storm.rdf | 6 +- external/flux/LICENSE | 202 --- .../apache/storm/flux/test/SimpleTopology.java | 17 ++ .../storm/flux/test/SimpleTopologySource.java | 17 ++ .../test/SimpleTopologyWithConfigParam.java | 17 ++ .../org/apache/storm/flux/test/TestBolt.java| 17 ++ .../storm/flux/test/TridentTopologySource.java | 17 ++ .../existing-topology-method-override.yaml | 15 ++ .../existing-topology-reflection-config.yaml| 15 ++ .../configs/existing-topology-reflection.yaml | 15 ++ .../configs/existing-topology-trident.yaml | 15 ++ .../resources/configs/existing-topology.yaml| 15 ++ .../configs/invalid-existing-topology.yaml | 16 ++ .../src/test/resources/configs/test.properties | 16 ++ .../src/main/resources/config.properties| 15 ++ external/storm-hbase/LICENSE| 202 --- .../storm/hive/trident/HiveStateFactory.java| 17 ++ .../apache/storm/hive/trident/HiveUpdater.java | 17 ++ external/storm-jdbc/LICENSE | 202 --- .../storm/jdbc/common/ConnectionProvider.java | 17 ++ .../jdbc/common/HikariCPConnectionProvider.java | 17 ++ .../storm/jdbc/mapper/JdbcLookupMapper.java | 17 ++ .../jdbc/mapper/SimpleJdbcLookupMapper.java | 17 ++ external/storm-jdbc/src/test/sql/test.sql | 17 ++ .../ExponentialBackoffMsgRetryManagerTest.java | 17 ++ external/storm-redis/LICENSE| 202 --- .../redis/trident/WordCountLookupMapper.java| 17 ++ .../redis/trident/WordCountStoreMapper.java | 17 ++ pom.xml | 17 +- storm-core/src/clj/backtype/storm/converter.clj | 15 ++ .../src/dev/drpc-simple-acl-test-scenario.yaml | 17 ++ .../storm/messaging/ConnectionWithStatus.java | 17 ++ .../auth/authorizer/DRPCAuthorizerBase.java | 17 ++ .../authorizer/DRPCSimpleACLAuthorizer.java | 18 ++ .../authorizer/ImpersonationAuthorizer.java | 17 ++ .../auth/kerberos/jaas_kerberos_cluster.conf| 20 +- .../auth/kerberos/jaas_kerberos_launcher.conf | 19 ++ .../worker-launcher/.deps/worker-launcher.Po| 16 ++ .../auth/DefaultHttpCredentialsPlugin_test.clj | 15 ++ .../authorizer/DRPCSimpleACLAuthorizer_test.clj | 15 ++ .../storm/security/auth/drpc-auth-alice.jaas| 17 ++ .../storm/security/auth/drpc-auth-bob.jaas | 17 ++ .../storm/security/auth/drpc-auth-charlie.jaas | 17 ++ .../storm/security/auth/drpc-auth-server.jaas | 17 ++ 48 files changed, 692 insertions(+), 817 deletions(-) --
[40/50] [abbrv] storm git commit: Merge branch 'java17'
Merge branch 'java17' Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d285d94d Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d285d94d Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d285d94d Branch: refs/heads/0.10.x-branch Commit: d285d94d418bede3376e743f882c157a83f68a7a Parents: e7818af fc73600 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue Jun 2 17:15:30 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue Jun 2 17:15:30 2015 -0400 -- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) --
[48/50] [abbrv] storm git commit: add missing license headers and clean up RAT report
add missing license headers and clean up RAT report Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2154048f Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2154048f Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2154048f Branch: refs/heads/0.10.x-branch Commit: 2154048fd123a0960a4f5854ec4dd436bb775329 Parents: 285d943 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 17:38:14 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 17:38:14 2015 -0400 -- .travis.yml | 11 + LICENSE | 4 +- conf/jaas_kerberos.conf | 17 ++ dev-tools/test-ns.py| 15 ++ doap_Storm.rdf | 6 +- external/flux/LICENSE | 202 --- .../apache/storm/flux/test/SimpleTopology.java | 17 ++ .../storm/flux/test/SimpleTopologySource.java | 17 ++ .../test/SimpleTopologyWithConfigParam.java | 17 ++ .../org/apache/storm/flux/test/TestBolt.java| 17 ++ .../storm/flux/test/TridentTopologySource.java | 17 ++ .../existing-topology-method-override.yaml | 15 ++ .../existing-topology-reflection-config.yaml| 15 ++ .../configs/existing-topology-reflection.yaml | 15 ++ .../configs/existing-topology-trident.yaml | 15 ++ .../resources/configs/existing-topology.yaml| 15 ++ .../configs/invalid-existing-topology.yaml | 16 ++ .../src/test/resources/configs/test.properties | 16 ++ .../src/main/resources/config.properties| 15 ++ external/storm-hbase/LICENSE| 202 --- .../storm/hive/trident/HiveStateFactory.java| 17 ++ .../apache/storm/hive/trident/HiveUpdater.java | 17 ++ external/storm-jdbc/LICENSE | 202 --- .../storm/jdbc/common/ConnectionProvider.java | 17 ++ .../jdbc/common/HikariCPConnectionProvider.java | 17 ++ .../storm/jdbc/mapper/JdbcLookupMapper.java | 17 ++ .../jdbc/mapper/SimpleJdbcLookupMapper.java | 17 ++ external/storm-jdbc/src/test/sql/test.sql | 17 ++ .../ExponentialBackoffMsgRetryManagerTest.java | 17 ++ external/storm-redis/LICENSE| 202 --- .../redis/trident/WordCountLookupMapper.java| 17 ++ .../redis/trident/WordCountStoreMapper.java | 17 ++ pom.xml | 17 +- storm-core/src/clj/backtype/storm/converter.clj | 15 ++ .../src/dev/drpc-simple-acl-test-scenario.yaml | 17 ++ .../storm/messaging/ConnectionWithStatus.java | 17 ++ .../auth/authorizer/DRPCAuthorizerBase.java | 17 ++ .../authorizer/DRPCSimpleACLAuthorizer.java | 18 ++ .../authorizer/ImpersonationAuthorizer.java | 17 ++ .../auth/kerberos/jaas_kerberos_cluster.conf| 20 +- .../auth/kerberos/jaas_kerberos_launcher.conf | 19 ++ .../worker-launcher/.deps/worker-launcher.Po| 16 ++ .../auth/DefaultHttpCredentialsPlugin_test.clj | 15 ++ .../authorizer/DRPCSimpleACLAuthorizer_test.clj | 15 ++ .../storm/security/auth/drpc-auth-alice.jaas| 17 ++ .../storm/security/auth/drpc-auth-bob.jaas | 17 ++ .../storm/security/auth/drpc-auth-charlie.jaas | 17 ++ .../storm/security/auth/drpc-auth-server.jaas | 17 ++ 48 files changed, 692 insertions(+), 817 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/2154048f/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index 99da952..27484eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,14 @@ +# Licensed under the Apache License, Version 2.0 (the License); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. language: java jdk: - oraclejdk7 http://git-wip-us.apache.org/repos/asf/storm/blob/2154048f/LICENSE -- diff --git a/LICENSE b/LICENSE index f468287..8755d1b 100644 --- a/LICENSE +++ b/LICENSE @@ -477,8 +477,8 @@ THE SOFTWARE. For jquery dataTables bootstrap integration -(storm-core/src/ui/public/js/dataTables.bootstrap.min.js -storm-core/src/ui/public/css/dataTables.bootstrap.css) +(storm-core/src/ui/public/js/jquery.dataTables.1.10.4.min.js
[17/50] [abbrv] storm git commit: document how to build from source
document how to build from source Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/601cee72 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/601cee72 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/601cee72 Branch: refs/heads/0.10.x-branch Commit: 601cee72aba6d9130bd02a08617f973e14c61eea Parents: f432abf Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue May 5 15:39:25 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue May 5 15:39:25 2015 -0400 -- README.md | 33 .../apache/storm/flux/model/TopologyDef.java| 1 - 2 files changed, 33 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/601cee72/README.md -- diff --git a/README.md b/README.md index 92fec10..6683848 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,39 @@ the layout and configuration of your topologies. To use Flux, add it as a dependency and package all your Storm components in a fat jar, then create a YAML document to define your topology (see below for YAML configuration options). +### Building from Source +The easiest way to use Flux, is to add it as a Maven dependency in you project as described below. + +If you would like to build Flux from source and run the unit/integration tests, you will need the following installed +on your system: + +* Python 2.6.x or later +* Node.js 0.10.x or later + + Building with unit tests enabled: + +``` +mvn clean install +``` + + Building with unit tests disabled: +If you would like to build Flux without installing Python or Node.js you can simply skip the unit tests: + +``` +mvn clean install -DskipTests=true +``` + +Note that if you plan on using Flux to deploy topologies to a remote cluster, you will still need to have Python +installed since it is required by Apache Storm. + + + Building with integration tests enabled: + +``` +mvn clean install -DskipIntegration=false +``` + + ### Packaging with Maven To enable Flux for your Storm components, you need to add it as a dependency such that it's included in the Storm topology jar. This can be accomplished with the Maven shade plugin (preferred) or the Maven assembly plugin (not http://git-wip-us.apache.org/repos/asf/storm/blob/601cee72/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java b/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java index 6c34018..a6ae450 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java +++ b/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java @@ -45,7 +45,6 @@ public class TopologyDef { private TopologySourceDef topologySource; // the following are required if we're defining a core storm topology DAG in YAML, etc. -//TODO if any of these are specified and `topologySource != null` it should be considered an error. private MapString, BoltDef boltMap = new LinkedHashMapString, BoltDef(); private MapString, SpoutDef spoutMap = new LinkedHashMapString, SpoutDef(); private ListStreamDef streams = new ArrayListStreamDef();
[04/50] [abbrv] storm git commit: lock down methods that don't need to be public
lock down methods that don't need to be public Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/11b01518 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/11b01518 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/11b01518 Branch: refs/heads/0.10.x-branch Commit: 11b01518f0935dedfd71d09b697d7189a29b91b5 Parents: df34930 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Thu Apr 2 00:08:12 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Thu Apr 2 00:08:12 2015 -0400 -- .../main/java/org/apache/storm/flux/FluxBuilder.java| 12 ++-- 1 file changed, 6 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/11b01518/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java b/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java index 31b6e64..964c62e 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java +++ b/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java @@ -61,7 +61,7 @@ public class FluxBuilder { * @throws NoSuchMethodException * @throws InvocationTargetException */ -public static StormTopology buildTopology(ExecutionContext context) throws IllegalAccessException, +static StormTopology buildTopology(ExecutionContext context) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException { StormTopology topology = null; @@ -206,7 +206,7 @@ public class FluxBuilder { } } -public static void applyProperties(ObjectDef bean, Object instance, ExecutionContext context) throws +private static void applyProperties(ObjectDef bean, Object instance, ExecutionContext context) throws IllegalAccessException, InvocationTargetException { ListPropertyDef props = bean.getProperties(); Class clazz = instance.getClass(); @@ -230,7 +230,7 @@ public class FluxBuilder { } } -public static Field findPublicField(Class clazz, String property, Object arg) { +private static Field findPublicField(Class clazz, String property, Object arg) { Field field = null; try { field = clazz.getField(property); @@ -240,7 +240,7 @@ public class FluxBuilder { return field; } -public static Method findSetter(Class clazz, String property, Object arg) { +private static Method findSetter(Class clazz, String property, Object arg) { String setterName = toSetterName(property); Method retval = null; Method[] methods = clazz.getMethods(); @@ -253,11 +253,11 @@ public class FluxBuilder { return retval; } -public static String toSetterName(String name) { +private static String toSetterName(String name) { return set + name.substring(0, 1).toUpperCase() + name.substring(1, name.length()); } -public static ListObject resolveReferences(ListObject args, ExecutionContext context) { +private static ListObject resolveReferences(ListObject args, ExecutionContext context) { LOG.debug(Checking arguments for references.); ListObject cArgs = new ArrayListObject(); // resolve references
[14/50] [abbrv] storm git commit: update hbase and hdfs examples
update hbase and hdfs examples Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9f5f8227 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9f5f8227 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9f5f8227 Branch: refs/heads/0.10.x-branch Commit: 9f5f822720544e356682d3c007c3407f5c2e5058 Parents: 01702dc Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Apr 8 12:50:59 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Apr 8 12:50:59 2015 -0400 -- .../storm/flux/examples/WordCountClient.java| 17 +++-- flux-examples/src/main/resources/hbase-site.xml | 36 .../src/main/resources/hbase_bolt.properties| 2 ++ .../src/main/resources/hdfs_bolt.properties | 6 ++-- .../src/main/resources/simple_hbase.yaml| 3 +- .../src/main/resources/simple_hdfs.yaml | 6 ++-- 6 files changed, 24 insertions(+), 46 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java -- diff --git a/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java b/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java index 55873d5..eb4fb7a 100644 --- a/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java +++ b/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java @@ -24,6 +24,9 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; +import java.io.FileInputStream; +import java.util.Properties; + /** * Connects to the 'WordCount' HBase table and prints counts for each word. * @@ -39,8 +42,17 @@ public class WordCountClient { public static void main(String[] args) throws Exception { Configuration config = HBaseConfiguration.create(); -if(args.length 0){ -config.set(hbase.rootdir, args[0]); +if(args.length == 1){ +Properties props = new Properties(); +props.load(new FileInputStream(args[0])); +System.out.println(HBase configuration:); +for(Object key : props.keySet()) { +System.out.println(key + = + props.get(key)); +config.set((String)key, props.getProperty((String)key)); +} +} else { +System.out.println(Usage: WordCountClient hbase_config.properties); +System.exit(1); } HTable table = new HTable(config, WordCount); @@ -54,7 +66,6 @@ public class WordCountClient { byte[] wordBytes = result.getValue(Bytes.toBytes(cf), Bytes.toBytes(word)); String wordStr = Bytes.toString(wordBytes); -System.out.println(wordStr); long count = Bytes.toLong(countBytes); System.out.println(Word: ' + wordStr + ', Count: + count); } http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/hbase-site.xml -- diff --git a/flux-examples/src/main/resources/hbase-site.xml b/flux-examples/src/main/resources/hbase-site.xml deleted file mode 100644 index 06c3031..000 --- a/flux-examples/src/main/resources/hbase-site.xml +++ /dev/null @@ -1,36 +0,0 @@ -?xml version=1.0? -?xml-stylesheet type=text/xsl href=configuration.xsl? -!-- -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- -configuration - property - namehbase.cluster.distributed/name - valuetrue/value - /property - property - namehbase.rootdir/name - valuehdfs://hadoop:54310/hbase/value - /property - property - namehbase.zookeeper.quorum/name - valuehadoop/value - /property -/configuration \ No newline at end of file
[16/50] [abbrv] storm git commit: fix validation
fix validation Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f432abf5 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f432abf5 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f432abf5 Branch: refs/heads/0.10.x-branch Commit: f432abf523a54806e4b927f676b93575bdf687f2 Parents: abf2924 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Fri Apr 10 15:47:39 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Fri Apr 10 15:47:39 2015 -0400 -- .../apache/storm/flux/model/TopologyDef.java| 23 +++- 1 file changed, 13 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/f432abf5/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java b/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java index 3be7dd6..6c34018 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java +++ b/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java @@ -196,19 +196,22 @@ public class TopologyDef { } public boolean isDslTopology(){ -boolean hasSpouts = this.spoutMap != null this.spoutMap.size() 0; -boolean hasBolts = this.boltMap != null this.boltMap.size() 0; -boolean hasStreams = this.streams != null this.streams.size() 0; -boolean isDslTopology = hasSpouts || hasBolts || hasStreams; - -return isDslTopology; +return this.topologySource == null; } public boolean validate(){ -// we can't have a topology source and spout/bolt/stream definitions at the same time -boolean isDslTopology = isDslTopology(); -boolean isTopologySource = this.topologySource != null; -return !(isDslTopology isTopologySource); +boolean hasSpouts = this.spoutMap != null this.spoutMap.size() 0; +boolean hasBolts = this.boltMap != null this.boltMap.size() 0; +boolean hasStreams = this.streams != null this.streams.size() 0; +boolean hasSpoutsBoltsStreams = hasStreams hasBolts hasSpouts; +// you cant define a topologySource and a DSL topology at the same time... +if (!isDslTopology() ((hasSpouts || hasBolts || hasStreams))) { +return false; +} +if(isDslTopology() (hasSpouts hasBolts hasStreams)) { +return true; +} +return true; } }
[26/50] [abbrv] storm git commit: merge flux into external/flux/
merge flux into external/flux/ Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b21a98dd Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b21a98dd Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b21a98dd Branch: refs/heads/0.10.x-branch Commit: b21a98dd87f82a06a6295ab2bfd832c2810ca57e Parents: ea0fe12 b372a11 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed May 6 13:31:04 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed May 6 13:31:04 2015 -0400 -- external/flux/.gitignore| 15 + external/flux/LICENSE | 202 + external/flux/README.md | 845 +++ external/flux/flux-core/pom.xml | 94 +++ .../main/java/org/apache/storm/flux/Flux.java | 263 ++ .../java/org/apache/storm/flux/FluxBuilder.java | 591 + .../apache/storm/flux/api/TopologySource.java | 39 + .../org/apache/storm/flux/model/BeanDef.java| 39 + .../apache/storm/flux/model/BeanReference.java | 39 + .../org/apache/storm/flux/model/BoltDef.java| 24 + .../storm/flux/model/ConfigMethodDef.java | 62 ++ .../storm/flux/model/ExecutionContext.java | 77 ++ .../apache/storm/flux/model/GroupingDef.java| 77 ++ .../org/apache/storm/flux/model/IncludeDef.java | 54 ++ .../org/apache/storm/flux/model/ObjectDef.java | 90 ++ .../apache/storm/flux/model/PropertyDef.java| 58 ++ .../org/apache/storm/flux/model/SpoutDef.java | 24 + .../org/apache/storm/flux/model/StreamDef.java | 64 ++ .../apache/storm/flux/model/TopologyDef.java| 216 + .../storm/flux/model/TopologySourceDef.java | 36 + .../org/apache/storm/flux/model/VertexDef.java | 36 + .../apache/storm/flux/parser/FluxParser.java| 202 + .../flux-core/src/main/resources/splash.txt | 9 + .../org/apache/storm/flux/FluxBuilderTest.java | 31 + .../org/apache/storm/flux/IntegrationTest.java | 41 + .../java/org/apache/storm/flux/TCKTest.java | 234 + .../multilang/MultilangEnvirontmentTest.java| 89 ++ .../apache/storm/flux/test/SimpleTopology.java | 42 + .../storm/flux/test/SimpleTopologySource.java | 35 + .../test/SimpleTopologyWithConfigParam.java | 38 + .../org/apache/storm/flux/test/TestBolt.java| 63 ++ .../storm/flux/test/TridentTopologySource.java | 54 ++ .../src/test/resources/configs/bad_hbase.yaml | 98 +++ .../resources/configs/config-methods-test.yaml | 70 ++ .../existing-topology-method-override.yaml | 10 + .../existing-topology-reflection-config.yaml| 9 + .../configs/existing-topology-reflection.yaml | 9 + .../configs/existing-topology-trident.yaml | 9 + .../resources/configs/existing-topology.yaml| 8 + .../src/test/resources/configs/hdfs_test.yaml | 97 +++ .../test/resources/configs/include_test.yaml| 25 + .../configs/invalid-existing-topology.yaml | 17 + .../src/test/resources/configs/kafka_test.yaml | 126 +++ .../src/test/resources/configs/shell_test.yaml | 104 +++ .../test/resources/configs/simple_hbase.yaml| 120 +++ .../resources/configs/substitution-test.yaml| 106 +++ .../src/test/resources/configs/tck.yaml | 95 +++ .../src/test/resources/configs/test.properties | 2 + .../flux-core/src/test/resources/logback.xml| 30 + external/flux/flux-examples/README.md | 68 ++ external/flux/flux-examples/pom.xml | 87 ++ .../storm/flux/examples/WordCountClient.java| 74 ++ .../apache/storm/flux/examples/WordCounter.java | 71 ++ .../src/main/resources/hbase_bolt.properties| 18 + .../src/main/resources/hdfs_bolt.properties | 26 + .../src/main/resources/kafka_spout.yaml | 136 +++ .../src/main/resources/multilang.yaml | 89 ++ .../src/main/resources/simple_hbase.yaml| 92 ++ .../src/main/resources/simple_hdfs.yaml | 105 +++ .../src/main/resources/simple_wordcount.yaml| 68 ++ external/flux/flux-ui/README.md | 3 + external/flux/flux-wrappers/pom.xml | 35 + .../flux/wrappers/bolts/FluxShellBolt.java | 56 ++ .../storm/flux/wrappers/bolts/LogInfoBolt.java | 44 + .../flux/wrappers/spouts/FluxShellSpout.java| 55 ++ .../main/resources/resources/randomsentence.js | 93 ++ .../main/resources/resources/splitsentence.py | 24 + .../src/main/resources/resources/storm.js | 373 .../src/main/resources/resources/storm.py | 260 ++ external/flux/pom.xml | 126 +++ 70 files changed, 6621 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/b21a98dd/external/flux/.gitignore
[06/50] [abbrv] storm git commit: add basic instructions for examples
add basic instructions for examples Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8b690e63 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8b690e63 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8b690e63 Branch: refs/heads/0.10.x-branch Commit: 8b690e63954b14aa8f079f904d42d2f4beb34cc6 Parents: 2e44c9e Author: P. Taylor Goetz ptgo...@gmail.com Authored: Thu Apr 2 12:19:13 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Thu Apr 2 12:19:13 2015 -0400 -- flux-examples/README.md | 28 1 file changed, 28 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/8b690e63/flux-examples/README.md -- diff --git a/flux-examples/README.md b/flux-examples/README.md new file mode 100644 index 000..2f107e7 --- /dev/null +++ b/flux-examples/README.md @@ -0,0 +1,28 @@ +# Flux Examples +A collection of examples illustrating various capabilities. + +## Building From Source and Running + +Checkout the projects source and perform a top level Maven build (i.e. from the `flux` directory): + +```bash +git clone https://github.com/ptgoetz/flux.git +cd flux +mvn install +``` + +This will create a shaded (i.e. fat or uber) jar in the `flux-examples/target` directory that can run/deployed with +the `storm` command: + +```bash +cd flux-examples +storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_wordcount.yaml +``` + +The example YAML files are also packaged in the examples jar, so they can also be referenced with Flux's `--resource` +command line switch: + +```bash +storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local --resource /sime_wordcount.yaml +``` +
[03/50] [abbrv] storm git commit: [maven-release-plugin] prepare for next development iteration
[maven-release-plugin] prepare for next development iteration Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/df349303 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/df349303 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/df349303 Branch: refs/heads/0.10.x-branch Commit: df349303e1e2635ac4e6bcab5d872348040a36f4 Parents: 54f5fb7 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Apr 1 13:27:35 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Apr 1 13:27:35 2015 -0400 -- flux-core/pom.xml | 2 +- flux-examples/pom.xml | 2 +- flux-wrappers/pom.xml | 2 +- pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/df349303/flux-core/pom.xml -- diff --git a/flux-core/pom.xml b/flux-core/pom.xml index 1ea353a..0d72ead 100644 --- a/flux-core/pom.xml +++ b/flux-core/pom.xml @@ -21,7 +21,7 @@ parent groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.2.2/version +version0.2.3-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/storm/blob/df349303/flux-examples/pom.xml -- diff --git a/flux-examples/pom.xml b/flux-examples/pom.xml index ac19e41..29a2e62 100644 --- a/flux-examples/pom.xml +++ b/flux-examples/pom.xml @@ -21,7 +21,7 @@ parent groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.2.2/version +version0.2.3-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/storm/blob/df349303/flux-wrappers/pom.xml -- diff --git a/flux-wrappers/pom.xml b/flux-wrappers/pom.xml index 7884af2..e571d56 100644 --- a/flux-wrappers/pom.xml +++ b/flux-wrappers/pom.xml @@ -21,7 +21,7 @@ parent groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.2.2/version +version0.2.3-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/storm/blob/df349303/pom.xml -- diff --git a/pom.xml b/pom.xml index b06103b..e79e25a 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.2.2/version +version0.2.3-SNAPSHOT/version packagingpom/packaging nameflux/name urlhttps://github.com/ptgoetz/flux/url
[29/50] [abbrv] storm git commit: cleanup github references and versions
cleanup github references and versions Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f7d2f7fa Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f7d2f7fa Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f7d2f7fa Branch: refs/heads/0.10.x-branch Commit: f7d2f7fa0c7a0511911e984f13c20fceed743b17 Parents: e1e1609 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Thu May 7 10:33:44 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Thu May 7 10:33:44 2015 -0400 -- external/flux/README.md | 8 external/flux/flux-core/pom.xml | 1 - external/flux/flux-examples/README.md | 14 +++--- external/flux/flux-examples/pom.xml | 1 - 4 files changed, 11 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/f7d2f7fa/external/flux/README.md -- diff --git a/external/flux/README.md b/external/flux/README.md index d09a73c..0387f3f 100644 --- a/external/flux/README.md +++ b/external/flux/README.md @@ -109,9 +109,9 @@ recommended). The current version of Flux is available in Maven Central at the following coordinates: ```xml dependency -groupIdcom.github.ptgoetz/groupId +groupIdorg.apache.storm/groupId artifactIdflux-core/artifactId -version0.3.0/version +version${storm.version}/version /dependency ``` @@ -123,9 +123,9 @@ The example below illustrates Flux usage with the Maven shade plugin: dependencies !-- Flux include -- dependency -groupIdcom.github.ptgoetz/groupId +groupIdorg.apache.storm/groupId artifactIdflux-core/artifactId -version0.3.0/version +version${storm.version}/version /dependency !-- add user dependencies here... -- http://git-wip-us.apache.org/repos/asf/storm/blob/f7d2f7fa/external/flux/flux-core/pom.xml -- diff --git a/external/flux/flux-core/pom.xml b/external/flux/flux-core/pom.xml index c3842bd..f74d034 100644 --- a/external/flux/flux-core/pom.xml +++ b/external/flux/flux-core/pom.xml @@ -29,7 +29,6 @@ packagingjar/packaging nameflux-core/name -urlhttps://github.com/ptgoetz/flux/url dependencies dependency http://git-wip-us.apache.org/repos/asf/storm/blob/f7d2f7fa/external/flux/flux-examples/README.md -- diff --git a/external/flux/flux-examples/README.md b/external/flux/flux-examples/README.md index b3798a6..fceebd8 100644 --- a/external/flux/flux-examples/README.md +++ b/external/flux/flux-examples/README.md @@ -6,24 +6,24 @@ A collection of examples illustrating various capabilities. Checkout the projects source and perform a top level Maven build (i.e. from the `flux` directory): ```bash -git clone https://github.com/ptgoetz/flux.git -cd flux -mvn install +git clone https://github.com/apache/storm.git +cd storm +mvn install -DskipTests=true ``` -This will create a shaded (i.e. fat or uber) jar in the `flux-examples/target` directory that can run/deployed with +This will create a shaded (i.e. fat or uber) jar in the `external/flux/flux-examples/target` directory that can run/deployed with the `storm` command: ```bash cd flux-examples -storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_wordcount.yaml +storm jar ./target/flux-examples-*-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_wordcount.yaml ``` The example YAML files are also packaged in the examples jar, so they can also be referenced with Flux's `--resource` command line switch: ```bash -storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local --resource /simple_wordcount.yaml +storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local --resource /simple_wordcount.yaml ``` ## Available Examples @@ -51,7 +51,7 @@ To run the `simple_hdfs.yaml` example, copy the `hdfs_bolt.properties` file to a least, the property `hdfs.url` to point to a HDFS cluster. Then you can run the example something like: ```bash -storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hdfs.yaml --filter my_hdfs_bolt.properties +storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hdfs.yaml --filter my_hdfs_bolt.properties ``` ### [simple_hbase.yaml](src/main/resources/simple_hbase.yaml) http://git-wip-us.apache.org/repos/asf/storm/blob/f7d2f7fa/external/flux/flux-examples/pom.xml --
[23/50] [abbrv] storm git commit: merge flux into external/flux/
http://git-wip-us.apache.org/repos/asf/storm/blob/b21a98dd/external/flux/flux-examples/src/main/resources/simple_hdfs.yaml -- diff --cc external/flux/flux-examples/src/main/resources/simple_hdfs.yaml index 000,000..9007869 new file mode 100644 --- /dev/null +++ b/external/flux/flux-examples/src/main/resources/simple_hdfs.yaml @@@ -1,0 -1,0 +1,105 @@@ ++# Licensed to the Apache Software Foundation (ASF) under one ++# or more contributor license agreements. See the NOTICE file ++# distributed with this work for additional information ++# regarding copyright ownership. The ASF licenses this file ++# to you under the Apache License, Version 2.0 (the ++# License); you may not use this file except in compliance ++# with the License. You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an AS IS BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++# Test ability to wire together shell spouts/bolts ++--- ++ ++# topology definition ++# name to be used when submitting ++name: hdfs-topology ++ ++# Components ++# Components are analagous to Spring beans. They are meant to be used as constructor, ++# property(setter), and builder arguments. ++# ++# for the time being, components must be declared in the order they are referenced ++components: ++ - id: syncPolicy ++className: org.apache.storm.hdfs.bolt.sync.CountSyncPolicy ++constructorArgs: ++ - 1000 ++ - id: rotationPolicy ++className: org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy ++constructorArgs: ++ - 30 ++ - SECONDS ++ ++ - id: fileNameFormat ++className: org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat ++configMethods: ++ - name: withPath ++args: [${hdfs.write.dir}] ++ - name: withExtension ++args: [.txt] ++ ++ - id: recordFormat ++className: org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat ++configMethods: ++ - name: withFieldDelimiter ++args: [|] ++ ++ - id: rotationAction ++className: org.apache.storm.hdfs.common.rotation.MoveFileAction ++configMethods: ++ - name: toDestination ++args: [${hdfs.dest.dir}] ++ ++# spout definitions ++spouts: ++ - id: spout-1 ++className: backtype.storm.testing.TestWordSpout ++parallelism: 1 ++# ... ++ ++# bolt definitions ++ ++bolts: ++ - id: bolt-1 ++className: org.apache.storm.hdfs.bolt.HdfsBolt ++configMethods: ++ - name: withConfigKey ++args: [hdfs.config] ++ - name: withFsUrl ++args: [${hdfs.url}] ++ - name: withFileNameFormat ++args: [ref: fileNameFormat] ++ - name: withRecordFormat ++args: [ref: recordFormat] ++ - name: withRotationPolicy ++args: [ref: rotationPolicy] ++ - name: withSyncPolicy ++args: [ref: syncPolicy] ++ - name: addRotationAction ++args: [ref: rotationAction] ++parallelism: 1 ++# ... ++ ++ - id: bolt-2 ++className: org.apache.storm.flux.wrappers.bolts.LogInfoBolt ++parallelism: 1 ++ ++streams: ++ - name: # name isn't used (placeholder for logging, UI, etc.) ++from: spout-1 ++to: bolt-1 ++grouping: ++ type: SHUFFLE ++ ++ - name: # name isn't used (placeholder for logging, UI, etc.) ++from: spout-1 ++to: bolt-2 ++grouping: ++ type: SHUFFLE http://git-wip-us.apache.org/repos/asf/storm/blob/b21a98dd/external/flux/flux-examples/src/main/resources/simple_wordcount.yaml -- diff --cc external/flux/flux-examples/src/main/resources/simple_wordcount.yaml index 000,000..380f9d2 new file mode 100644 --- /dev/null +++ b/external/flux/flux-examples/src/main/resources/simple_wordcount.yaml @@@ -1,0 -1,0 +1,68 @@@ ++# Licensed to the Apache Software Foundation (ASF) under one ++# or more contributor license agreements. See the NOTICE file ++# distributed with this work for additional information ++# regarding copyright ownership. The ASF licenses this file ++# to you under the Apache License, Version 2.0 (the ++# License); you may not use this file except in compliance ++# with the License. You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an AS IS BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++--- ++ ++# topology definition ++# name to be used when
[12/50] [abbrv] storm git commit: use a better name
use a better name Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ae305c72 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ae305c72 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ae305c72 Branch: refs/heads/0.10.x-branch Commit: ae305c722c6252cd33530ebb74e56ef991451aba Parents: edc5744 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Apr 8 00:00:51 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Apr 8 00:00:51 2015 -0400 -- flux-examples/src/main/resources/kafka_spout.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/ae305c72/flux-examples/src/main/resources/kafka_spout.yaml -- diff --git a/flux-examples/src/main/resources/kafka_spout.yaml b/flux-examples/src/main/resources/kafka_spout.yaml index 136f62d..8ffddc5 100644 --- a/flux-examples/src/main/resources/kafka_spout.yaml +++ b/flux-examples/src/main/resources/kafka_spout.yaml @@ -20,7 +20,7 @@ # topology definition # name to be used when submitting -name: shell-topology +name: kafka-topology # Components # Components are analagous to Spring beans. They are meant to be used as constructor,
[01/50] [abbrv] storm git commit: prep for 0.2.2 release
Repository: storm Updated Branches: refs/heads/0.10.x-branch efdf94966 - b26429621 prep for 0.2.2 release Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/14970de2 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/14970de2 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/14970de2 Branch: refs/heads/0.10.x-branch Commit: 14970de2091485f4ca777606735788657f7bec84 Parents: c3cffa0 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Apr 1 13:25:52 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Apr 1 13:25:52 2015 -0400 -- README.md | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/14970de2/README.md -- diff --git a/README.md b/README.md index 78a2723..92fec10 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ The current version of Flux is available in Maven Central at the following coord dependency groupIdcom.github.ptgoetz/groupId artifactIdflux-core/artifactId -version0.2.2-SNAPSHOT/version +version0.2.2/version /dependency ``` @@ -92,7 +92,7 @@ The example below illustrates Flux usage with the Maven shade plugin: dependency groupIdcom.github.ptgoetz/groupId artifactIdflux-core/artifactId -version0.2.2-SNAPSHOT/version +version0.2.2/version /dependency !-- add user dependencies here... -- @@ -171,6 +171,7 @@ usage: storm jar my_topology_uber_jar.jar org.apache.storm.flux.Flux -z,--zookeeper host:port When running in local mode, use the ZooKeeper at the specified host:port instead of the in-process ZooKeeper. + (requires Storm 0.9.3 or later) ``` **NOTE:** Flux tries to avoid command line switch collision with the `storm` command, and allows any other command line @@ -193,7 +194,7 @@ storm jar myTopology-0.1.0-SNAPSHOT.jar org.apache.storm.flux.Flux --remote my_c âââ ââââââââ âââââââ âââ âââ +- Apache Storm-+ +- data FLow User eXperience -+ -Version: 0.2.2-SNAPSHOT +Version: 0.2.2 Parsing file: /Users/hsimpson/Projects/donut_domination/storm/shell_test.yaml -- TOPOLOGY DETAILS -- Name: shell-topology
[31/50] [abbrv] storm git commit: add missing kafka dependency to flux-examples
add missing kafka dependency to flux-examples Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8c9e6cee Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8c9e6cee Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8c9e6cee Branch: refs/heads/0.10.x-branch Commit: 8c9e6ceeafc1f50eebc519c8f03d5821813ee99b Parents: 9fad816 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Fri May 8 15:35:11 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Fri May 8 15:35:11 2015 -0400 -- .../org/apache/storm/flux/IntegrationTest.java | 2 -- external/flux/flux-examples/pom.xml | 20 2 files changed, 20 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/8c9e6cee/external/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java -- diff --git a/external/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java b/external/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java index 5e17f5e..c5807f8 100644 --- a/external/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java +++ b/external/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java @@ -30,8 +30,6 @@ public class IntegrationTest { } } - - @Test public void testRunTopologySource() throws Exception { if(!skipTest) { http://git-wip-us.apache.org/repos/asf/storm/blob/8c9e6cee/external/flux/flux-examples/pom.xml -- diff --git a/external/flux/flux-examples/pom.xml b/external/flux/flux-examples/pom.xml index 571f302..e3996e5 100644 --- a/external/flux/flux-examples/pom.xml +++ b/external/flux/flux-examples/pom.xml @@ -52,6 +52,26 @@ artifactIdstorm-hbase/artifactId version${project.version}/version /dependency +dependency +groupIdorg.apache.storm/groupId +artifactIdstorm-kafka/artifactId +version${project.version}/version +/dependency +dependency +groupIdorg.apache.kafka/groupId +artifactIdkafka_2.10/artifactId +version0.8.1.1/version +exclusions +exclusion +groupIdorg.apache.zookeeper/groupId +artifactIdzookeeper/artifactId +/exclusion +exclusion +groupIdlog4j/groupId +artifactIdlog4j/artifactId +/exclusion +/exclusions +/dependency /dependencies build
[32/50] [abbrv] storm git commit: fix typo in README
fix typo in README Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/91369ac2 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/91369ac2 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/91369ac2 Branch: refs/heads/0.10.x-branch Commit: 91369ac2f15d4a2dade5d03ee2d0970d99e4d0bd Parents: 8c9e6ce Author: P. Taylor Goetz ptgo...@gmail.com Authored: Fri May 8 15:43:00 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Fri May 8 15:43:00 2015 -0400 -- external/flux/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/91369ac2/external/flux/README.md -- diff --git a/external/flux/README.md b/external/flux/README.md index 0387f3f..2c5127e 100644 --- a/external/flux/README.md +++ b/external/flux/README.md @@ -183,7 +183,7 @@ usage: storm jar my_topology_uber_jar.jar org.apache.storm.flux.Flux build, validate, and print information about the topology. -e,--env-filter Perform environment variable substitution. - Replace keysidentified with `${ENV-[NAME]}` + Replace keys identified with `${ENV-[NAME]}` will be replaced with the corresponding `NAME` environment value -f,--filter file Perform property substitution. Use the
[18/50] [abbrv] storm git commit: force source/target to use Java 1.6
force source/target to use Java 1.6 Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c48f63e3 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c48f63e3 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c48f63e3 Branch: refs/heads/0.10.x-branch Commit: c48f63e3f361e9416669d4d791de3b3a67c461d0 Parents: 601cee7 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue May 5 15:56:47 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue May 5 15:56:47 2015 -0400 -- flux-core/pom.xml | 2 +- pom.xml | 11 ++- 2 files changed, 11 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/c48f63e3/flux-core/pom.xml -- diff --git a/flux-core/pom.xml b/flux-core/pom.xml index fe2e301..2d03ea4 100644 --- a/flux-core/pom.xml +++ b/flux-core/pom.xml @@ -53,7 +53,7 @@ dependency groupIdorg.apache.storm/groupId artifactIdstorm-hbase/artifactId -version0.11.0-SNAPSHOT/version +version${storm.version}/version scopetest/scope /dependency /dependencies http://git-wip-us.apache.org/repos/asf/storm/blob/c48f63e3/pom.xml -- diff --git a/pom.xml b/pom.xml index e79e25a..de48f7b 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,16 @@ /resources plugins - +plugin +groupIdorg.apache.maven.plugins/groupId +artifactIdmaven-compiler-plugin/artifactId +version3.3/version +configuration +source1.6/source +target1.6/target +encodingUTF-8/encoding +/configuration +/plugin /plugins /build /project
[46/50] [abbrv] storm git commit: Merge branch 'master' into 0.10.x-branch
Merge branch 'master' into 0.10.x-branch Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ea81f96d Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ea81f96d Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ea81f96d Branch: refs/heads/0.10.x-branch Commit: ea81f96df6eedffe668b44201c441095760c74da Parents: efdf949 285d943 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 13:38:51 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 13:38:51 2015 -0400 -- CHANGELOG.md| 3 + external/flux/.gitignore| 15 + external/flux/LICENSE | 202 + external/flux/README.md | 834 +++ external/flux/flux-core/pom.xml | 92 ++ .../main/java/org/apache/storm/flux/Flux.java | 263 ++ .../java/org/apache/storm/flux/FluxBuilder.java | 591 + .../apache/storm/flux/api/TopologySource.java | 39 + .../org/apache/storm/flux/model/BeanDef.java| 39 + .../apache/storm/flux/model/BeanReference.java | 39 + .../org/apache/storm/flux/model/BoltDef.java| 24 + .../storm/flux/model/ConfigMethodDef.java | 62 ++ .../storm/flux/model/ExecutionContext.java | 77 ++ .../apache/storm/flux/model/GroupingDef.java| 77 ++ .../org/apache/storm/flux/model/IncludeDef.java | 54 ++ .../org/apache/storm/flux/model/ObjectDef.java | 90 ++ .../apache/storm/flux/model/PropertyDef.java| 58 ++ .../org/apache/storm/flux/model/SpoutDef.java | 24 + .../org/apache/storm/flux/model/StreamDef.java | 64 ++ .../apache/storm/flux/model/TopologyDef.java| 216 + .../storm/flux/model/TopologySourceDef.java | 36 + .../org/apache/storm/flux/model/VertexDef.java | 36 + .../apache/storm/flux/parser/FluxParser.java| 202 + .../flux-core/src/main/resources/splash.txt | 9 + .../org/apache/storm/flux/FluxBuilderTest.java | 31 + .../org/apache/storm/flux/IntegrationTest.java | 39 + .../java/org/apache/storm/flux/TCKTest.java | 234 ++ .../multilang/MultilangEnvirontmentTest.java| 89 ++ .../apache/storm/flux/test/SimpleTopology.java | 42 + .../storm/flux/test/SimpleTopologySource.java | 35 + .../test/SimpleTopologyWithConfigParam.java | 38 + .../org/apache/storm/flux/test/TestBolt.java| 63 ++ .../storm/flux/test/TridentTopologySource.java | 54 ++ .../src/test/resources/configs/bad_hbase.yaml | 98 +++ .../resources/configs/config-methods-test.yaml | 70 ++ .../existing-topology-method-override.yaml | 10 + .../existing-topology-reflection-config.yaml| 9 + .../configs/existing-topology-reflection.yaml | 9 + .../configs/existing-topology-trident.yaml | 9 + .../resources/configs/existing-topology.yaml| 8 + .../src/test/resources/configs/hdfs_test.yaml | 97 +++ .../test/resources/configs/include_test.yaml| 25 + .../configs/invalid-existing-topology.yaml | 17 + .../src/test/resources/configs/kafka_test.yaml | 126 +++ .../src/test/resources/configs/shell_test.yaml | 104 +++ .../test/resources/configs/simple_hbase.yaml| 120 +++ .../resources/configs/substitution-test.yaml| 106 +++ .../src/test/resources/configs/tck.yaml | 95 +++ .../src/test/resources/configs/test.properties | 2 + .../flux-core/src/test/resources/logback.xml| 30 + external/flux/flux-examples/README.md | 66 ++ external/flux/flux-examples/pom.xml | 105 +++ .../storm/flux/examples/WordCountClient.java| 74 ++ .../apache/storm/flux/examples/WordCounter.java | 71 ++ .../src/main/resources/hbase_bolt.properties| 18 + .../src/main/resources/hdfs_bolt.properties | 26 + .../src/main/resources/kafka_spout.yaml | 136 +++ .../src/main/resources/multilang.yaml | 89 ++ .../src/main/resources/simple_hbase.yaml| 92 ++ .../src/main/resources/simple_hdfs.yaml | 105 +++ .../src/main/resources/simple_wordcount.yaml| 68 ++ external/flux/flux-ui/README.md | 3 + external/flux/flux-wrappers/pom.xml | 51 ++ .../flux/wrappers/bolts/FluxShellBolt.java | 56 ++ .../storm/flux/wrappers/bolts/LogInfoBolt.java | 44 + .../flux/wrappers/spouts/FluxShellSpout.java| 55 ++ .../main/resources/resources/randomsentence.js | 93 +++ .../main/resources/resources/splitsentence.py | 24 + external/flux/pom.xml | 119 +++ external/storm-eventhubs/pom.xml| 38 +- .../eventhubs/bolt/DefaultEventDataFormat.java | 47 ++ .../storm/eventhubs/bolt/EventHubBolt.java | 56 +- .../eventhubs/bolt/EventHubBoltConfig.java | 109 +++ .../storm/eventhubs/bolt/IEventDataFormat.java | 28 +
[44/50] [abbrv] storm git commit: Merge branch 'STORM-561'
Merge branch 'STORM-561' Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cb370a99 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cb370a99 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cb370a99 Branch: refs/heads/0.10.x-branch Commit: cb370a99a12a8466b4acd644bfe4eba00ee26107 Parents: 2313775 b90ec78 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 13:21:16 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 13:21:16 2015 -0400 -- external/flux/.gitignore| 15 + external/flux/LICENSE | 202 + external/flux/README.md | 834 +++ external/flux/flux-core/pom.xml | 92 ++ .../main/java/org/apache/storm/flux/Flux.java | 263 ++ .../java/org/apache/storm/flux/FluxBuilder.java | 591 + .../apache/storm/flux/api/TopologySource.java | 39 + .../org/apache/storm/flux/model/BeanDef.java| 39 + .../apache/storm/flux/model/BeanReference.java | 39 + .../org/apache/storm/flux/model/BoltDef.java| 24 + .../storm/flux/model/ConfigMethodDef.java | 62 ++ .../storm/flux/model/ExecutionContext.java | 77 ++ .../apache/storm/flux/model/GroupingDef.java| 77 ++ .../org/apache/storm/flux/model/IncludeDef.java | 54 ++ .../org/apache/storm/flux/model/ObjectDef.java | 90 ++ .../apache/storm/flux/model/PropertyDef.java| 58 ++ .../org/apache/storm/flux/model/SpoutDef.java | 24 + .../org/apache/storm/flux/model/StreamDef.java | 64 ++ .../apache/storm/flux/model/TopologyDef.java| 216 + .../storm/flux/model/TopologySourceDef.java | 36 + .../org/apache/storm/flux/model/VertexDef.java | 36 + .../apache/storm/flux/parser/FluxParser.java| 202 + .../flux-core/src/main/resources/splash.txt | 9 + .../org/apache/storm/flux/FluxBuilderTest.java | 31 + .../org/apache/storm/flux/IntegrationTest.java | 39 + .../java/org/apache/storm/flux/TCKTest.java | 234 ++ .../multilang/MultilangEnvirontmentTest.java| 89 ++ .../apache/storm/flux/test/SimpleTopology.java | 42 + .../storm/flux/test/SimpleTopologySource.java | 35 + .../test/SimpleTopologyWithConfigParam.java | 38 + .../org/apache/storm/flux/test/TestBolt.java| 63 ++ .../storm/flux/test/TridentTopologySource.java | 54 ++ .../src/test/resources/configs/bad_hbase.yaml | 98 +++ .../resources/configs/config-methods-test.yaml | 70 ++ .../existing-topology-method-override.yaml | 10 + .../existing-topology-reflection-config.yaml| 9 + .../configs/existing-topology-reflection.yaml | 9 + .../configs/existing-topology-trident.yaml | 9 + .../resources/configs/existing-topology.yaml| 8 + .../src/test/resources/configs/hdfs_test.yaml | 97 +++ .../test/resources/configs/include_test.yaml| 25 + .../configs/invalid-existing-topology.yaml | 17 + .../src/test/resources/configs/kafka_test.yaml | 126 +++ .../src/test/resources/configs/shell_test.yaml | 104 +++ .../test/resources/configs/simple_hbase.yaml| 120 +++ .../resources/configs/substitution-test.yaml| 106 +++ .../src/test/resources/configs/tck.yaml | 95 +++ .../src/test/resources/configs/test.properties | 2 + .../flux-core/src/test/resources/logback.xml| 30 + external/flux/flux-examples/README.md | 66 ++ external/flux/flux-examples/pom.xml | 105 +++ .../storm/flux/examples/WordCountClient.java| 74 ++ .../apache/storm/flux/examples/WordCounter.java | 71 ++ .../src/main/resources/hbase_bolt.properties| 18 + .../src/main/resources/hdfs_bolt.properties | 26 + .../src/main/resources/kafka_spout.yaml | 136 +++ .../src/main/resources/multilang.yaml | 89 ++ .../src/main/resources/simple_hbase.yaml| 92 ++ .../src/main/resources/simple_hdfs.yaml | 105 +++ .../src/main/resources/simple_wordcount.yaml| 68 ++ external/flux/flux-ui/README.md | 3 + external/flux/flux-wrappers/pom.xml | 51 ++ .../flux/wrappers/bolts/FluxShellBolt.java | 56 ++ .../storm/flux/wrappers/bolts/LogInfoBolt.java | 44 + .../flux/wrappers/spouts/FluxShellSpout.java| 55 ++ .../main/resources/resources/randomsentence.js | 93 +++ .../main/resources/resources/splitsentence.py | 24 + external/flux/pom.xml | 119 +++ pom.xml | 1 + storm-dist/binary/src/main/assembly/binary.xml | 44 + 70 files changed, 6043 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/cb370a99/pom.xml --
[09/50] [abbrv] storm git commit: link examples to source
link examples to source Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a791604a Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a791604a Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a791604a Branch: refs/heads/0.10.x-branch Commit: a791604ac5d274cededae8a01362dac4dfd82d84 Parents: 0c1e0aa Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue Apr 7 00:07:49 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue Apr 7 00:07:49 2015 -0400 -- .../src/test/resources/configs/hdfs_test.yaml | 97 flux-examples/README.md | 8 +- 2 files changed, 101 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/a791604a/flux-core/src/test/resources/configs/hdfs_test.yaml -- diff --git a/flux-core/src/test/resources/configs/hdfs_test.yaml b/flux-core/src/test/resources/configs/hdfs_test.yaml new file mode 100644 index 000..8fe0a9a --- /dev/null +++ b/flux-core/src/test/resources/configs/hdfs_test.yaml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# License); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test ability to wire together shell spouts/bolts +--- + +# topology definition +# name to be used when submitting +name: hdfs-topology + +# Components +# Components are analagous to Spring beans. They are meant to be used as constructor, +# property(setter), and builder arguments. +# +# for the time being, components must be declared in the order they are referenced +components: + - id: syncPolicy +className: org.apache.storm.hdfs.bolt.sync.CountSyncPolicy +constructorArgs: + - 1000 + - id: rotationPolicy +className: org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy +constructorArgs: + - 5.0 + - MB + + - id: fileNameFormat +className: org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat +configMethods: + - name: withPath +args: [/tmp/foo/] + - name: withExtension +args: [.txt] + + - id: recordFormat +className: org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat +configMethods: + - name: withFieldDelimiter +args: [|] + + - id: rotationAction +className: org.apache.storm.hdfs.common.rotation.MoveFileAction +configMethods: + - name: toDestination +args: [/tmp/dest2] + +# spout definitions +spouts: + - id: spout-1 +className: backtype.storm.testing.TestWordSpout +parallelism: 1 +# ... + +# bolt definitions + +#HdfsBolt bolt = new HdfsBolt() +#.withConfigKey(hdfs.config) +#.withFsUrl(args[0]) +#.withFileNameFormat(fileNameFormat) +#.withRecordFormat(format) +#.withRotationPolicy(rotationPolicy) +#.withSyncPolicy(syncPolicy) +#.addRotationAction(new MoveFileAction().toDestination(/tmp/dest2/)); +bolts: + - id: bolt-1 +className: org.apache.storm.hdfs.bolt.HdfsBolt +configMethods: + - name: withConfigKey +args: [hdfs.config] + - name: withFsUrl +args: [hdfs://hadoop:54310] + - name: withFileNameFormat +args: [ref: fileNameFormat] + - name: withRecordFormat +args: [ref: recordFormat] + - name: withRotationPolicy +args: [ref: rotationPolicy] + - name: withSyncPolicy +args: [ref: syncPolicy] + - name: addRotationAction +args: [ref: rotationAction] +parallelism: 1 +# ... + http://git-wip-us.apache.org/repos/asf/storm/blob/a791604a/flux-examples/README.md -- diff --git a/flux-examples/README.md b/flux-examples/README.md index 9f5682e..425ad98 100644 --- a/flux-examples/README.md +++ b/flux-examples/README.md @@ -28,21 +28,21 @@ storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux - ## Available Examples -### simple_wordcount.yaml +###
[4/6] storm git commit: Merge branch 'add-storm-redis-to-storm-dist' of github.com:HeartSaVioR/storm into 0.10.x-branch
Merge branch 'add-storm-redis-to-storm-dist' of github.com:HeartSaVioR/storm into 0.10.x-branch Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7258f94f Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7258f94f Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7258f94f Branch: refs/heads/0.10.x-branch Commit: 7258f94fa1ca8ff94a4a7701967dcf86a2bce2df Parents: 53cc3da b01a8c9 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 22:28:13 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 22:28:13 2015 -0400 -- storm-dist/binary/src/main/assembly/binary.xml | 14 ++ 1 file changed, 14 insertions(+) --
Git Push Summary
Repository: storm Updated Tags: refs/tags/v0.10.0 [created] 54d54e181
[35/50] [abbrv] storm git commit: Remove client adding reference eventhubs-client package and use ResilientEventHubReceiver
Remove client adding reference eventhubs-client package and use ResilientEventHubReceiver Signed-off-by: Shanyu Zhao shz...@microsoft.com Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/85aeb3d4 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/85aeb3d4 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/85aeb3d4 Branch: refs/heads/0.10.x-branch Commit: 85aeb3d48efcaa28d6fc5dbfe6ce87af9f3e2615 Parents: 1f13f15 Author: Shanyu Zhao shz...@microsoft.com Authored: Sun May 17 01:14:01 2015 -0700 Committer: Shanyu Zhao shz...@microsoft.com Committed: Sun May 17 01:14:01 2015 -0700 -- external/storm-eventhubs/pom.xml| 13 +- .../storm/eventhubs/bolt/EventHubBolt.java | 6 +- .../eventhubs/bolt/EventHubBoltConfig.java | 4 +- .../client/ConnectionStringBuilder.java | 116 .../storm/eventhubs/client/Constants.java | 32 - .../storm/eventhubs/client/EventHubClient.java | 95 - .../eventhubs/client/EventHubConsumerGroup.java | 72 -- .../eventhubs/client/EventHubException.java | 37 - .../eventhubs/client/EventHubReceiver.java | 139 --- .../eventhubs/client/EventHubSendClient.java| 70 -- .../storm/eventhubs/client/EventHubSender.java | 99 - .../storm/eventhubs/client/SelectorFilter.java | 38 - .../eventhubs/client/SelectorFilterWriter.java | 64 - .../eventhubs/spout/EventHubReceiverFilter.java | 56 .../eventhubs/spout/EventHubReceiverImpl.java | 50 +++ .../eventhubs/spout/EventHubSpoutConfig.java| 31 + .../eventhubs/spout/IEventHubReceiver.java | 5 +- .../spout/IEventHubReceiverFilter.java | 35 - .../eventhubs/spout/SimplePartitionManager.java | 11 +- .../spout/StaticPartitionCoordinator.java | 2 +- .../TransactionalTridentEventHubEmitter.java| 2 +- .../trident/TridentPartitionManager.java| 12 +- .../eventhubs/spout/EventHubReceiverMock.java | 18 +-- 23 files changed, 56 insertions(+), 951 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/85aeb3d4/external/storm-eventhubs/pom.xml -- diff --git a/external/storm-eventhubs/pom.xml b/external/storm-eventhubs/pom.xml index 2dfb739..6d4a47b 100755 --- a/external/storm-eventhubs/pom.xml +++ b/external/storm-eventhubs/pom.xml @@ -33,7 +33,7 @@ properties project.build.sourceEncodingUTF-8/project.build.sourceEncoding -qpid.version0.32/qpid.version +eventhubs.client.version0.9/eventhubs.client.version /properties build plugins @@ -77,14 +77,9 @@ /build dependencies dependency -groupIdorg.apache.qpid/groupId -artifactIdqpid-client/artifactId -version${qpid.version}/version -/dependency -dependency -groupIdorg.apache.qpid/groupId -artifactIdqpid-amqp-1-0-client-jms/artifactId -version${qpid.version}/version +groupIdcom.microsoft.eventhubs.client/groupId +artifactIdeventhubs-client/artifactId +version${eventhubs.client.version}/version /dependency dependency groupIdorg.apache.storm/groupId http://git-wip-us.apache.org/repos/asf/storm/blob/85aeb3d4/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java -- diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java index a817744..9acf7fa 100755 --- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java +++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java @@ -22,9 +22,9 @@ import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.storm.eventhubs.client.EventHubClient; -import org.apache.storm.eventhubs.client.EventHubException; -import org.apache.storm.eventhubs.client.EventHubSender; +import com.microsoft.eventhubs.client.EventHubClient; +import com.microsoft.eventhubs.client.EventHubException; +import com.microsoft.eventhubs.client.EventHubSender; import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; http://git-wip-us.apache.org/repos/asf/storm/blob/85aeb3d4/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBoltConfig.java -- diff --git
[33/50] [abbrv] storm git commit: storm-eventhubs improvement
storm-eventhubs improvement EventHubBolt add event formatter to format tuples into bytes Refactor EventHubSpoutConfig Add support for specifying consumer group name Workaround for Qpid issue that in rare cases messages cannot be received Signed-off-by: Shanyu Zhao shz...@microsoft.com Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e5154928 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e5154928 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e5154928 Branch: refs/heads/0.10.x-branch Commit: e515492864092c16c2ecc86577a3be73ec79fa56 Parents: 847958c Author: Shanyu Zhao shz...@microsoft.com Authored: Wed May 13 12:26:53 2015 -0700 Committer: Shanyu Zhao shz...@microsoft.com Committed: Wed May 13 12:26:53 2015 -0700 -- external/storm-eventhubs/pom.xml| 239 +-- .../eventhubs/bolt/DefaultEventDataFormat.java | 47 .../storm/eventhubs/bolt/EventHubBolt.java | 182 +++--- .../eventhubs/bolt/EventHubBoltConfig.java | 107 + .../storm/eventhubs/bolt/IEventDataFormat.java | 28 +++ .../storm/eventhubs/client/EventHubClient.java | 187 --- .../storm/eventhubs/client/EventHubSender.java | 194 +++ .../storm/eventhubs/samples/EventCount.java | 5 +- .../storm/eventhubs/samples/EventHubLoop.java | 103 .../eventhubs/spout/EventHubReceiverImpl.java | 20 +- .../storm/eventhubs/spout/EventHubSpout.java| 5 + .../eventhubs/spout/EventHubSpoutConfig.java| 105 +--- .../src/main/resources/config.properties| 5 +- .../eventhubs/spout/TestEventHubSpout.java | 4 +- 14 files changed, 750 insertions(+), 481 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/e5154928/external/storm-eventhubs/pom.xml -- diff --git a/external/storm-eventhubs/pom.xml b/external/storm-eventhubs/pom.xml index 5ed65c7..2ceed09 100755 --- a/external/storm-eventhubs/pom.xml +++ b/external/storm-eventhubs/pom.xml @@ -1,122 +1,119 @@ -?xml version=1.0 encoding=UTF-8? -!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the License); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an AS IS BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. --- -project xmlns=http://maven.apache.org/POM/4.0.0; xmlns:xsi=http://www.w3.org/2001/XMLSchema-instance; xsi:schemaLocation=http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd; -modelVersion4.0.0/modelVersion - -parent -artifactIdstorm/artifactId -groupIdorg.apache.storm/groupId -version0.11.0-SNAPSHOT/version -relativePath../../pom.xml/relativePath -/parent - -artifactIdstorm-eventhubs/artifactId -version0.11.0-SNAPSHOT/version -packagingjar/packaging -namestorm-eventhubs/name -descriptionEventHubs Storm Spout/description - -properties -project.build.sourceEncodingUTF-8/project.build.sourceEncoding -qpid.version0.28/qpid.version -/properties -build -plugins -plugin -groupIdorg.apache.maven.plugins/groupId -artifactIdmaven-assembly-plugin/artifactId -version2.4.1/version -executions -execution -goals -goalattached/goal -/goals -phasepackage/phase -configuration -descriptorRefs - descriptorRefjar-with-dependencies/descriptorRef -/descriptorRefs -archive -manifest - mainClassorg.apache.storm.eventhubs.samples.EventCount/mainClass -/manifest -/archive -/configuration -/execution -/executions -/plugin -plugin - artifactIdmaven-antrun-plugin/artifactId -
[39/50] [abbrv] storm git commit: fix docs for hbase example
fix docs for hbase example Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b90ec781 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b90ec781 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b90ec781 Branch: refs/heads/0.10.x-branch Commit: b90ec781c7a7ba77f6598fe20e7842fa4a228098 Parents: 91369ac Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue Jun 2 17:14:34 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue Jun 2 17:14:34 2015 -0400 -- external/flux/flux-examples/README.md | 10 -- 1 file changed, 4 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/b90ec781/external/flux/flux-examples/README.md -- diff --git a/external/flux/flux-examples/README.md b/external/flux/flux-examples/README.md index fceebd8..0a7085e 100644 --- a/external/flux/flux-examples/README.md +++ b/external/flux/flux-examples/README.md @@ -58,11 +58,9 @@ storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/ This example illustrates how to use Flux to setup a storm-hbase bolt to write to HBase. -In order to use this example, you will need to edit the `src/main resrouces/hbase-site.xml` file to reflect your HBase -environment, and then rebuild the topology jar. - -You can do so by running the following Maven command in the `flux-examples` directory: +To run the `simple_hbase.yaml` example, copy the `hbase_bolt.properties` file to a convenient location and change the properties + `hbase.rootdir` and `hbase.zookeeper.quorum`. Then you can run the example something like: ```bash -mvn clean install -``` \ No newline at end of file +storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hbase.yaml --filter my_hbase_bolt.properties +```
[21/50] [abbrv] storm git commit: [maven-release-plugin] prepare release flux-0.3.0
[maven-release-plugin] prepare release flux-0.3.0 Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/20a30116 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/20a30116 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/20a30116 Branch: refs/heads/0.10.x-branch Commit: 20a30116935eb3bcc4a255c1617bcb8e50331c5e Parents: 8e0f167 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue May 5 16:54:01 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue May 5 16:54:01 2015 -0400 -- flux-core/pom.xml | 2 +- flux-examples/pom.xml | 11 --- flux-wrappers/pom.xml | 2 +- pom.xml | 2 +- 4 files changed, 7 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/20a30116/flux-core/pom.xml -- diff --git a/flux-core/pom.xml b/flux-core/pom.xml index 2d03ea4..12312f5 100644 --- a/flux-core/pom.xml +++ b/flux-core/pom.xml @@ -21,7 +21,7 @@ parent groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.2.3-SNAPSHOT/version +version0.3.0/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/storm/blob/20a30116/flux-examples/pom.xml -- diff --git a/flux-examples/pom.xml b/flux-examples/pom.xml index 2321074..e186b9c 100644 --- a/flux-examples/pom.xml +++ b/flux-examples/pom.xml @@ -15,14 +15,13 @@ See the License for the specific language governing permissions and limitations under the License. -- -project xmlns=http://maven.apache.org/POM/4.0.0; xmlns:xsi=http://www.w3.org/2001/XMLSchema-instance; - xsi:schemaLocation=http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd; +project xmlns=http://maven.apache.org/POM/4.0.0; xmlns:xsi=http://www.w3.org/2001/XMLSchema-instance; xsi:schemaLocation=http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd; modelVersion4.0.0/modelVersion parent groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.2.3-SNAPSHOT/version +version0.3.0/version relativePath../pom.xml/relativePath /parent @@ -74,10 +73,8 @@ /goals configuration transformers -transformer - implementation=org.apache.maven.plugins.shade.resource.ServicesResourceTransformer/ -transformer - implementation=org.apache.maven.plugins.shade.resource.ManifestResourceTransformer +transformer implementation=org.apache.maven.plugins.shade.resource.ServicesResourceTransformer / +transformer implementation=org.apache.maven.plugins.shade.resource.ManifestResourceTransformer mainClassorg.apache.storm.flux.Flux/mainClass /transformer /transformers http://git-wip-us.apache.org/repos/asf/storm/blob/20a30116/flux-wrappers/pom.xml -- diff --git a/flux-wrappers/pom.xml b/flux-wrappers/pom.xml index e571d56..532da15 100644 --- a/flux-wrappers/pom.xml +++ b/flux-wrappers/pom.xml @@ -21,7 +21,7 @@ parent groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.2.3-SNAPSHOT/version +version0.3.0/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/storm/blob/20a30116/pom.xml -- diff --git a/pom.xml b/pom.xml index de48f7b..942cbeb 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.2.3-SNAPSHOT/version +version0.3.0/version packagingpom/packaging nameflux/name urlhttps://github.com/ptgoetz/flux/url
[24/50] [abbrv] storm git commit: merge flux into external/flux/
http://git-wip-us.apache.org/repos/asf/storm/blob/b21a98dd/external/flux/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java -- diff --cc external/flux/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java index 000,000..9456d1b new file mode 100644 --- /dev/null +++ b/external/flux/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java @@@ -1,0 -1,0 +1,234 @@@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one ++ * or more contributor license agreements. See the NOTICE file ++ * distributed with this work for additional information ++ * regarding copyright ownership. The ASF licenses this file ++ * to you under the Apache License, Version 2.0 (the ++ * License); you may not use this file except in compliance ++ * with the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an AS IS BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.storm.flux; ++ ++import backtype.storm.Config; ++import backtype.storm.generated.StormTopology; ++import org.apache.storm.flux.model.ExecutionContext; ++import org.apache.storm.flux.model.TopologyDef; ++import org.apache.storm.flux.parser.FluxParser; ++import org.apache.storm.flux.test.TestBolt; ++import org.junit.Test; ++ ++import java.io.File; ++ ++import static org.junit.Assert.*; ++ ++public class TCKTest { ++@Test ++public void testTCK() throws Exception { ++TopologyDef topologyDef = FluxParser.parseResource(/configs/tck.yaml, false, true, null, false); ++Config conf = FluxBuilder.buildConfig(topologyDef); ++ExecutionContext context = new ExecutionContext(topologyDef, conf); ++StormTopology topology = FluxBuilder.buildTopology(context); ++assertNotNull(topology); ++topology.validate(); ++} ++ ++@Test ++public void testShellComponents() throws Exception { ++TopologyDef topologyDef = FluxParser.parseResource(/configs/shell_test.yaml, false, true, null, false); ++Config conf = FluxBuilder.buildConfig(topologyDef); ++ExecutionContext context = new ExecutionContext(topologyDef, conf); ++StormTopology topology = FluxBuilder.buildTopology(context); ++assertNotNull(topology); ++topology.validate(); ++} ++ ++@Test ++public void testKafkaSpoutConfig() throws Exception { ++TopologyDef topologyDef = FluxParser.parseResource(/configs/kafka_test.yaml, false, true, null, false); ++Config conf = FluxBuilder.buildConfig(topologyDef); ++ExecutionContext context = new ExecutionContext(topologyDef, conf); ++StormTopology topology = FluxBuilder.buildTopology(context); ++assertNotNull(topology); ++topology.validate(); ++} ++ ++@Test ++public void testLoadFromResource() throws Exception { ++TopologyDef topologyDef = FluxParser.parseResource(/configs/kafka_test.yaml, false, true, null, false); ++Config conf = FluxBuilder.buildConfig(topologyDef); ++ExecutionContext context = new ExecutionContext(topologyDef, conf); ++StormTopology topology = FluxBuilder.buildTopology(context); ++assertNotNull(topology); ++topology.validate(); ++} ++ ++ ++@Test ++public void testHdfs() throws Exception { ++TopologyDef topologyDef = FluxParser.parseResource(/configs/hdfs_test.yaml, false, true, null, false); ++Config conf = FluxBuilder.buildConfig(topologyDef); ++ExecutionContext context = new ExecutionContext(topologyDef, conf); ++StormTopology topology = FluxBuilder.buildTopology(context); ++assertNotNull(topology); ++topology.validate(); ++} ++ ++@Test ++public void testHbase() throws Exception { ++TopologyDef topologyDef = FluxParser.parseResource(/configs/simple_hbase.yaml, false, true, null, false); ++Config conf = FluxBuilder.buildConfig(topologyDef); ++ExecutionContext context = new ExecutionContext(topologyDef, conf); ++StormTopology topology = FluxBuilder.buildTopology(context); ++assertNotNull(topology); ++topology.validate(); ++} ++ ++@Test(expected = IllegalArgumentException.class) ++public void testBadHbase() throws Exception { ++TopologyDef topologyDef = FluxParser.parseResource(/configs/bad_hbase.yaml, false, true, null, false); ++Config conf = FluxBuilder.buildConfig(topologyDef); ++ExecutionContext context = new ExecutionContext(topologyDef, conf); ++StormTopology topology = FluxBuilder.buildTopology(context); ++
[19/50] [abbrv] storm git commit: add missing license headers
add missing license headers Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/01a66bf5 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/01a66bf5 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/01a66bf5 Branch: refs/heads/0.10.x-branch Commit: 01a66bf5b17a9fce94fcdac4fbaa2f28c2df9189 Parents: c48f63e Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue May 5 16:14:33 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue May 5 16:14:33 2015 -0400 -- .../apache/storm/flux/api/TopologySource.java | 17 +++ .../storm/flux/model/ConfigMethodDef.java | 17 +++ .../storm/flux/model/TopologySourceDef.java | 17 +++ .../java/org/apache/storm/flux/FrankenBean.java | 138 --- .../org/apache/storm/flux/IntegrationTest.java | 17 +++ .../src/main/resources/hbase_bolt.properties| 16 +++ .../src/main/resources/hdfs_bolt.properties | 17 +++ .../storm/flux/wrappers/bolts/LogInfoBolt.java | 18 +++ 8 files changed, 119 insertions(+), 138 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/01a66bf5/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java b/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java index 779e676..fbccfb7 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java +++ b/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.storm.flux.api; http://git-wip-us.apache.org/repos/asf/storm/blob/01a66bf5/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java b/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java index e274915..6f7e4d4 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java +++ b/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.storm.flux.model; import java.util.ArrayList; http://git-wip-us.apache.org/repos/asf/storm/blob/01a66bf5/flux-core/src/main/java/org/apache/storm/flux/model/TopologySourceDef.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/model/TopologySourceDef.java b/flux-core/src/main/java/org/apache/storm/flux/model/TopologySourceDef.java index 2949659..d6a2f57 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/model/TopologySourceDef.java +++ b/flux-core/src/main/java/org/apache/storm/flux/model/TopologySourceDef.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the
[39/50] [abbrv] storm git commit: add missing kafka dependency to flux-examples
add missing kafka dependency to flux-examples Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8c9e6cee Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8c9e6cee Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8c9e6cee Branch: refs/heads/master Commit: 8c9e6ceeafc1f50eebc519c8f03d5821813ee99b Parents: 9fad816 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Fri May 8 15:35:11 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Fri May 8 15:35:11 2015 -0400 -- .../org/apache/storm/flux/IntegrationTest.java | 2 -- external/flux/flux-examples/pom.xml | 20 2 files changed, 20 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/8c9e6cee/external/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java -- diff --git a/external/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java b/external/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java index 5e17f5e..c5807f8 100644 --- a/external/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java +++ b/external/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java @@ -30,8 +30,6 @@ public class IntegrationTest { } } - - @Test public void testRunTopologySource() throws Exception { if(!skipTest) { http://git-wip-us.apache.org/repos/asf/storm/blob/8c9e6cee/external/flux/flux-examples/pom.xml -- diff --git a/external/flux/flux-examples/pom.xml b/external/flux/flux-examples/pom.xml index 571f302..e3996e5 100644 --- a/external/flux/flux-examples/pom.xml +++ b/external/flux/flux-examples/pom.xml @@ -52,6 +52,26 @@ artifactIdstorm-hbase/artifactId version${project.version}/version /dependency +dependency +groupIdorg.apache.storm/groupId +artifactIdstorm-kafka/artifactId +version${project.version}/version +/dependency +dependency +groupIdorg.apache.kafka/groupId +artifactIdkafka_2.10/artifactId +version0.8.1.1/version +exclusions +exclusion +groupIdorg.apache.zookeeper/groupId +artifactIdzookeeper/artifactId +/exclusion +exclusion +groupIdlog4j/groupId +artifactIdlog4j/artifactId +/exclusion +/exclusions +/dependency /dependencies build
[17/50] [abbrv] storm git commit: link examples to source
link examples to source Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a791604a Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a791604a Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a791604a Branch: refs/heads/master Commit: a791604ac5d274cededae8a01362dac4dfd82d84 Parents: 0c1e0aa Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue Apr 7 00:07:49 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue Apr 7 00:07:49 2015 -0400 -- .../src/test/resources/configs/hdfs_test.yaml | 97 flux-examples/README.md | 8 +- 2 files changed, 101 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/a791604a/flux-core/src/test/resources/configs/hdfs_test.yaml -- diff --git a/flux-core/src/test/resources/configs/hdfs_test.yaml b/flux-core/src/test/resources/configs/hdfs_test.yaml new file mode 100644 index 000..8fe0a9a --- /dev/null +++ b/flux-core/src/test/resources/configs/hdfs_test.yaml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# License); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test ability to wire together shell spouts/bolts +--- + +# topology definition +# name to be used when submitting +name: hdfs-topology + +# Components +# Components are analagous to Spring beans. They are meant to be used as constructor, +# property(setter), and builder arguments. +# +# for the time being, components must be declared in the order they are referenced +components: + - id: syncPolicy +className: org.apache.storm.hdfs.bolt.sync.CountSyncPolicy +constructorArgs: + - 1000 + - id: rotationPolicy +className: org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy +constructorArgs: + - 5.0 + - MB + + - id: fileNameFormat +className: org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat +configMethods: + - name: withPath +args: [/tmp/foo/] + - name: withExtension +args: [.txt] + + - id: recordFormat +className: org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat +configMethods: + - name: withFieldDelimiter +args: [|] + + - id: rotationAction +className: org.apache.storm.hdfs.common.rotation.MoveFileAction +configMethods: + - name: toDestination +args: [/tmp/dest2] + +# spout definitions +spouts: + - id: spout-1 +className: backtype.storm.testing.TestWordSpout +parallelism: 1 +# ... + +# bolt definitions + +#HdfsBolt bolt = new HdfsBolt() +#.withConfigKey(hdfs.config) +#.withFsUrl(args[0]) +#.withFileNameFormat(fileNameFormat) +#.withRecordFormat(format) +#.withRotationPolicy(rotationPolicy) +#.withSyncPolicy(syncPolicy) +#.addRotationAction(new MoveFileAction().toDestination(/tmp/dest2/)); +bolts: + - id: bolt-1 +className: org.apache.storm.hdfs.bolt.HdfsBolt +configMethods: + - name: withConfigKey +args: [hdfs.config] + - name: withFsUrl +args: [hdfs://hadoop:54310] + - name: withFileNameFormat +args: [ref: fileNameFormat] + - name: withRecordFormat +args: [ref: recordFormat] + - name: withRotationPolicy +args: [ref: rotationPolicy] + - name: withSyncPolicy +args: [ref: syncPolicy] + - name: addRotationAction +args: [ref: rotationAction] +parallelism: 1 +# ... + http://git-wip-us.apache.org/repos/asf/storm/blob/a791604a/flux-examples/README.md -- diff --git a/flux-examples/README.md b/flux-examples/README.md index 9f5682e..425ad98 100644 --- a/flux-examples/README.md +++ b/flux-examples/README.md @@ -28,21 +28,21 @@ storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux - ## Available Examples -### simple_wordcount.yaml +###
[47/50] [abbrv] storm git commit: Merge branch 'ehimprove' of github.com:shanyu/storm
Merge branch 'ehimprove' of github.com:shanyu/storm Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6978b585 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6978b585 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6978b585 Branch: refs/heads/master Commit: 6978b58574a4de67ef12ded945e5183d96680618 Parents: a55bbbe 9c2972a Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 13:03:48 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 13:03:48 2015 -0400 -- external/storm-eventhubs/pom.xml| 38 ++--- .../eventhubs/bolt/DefaultEventDataFormat.java | 47 +++ .../storm/eventhubs/bolt/EventHubBolt.java | 56 +--- .../eventhubs/bolt/EventHubBoltConfig.java | 109 +++ .../storm/eventhubs/bolt/IEventDataFormat.java | 28 .../client/ConnectionStringBuilder.java | 116 .../storm/eventhubs/client/Constants.java | 32 - .../storm/eventhubs/client/EventHubClient.java | 92 .../eventhubs/client/EventHubConsumerGroup.java | 72 -- .../eventhubs/client/EventHubException.java | 37 - .../eventhubs/client/EventHubReceiver.java | 139 --- .../eventhubs/client/EventHubSendClient.java| 70 -- .../storm/eventhubs/client/EventHubSender.java | 95 - .../storm/eventhubs/client/SelectorFilter.java | 38 - .../eventhubs/client/SelectorFilterWriter.java | 64 - .../storm/eventhubs/samples/EventCount.java | 5 +- .../storm/eventhubs/samples/EventHubLoop.java | 9 +- .../eventhubs/spout/EventHubReceiverFilter.java | 56 .../eventhubs/spout/EventHubReceiverImpl.java | 49 --- .../storm/eventhubs/spout/EventHubSpout.java| 5 + .../eventhubs/spout/EventHubSpoutConfig.java| 126 + .../eventhubs/spout/IEventHubReceiver.java | 5 +- .../spout/IEventHubReceiverFilter.java | 35 - .../eventhubs/spout/SimplePartitionManager.java | 11 +- .../spout/StaticPartitionCoordinator.java | 2 +- .../TransactionalTridentEventHubEmitter.java| 2 +- .../trident/TridentPartitionManager.java| 12 +- .../src/main/resources/config.properties| 5 +- .../eventhubs/spout/EventHubReceiverMock.java | 18 +-- .../eventhubs/spout/TestEventHubSpout.java | 4 +- 30 files changed, 376 insertions(+), 1001 deletions(-) --
[34/50] [abbrv] storm git commit: merge flux into external/flux/
merge flux into external/flux/ Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b21a98dd Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b21a98dd Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b21a98dd Branch: refs/heads/master Commit: b21a98dd87f82a06a6295ab2bfd832c2810ca57e Parents: ea0fe12 b372a11 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed May 6 13:31:04 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed May 6 13:31:04 2015 -0400 -- external/flux/.gitignore| 15 + external/flux/LICENSE | 202 + external/flux/README.md | 845 +++ external/flux/flux-core/pom.xml | 94 +++ .../main/java/org/apache/storm/flux/Flux.java | 263 ++ .../java/org/apache/storm/flux/FluxBuilder.java | 591 + .../apache/storm/flux/api/TopologySource.java | 39 + .../org/apache/storm/flux/model/BeanDef.java| 39 + .../apache/storm/flux/model/BeanReference.java | 39 + .../org/apache/storm/flux/model/BoltDef.java| 24 + .../storm/flux/model/ConfigMethodDef.java | 62 ++ .../storm/flux/model/ExecutionContext.java | 77 ++ .../apache/storm/flux/model/GroupingDef.java| 77 ++ .../org/apache/storm/flux/model/IncludeDef.java | 54 ++ .../org/apache/storm/flux/model/ObjectDef.java | 90 ++ .../apache/storm/flux/model/PropertyDef.java| 58 ++ .../org/apache/storm/flux/model/SpoutDef.java | 24 + .../org/apache/storm/flux/model/StreamDef.java | 64 ++ .../apache/storm/flux/model/TopologyDef.java| 216 + .../storm/flux/model/TopologySourceDef.java | 36 + .../org/apache/storm/flux/model/VertexDef.java | 36 + .../apache/storm/flux/parser/FluxParser.java| 202 + .../flux-core/src/main/resources/splash.txt | 9 + .../org/apache/storm/flux/FluxBuilderTest.java | 31 + .../org/apache/storm/flux/IntegrationTest.java | 41 + .../java/org/apache/storm/flux/TCKTest.java | 234 + .../multilang/MultilangEnvirontmentTest.java| 89 ++ .../apache/storm/flux/test/SimpleTopology.java | 42 + .../storm/flux/test/SimpleTopologySource.java | 35 + .../test/SimpleTopologyWithConfigParam.java | 38 + .../org/apache/storm/flux/test/TestBolt.java| 63 ++ .../storm/flux/test/TridentTopologySource.java | 54 ++ .../src/test/resources/configs/bad_hbase.yaml | 98 +++ .../resources/configs/config-methods-test.yaml | 70 ++ .../existing-topology-method-override.yaml | 10 + .../existing-topology-reflection-config.yaml| 9 + .../configs/existing-topology-reflection.yaml | 9 + .../configs/existing-topology-trident.yaml | 9 + .../resources/configs/existing-topology.yaml| 8 + .../src/test/resources/configs/hdfs_test.yaml | 97 +++ .../test/resources/configs/include_test.yaml| 25 + .../configs/invalid-existing-topology.yaml | 17 + .../src/test/resources/configs/kafka_test.yaml | 126 +++ .../src/test/resources/configs/shell_test.yaml | 104 +++ .../test/resources/configs/simple_hbase.yaml| 120 +++ .../resources/configs/substitution-test.yaml| 106 +++ .../src/test/resources/configs/tck.yaml | 95 +++ .../src/test/resources/configs/test.properties | 2 + .../flux-core/src/test/resources/logback.xml| 30 + external/flux/flux-examples/README.md | 68 ++ external/flux/flux-examples/pom.xml | 87 ++ .../storm/flux/examples/WordCountClient.java| 74 ++ .../apache/storm/flux/examples/WordCounter.java | 71 ++ .../src/main/resources/hbase_bolt.properties| 18 + .../src/main/resources/hdfs_bolt.properties | 26 + .../src/main/resources/kafka_spout.yaml | 136 +++ .../src/main/resources/multilang.yaml | 89 ++ .../src/main/resources/simple_hbase.yaml| 92 ++ .../src/main/resources/simple_hdfs.yaml | 105 +++ .../src/main/resources/simple_wordcount.yaml| 68 ++ external/flux/flux-ui/README.md | 3 + external/flux/flux-wrappers/pom.xml | 35 + .../flux/wrappers/bolts/FluxShellBolt.java | 56 ++ .../storm/flux/wrappers/bolts/LogInfoBolt.java | 44 + .../flux/wrappers/spouts/FluxShellSpout.java| 55 ++ .../main/resources/resources/randomsentence.js | 93 ++ .../main/resources/resources/splitsentence.py | 24 + .../src/main/resources/resources/storm.js | 373 .../src/main/resources/resources/storm.py | 260 ++ external/flux/pom.xml | 126 +++ 70 files changed, 6621 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/b21a98dd/external/flux/.gitignore -- diff
[36/50] [abbrv] storm git commit: add flux to binary distribution
add flux to binary distribution Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e1e1609d Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e1e1609d Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e1e1609d Branch: refs/heads/master Commit: e1e1609dd3642b820fe6f514b25454dedd446836 Parents: 2094a08 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed May 6 14:30:38 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed May 6 14:30:38 2015 -0400 -- storm-dist/binary/src/main/assembly/binary.xml | 44 + 1 file changed, 44 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/e1e1609d/storm-dist/binary/src/main/assembly/binary.xml -- diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml index fef56bb..31d8b9e 100644 --- a/storm-dist/binary/src/main/assembly/binary.xml +++ b/storm-dist/binary/src/main/assembly/binary.xml @@ -159,6 +159,50 @@ /includes /fileSet +fileSet + directory${project.basedir}/../../external/flux/flux-core/target/directory +outputDirectoryexternal/flux/outputDirectory +includes +includeflux*jar/include +/includes +/fileSet +fileSet + directory${project.basedir}/../../external/flux/flux-wrappers/target/directory +outputDirectoryexternal/flux/outputDirectory +includes +includeflux*jar/include +/includes +/fileSet +fileSet + directory${project.basedir}/../../external/flux/flux-examples/target/directory +outputDirectoryexternal/flux/outputDirectory +includes +includeflux*jar/include +/includes +/fileSet +fileSet + directory${project.basedir}/../../external/flux/flux-examples/directory +outputDirectoryexternal/flux/examples/outputDirectory +includes +includeREADME.*/include +/includes +/fileSet +fileSet + directory${project.basedir}/../../external/flux/flux-examples/src/main/resources/directory +outputDirectoryexternal/flux/examples/outputDirectory +includes +include*/include +/includes +/fileSet +fileSet +directory${project.basedir}/../../external/flux/directory +outputDirectoryexternal/flux/outputDirectory +includes +includeREADME.*/include +/includes +/fileSet + + !-- $STORM_HOME/extlib -- fileSet directory/directory
[33/50] [abbrv] storm git commit: merge flux into external/flux/
http://git-wip-us.apache.org/repos/asf/storm/blob/b21a98dd/external/flux/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java -- diff --cc external/flux/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java index 000,000..57237b6 new file mode 100644 --- /dev/null +++ b/external/flux/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java @@@ -1,0 -1,0 +1,591 @@@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one ++ * or more contributor license agreements. See the NOTICE file ++ * distributed with this work for additional information ++ * regarding copyright ownership. The ASF licenses this file ++ * to you under the Apache License, Version 2.0 (the ++ * License); you may not use this file except in compliance ++ * with the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an AS IS BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.storm.flux; ++ ++import backtype.storm.Config; ++import backtype.storm.generated.StormTopology; ++import backtype.storm.grouping.CustomStreamGrouping; ++import backtype.storm.topology.*; ++import backtype.storm.tuple.Fields; ++import backtype.storm.utils.Utils; ++import org.apache.storm.flux.api.TopologySource; ++import org.apache.storm.flux.model.*; ++import org.slf4j.Logger; ++import org.slf4j.LoggerFactory; ++ ++import java.lang.reflect.*; ++import java.util.ArrayList; ++import java.util.Collection; ++import java.util.List; ++import java.util.Map; ++ ++public class FluxBuilder { ++private static Logger LOG = LoggerFactory.getLogger(FluxBuilder.class); ++ ++/** ++ * Given a topology definition, return a populated `backtype.storm.Config` instance. ++ * ++ * @param topologyDef ++ * @return ++ */ ++public static Config buildConfig(TopologyDef topologyDef) { ++// merge contents of `config` into topology config ++Config conf = new Config(); ++conf.putAll(topologyDef.getConfig()); ++return conf; ++} ++ ++/** ++ * Given a topology definition, return a Storm topology that can be run either locally or remotely. ++ * ++ * @param context ++ * @return ++ * @throws IllegalAccessException ++ * @throws InstantiationException ++ * @throws ClassNotFoundException ++ * @throws NoSuchMethodException ++ * @throws InvocationTargetException ++ */ ++static StormTopology buildTopology(ExecutionContext context) throws IllegalAccessException, ++InstantiationException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException { ++ ++StormTopology topology = null; ++TopologyDef topologyDef = context.getTopologyDef(); ++ ++if(!topologyDef.validate()){ ++throw new IllegalArgumentException(Invalid topology config. Spouts, bolts and streams cannot be + ++defined in the same configuration as a topologySource.); ++} ++ ++// build components that may be referenced by spouts, bolts, etc. ++// the map will be a String -- Object where the object is a fully ++// constructed class instance ++buildComponents(context); ++ ++if(topologyDef.isDslTopology()) { ++// This is a DSL (YAML, etc.) topology... ++LOG.info(Detected DSL topology...); ++ ++TopologyBuilder builder = new TopologyBuilder(); ++ ++// create spouts ++buildSpouts(context, builder); ++ ++// we need to be able to lookup bolts by id, then switch based ++// on whether they are IBasicBolt or IRichBolt instances ++buildBolts(context); ++ ++// process stream definitions ++buildStreamDefinitions(context, builder); ++ ++topology = builder.createTopology(); ++} else { ++// user class supplied... ++// this also provides a bridge to Trident... ++LOG.info(A topology source has been specified...); ++ObjectDef def = topologyDef.getTopologySource(); ++topology = buildExternalTopology(def, context); ++} ++return topology; ++} ++ ++/** ++ * Given a `java.lang.Object` instance and a method name, attempt to find a method that matches the input ++ * parameter: `java.util.Map` or `backtype.storm.Config`. ++ * ++ * @param topologySource object to inspect for the specified method ++ * @param methodName name of the method to look for ++ * @return ++ * @throws NoSuchMethodException ++ */ ++private static
[07/50] [abbrv] storm git commit: update doc version to 0.2.2-SNAPSHOT
update doc version to 0.2.2-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b931d1b3 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b931d1b3 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b931d1b3 Branch: refs/heads/master Commit: b931d1b3da7008c0c278cb6bab9a5ed366333d8d Parents: 44d5b5b Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue Mar 31 00:19:47 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue Mar 31 00:19:47 2015 -0400 -- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/b931d1b3/README.md -- diff --git a/README.md b/README.md index 71403e0..78a2723 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ The current version of Flux is available in Maven Central at the following coord dependency groupIdcom.github.ptgoetz/groupId artifactIdflux-core/artifactId -version0.2.1/version +version0.2.2-SNAPSHOT/version /dependency ``` @@ -92,7 +92,7 @@ The example below illustrates Flux usage with the Maven shade plugin: dependency groupIdcom.github.ptgoetz/groupId artifactIdflux-core/artifactId -version0.2.1/version +version0.2.2-SNAPSHOT/version /dependency !-- add user dependencies here... -- @@ -193,7 +193,7 @@ storm jar myTopology-0.1.0-SNAPSHOT.jar org.apache.storm.flux.Flux --remote my_c âââ ââââââââ âââââââ âââ âââ +- Apache Storm-+ +- data FLow User eXperience -+ -Version: 0.2.1 +Version: 0.2.2-SNAPSHOT Parsing file: /Users/hsimpson/Projects/donut_domination/storm/shell_test.yaml -- TOPOLOGY DETAILS -- Name: shell-topology
[30/50] [abbrv] storm git commit: [maven-release-plugin] prepare for next development iteration
[maven-release-plugin] prepare for next development iteration Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b372a119 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b372a119 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b372a119 Branch: refs/heads/master Commit: b372a1198f2227ccc2951b1f5ffccd21a98b589b Parents: 20a3011 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue May 5 16:54:05 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue May 5 16:54:05 2015 -0400 -- flux-core/pom.xml | 2 +- flux-examples/pom.xml | 2 +- flux-wrappers/pom.xml | 2 +- pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/b372a119/flux-core/pom.xml -- diff --git a/flux-core/pom.xml b/flux-core/pom.xml index 12312f5..600613d 100644 --- a/flux-core/pom.xml +++ b/flux-core/pom.xml @@ -21,7 +21,7 @@ parent groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.3.0/version +version0.3.1-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/storm/blob/b372a119/flux-examples/pom.xml -- diff --git a/flux-examples/pom.xml b/flux-examples/pom.xml index e186b9c..0b9796e 100644 --- a/flux-examples/pom.xml +++ b/flux-examples/pom.xml @@ -21,7 +21,7 @@ parent groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.3.0/version +version0.3.1-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/storm/blob/b372a119/flux-wrappers/pom.xml -- diff --git a/flux-wrappers/pom.xml b/flux-wrappers/pom.xml index 532da15..6784141 100644 --- a/flux-wrappers/pom.xml +++ b/flux-wrappers/pom.xml @@ -21,7 +21,7 @@ parent groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.3.0/version +version0.3.1-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/storm/blob/b372a119/pom.xml -- diff --git a/pom.xml b/pom.xml index 942cbeb..5ea1b40 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ groupIdcom.github.ptgoetz/groupId artifactIdflux/artifactId -version0.3.0/version +version0.3.1-SNAPSHOT/version packagingpom/packaging nameflux/name urlhttps://github.com/ptgoetz/flux/url
[12/50] [abbrv] storm git commit: lock down methods that don't need to be public
lock down methods that don't need to be public Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/11b01518 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/11b01518 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/11b01518 Branch: refs/heads/master Commit: 11b01518f0935dedfd71d09b697d7189a29b91b5 Parents: df34930 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Thu Apr 2 00:08:12 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Thu Apr 2 00:08:12 2015 -0400 -- .../main/java/org/apache/storm/flux/FluxBuilder.java| 12 ++-- 1 file changed, 6 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/11b01518/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java b/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java index 31b6e64..964c62e 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java +++ b/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java @@ -61,7 +61,7 @@ public class FluxBuilder { * @throws NoSuchMethodException * @throws InvocationTargetException */ -public static StormTopology buildTopology(ExecutionContext context) throws IllegalAccessException, +static StormTopology buildTopology(ExecutionContext context) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException { StormTopology topology = null; @@ -206,7 +206,7 @@ public class FluxBuilder { } } -public static void applyProperties(ObjectDef bean, Object instance, ExecutionContext context) throws +private static void applyProperties(ObjectDef bean, Object instance, ExecutionContext context) throws IllegalAccessException, InvocationTargetException { ListPropertyDef props = bean.getProperties(); Class clazz = instance.getClass(); @@ -230,7 +230,7 @@ public class FluxBuilder { } } -public static Field findPublicField(Class clazz, String property, Object arg) { +private static Field findPublicField(Class clazz, String property, Object arg) { Field field = null; try { field = clazz.getField(property); @@ -240,7 +240,7 @@ public class FluxBuilder { return field; } -public static Method findSetter(Class clazz, String property, Object arg) { +private static Method findSetter(Class clazz, String property, Object arg) { String setterName = toSetterName(property); Method retval = null; Method[] methods = clazz.getMethods(); @@ -253,11 +253,11 @@ public class FluxBuilder { return retval; } -public static String toSetterName(String name) { +private static String toSetterName(String name) { return set + name.substring(0, 1).toUpperCase() + name.substring(1, name.length()); } -public static ListObject resolveReferences(ListObject args, ExecutionContext context) { +private static ListObject resolveReferences(ListObject args, ExecutionContext context) { LOG.debug(Checking arguments for references.); ListObject cArgs = new ArrayListObject(); // resolve references
[01/50] [abbrv] storm git commit: preliminary implementation for variable substitution
Repository: storm Updated Branches: refs/heads/master a55bbbea8 - 285d943b8 preliminary implementation for variable substitution Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5b1d10ab Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5b1d10ab Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5b1d10ab Branch: refs/heads/master Commit: 5b1d10aba4118d854ca06840100b1e8da98bea34 Parents: adf91d9 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Mon Mar 30 01:40:32 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Mon Mar 30 01:40:32 2015 -0400 -- .../main/java/org/apache/storm/flux/Flux.java | 18 +++- .../apache/storm/flux/parser/FluxParser.java| 45 +++- .../java/org/apache/storm/flux/TCKTest.java | 28 ++-- .../src/test/resources/configs/hdfs_test.yaml | 8 4 files changed, 72 insertions(+), 27 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/5b1d10ab/flux-core/src/main/java/org/apache/storm/flux/Flux.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/Flux.java b/flux-core/src/main/java/org/apache/storm/flux/Flux.java index 4583bae..3910e10 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/Flux.java +++ b/flux-core/src/main/java/org/apache/storm/flux/Flux.java @@ -31,6 +31,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; +import java.util.Map; +import java.util.Properties; /** * Flux entry point. @@ -52,6 +54,7 @@ public class Flux { private static final String OPTION_NO_SPLASH = no-splash; private static final String OPTION_INACTIVE = inactive; private static final String OPTION_ZOOKEEPER = zookeeper; +private static final String OPTION_FILTER = filter; public static void main(String[] args) throws Exception { Options options = new Options(); @@ -77,6 +80,9 @@ public class Flux { options.addOption(option(1, z, OPTION_ZOOKEEPER, host:port, When running in local mode, use the ZooKeeper at the + specified host:port instead of the in-process ZooKeeper.)); +options.addOption(option(1, f, OPTION_FILTER, file, Use the specified file as a source of properties, and + +perform variable substitution.)); + CommandLineParser parser = new BasicParser(); CommandLine cmd = parser.parse(options, args); @@ -116,13 +122,21 @@ public class Flux { TopologyDef topologyDef = null; String filePath = (String)cmd.getArgList().get(0); + +// TODO conditionally load properties from a file our resource +Properties props = new Properties(); +if(cmd.hasOption(OPTION_FILTER)){ +//InputStream in = new FileInputStream(); +props.load((InputStream)null); +} + if(cmd.hasOption(OPTION_RESOURCE)){ printf(Parsing classpath resource: %s, filePath); -topologyDef = FluxParser.parseResource(filePath, dumpYaml, true); +topologyDef = FluxParser.parseResource(filePath, dumpYaml, true, props); } else { printf(Parsing file: %s, new File(filePath).getAbsolutePath()); -topologyDef = FluxParser.parseFile(filePath, dumpYaml, true); +topologyDef = FluxParser.parseFile(filePath, dumpYaml, true, props); } http://git-wip-us.apache.org/repos/asf/storm/blob/5b1d10ab/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java b/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java index baeb4c4..40cfe57 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java +++ b/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java @@ -17,6 +17,7 @@ */ package org.apache.storm.flux.parser; +import org.apache.storm.flux.api.TopologySource; import org.apache.storm.flux.model.BoltDef; import org.apache.storm.flux.model.IncludeDef; import org.apache.storm.flux.model.SpoutDef; @@ -27,48 +28,70 @@ import org.yaml.snakeyaml.TypeDescription; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.Constructor; +import java.io.ByteArrayOutputStream; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; import java.util.Map; +import java.util.Properties; public class FluxParser { private static final Logger LOG = LoggerFactory.getLogger(FluxParser.class); private FluxParser(){} -public static TopologyDef
[43/50] [abbrv] storm git commit: Remove client adding reference eventhubs-client package and use ResilientEventHubReceiver
Remove client adding reference eventhubs-client package and use ResilientEventHubReceiver Signed-off-by: Shanyu Zhao shz...@microsoft.com Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/85aeb3d4 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/85aeb3d4 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/85aeb3d4 Branch: refs/heads/master Commit: 85aeb3d48efcaa28d6fc5dbfe6ce87af9f3e2615 Parents: 1f13f15 Author: Shanyu Zhao shz...@microsoft.com Authored: Sun May 17 01:14:01 2015 -0700 Committer: Shanyu Zhao shz...@microsoft.com Committed: Sun May 17 01:14:01 2015 -0700 -- external/storm-eventhubs/pom.xml| 13 +- .../storm/eventhubs/bolt/EventHubBolt.java | 6 +- .../eventhubs/bolt/EventHubBoltConfig.java | 4 +- .../client/ConnectionStringBuilder.java | 116 .../storm/eventhubs/client/Constants.java | 32 - .../storm/eventhubs/client/EventHubClient.java | 95 - .../eventhubs/client/EventHubConsumerGroup.java | 72 -- .../eventhubs/client/EventHubException.java | 37 - .../eventhubs/client/EventHubReceiver.java | 139 --- .../eventhubs/client/EventHubSendClient.java| 70 -- .../storm/eventhubs/client/EventHubSender.java | 99 - .../storm/eventhubs/client/SelectorFilter.java | 38 - .../eventhubs/client/SelectorFilterWriter.java | 64 - .../eventhubs/spout/EventHubReceiverFilter.java | 56 .../eventhubs/spout/EventHubReceiverImpl.java | 50 +++ .../eventhubs/spout/EventHubSpoutConfig.java| 31 + .../eventhubs/spout/IEventHubReceiver.java | 5 +- .../spout/IEventHubReceiverFilter.java | 35 - .../eventhubs/spout/SimplePartitionManager.java | 11 +- .../spout/StaticPartitionCoordinator.java | 2 +- .../TransactionalTridentEventHubEmitter.java| 2 +- .../trident/TridentPartitionManager.java| 12 +- .../eventhubs/spout/EventHubReceiverMock.java | 18 +-- 23 files changed, 56 insertions(+), 951 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/85aeb3d4/external/storm-eventhubs/pom.xml -- diff --git a/external/storm-eventhubs/pom.xml b/external/storm-eventhubs/pom.xml index 2dfb739..6d4a47b 100755 --- a/external/storm-eventhubs/pom.xml +++ b/external/storm-eventhubs/pom.xml @@ -33,7 +33,7 @@ properties project.build.sourceEncodingUTF-8/project.build.sourceEncoding -qpid.version0.32/qpid.version +eventhubs.client.version0.9/eventhubs.client.version /properties build plugins @@ -77,14 +77,9 @@ /build dependencies dependency -groupIdorg.apache.qpid/groupId -artifactIdqpid-client/artifactId -version${qpid.version}/version -/dependency -dependency -groupIdorg.apache.qpid/groupId -artifactIdqpid-amqp-1-0-client-jms/artifactId -version${qpid.version}/version +groupIdcom.microsoft.eventhubs.client/groupId +artifactIdeventhubs-client/artifactId +version${eventhubs.client.version}/version /dependency dependency groupIdorg.apache.storm/groupId http://git-wip-us.apache.org/repos/asf/storm/blob/85aeb3d4/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java -- diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java index a817744..9acf7fa 100755 --- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java +++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java @@ -22,9 +22,9 @@ import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.storm.eventhubs.client.EventHubClient; -import org.apache.storm.eventhubs.client.EventHubException; -import org.apache.storm.eventhubs.client.EventHubSender; +import com.microsoft.eventhubs.client.EventHubClient; +import com.microsoft.eventhubs.client.EventHubException; +import com.microsoft.eventhubs.client.EventHubSender; import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; http://git-wip-us.apache.org/repos/asf/storm/blob/85aeb3d4/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBoltConfig.java -- diff --git
[48/50] [abbrv] storm git commit: add STORM-818 to changelog
add STORM-818 to changelog Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/23137753 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/23137753 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/23137753 Branch: refs/heads/master Commit: 23137753250b4d8e8e15cfe661a9ba6cdc037273 Parents: 6978b58 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 13:08:46 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 13:08:46 2015 -0400 -- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/23137753/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index 6329872..02aa95d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 0.11.0 ## 0.10.0 + * STORM-818: storm-eventhubs configuration improvement and refactoring * STORM-842: Drop Support for Java 1.6 * STORM-835: Netty Client hold batch object until io operation complete * STORM-827: Allow AutoTGT to work with storm-hdfs too.
[44/50] [abbrv] storm git commit: update EventHubClient library version to 0.9.1
update EventHubClient library version to 0.9.1 Signed-off-by: Shanyu Zhao shz...@microsoft.com Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/86f326a3 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/86f326a3 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/86f326a3 Branch: refs/heads/master Commit: 86f326a3c0b40fc1b0eab65fcb17b438e433cfd6 Parents: 85aeb3d Author: Shanyu Zhao shz...@microsoft.com Authored: Fri May 29 14:05:28 2015 -0700 Committer: Shanyu Zhao shz...@microsoft.com Committed: Fri May 29 14:05:28 2015 -0700 -- external/storm-eventhubs/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/86f326a3/external/storm-eventhubs/pom.xml -- diff --git a/external/storm-eventhubs/pom.xml b/external/storm-eventhubs/pom.xml index 6d4a47b..5de412f 100755 --- a/external/storm-eventhubs/pom.xml +++ b/external/storm-eventhubs/pom.xml @@ -33,7 +33,7 @@ properties project.build.sourceEncodingUTF-8/project.build.sourceEncoding -eventhubs.client.version0.9/eventhubs.client.version +eventhubs.client.version0.9.1/eventhubs.client.version /properties build plugins
[40/50] [abbrv] storm git commit: fix typo in README
fix typo in README Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/91369ac2 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/91369ac2 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/91369ac2 Branch: refs/heads/master Commit: 91369ac2f15d4a2dade5d03ee2d0970d99e4d0bd Parents: 8c9e6ce Author: P. Taylor Goetz ptgo...@gmail.com Authored: Fri May 8 15:43:00 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Fri May 8 15:43:00 2015 -0400 -- external/flux/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/91369ac2/external/flux/README.md -- diff --git a/external/flux/README.md b/external/flux/README.md index 0387f3f..2c5127e 100644 --- a/external/flux/README.md +++ b/external/flux/README.md @@ -183,7 +183,7 @@ usage: storm jar my_topology_uber_jar.jar org.apache.storm.flux.Flux build, validate, and print information about the topology. -e,--env-filter Perform environment variable substitution. - Replace keysidentified with `${ENV-[NAME]}` + Replace keys identified with `${ENV-[NAME]}` will be replaced with the corresponding `NAME` environment value -f,--filter file Perform property substitution. Use the
[08/50] [abbrv] storm git commit: fix to make flux compatible with Storm versions earlier than 0.9.3
fix to make flux compatible with Storm versions earlier than 0.9.3 Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c3cffa08 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c3cffa08 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c3cffa08 Branch: refs/heads/master Commit: c3cffa08bbdb3b4748202f94ec802858df45e1fd Parents: b931d1b Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Apr 1 12:04:55 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Apr 1 12:04:55 2015 -0400 -- flux-core/src/main/java/org/apache/storm/flux/Flux.java | 12 +--- .../main/java/org/apache/storm/flux/FluxBuilder.java| 8 +--- 2 files changed, 14 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/c3cffa08/flux-core/src/main/java/org/apache/storm/flux/Flux.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/Flux.java b/flux-core/src/main/java/org/apache/storm/flux/Flux.java index 2c2105c..6300631 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/Flux.java +++ b/flux-core/src/main/java/org/apache/storm/flux/Flux.java @@ -79,7 +79,7 @@ public class Flux { options.addOption(option(0, i, OPTION_INACTIVE, Deploy the topology, but do not activate it.)); options.addOption(option(1, z, OPTION_ZOOKEEPER, host:port, When running in local mode, use the ZooKeeper at the + -specified host:port instead of the in-process ZooKeeper.)); +specified host:port instead of the in-process ZooKeeper. (requires Storm 0.9.3 or later))); options.addOption(option(1, f, OPTION_FILTER, file, Perform property substitution. Use the specified file + as a source of properties, and replace keys identified with {$[property name]} with the value defined + @@ -169,7 +169,7 @@ public class Flux { LOG.info(Deploying topology in an ACTIVE state...); submitOptions = new SubmitOptions(TopologyInitialStatus.ACTIVE); } -StormSubmitter.submitTopology(topologyName, conf, topology, submitOptions); +StormSubmitter.submitTopology(topologyName, conf, topology, submitOptions, null); } catch (Exception e) { LOG.warn(Unable to deploy topology to remote cluster., e); } @@ -198,7 +198,13 @@ public class Flux { } else { zkHost = zkStr; } -cluster = new LocalCluster(zkHost, zkPort); +// the following constructor is only available in 0.9.3 and later +try { +cluster = new LocalCluster(zkHost, zkPort); +} catch (NoSuchMethodError e){ +LOG.error(The --zookeeper option can only be used with Apache Storm 0.9.3 and later.); +System.exit(1); +} } else { cluster = new LocalCluster(); } http://git-wip-us.apache.org/repos/asf/storm/blob/c3cffa08/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java b/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java index c576855..31b6e64 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java +++ b/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java @@ -419,7 +419,8 @@ public class FluxBuilder { Object[] methodArgs = getArgsWithListCoercian(args, method.getParameterTypes()); method.invoke(instance, methodArgs); } else { -LOG.warn(Unable to find method '{}' in class '{}' with arguments {}., methodName, clazz.getName(), args); +LOG.warn(Unable to find method '{}' in class '{}' with arguments {}., +new Object[]{methodName, clazz.getName(), args}); throw new IllegalArgumentException(Configuration method not found.); } } @@ -447,8 +448,9 @@ public class FluxBuilder { } } if (eligibleCount 1) { -LOG.warn(Found multiple invokable methods for class {}, method {}, given arguments {}. Using the last one found., -target, methodName, args); +LOG.warn(Found multiple invokable methods for class {}, method {}, given arguments {}. + +Using the last one found., +new
[24/50] [abbrv] storm git commit: fix validation
fix validation Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f432abf5 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f432abf5 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f432abf5 Branch: refs/heads/master Commit: f432abf523a54806e4b927f676b93575bdf687f2 Parents: abf2924 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Fri Apr 10 15:47:39 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Fri Apr 10 15:47:39 2015 -0400 -- .../apache/storm/flux/model/TopologyDef.java| 23 +++- 1 file changed, 13 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/f432abf5/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java b/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java index 3be7dd6..6c34018 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java +++ b/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java @@ -196,19 +196,22 @@ public class TopologyDef { } public boolean isDslTopology(){ -boolean hasSpouts = this.spoutMap != null this.spoutMap.size() 0; -boolean hasBolts = this.boltMap != null this.boltMap.size() 0; -boolean hasStreams = this.streams != null this.streams.size() 0; -boolean isDslTopology = hasSpouts || hasBolts || hasStreams; - -return isDslTopology; +return this.topologySource == null; } public boolean validate(){ -// we can't have a topology source and spout/bolt/stream definitions at the same time -boolean isDslTopology = isDslTopology(); -boolean isTopologySource = this.topologySource != null; -return !(isDslTopology isTopologySource); +boolean hasSpouts = this.spoutMap != null this.spoutMap.size() 0; +boolean hasBolts = this.boltMap != null this.boltMap.size() 0; +boolean hasStreams = this.streams != null this.streams.size() 0; +boolean hasSpoutsBoltsStreams = hasStreams hasBolts hasSpouts; +// you cant define a topologySource and a DSL topology at the same time... +if (!isDslTopology() ((hasSpouts || hasBolts || hasStreams))) { +return false; +} +if(isDslTopology() (hasSpouts hasBolts hasStreams)) { +return true; +} +return true; } }
[31/50] [abbrv] storm git commit: merge flux into external/flux/
http://git-wip-us.apache.org/repos/asf/storm/blob/b21a98dd/external/flux/flux-examples/src/main/resources/simple_hdfs.yaml -- diff --cc external/flux/flux-examples/src/main/resources/simple_hdfs.yaml index 000,000..9007869 new file mode 100644 --- /dev/null +++ b/external/flux/flux-examples/src/main/resources/simple_hdfs.yaml @@@ -1,0 -1,0 +1,105 @@@ ++# Licensed to the Apache Software Foundation (ASF) under one ++# or more contributor license agreements. See the NOTICE file ++# distributed with this work for additional information ++# regarding copyright ownership. The ASF licenses this file ++# to you under the Apache License, Version 2.0 (the ++# License); you may not use this file except in compliance ++# with the License. You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an AS IS BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++# Test ability to wire together shell spouts/bolts ++--- ++ ++# topology definition ++# name to be used when submitting ++name: hdfs-topology ++ ++# Components ++# Components are analagous to Spring beans. They are meant to be used as constructor, ++# property(setter), and builder arguments. ++# ++# for the time being, components must be declared in the order they are referenced ++components: ++ - id: syncPolicy ++className: org.apache.storm.hdfs.bolt.sync.CountSyncPolicy ++constructorArgs: ++ - 1000 ++ - id: rotationPolicy ++className: org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy ++constructorArgs: ++ - 30 ++ - SECONDS ++ ++ - id: fileNameFormat ++className: org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat ++configMethods: ++ - name: withPath ++args: [${hdfs.write.dir}] ++ - name: withExtension ++args: [.txt] ++ ++ - id: recordFormat ++className: org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat ++configMethods: ++ - name: withFieldDelimiter ++args: [|] ++ ++ - id: rotationAction ++className: org.apache.storm.hdfs.common.rotation.MoveFileAction ++configMethods: ++ - name: toDestination ++args: [${hdfs.dest.dir}] ++ ++# spout definitions ++spouts: ++ - id: spout-1 ++className: backtype.storm.testing.TestWordSpout ++parallelism: 1 ++# ... ++ ++# bolt definitions ++ ++bolts: ++ - id: bolt-1 ++className: org.apache.storm.hdfs.bolt.HdfsBolt ++configMethods: ++ - name: withConfigKey ++args: [hdfs.config] ++ - name: withFsUrl ++args: [${hdfs.url}] ++ - name: withFileNameFormat ++args: [ref: fileNameFormat] ++ - name: withRecordFormat ++args: [ref: recordFormat] ++ - name: withRotationPolicy ++args: [ref: rotationPolicy] ++ - name: withSyncPolicy ++args: [ref: syncPolicy] ++ - name: addRotationAction ++args: [ref: rotationAction] ++parallelism: 1 ++# ... ++ ++ - id: bolt-2 ++className: org.apache.storm.flux.wrappers.bolts.LogInfoBolt ++parallelism: 1 ++ ++streams: ++ - name: # name isn't used (placeholder for logging, UI, etc.) ++from: spout-1 ++to: bolt-1 ++grouping: ++ type: SHUFFLE ++ ++ - name: # name isn't used (placeholder for logging, UI, etc.) ++from: spout-1 ++to: bolt-2 ++grouping: ++ type: SHUFFLE http://git-wip-us.apache.org/repos/asf/storm/blob/b21a98dd/external/flux/flux-examples/src/main/resources/simple_wordcount.yaml -- diff --cc external/flux/flux-examples/src/main/resources/simple_wordcount.yaml index 000,000..380f9d2 new file mode 100644 --- /dev/null +++ b/external/flux/flux-examples/src/main/resources/simple_wordcount.yaml @@@ -1,0 -1,0 +1,68 @@@ ++# Licensed to the Apache Software Foundation (ASF) under one ++# or more contributor license agreements. See the NOTICE file ++# distributed with this work for additional information ++# regarding copyright ownership. The ASF licenses this file ++# to you under the Apache License, Version 2.0 (the ++# License); you may not use this file except in compliance ++# with the License. You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an AS IS BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++--- ++ ++# topology definition ++# name to be used when
[50/50] [abbrv] storm git commit: add STORM-561 to changelog
add STORM-561 to changelog Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/285d943b Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/285d943b Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/285d943b Branch: refs/heads/master Commit: 285d943b87f5037f0eaf3d45af64622f13017dea Parents: cb370a9 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 13:24:03 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 13:24:03 2015 -0400 -- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/285d943b/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index 02aa95d..aa390f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,11 @@ ## 0.11.0 ## 0.10.0 - * STORM-818: storm-eventhubs configuration improvement and refactoring * STORM-842: Drop Support for Java 1.6 * STORM-835: Netty Client hold batch object until io operation complete * STORM-827: Allow AutoTGT to work with storm-hdfs too. * STORM-821: Adding connection provider interface to decouple jdbc connector from a single connection pooling implementation. + * STORM-818: storm-eventhubs configuration improvement and refactoring * STORM-816: maven-gpg-plugin does not work with gpg 2.1 * STORM-811: remove old metastor_db before running tests again. * STORM-808: allow null to be parsed as null @@ -124,6 +124,7 @@ * STORM-567: Move Storm Documentation/Website from SVN to git * STORM-565: Fix NPE when topology.groups is null. * STORM-563: Kafka Spout doesn't pick up from the beginning of the queue unless forceFromStart specified. + * STORM-561: Add flux as an external module * STORM-557: High Quality Images for presentations * STORM-554: the type of first param topology should be ^StormTopology not ^TopologyContext * STORM-552: Add netty socket backlog config
[06/50] [abbrv] storm git commit: implement/document environment variable substitution; document command line switch pass-through;
implement/document environment variable substitution; document command line switch pass-through; Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/44d5b5be Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/44d5b5be Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/44d5b5be Branch: refs/heads/master Commit: 44d5b5be2dc0e2334fa1e4bd5934eb1f57985084 Parents: d8d6cfa Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue Mar 31 00:16:37 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue Mar 31 00:16:37 2015 -0400 -- README.md | 35 ++-- .../main/java/org/apache/storm/flux/Flux.java | 15 +-- .../apache/storm/flux/parser/FluxParser.java| 35 +++- .../java/org/apache/storm/flux/TCKTest.java | 43 .../resources/configs/substitution-test.yaml| 2 + 5 files changed, 97 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/44d5b5be/README.md -- diff --git a/README.md b/README.md index ebf9d5a..71403e0 100644 --- a/README.md +++ b/README.md @@ -149,9 +149,15 @@ usage: storm jar my_topology_uber_jar.jar org.apache.storm.flux.Flux -d,--dry-run Do not run or deploy the topology. Just build, validate, and print information about the topology. - -f,--filter file Use the specified file as a source of - properties, and perform variable - substitution. + -e,--env-filter Perform environment variable substitution. + Replace keysidentified with `${ENV-[NAME]}` + will be replaced with the corresponding + `NAME` environment value + -f,--filter file Perform property substitution. Use the + specified file as a source of properties, + and replace keys identified with {$[property + name]} with the value defined in the + properties file. -i,--inactiveDeploy the topology, but do not activate it. -l,--local Run the topology in local mode. -n,--no-splash Suppress the printing of the splash screen. @@ -167,6 +173,16 @@ usage: storm jar my_topology_uber_jar.jar org.apache.storm.flux.Flux instead of the in-process ZooKeeper. ``` +**NOTE:** Flux tries to avoid command line switch collision with the `storm` command, and allows any other command line +switches to pass through to the `storm` command. + +For example, you can use the `storm` command switch `-c` to override a topology configuration property. The following +example command will run Flux and override the `nimus.host` configuration: + +```bash +storm jar myTopology-0.1.0-SNAPSHOT.jar org.apache.storm.flux.Flux --remote my_config.yaml -c nimbus.host=localhost +``` + ### Sample output ``` âââââââââââ âââ ââââââ âââ @@ -281,6 +297,13 @@ You would then be able to reference those properties by key in your `.yaml` file In this case, Flux would replace `${kafka.zookeeper.hosts}` with `localhost:2181` before parsing the YAML contents. +### Environment Variable Substitution/Filtering +Flux also allows environment variable substitution. For example, if an environment variable named `ZK_HOSTS` if defined, +you can reference it in a Flux YAML file with the following syntax: + +``` +${ENV-ZK_HOSTS} +``` ## Components Components are essentially named object instances that are made available as configuration options for spouts and @@ -772,6 +795,12 @@ topologySource: methodName: getTopologyWithDifferentMethodName ``` +## Author +P. Taylor Goetz + +## Contributors + + ## Contributing Contributions in any form are more than welcome. http://git-wip-us.apache.org/repos/asf/storm/blob/44d5b5be/flux-core/src/main/java/org/apache/storm/flux/Flux.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/Flux.java b/flux-core/src/main/java/org/apache/storm/flux/Flux.java index dcd3953..2c2105c 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/Flux.java +++ b/flux-core/src/main/java/org/apache/storm/flux/Flux.java @@ -55,6 +55,7 @@ public class Flux { private static final String OPTION_INACTIVE = inactive; private static final String OPTION_ZOOKEEPER = zookeeper; private static final String OPTION_FILTER = filter; +private static
[14/50] [abbrv] storm git commit: add basic instructions for examples
add basic instructions for examples Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8b690e63 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8b690e63 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8b690e63 Branch: refs/heads/master Commit: 8b690e63954b14aa8f079f904d42d2f4beb34cc6 Parents: 2e44c9e Author: P. Taylor Goetz ptgo...@gmail.com Authored: Thu Apr 2 12:19:13 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Thu Apr 2 12:19:13 2015 -0400 -- flux-examples/README.md | 28 1 file changed, 28 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/8b690e63/flux-examples/README.md -- diff --git a/flux-examples/README.md b/flux-examples/README.md new file mode 100644 index 000..2f107e7 --- /dev/null +++ b/flux-examples/README.md @@ -0,0 +1,28 @@ +# Flux Examples +A collection of examples illustrating various capabilities. + +## Building From Source and Running + +Checkout the projects source and perform a top level Maven build (i.e. from the `flux` directory): + +```bash +git clone https://github.com/ptgoetz/flux.git +cd flux +mvn install +``` + +This will create a shaded (i.e. fat or uber) jar in the `flux-examples/target` directory that can run/deployed with +the `storm` command: + +```bash +cd flux-examples +storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_wordcount.yaml +``` + +The example YAML files are also packaged in the examples jar, so they can also be referenced with Flux's `--resource` +command line switch: + +```bash +storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local --resource /sime_wordcount.yaml +``` +
[49/50] [abbrv] storm git commit: Merge branch 'STORM-561'
Merge branch 'STORM-561' Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cb370a99 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cb370a99 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cb370a99 Branch: refs/heads/master Commit: cb370a99a12a8466b4acd644bfe4eba00ee26107 Parents: 2313775 b90ec78 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Jun 3 13:21:16 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Jun 3 13:21:16 2015 -0400 -- external/flux/.gitignore| 15 + external/flux/LICENSE | 202 + external/flux/README.md | 834 +++ external/flux/flux-core/pom.xml | 92 ++ .../main/java/org/apache/storm/flux/Flux.java | 263 ++ .../java/org/apache/storm/flux/FluxBuilder.java | 591 + .../apache/storm/flux/api/TopologySource.java | 39 + .../org/apache/storm/flux/model/BeanDef.java| 39 + .../apache/storm/flux/model/BeanReference.java | 39 + .../org/apache/storm/flux/model/BoltDef.java| 24 + .../storm/flux/model/ConfigMethodDef.java | 62 ++ .../storm/flux/model/ExecutionContext.java | 77 ++ .../apache/storm/flux/model/GroupingDef.java| 77 ++ .../org/apache/storm/flux/model/IncludeDef.java | 54 ++ .../org/apache/storm/flux/model/ObjectDef.java | 90 ++ .../apache/storm/flux/model/PropertyDef.java| 58 ++ .../org/apache/storm/flux/model/SpoutDef.java | 24 + .../org/apache/storm/flux/model/StreamDef.java | 64 ++ .../apache/storm/flux/model/TopologyDef.java| 216 + .../storm/flux/model/TopologySourceDef.java | 36 + .../org/apache/storm/flux/model/VertexDef.java | 36 + .../apache/storm/flux/parser/FluxParser.java| 202 + .../flux-core/src/main/resources/splash.txt | 9 + .../org/apache/storm/flux/FluxBuilderTest.java | 31 + .../org/apache/storm/flux/IntegrationTest.java | 39 + .../java/org/apache/storm/flux/TCKTest.java | 234 ++ .../multilang/MultilangEnvirontmentTest.java| 89 ++ .../apache/storm/flux/test/SimpleTopology.java | 42 + .../storm/flux/test/SimpleTopologySource.java | 35 + .../test/SimpleTopologyWithConfigParam.java | 38 + .../org/apache/storm/flux/test/TestBolt.java| 63 ++ .../storm/flux/test/TridentTopologySource.java | 54 ++ .../src/test/resources/configs/bad_hbase.yaml | 98 +++ .../resources/configs/config-methods-test.yaml | 70 ++ .../existing-topology-method-override.yaml | 10 + .../existing-topology-reflection-config.yaml| 9 + .../configs/existing-topology-reflection.yaml | 9 + .../configs/existing-topology-trident.yaml | 9 + .../resources/configs/existing-topology.yaml| 8 + .../src/test/resources/configs/hdfs_test.yaml | 97 +++ .../test/resources/configs/include_test.yaml| 25 + .../configs/invalid-existing-topology.yaml | 17 + .../src/test/resources/configs/kafka_test.yaml | 126 +++ .../src/test/resources/configs/shell_test.yaml | 104 +++ .../test/resources/configs/simple_hbase.yaml| 120 +++ .../resources/configs/substitution-test.yaml| 106 +++ .../src/test/resources/configs/tck.yaml | 95 +++ .../src/test/resources/configs/test.properties | 2 + .../flux-core/src/test/resources/logback.xml| 30 + external/flux/flux-examples/README.md | 66 ++ external/flux/flux-examples/pom.xml | 105 +++ .../storm/flux/examples/WordCountClient.java| 74 ++ .../apache/storm/flux/examples/WordCounter.java | 71 ++ .../src/main/resources/hbase_bolt.properties| 18 + .../src/main/resources/hdfs_bolt.properties | 26 + .../src/main/resources/kafka_spout.yaml | 136 +++ .../src/main/resources/multilang.yaml | 89 ++ .../src/main/resources/simple_hbase.yaml| 92 ++ .../src/main/resources/simple_hdfs.yaml | 105 +++ .../src/main/resources/simple_wordcount.yaml| 68 ++ external/flux/flux-ui/README.md | 3 + external/flux/flux-wrappers/pom.xml | 51 ++ .../flux/wrappers/bolts/FluxShellBolt.java | 56 ++ .../storm/flux/wrappers/bolts/LogInfoBolt.java | 44 + .../flux/wrappers/spouts/FluxShellSpout.java| 55 ++ .../main/resources/resources/randomsentence.js | 93 +++ .../main/resources/resources/splitsentence.py | 24 + external/flux/pom.xml | 119 +++ pom.xml | 1 + storm-dist/binary/src/main/assembly/binary.xml | 44 + 70 files changed, 6043 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/cb370a99/pom.xml --
[19/50] [abbrv] storm git commit: add basic docs for HBase example
add basic docs for HBase example Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/edc5744c Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/edc5744c Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/edc5744c Branch: refs/heads/master Commit: edc5744c48b65c3d7298ec5a61affb6164fc4d96 Parents: 4a1db96 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue Apr 7 23:57:53 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue Apr 7 23:57:53 2015 -0400 -- flux-examples/README.md | 12 1 file changed, 12 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/edc5744c/flux-examples/README.md -- diff --git a/flux-examples/README.md b/flux-examples/README.md index 425ad98..b3798a6 100644 --- a/flux-examples/README.md +++ b/flux-examples/README.md @@ -54,3 +54,15 @@ least, the property `hdfs.url` to point to a HDFS cluster. Then you can run the storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hdfs.yaml --filter my_hdfs_bolt.properties ``` +### [simple_hbase.yaml](src/main/resources/simple_hbase.yaml) + +This example illustrates how to use Flux to setup a storm-hbase bolt to write to HBase. + +In order to use this example, you will need to edit the `src/main resrouces/hbase-site.xml` file to reflect your HBase +environment, and then rebuild the topology jar. + +You can do so by running the following Maven command in the `flux-examples` directory: + +```bash +mvn clean install +``` \ No newline at end of file
[13/50] [abbrv] storm git commit: when a resource can't be found, log an error and exit rather than throw NPE
when a resource can't be found, log an error and exit rather than throw NPE Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2e44c9e6 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2e44c9e6 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2e44c9e6 Branch: refs/heads/master Commit: 2e44c9e6a2fb4382950da463fad9f392f8991468 Parents: 11b0151 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Thu Apr 2 12:18:12 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Thu Apr 2 12:18:12 2015 -0400 -- .../src/main/java/org/apache/storm/flux/parser/FluxParser.java | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/2e44c9e6/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java b/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java index 78c52d5..72f8a8e 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java +++ b/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java @@ -63,6 +63,10 @@ public class FluxParser { String propertiesFile, boolean envSub) throws IOException { Yaml yaml = yaml(); InputStream in = FluxParser.class.getResourceAsStream(resource); +if(in == null){ +LOG.error(Unable to load classpath resource: + resource); +System.exit(1); +} TopologyDef topology = loadYaml(yaml, in, propertiesFile, envSub); in.close(); if(dumpYaml){
[16/50] [abbrv] storm git commit: update examples and docs for HDFS example
update examples and docs for HDFS example Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0c1e0aa8 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0c1e0aa8 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0c1e0aa8 Branch: refs/heads/master Commit: 0c1e0aa81f39e473bcf3482448813d78065e1212 Parents: 3411bc7 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Mon Apr 6 23:48:38 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Mon Apr 6 23:48:38 2015 -0400 -- .../src/test/resources/configs/hdfs_test.yaml | 97 - flux-examples/README.md | 30 +- flux-examples/pom.xml | 6 ++ .../src/main/resources/hdfs_bolt.properties | 9 ++ flux-examples/src/main/resources/multilang.yaml | 89 flux-examples/src/main/resources/shell.yaml | 89 .../src/main/resources/simple_hdfs.yaml | 105 +++ 7 files changed, 238 insertions(+), 187 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/0c1e0aa8/flux-core/src/test/resources/configs/hdfs_test.yaml -- diff --git a/flux-core/src/test/resources/configs/hdfs_test.yaml b/flux-core/src/test/resources/configs/hdfs_test.yaml deleted file mode 100644 index c1d28d2..000 --- a/flux-core/src/test/resources/configs/hdfs_test.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# License); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an AS IS BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Test ability to wire together shell spouts/bolts - -# topology definition -# name to be used when submitting -name: hdfs-topology - -# Components -# Components are analagous to Spring beans. They are meant to be used as constructor, -# property(setter), and builder arguments. -# -# for the time being, components must be declared in the order they are referenced -components: - - id: syncPolicy -className: org.apache.storm.hdfs.bolt.sync.CountSyncPolicy -constructorArgs: - - 1000 - - id: rotationPolicy -className: org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy -constructorArgs: - - 5.0 - - MB - - - id: fileNameFormat -className: org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat -configMethods: - - name: withPath -args: [/tmp/foo/] - - name: withExtension -args: [.txt] - - - id: recordFormat -className: org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat -configMethods: - - name: withFieldDelimiter -args: [|] - - - id: rotationAction -className: org.apache.storm.hdfs.common.rotation.MoveFileAction -configMethods: - - name: toDestination -args: [/tmp/dest2] - -# spout definitions -spouts: - - id: spout-1 -className: backtype.storm.testing.TestWordSpout -parallelism: 1 -# ... - -# bolt definitions - -#HdfsBolt bolt = new HdfsBolt() -#.withConfigKey(hdfs.config) -#.withFsUrl(args[0]) -#.withFileNameFormat(fileNameFormat) -#.withRecordFormat(format) -#.withRotationPolicy(rotationPolicy) -#.withSyncPolicy(syncPolicy) -#.addRotationAction(new MoveFileAction().toDestination(/tmp/dest2/)); -bolts: - - id: bolt-1 -className: org.apache.storm.hdfs.bolt.HdfsBolt -configMethods: - - name: withConfigKey -args: [hdfs.config] - - name: withFsUrl -args: [hdfs://localhost:1234] - - name: withFileNameFormat -args: [ref: fileNameFormat] - - name: withRecordFormat -args: [ref: recordFormat] - - name: withRotationPolicy -args: [ref: rotationPolicy] - - name: withSyncPolicy -args: [ref: syncPolicy] - - name: addRotationAction -args: [ref: rotationAction] -parallelism: 1 -# ... - http://git-wip-us.apache.org/repos/asf/storm/blob/0c1e0aa8/flux-examples/README.md -- diff
[37/50] [abbrv] storm git commit: cleanup github references and versions
cleanup github references and versions Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f7d2f7fa Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f7d2f7fa Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f7d2f7fa Branch: refs/heads/master Commit: f7d2f7fa0c7a0511911e984f13c20fceed743b17 Parents: e1e1609 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Thu May 7 10:33:44 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Thu May 7 10:33:44 2015 -0400 -- external/flux/README.md | 8 external/flux/flux-core/pom.xml | 1 - external/flux/flux-examples/README.md | 14 +++--- external/flux/flux-examples/pom.xml | 1 - 4 files changed, 11 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/f7d2f7fa/external/flux/README.md -- diff --git a/external/flux/README.md b/external/flux/README.md index d09a73c..0387f3f 100644 --- a/external/flux/README.md +++ b/external/flux/README.md @@ -109,9 +109,9 @@ recommended). The current version of Flux is available in Maven Central at the following coordinates: ```xml dependency -groupIdcom.github.ptgoetz/groupId +groupIdorg.apache.storm/groupId artifactIdflux-core/artifactId -version0.3.0/version +version${storm.version}/version /dependency ``` @@ -123,9 +123,9 @@ The example below illustrates Flux usage with the Maven shade plugin: dependencies !-- Flux include -- dependency -groupIdcom.github.ptgoetz/groupId +groupIdorg.apache.storm/groupId artifactIdflux-core/artifactId -version0.3.0/version +version${storm.version}/version /dependency !-- add user dependencies here... -- http://git-wip-us.apache.org/repos/asf/storm/blob/f7d2f7fa/external/flux/flux-core/pom.xml -- diff --git a/external/flux/flux-core/pom.xml b/external/flux/flux-core/pom.xml index c3842bd..f74d034 100644 --- a/external/flux/flux-core/pom.xml +++ b/external/flux/flux-core/pom.xml @@ -29,7 +29,6 @@ packagingjar/packaging nameflux-core/name -urlhttps://github.com/ptgoetz/flux/url dependencies dependency http://git-wip-us.apache.org/repos/asf/storm/blob/f7d2f7fa/external/flux/flux-examples/README.md -- diff --git a/external/flux/flux-examples/README.md b/external/flux/flux-examples/README.md index b3798a6..fceebd8 100644 --- a/external/flux/flux-examples/README.md +++ b/external/flux/flux-examples/README.md @@ -6,24 +6,24 @@ A collection of examples illustrating various capabilities. Checkout the projects source and perform a top level Maven build (i.e. from the `flux` directory): ```bash -git clone https://github.com/ptgoetz/flux.git -cd flux -mvn install +git clone https://github.com/apache/storm.git +cd storm +mvn install -DskipTests=true ``` -This will create a shaded (i.e. fat or uber) jar in the `flux-examples/target` directory that can run/deployed with +This will create a shaded (i.e. fat or uber) jar in the `external/flux/flux-examples/target` directory that can run/deployed with the `storm` command: ```bash cd flux-examples -storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_wordcount.yaml +storm jar ./target/flux-examples-*-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_wordcount.yaml ``` The example YAML files are also packaged in the examples jar, so they can also be referenced with Flux's `--resource` command line switch: ```bash -storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local --resource /simple_wordcount.yaml +storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local --resource /simple_wordcount.yaml ``` ## Available Examples @@ -51,7 +51,7 @@ To run the `simple_hdfs.yaml` example, copy the `hdfs_bolt.properties` file to a least, the property `hdfs.url` to point to a HDFS cluster. Then you can run the example something like: ```bash -storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hdfs.yaml --filter my_hdfs_bolt.properties +storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hdfs.yaml --filter my_hdfs_bolt.properties ``` ### [simple_hbase.yaml](src/main/resources/simple_hbase.yaml) http://git-wip-us.apache.org/repos/asf/storm/blob/f7d2f7fa/external/flux/flux-examples/pom.xml -- diff
[22/50] [abbrv] storm git commit: update hbase and hdfs examples
update hbase and hdfs examples Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9f5f8227 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9f5f8227 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9f5f8227 Branch: refs/heads/master Commit: 9f5f822720544e356682d3c007c3407f5c2e5058 Parents: 01702dc Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Apr 8 12:50:59 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Apr 8 12:50:59 2015 -0400 -- .../storm/flux/examples/WordCountClient.java| 17 +++-- flux-examples/src/main/resources/hbase-site.xml | 36 .../src/main/resources/hbase_bolt.properties| 2 ++ .../src/main/resources/hdfs_bolt.properties | 6 ++-- .../src/main/resources/simple_hbase.yaml| 3 +- .../src/main/resources/simple_hdfs.yaml | 6 ++-- 6 files changed, 24 insertions(+), 46 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java -- diff --git a/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java b/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java index 55873d5..eb4fb7a 100644 --- a/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java +++ b/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java @@ -24,6 +24,9 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; +import java.io.FileInputStream; +import java.util.Properties; + /** * Connects to the 'WordCount' HBase table and prints counts for each word. * @@ -39,8 +42,17 @@ public class WordCountClient { public static void main(String[] args) throws Exception { Configuration config = HBaseConfiguration.create(); -if(args.length 0){ -config.set(hbase.rootdir, args[0]); +if(args.length == 1){ +Properties props = new Properties(); +props.load(new FileInputStream(args[0])); +System.out.println(HBase configuration:); +for(Object key : props.keySet()) { +System.out.println(key + = + props.get(key)); +config.set((String)key, props.getProperty((String)key)); +} +} else { +System.out.println(Usage: WordCountClient hbase_config.properties); +System.exit(1); } HTable table = new HTable(config, WordCount); @@ -54,7 +66,6 @@ public class WordCountClient { byte[] wordBytes = result.getValue(Bytes.toBytes(cf), Bytes.toBytes(word)); String wordStr = Bytes.toString(wordBytes); -System.out.println(wordStr); long count = Bytes.toLong(countBytes); System.out.println(Word: ' + wordStr + ', Count: + count); } http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/hbase-site.xml -- diff --git a/flux-examples/src/main/resources/hbase-site.xml b/flux-examples/src/main/resources/hbase-site.xml deleted file mode 100644 index 06c3031..000 --- a/flux-examples/src/main/resources/hbase-site.xml +++ /dev/null @@ -1,36 +0,0 @@ -?xml version=1.0? -?xml-stylesheet type=text/xsl href=configuration.xsl? -!-- -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- -configuration - property - namehbase.cluster.distributed/name - valuetrue/value - /property - property - namehbase.rootdir/name - valuehdfs://hadoop:54310/hbase/value - /property - property - namehbase.zookeeper.quorum/name - valuehadoop/value - /property -/configuration \ No newline at end of file
[38/50] [abbrv] storm git commit: add back missing multi-lang scripts
add back missing multi-lang scripts Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9fad816f Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9fad816f Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9fad816f Branch: refs/heads/master Commit: 9fad816fd5840d5ad7d2727db4c7dafe4f1355ba Parents: f7d2f7f Author: P. Taylor Goetz ptgo...@gmail.com Authored: Fri May 8 15:23:54 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Fri May 8 15:23:54 2015 -0400 -- .../main/resources/resources/randomsentence.js | 93 .../main/resources/resources/splitsentence.py | 24 + 2 files changed, 117 insertions(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/9fad816f/external/flux/flux-wrappers/src/main/resources/resources/randomsentence.js -- diff --git a/external/flux/flux-wrappers/src/main/resources/resources/randomsentence.js b/external/flux/flux-wrappers/src/main/resources/resources/randomsentence.js new file mode 100644 index 000..36fc5f5 --- /dev/null +++ b/external/flux/flux-wrappers/src/main/resources/resources/randomsentence.js @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Example for storm spout. Emits random sentences. + * The original class in java - storm.starter.spout.RandomSentenceSpout. + * + */ + +var storm = require('./storm'); +var Spout = storm.Spout; + + +var SENTENCES = [ +the cow jumped over the moon, +an apple a day keeps the doctor away, +four score and seven years ago, +snow white and the seven dwarfs, +i am at two with nature] + +function RandomSentenceSpout(sentences) { +Spout.call(this); +this.runningTupleId = 0; +this.sentences = sentences; +this.pending = {}; +}; + +RandomSentenceSpout.prototype = Object.create(Spout.prototype); +RandomSentenceSpout.prototype.constructor = RandomSentenceSpout; + +RandomSentenceSpout.prototype.getRandomSentence = function() { +return this.sentences[getRandomInt(0, this.sentences.length - 1)]; +} + +RandomSentenceSpout.prototype.nextTuple = function(done) { +var self = this; +var sentence = this.getRandomSentence(); +var tup = [sentence]; +var id = this.createNextTupleId(); +this.pending[id] = tup; +//This timeout can be removed if TOPOLOGY_SLEEP_SPOUT_WAIT_STRATEGY_TIME_MS is configured to 100 +setTimeout(function() { +self.emit({tuple: tup, id: id}, function(taskIds) { +self.log(tup + ' sent to task ids - ' + taskIds); +}); +done(); +},100); +} + +RandomSentenceSpout.prototype.createNextTupleId = function() { +var id = this.runningTupleId; +this.runningTupleId++; +return id; +} + +RandomSentenceSpout.prototype.ack = function(id, done) { +this.log('Received ack for - ' + id); +delete this.pending[id]; +done(); +} + +RandomSentenceSpout.prototype.fail = function(id, done) { +var self = this; +this.log('Received fail for - ' + id + '. Retrying.'); +this.emit({tuple: this.pending[id], id:id}, function(taskIds) { +self.log(self.pending[id] + ' sent to task ids - ' + taskIds); +}); +done(); +} + +/** + * Returns a random integer between min (inclusive) and max (inclusive) + */ +function getRandomInt(min, max) { +return Math.floor(Math.random() * (max - min + 1)) + min; +} + +new RandomSentenceSpout(SENTENCES).run(); http://git-wip-us.apache.org/repos/asf/storm/blob/9fad816f/external/flux/flux-wrappers/src/main/resources/resources/splitsentence.py -- diff --git a/external/flux/flux-wrappers/src/main/resources/resources/splitsentence.py b/external/flux/flux-wrappers/src/main/resources/resources/splitsentence.py new file mode 100644 index 000..300105f --- /dev/null +++ b/external/flux/flux-wrappers/src/main/resources/resources/splitsentence.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation
[23/50] [abbrv] storm git commit: set default parallelism of spouts/bolts to 1
set default parallelism of spouts/bolts to 1 Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/abf2924e Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/abf2924e Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/abf2924e Branch: refs/heads/master Commit: abf2924e8a89d9b9fc28037a4ce079c607f62939 Parents: 9f5f822 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Fri Apr 10 15:47:16 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Fri Apr 10 15:47:16 2015 -0400 -- .../src/main/java/org/apache/storm/flux/model/VertexDef.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/abf2924e/flux-core/src/main/java/org/apache/storm/flux/model/VertexDef.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/model/VertexDef.java b/flux-core/src/main/java/org/apache/storm/flux/model/VertexDef.java index f8bf607..e71bcc2 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/model/VertexDef.java +++ b/flux-core/src/main/java/org/apache/storm/flux/model/VertexDef.java @@ -23,7 +23,8 @@ package org.apache.storm.flux.model; */ public abstract class VertexDef extends BeanDef { -private int parallelism; +// default parallelism to 1 so if it's ommitted, the topology will still function. +private int parallelism = 1; public int getParallelism() { return parallelism;
[15/50] [abbrv] storm git commit: make examples produce a shaded jar
make examples produce a shaded jar Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3411bc7d Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3411bc7d Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3411bc7d Branch: refs/heads/master Commit: 3411bc7d296c948e3ca6fc1a02ef8a118df39e65 Parents: 8b690e6 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Thu Apr 2 12:20:59 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Thu Apr 2 12:20:59 2015 -0400 -- flux-examples/pom.xml | 46 +- 1 file changed, 45 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/3411bc7d/flux-examples/pom.xml -- diff --git a/flux-examples/pom.xml b/flux-examples/pom.xml index 29a2e62..63bc312 100644 --- a/flux-examples/pom.xml +++ b/flux-examples/pom.xml @@ -15,7 +15,8 @@ See the License for the specific language governing permissions and limitations under the License. -- -project xmlns=http://maven.apache.org/POM/4.0.0; xmlns:xsi=http://www.w3.org/2001/XMLSchema-instance; xsi:schemaLocation=http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd; +project xmlns=http://maven.apache.org/POM/4.0.0; xmlns:xsi=http://www.w3.org/2001/XMLSchema-instance; + xsi:schemaLocation=http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd; modelVersion4.0.0/modelVersion parent @@ -32,4 +33,47 @@ nameflux-examples/name urlhttps://github.com/ptgoetz/flux/url +dependencies +dependency +groupIdcom.github.ptgoetz/groupId +artifactIdflux-core/artifactId +version${project.version}/version +/dependency +dependency +groupIdcom.github.ptgoetz/groupId +artifactIdflux-wrappers/artifactId +version${project.version}/version +/dependency +/dependencies + +build +plugins +plugin +groupIdorg.apache.maven.plugins/groupId +artifactIdmaven-shade-plugin/artifactId +version1.4/version +configuration + createDependencyReducedPomtrue/createDependencyReducedPom +/configuration +executions +execution +phasepackage/phase +goals +goalshade/goal +/goals +configuration +transformers +transformer + implementation=org.apache.maven.plugins.shade.resource.ServicesResourceTransformer/ +transformer + implementation=org.apache.maven.plugins.shade.resource.ManifestResourceTransformer + mainClassorg.apache.storm.flux.Flux/mainClass +/transformer +/transformers +/configuration +/execution +/executions +/plugin +/plugins +/build /project
[27/50] [abbrv] storm git commit: add missing license headers
add missing license headers Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/01a66bf5 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/01a66bf5 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/01a66bf5 Branch: refs/heads/master Commit: 01a66bf5b17a9fce94fcdac4fbaa2f28c2df9189 Parents: c48f63e Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue May 5 16:14:33 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue May 5 16:14:33 2015 -0400 -- .../apache/storm/flux/api/TopologySource.java | 17 +++ .../storm/flux/model/ConfigMethodDef.java | 17 +++ .../storm/flux/model/TopologySourceDef.java | 17 +++ .../java/org/apache/storm/flux/FrankenBean.java | 138 --- .../org/apache/storm/flux/IntegrationTest.java | 17 +++ .../src/main/resources/hbase_bolt.properties| 16 +++ .../src/main/resources/hdfs_bolt.properties | 17 +++ .../storm/flux/wrappers/bolts/LogInfoBolt.java | 18 +++ 8 files changed, 119 insertions(+), 138 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/01a66bf5/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java b/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java index 779e676..fbccfb7 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java +++ b/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.storm.flux.api; http://git-wip-us.apache.org/repos/asf/storm/blob/01a66bf5/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java b/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java index e274915..6f7e4d4 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java +++ b/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.storm.flux.model; import java.util.ArrayList; http://git-wip-us.apache.org/repos/asf/storm/blob/01a66bf5/flux-core/src/main/java/org/apache/storm/flux/model/TopologySourceDef.java -- diff --git a/flux-core/src/main/java/org/apache/storm/flux/model/TopologySourceDef.java b/flux-core/src/main/java/org/apache/storm/flux/model/TopologySourceDef.java index 2949659..d6a2f57 100644 --- a/flux-core/src/main/java/org/apache/storm/flux/model/TopologySourceDef.java +++ b/flux-core/src/main/java/org/apache/storm/flux/model/TopologySourceDef.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + *
[09/50] [abbrv] storm git commit: prep for 0.2.2 release
prep for 0.2.2 release Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/14970de2 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/14970de2 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/14970de2 Branch: refs/heads/master Commit: 14970de2091485f4ca777606735788657f7bec84 Parents: c3cffa0 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Wed Apr 1 13:25:52 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Wed Apr 1 13:25:52 2015 -0400 -- README.md | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/14970de2/README.md -- diff --git a/README.md b/README.md index 78a2723..92fec10 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ The current version of Flux is available in Maven Central at the following coord dependency groupIdcom.github.ptgoetz/groupId artifactIdflux-core/artifactId -version0.2.2-SNAPSHOT/version +version0.2.2/version /dependency ``` @@ -92,7 +92,7 @@ The example below illustrates Flux usage with the Maven shade plugin: dependency groupIdcom.github.ptgoetz/groupId artifactIdflux-core/artifactId -version0.2.2-SNAPSHOT/version +version0.2.2/version /dependency !-- add user dependencies here... -- @@ -171,6 +171,7 @@ usage: storm jar my_topology_uber_jar.jar org.apache.storm.flux.Flux -z,--zookeeper host:port When running in local mode, use the ZooKeeper at the specified host:port instead of the in-process ZooKeeper. + (requires Storm 0.9.3 or later) ``` **NOTE:** Flux tries to avoid command line switch collision with the `storm` command, and allows any other command line @@ -193,7 +194,7 @@ storm jar myTopology-0.1.0-SNAPSHOT.jar org.apache.storm.flux.Flux --remote my_c âââ ââââââââ âââââââ âââ âââ +- Apache Storm-+ +- data FLow User eXperience -+ -Version: 0.2.2-SNAPSHOT +Version: 0.2.2 Parsing file: /Users/hsimpson/Projects/donut_domination/storm/shell_test.yaml -- TOPOLOGY DETAILS -- Name: shell-topology
[46/50] [abbrv] storm git commit: fix docs for hbase example
fix docs for hbase example Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b90ec781 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b90ec781 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b90ec781 Branch: refs/heads/master Commit: b90ec781c7a7ba77f6598fe20e7842fa4a228098 Parents: 91369ac Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue Jun 2 17:14:34 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue Jun 2 17:14:34 2015 -0400 -- external/flux/flux-examples/README.md | 10 -- 1 file changed, 4 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/b90ec781/external/flux/flux-examples/README.md -- diff --git a/external/flux/flux-examples/README.md b/external/flux/flux-examples/README.md index fceebd8..0a7085e 100644 --- a/external/flux/flux-examples/README.md +++ b/external/flux/flux-examples/README.md @@ -58,11 +58,9 @@ storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/ This example illustrates how to use Flux to setup a storm-hbase bolt to write to HBase. -In order to use this example, you will need to edit the `src/main resrouces/hbase-site.xml` file to reflect your HBase -environment, and then rebuild the topology jar. - -You can do so by running the following Maven command in the `flux-examples` directory: +To run the `simple_hbase.yaml` example, copy the `hbase_bolt.properties` file to a convenient location and change the properties + `hbase.rootdir` and `hbase.zookeeper.quorum`. Then you can run the example something like: ```bash -mvn clean install -``` \ No newline at end of file +storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hbase.yaml --filter my_hbase_bolt.properties +```
[05/50] [abbrv] storm git commit: add feature list to README
add feature list to README Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d8d6cfa0 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d8d6cfa0 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d8d6cfa0 Branch: refs/heads/master Commit: d8d6cfa073fc5566444675581f3559ea3f3945e4 Parents: 85e53e9 Author: P. Taylor Goetz ptgo...@gmail.com Authored: Mon Mar 30 22:26:07 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Mon Mar 30 22:26:07 2015 -0400 -- README.md | 16 +++- 1 file changed, 11 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/d8d6cfa0/README.md -- diff --git a/README.md b/README.md index 3f6bdc1..ebf9d5a 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,6 @@ Have you ever found yourself repeating this pattern?: public static void main(String[] args) throws Exception { // logic to determine if we're running locally or not... // create necessary config options... - boolean runLocal = shouldRunLocal(); if(runLocal){ LocalCluster cluster = new LocalCluster(); @@ -52,9 +51,16 @@ and that any changes require recompilation and repackaging of the topology jar f pain by allowing you to package all your Storm components in a single jar, and use an external text file to define the layout and configuration of your topologies. -## Disclaimer +## Features -This is an experimental project that is rapidly changing. Documentation may be out of date. + * Easily configure and deploy Storm topologies (Both Storm core and Microbatch API) without embedding configuration + in your topology code + * Support for existing topology code (see below) + * Define Storm Core API (Spouts/Bolts) using a flexible YAML DSL + * YAML DSL support for most Storm components (storm-kafka, storm-hdfs, storm-hbase, etc.) + * Convenient support for multi-lang components + * External property substitution/filtering for easily switching between configurations/environments (similar to Maven-style + `${variable.name}` substitution) ## Usage @@ -749,8 +755,8 @@ streams: ``` -## Trident (Micro-Batching API) Support -Currenty, the Flux YAML DSL only supports the Core Storm API, but support for Trident is planned. +## Micro-Batching (Trident) API Support +Currenty, the Flux YAML DSL only supports the Core Storm API, but support for Storm's micro-batching API is planned. To use Flux with a Trident topology, define a topology getter method and reference it in your YAML config:
[42/50] [abbrv] storm git commit: Fix line endings so that the diff is meaningful.
Fix line endings so that the diff is meaningful. Signed-off-by: Shanyu Zhao shz...@microsoft.com Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1f13f15d Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1f13f15d Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1f13f15d Branch: refs/heads/master Commit: 1f13f15d0c0de233fd4cc4ff6d6de586c4736142 Parents: e515492 Author: Shanyu Zhao shz...@microsoft.com Authored: Wed May 13 14:50:02 2015 -0700 Committer: Shanyu Zhao shz...@microsoft.com Committed: Wed May 13 14:50:02 2015 -0700 -- external/storm-eventhubs/pom.xml| 236 +-- .../eventhubs/bolt/DefaultEventDataFormat.java | 94 .../storm/eventhubs/bolt/EventHubBolt.java | 202 .../eventhubs/bolt/EventHubBoltConfig.java | 214 - .../storm/eventhubs/bolt/IEventDataFormat.java | 56 ++--- .../storm/eventhubs/client/EventHubClient.java | 190 +++ .../storm/eventhubs/client/EventHubSender.java | 198 .../storm/eventhubs/samples/EventHubLoop.java | 104 8 files changed, 647 insertions(+), 647 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/1f13f15d/external/storm-eventhubs/pom.xml -- diff --git a/external/storm-eventhubs/pom.xml b/external/storm-eventhubs/pom.xml index 2ceed09..2dfb739 100755 --- a/external/storm-eventhubs/pom.xml +++ b/external/storm-eventhubs/pom.xml @@ -1,119 +1,119 @@ -?xml version=1.0 encoding=UTF-8? -!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the License); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an AS IS BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. --- -project xmlns=http://maven.apache.org/POM/4.0.0; xmlns:xsi=http://www.w3.org/2001/XMLSchema-instance; xsi:schemaLocation=http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd; -modelVersion4.0.0/modelVersion - -parent -artifactIdstorm/artifactId -groupIdorg.apache.storm/groupId -version0.11.0-SNAPSHOT/version -relativePath../../pom.xml/relativePath -/parent - -artifactIdstorm-eventhubs/artifactId -version0.11.0-SNAPSHOT/version -packagingjar/packaging -namestorm-eventhubs/name -descriptionEventHubs Storm Spout/description - -properties -project.build.sourceEncodingUTF-8/project.build.sourceEncoding -qpid.version0.32/qpid.version -/properties -build -plugins -plugin -groupIdorg.apache.maven.plugins/groupId -artifactIdmaven-shade-plugin/artifactId -version2.3/version -executions -execution -goals -goalshade/goal -/goals -phasepackage/phase -/execution -/executions -configuration -transformers -transformer implementation=org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer -/transformer -/transformers - outputFiletarget/${project.artifactId}-${project.version}-jar-with-dependencies.jar/outputFile -/configuration - /plugin -plugin - artifactIdmaven-antrun-plugin/artifactId - executions - execution - phasepackage/phase - configuration - tasks - copy file=src/main/resources/config.properties tofile=target/eventhubs-config.properties/ -/tasks - /configuration - goals - goalrun/goal - /goals - /execution - /executions - /plugin -/plugins -/build -dependencies -dependency -
[28/50] [abbrv] storm git commit: update readme for release 0.3.0
update readme for release 0.3.0 Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8e0f167a Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8e0f167a Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8e0f167a Branch: refs/heads/master Commit: 8e0f167adf3155f55a7c24875e5e7542615e0c5a Parents: 01a66bf Author: P. Taylor Goetz ptgo...@gmail.com Authored: Tue May 5 16:52:03 2015 -0400 Committer: P. Taylor Goetz ptgo...@gmail.com Committed: Tue May 5 16:52:03 2015 -0400 -- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/8e0f167a/README.md -- diff --git a/README.md b/README.md index 6683848..6f27219 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ The current version of Flux is available in Maven Central at the following coord dependency groupIdcom.github.ptgoetz/groupId artifactIdflux-core/artifactId -version0.2.2/version +version0.3.0/version /dependency ``` @@ -125,7 +125,7 @@ The example below illustrates Flux usage with the Maven shade plugin: dependency groupIdcom.github.ptgoetz/groupId artifactIdflux-core/artifactId -version0.2.2/version +version0.3.0/version /dependency !-- add user dependencies here... -- @@ -227,7 +227,7 @@ storm jar myTopology-0.1.0-SNAPSHOT.jar org.apache.storm.flux.Flux --remote my_c âââ ââââââââ âââââââ âââ âââ +- Apache Storm-+ +- data FLow User eXperience -+ -Version: 0.2.2 +Version: 0.3.0 Parsing file: /Users/hsimpson/Projects/donut_domination/storm/shell_test.yaml -- TOPOLOGY DETAILS -- Name: shell-topology