svn commit: r12972 - in /dev/storm: apache-storm-0.10.0/ apache-storm-0.9.6-rc2/ apache-storm-1.0.0/
Author: ptgoetz Date: Fri Apr 1 23:43:16 2016 New Revision: 12972 Log: stage Storm release 1.0.0 Added: dev/storm/apache-storm-1.0.0/ dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz (with props) dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.asc dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.md5 dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.sha dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.zip (with props) dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.zip.asc dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.zip.md5 dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.zip.sha dev/storm/apache-storm-1.0.0/apache-storm-1.0.0.pom dev/storm/apache-storm-1.0.0/apache-storm-1.0.0.pom.asc dev/storm/apache-storm-1.0.0/apache-storm-1.0.0.tar.gz (with props) dev/storm/apache-storm-1.0.0/apache-storm-1.0.0.tar.gz.asc dev/storm/apache-storm-1.0.0/apache-storm-1.0.0.tar.gz.md5 dev/storm/apache-storm-1.0.0/apache-storm-1.0.0.tar.gz.sha dev/storm/apache-storm-1.0.0/apache-storm-1.0.0.zip (with props) dev/storm/apache-storm-1.0.0/apache-storm-1.0.0.zip.asc dev/storm/apache-storm-1.0.0/apache-storm-1.0.0.zip.md5 dev/storm/apache-storm-1.0.0/apache-storm-1.0.0.zip.sha Removed: dev/storm/apache-storm-0.10.0/ dev/storm/apache-storm-0.9.6-rc2/ Added: dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz == Binary file - no diff available. Propchange: dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz -- svn:mime-type = application/octet-stream Added: dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.asc == --- dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.asc (added) +++ dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.asc Fri Apr 1 23:43:16 2016 @@ -0,0 +1,11 @@ +-BEGIN PGP SIGNATURE- +Comment: GPGTools - https://gpgtools.org + +iQEcBAABCgAGBQJW/wQlAAoJEI3gOWLoC4/9B8cIAKRt8W+EknPmz1DrWAlDEDGw +xvePli4+JcPebXdxKX3Ibu0nBcB9+BXjIkz6jpds7uF16wIQG7KK6bCqz5n6M4FB +kkAXALxvbp09ZlNLo5xxK3Uw0RzZ3cYWcsVyFYxf/FLvsFHxB7OAGIt+ZNbTvZ2N +Rv8CLyHeS78hw1zhufFjcQLsOWFaE6nup6u6KT4m8LQBDWDC4WmBiNTPXbrg7lJD +hHQ8ke1aDx7v4+BFTyQtm3yrPjTbXtdBi3AyoqU5x7/0779mTY0jMEcYXOm34Yim +X+9t++BATXCzsSKtpiexdzXWQJbe4QEzoIJsEXPzEugdAIH9IG0oEhgWtuPrIRs= +=soUX +-END PGP SIGNATURE- Added: dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.md5 == --- dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.md5 (added) +++ dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.md5 Fri Apr 1 23:43:16 2016 @@ -0,0 +1 @@ +apache-storm-1.0.0-src.tar.gz: CA 41 51 D7 60 B8 A7 E3 D1 B5 AF 6D 5F DF 7E 13 Added: dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.sha == --- dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.sha (added) +++ dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.tar.gz.sha Fri Apr 1 23:43:16 2016 @@ -0,0 +1,4 @@ +apache-storm-1.0.0-src.tar.gz: 25F419E7 35F38505 C6C9E5B4 B3B3B467 6425CE32 + B0F05892 F8D4CF94 A75A56A5 FE9187E8 BF78C232 + 4FF53399 1DA89D34 B2B6178E 7F0526E6 F4EC1F8C + 376E9496 Added: dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.zip == Binary file - no diff available. Propchange: dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.zip -- svn:mime-type = application/octet-stream Added: dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.zip.asc == --- dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.zip.asc (added) +++ dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.zip.asc Fri Apr 1 23:43:16 2016 @@ -0,0 +1,11 @@ +-BEGIN PGP SIGNATURE- +Comment: GPGTools - https://gpgtools.org + +iQEcBAABCgAGBQJW/wQmAAoJEI3gOWLoC4/9X10IAKaZKORgSH4eeP36s2GcCPyP +Nn+7mV+8Zg5Dk+rUPeo1deLAdy3pIzUg7piHmnlfeQe+hn3yVv8XbS5wrH2cXAJ8 +/uCUB8fk5VrbBXzvFAidfnEUn819Xf14cJ5GNhrBiC2e2xAWC/0F+F1yIUbWlkkh +YgNJJ+7UU7o+DmPyMbvqymr+TcJlS3LHk2KI1wXZCKuqbhH1bQM8FdnBZ9D3ithK +9Yn4c1Pj88cgxzeq0Riysrr4CEnrKaCpGkGUZira85hnBotKW2dm6hW8lyqjs5Db +6ou2GU3+J/DZVHqlmohv0oS21/RmX0G7gu5mzgz8KjMNl03JKbKlj2lcJTKdAdU= +=3v75 +-END PGP SIGNATURE- Added: dev/storm/apache-storm-1.0.0/apache-storm-1.0.0-src.zip.md5 == --- dev/storm/a
storm git commit: [maven-release-plugin] prepare for next development iteration
Repository: storm Updated Branches: refs/heads/1.x-branch 787e4a6c3 -> a240df581 [maven-release-plugin] prepare for next development iteration Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a240df58 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a240df58 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a240df58 Branch: refs/heads/1.x-branch Commit: a240df5813f6821941d65c401c312660ddf27081 Parents: 787e4a6 Author: P. Taylor Goetz Authored: Fri Apr 1 17:05:53 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 17:05:53 2016 -0400 -- examples/storm-starter/pom.xml | 2 +- external/flux/flux-core/pom.xml | 2 +- external/flux/flux-examples/pom.xml | 2 +- external/flux/flux-wrappers/pom.xml | 2 +- external/flux/pom.xml| 2 +- external/sql/pom.xml | 2 +- external/sql/storm-sql-core/pom.xml | 2 +- external/sql/storm-sql-kafka/pom.xml | 2 +- external/sql/storm-sql-runtime/pom.xml | 2 +- external/storm-cassandra/pom.xml | 2 +- external/storm-elasticsearch/pom.xml | 2 +- external/storm-eventhubs/pom.xml | 4 ++-- external/storm-hbase/pom.xml | 2 +- external/storm-hdfs/pom.xml | 2 +- external/storm-hive/pom.xml | 2 +- external/storm-jdbc/pom.xml | 2 +- external/storm-kafka-client/pom.xml | 2 +- external/storm-kafka/pom.xml | 2 +- external/storm-metrics/pom.xml | 2 +- external/storm-mongodb/pom.xml | 2 +- external/storm-mqtt/core/pom.xml | 2 +- external/storm-mqtt/examples/pom.xml | 2 +- external/storm-mqtt/pom.xml | 2 +- external/storm-redis/pom.xml | 2 +- external/storm-solr/pom.xml | 2 +- pom.xml | 4 ++-- storm-buildtools/maven-shade-clojure-transformer/pom.xml | 2 +- storm-buildtools/storm-maven-plugins/pom.xml | 2 +- storm-core/pom.xml | 2 +- storm-dist/binary/pom.xml| 2 +- storm-dist/source/pom.xml| 2 +- storm-multilang/javascript/pom.xml | 2 +- storm-multilang/python/pom.xml | 2 +- storm-multilang/ruby/pom.xml | 2 +- storm-rename-hack/pom.xml| 2 +- 35 files changed, 37 insertions(+), 37 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/a240df58/examples/storm-starter/pom.xml -- diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml index 552adca..8e2ebfe 100644 --- a/examples/storm-starter/pom.xml +++ b/examples/storm-starter/pom.xml @@ -20,7 +20,7 @@ storm org.apache.storm - 1.0.0 + 1.0.1-SNAPSHOT ../../pom.xml http://git-wip-us.apache.org/repos/asf/storm/blob/a240df58/external/flux/flux-core/pom.xml -- diff --git a/external/flux/flux-core/pom.xml b/external/flux/flux-core/pom.xml index 6637ed6..135ea56 100644 --- a/external/flux/flux-core/pom.xml +++ b/external/flux/flux-core/pom.xml @@ -21,7 +21,7 @@ org.apache.storm flux -1.0.0 +1.0.1-SNAPSHOT ../pom.xml http://git-wip-us.apache.org/repos/asf/storm/blob/a240df58/external/flux/flux-examples/pom.xml -- diff --git a/external/flux/flux-examples/pom.xml b/external/flux/flux-examples/pom.xml index 4b7525d..2ba72df 100644 --- a/external/flux/flux-examples/pom.xml +++ b/external/flux/flux-examples/pom.xml @@ -21,7 +21,7 @@ org.apache.storm flux -1.0.0 +1.0.1-SNAPSHOT ../pom.xml http://git-wip-us.apache.org/repos/asf/storm/blob/a240df58/external/flux/flux-wrappers/pom.xml -- diff --git a/external/flux/flux-wrappers/pom.xml b/external/flux/flux-wrappers/pom.xml index e81e55e..516f8a0 100644 --- a/external/flux/flux-wrappers/pom.xml +++ b/external/flux/flux-wrappers/pom.xml @@ -21,7 +21,7 @@ org.apache.storm flux -
[2/2] storm git commit: [maven-release-plugin] prepare release v1.0.0
[maven-release-plugin] prepare release v1.0.0 Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/787e4a6c Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/787e4a6c Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/787e4a6c Branch: refs/heads/1.x-branch Commit: 787e4a6c375d290f724e59b3d8ebe34806ccd0d5 Parents: 51de47b Author: P. Taylor Goetz Authored: Fri Apr 1 17:05:44 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 17:05:44 2016 -0400 -- examples/storm-starter/pom.xml | 8 +- external/flux/flux-core/pom.xml | 2 +- external/flux/flux-examples/pom.xml | 2 +- external/flux/flux-wrappers/pom.xml | 2 +- external/flux/pom.xml | 2 +- external/sql/pom.xml| 2 +- external/sql/storm-sql-core/pom.xml | 2 +- external/sql/storm-sql-kafka/pom.xml| 2 +- external/sql/storm-sql-runtime/pom.xml | 2 +- external/storm-cassandra/pom.xml| 2 +- external/storm-elasticsearch/pom.xml| 5 +- external/storm-eventhubs/pom.xml| 226 +-- external/storm-hbase/pom.xml| 2 +- external/storm-hdfs/pom.xml | 2 +- external/storm-hive/pom.xml | 5 +- external/storm-jdbc/pom.xml | 4 +- external/storm-kafka-client/pom.xml | 5 +- external/storm-kafka/pom.xml| 2 +- external/storm-metrics/pom.xml | 21 +- external/storm-mongodb/pom.xml | 148 ++-- external/storm-mqtt/core/pom.xml| 5 +- external/storm-mqtt/examples/pom.xml| 5 +- external/storm-mqtt/pom.xml | 5 +- external/storm-redis/pom.xml| 2 +- external/storm-solr/pom.xml | 6 +- pom.xml | 8 +- .../maven-shade-clojure-transformer/pom.xml | 2 +- storm-buildtools/storm-maven-plugins/pom.xml| 5 +- storm-core/pom.xml | 6 +- storm-dist/binary/pom.xml | 2 +- storm-dist/source/pom.xml | 2 +- storm-multilang/javascript/pom.xml | 5 +- storm-multilang/python/pom.xml | 2 +- storm-multilang/ruby/pom.xml| 2 +- storm-rename-hack/pom.xml | 8 +- 35 files changed, 247 insertions(+), 264 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/787e4a6c/examples/storm-starter/pom.xml -- diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml index fb2b2bb..552adca 100644 --- a/examples/storm-starter/pom.xml +++ b/examples/storm-starter/pom.xml @@ -20,7 +20,7 @@ storm org.apache.storm - 1.0.0-SNAPSHOT + 1.0.0 ../../pom.xml @@ -186,10 +186,8 @@ - - + + http://git-wip-us.apache.org/repos/asf/storm/blob/787e4a6c/external/flux/flux-core/pom.xml -- diff --git a/external/flux/flux-core/pom.xml b/external/flux/flux-core/pom.xml index 538a37c..6637ed6 100644 --- a/external/flux/flux-core/pom.xml +++ b/external/flux/flux-core/pom.xml @@ -21,7 +21,7 @@ org.apache.storm flux -1.0.0-SNAPSHOT +1.0.0 ../pom.xml http://git-wip-us.apache.org/repos/asf/storm/blob/787e4a6c/external/flux/flux-examples/pom.xml -- diff --git a/external/flux/flux-examples/pom.xml b/external/flux/flux-examples/pom.xml index c72f6d5..4b7525d 100644 --- a/external/flux/flux-examples/pom.xml +++ b/external/flux/flux-examples/pom.xml @@ -21,7 +21,7 @@ org.apache.storm flux -1.0.0-SNAPSHOT +1.0.0 ../pom.xml http://git-wip-us.apache.org/repos/asf/storm/blob/787e4a6c/external/flux/flux-wrappers/pom.xml -- diff --git a/external/flux/flux-wrappers/pom.xml b/external/flux/flux-wrappers/pom.xml index c34a53a..e81e55e 100644 --- a/external/flux/flux-wrappers/pom.xml +++ b/external/flux/flux-wrappers/pom.xml @@ -21,7 +21,7 @@ org.apache.storm flux -1.0.0-SNAPSHOT +1.0.0
[1/2] storm git commit: fix version in storm-kafka-client pom
Repository: storm Updated Branches: refs/heads/1.x-branch f13f8ad0e -> 787e4a6c3 fix version in storm-kafka-client pom Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/51de47b5 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/51de47b5 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/51de47b5 Branch: refs/heads/1.x-branch Commit: 51de47b505d88337d877517177f0ab3a3b78cc70 Parents: f13f8ad Author: P. Taylor Goetz Authored: Fri Apr 1 16:53:40 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 16:53:40 2016 -0400 -- external/storm-kafka-client/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/51de47b5/external/storm-kafka-client/pom.xml -- diff --git a/external/storm-kafka-client/pom.xml b/external/storm-kafka-client/pom.xml index 6c82b6a..fdd6e70 100644 --- a/external/storm-kafka-client/pom.xml +++ b/external/storm-kafka-client/pom.xml @@ -23,7 +23,7 @@ storm org.apache.storm -2.0.0-SNAPSHOT +1.0.0-SNAPSHOT ../../pom.xml
[storm] Git Push Summary
Repository: storm Updated Tags: refs/tags/v1.0.0 [created] e1a73f93f
storm git commit: STORM-1671: apply test changes
Repository: storm
Updated Branches:
refs/heads/1.x-branch 7670c6350 -> f13f8ad0e
STORM-1671: apply test changes
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f13f8ad0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f13f8ad0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f13f8ad0
Branch: refs/heads/1.x-branch
Commit: f13f8ad0e3f65e0c130d447373c48409cb9488bb
Parents: 7670c63
Author: P. Taylor Goetz
Authored: Fri Apr 1 15:52:25 2016 -0400
Committer: P. Taylor Goetz
Committed: Fri Apr 1 15:52:25 2016 -0400
--
.../clj/org/apache/storm/logviewer_test.clj | 26 +++-
1 file changed, 14 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/f13f8ad0/storm-core/test/clj/org/apache/storm/logviewer_test.clj
--
diff --git a/storm-core/test/clj/org/apache/storm/logviewer_test.clj
b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
index c13e869..0abcda6 100644
--- a/storm-core/test/clj/org/apache/storm/logviewer_test.clj
+++ b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
@@ -238,18 +238,20 @@
(is (= expected (logviewer/identify-worker-log-dirs [port1-dir])))
(deftest test-get-dead-worker-dirs
- (testing "removes any files of workers that are still alive"
-(let [conf {SUPERVISOR-WORKER-TIMEOUT-SECS 5}
- id->hb {"42" {:time-secs 1}}
- now-secs 2
- unexpected-dir (mk-mock-File {:name "dir1" :type :directory})
- expected-dir (mk-mock-File {:name "dir2" :type :directory})
- log-dirs #{unexpected-dir expected-dir}]
- (stubbing [logviewer/identify-worker-log-dirs {"42" unexpected-dir,
- "007" expected-dir}
- supervisor/read-worker-heartbeats id->hb]
-(is (= #{expected-dir}
- (logviewer/get-dead-worker-dirs conf now-secs log-dirs)))
+ (testing "return directories for workers that are not alive"
+ (let [conf {SUPERVISOR-WORKER-TIMEOUT-SECS 5}
+id->hb {"42" {:time-secs 1}} ;; map for alive ids
+now-secs 2
+unexpected-dir1 (mk-mock-File {:name "dir1" :type
:directory})
+expected-dir2 (mk-mock-File {:name "dir2" :type
:directory})
+expected-dir3 (mk-mock-File {:name "dir3" :type
:directory})
+log-dirs #{unexpected-dir1 expected-dir2
expected-dir3}]
+ (stubbing [logviewer/identify-worker-log-dirs {"42"
unexpected-dir1,
+ "007"
expected-dir2,
+ ""
expected-dir3} ;; this tests a directory with no yaml file thus no worker id
+ supervisor/read-worker-heartbeats id->hb]
+ (is (= #{expected-dir2 expected-dir3}
+(logviewer/get-dead-worker-dirs conf
now-secs log-dirs)))
(deftest test-cleanup-fn
(testing "cleanup function rmr's files of dead workers"
[3/3] storm git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/storm
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/storm Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/69ec8ffa Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/69ec8ffa Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/69ec8ffa Branch: refs/heads/master Commit: 69ec8ffa1f9bb043291b94d3fd3c1b9b2a74f598 Parents: 3f11cca 68cc705 Author: P. Taylor Goetz Authored: Fri Apr 1 15:29:01 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 15:29:01 2016 -0400 -- --
[1/3] storm git commit: Merge branch '1671' of github.com:zhuoliu/storm
Repository: storm Updated Branches: refs/heads/master 68cc70518 -> 69ec8ffa1 Merge branch '1671' of github.com:zhuoliu/storm Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/224d156e Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/224d156e Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/224d156e Branch: refs/heads/master Commit: 224d156ec0a301591676d129105bc533b86d00c7 Parents: 175beec 8210283 Author: P. Taylor Goetz Authored: Fri Apr 1 14:45:23 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 14:45:23 2016 -0400 -- .../clj/org/apache/storm/daemon/logviewer.clj | 7 --- .../apache/storm/daemon/DirectoryCleaner.java | 2 +- .../clj/org/apache/storm/logviewer_test.clj | 20 ++-- 3 files changed, 15 insertions(+), 14 deletions(-) --
[1/2] storm git commit: [STORM-1671] Enable logviewer to delete a dir without yaml
Repository: storm
Updated Branches:
refs/heads/1.x-branch 96d190e8c -> 7670c6350
[STORM-1671] Enable logviewer to delete a dir without yaml
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9a369dea
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9a369dea
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9a369dea
Branch: refs/heads/1.x-branch
Commit: 9a369deacbdefea5b16cfc51950cab952a2e4b27
Parents: 96d190e
Author: P. Taylor Goetz
Authored: Fri Apr 1 15:27:23 2016 -0400
Committer: P. Taylor Goetz
Committed: Fri Apr 1 15:27:23 2016 -0400
--
storm-core/src/clj/org/apache/storm/daemon/logviewer.clj | 7 ---
.../src/jvm/org/apache/storm/daemon/DirectoryCleaner.java | 2 +-
2 files changed, 5 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/9a369dea/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
--
diff --git a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
index 1fcb5d5..7baeae7 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
@@ -149,9 +149,10 @@
(defn identify-worker-log-dirs [log-dirs]
"return the workerid to worker-log-dir map"
(into {} (for [logdir log-dirs
- :let [metaFile (get-metadata-file-for-wroker-logdir logdir)]
- :when metaFile]
- {(get-worker-id-from-metadata-file metaFile) logdir})))
+ :let [metaFile (get-metadata-file-for-wroker-logdir logdir)]]
+ (if metaFile
+ {(get-worker-id-from-metadata-file metaFile) logdir}
+ {"" logdir} ;; an old directory that has no yaml file will
be treated as a dead dir for deleting
(defn get-alive-ids
[conf now-secs]
http://git-wip-us.apache.org/repos/asf/storm/blob/9a369dea/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
--
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
b/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
index a51a49e..047e024 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
@@ -138,7 +138,7 @@ public class DirectoryCleaner {
while (!stack.isEmpty() && toDeleteSize > 0) {
File file = stack.pop();
toDeleteSize -= file.length();
-LOG.info("Delete file: {}, size: {}, lastModified: {}",
file.getName(), file.length(), file.lastModified());
+LOG.info("Delete file: {}, size: {}, lastModified: {}",
file.getCanonicalPath(), file.length(), file.lastModified());
file.delete();
deletedFiles++;
}
[2/3] storm git commit: add STORM-1671 to changelog
add STORM-1671 to changelog Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3f11cca9 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3f11cca9 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3f11cca9 Branch: refs/heads/master Commit: 3f11cca9355cebb6054b56ec28ad54cd13e453e8 Parents: 224d156 Author: P. Taylor Goetz Authored: Fri Apr 1 14:46:32 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 14:46:32 2016 -0400 -- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/3f11cca9/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index 120aae0..50129b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,7 @@ * STORM-1521: When using Kerberos login from keytab with multiple bolts/executors ticket is not renewed in hbase bolt. ## 1.0.0 + * STORM-1671: Enable logviewer to delete a dir without yaml * STORM-1673: log4j2/worker.xml refers old package of LoggerMetricsConsumer * STORM-1669: Fix SolrUpdateBolt flush bug * STORM-1573: Add batch support for MongoInsertBolt
[2/2] storm git commit: add STORM-1671 to changelog
add STORM-1671 to changelog Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7670c635 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7670c635 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7670c635 Branch: refs/heads/1.x-branch Commit: 7670c6350fc2def68caf4c622e3674b8eb7e3034 Parents: 9a369de Author: P. Taylor Goetz Authored: Fri Apr 1 15:28:19 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 15:28:19 2016 -0400 -- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/7670c635/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index 9106fbb..a232f2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 1.0.0 + * STORM-1671: Enable logviewer to delete a dir without yaml * STORM-822: Kafka Spout New Consumer API * STORM-1673: log4j2/worker.xml refers old package of LoggerMetricsConsumer * STORM-1632 Disable event logging by default
[2/2] storm git commit: Merge branch '1671' of https://github.com/zhuoliu/storm into STORM-1671
Merge branch '1671' of https://github.com/zhuoliu/storm into STORM-1671 Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/68cc7051 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/68cc7051 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/68cc7051 Branch: refs/heads/master Commit: 68cc70518f10a9e0ec75fe72aeb2e98624a7b330 Parents: 175beec 8210283 Author: zhuol Authored: Fri Apr 1 14:04:05 2016 -0500 Committer: zhuol Committed: Fri Apr 1 14:04:05 2016 -0500 -- .../clj/org/apache/storm/daemon/logviewer.clj | 7 --- .../apache/storm/daemon/DirectoryCleaner.java | 2 +- .../clj/org/apache/storm/logviewer_test.clj | 20 ++-- 3 files changed, 15 insertions(+), 14 deletions(-) --
[1/2] storm git commit: [STORM-1671] Enable logviewer to delete a dir without yaml
Repository: storm
Updated Branches:
refs/heads/master 175beec7e -> 68cc70518
[STORM-1671] Enable logviewer to delete a dir without yaml
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/82102837
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/82102837
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/82102837
Branch: refs/heads/master
Commit: 821028374fd09d88d97b3ca3e283da2192f0f2ae
Parents: 3de3afc
Author: zhuol
Authored: Thu Mar 31 13:25:58 2016 -0500
Committer: zhuol
Committed: Fri Apr 1 11:07:09 2016 -0500
--
.../clj/org/apache/storm/daemon/logviewer.clj | 7 ---
.../apache/storm/daemon/DirectoryCleaner.java | 2 +-
.../clj/org/apache/storm/logviewer_test.clj | 20 ++--
3 files changed, 15 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/82102837/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
--
diff --git a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
index 53e8527..2b80fd8 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
@@ -151,9 +151,10 @@
(defn identify-worker-log-dirs [log-dirs]
"return the workerid to worker-log-dir map"
(into {} (for [logdir log-dirs
- :let [metaFile (get-metadata-file-for-wroker-logdir logdir)]
- :when metaFile]
- {(get-worker-id-from-metadata-file metaFile) logdir})))
+ :let [metaFile (get-metadata-file-for-wroker-logdir logdir)]]
+ (if metaFile
+ {(get-worker-id-from-metadata-file metaFile) logdir}
+ {"" logdir} ;; an old directory that has no yaml file will
be treated as a dead dir for deleting
(defn get-alive-ids
[conf now-secs]
http://git-wip-us.apache.org/repos/asf/storm/blob/82102837/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
--
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
b/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
index a51a49e..047e024 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
@@ -138,7 +138,7 @@ public class DirectoryCleaner {
while (!stack.isEmpty() && toDeleteSize > 0) {
File file = stack.pop();
toDeleteSize -= file.length();
-LOG.info("Delete file: {}, size: {}, lastModified: {}",
file.getName(), file.length(), file.lastModified());
+LOG.info("Delete file: {}, size: {}, lastModified: {}",
file.getCanonicalPath(), file.length(), file.lastModified());
file.delete();
deletedFiles++;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/82102837/storm-core/test/clj/org/apache/storm/logviewer_test.clj
--
diff --git a/storm-core/test/clj/org/apache/storm/logviewer_test.clj
b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
index d06c11c..d161f3f 100644
--- a/storm-core/test/clj/org/apache/storm/logviewer_test.clj
+++ b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
@@ -246,24 +246,24 @@
(. (Mockito/when (.readWorkerHeartbeatsImpl supervisor-util
(Mockito/any))) (thenReturn nil))
(is (= expected (logviewer/identify-worker-log-dirs
[port1-dir]
-
-
(deftest test-get-dead-worker-dirs
- (testing "removes any files of workers that are still alive"
+ (testing "return directories for workers that are not alive"
(let [conf {SUPERVISOR-WORKER-TIMEOUT-SECS 5}
- hb (let[lwb (LSWorkerHeartbeat.)]
+ hb (let [lwb (LSWorkerHeartbeat.)]
(.set_time_secs lwb (int 1)) lwb)
id->hb {"42" hb}
now-secs 2
- unexpected-dir (mk-mock-File {:name "dir1" :type :directory})
- expected-dir (mk-mock-File {:name "dir2" :type :directory})
- log-dirs #{unexpected-dir expected-dir}
+ unexpected-dir1 (mk-mock-File {:name "dir1" :type :directory})
+ expected-dir2 (mk-mock-File {:name "dir2" :type :directory})
+ expected-dir3 (mk-mock-File {:name "dir3" :type :directory})
+ log-dirs #{unexpected-dir1 expected-dir2 expected-dir3}
supervisor-util (Mockito/mock SupervisorUtils)]
(with-open [_ (MockedSupervisorUtils. supervisor-util)]
- (stubbing [logviewer/identify-worker-log-dirs {"42" unexpected-dir,
-
[4/4] storm git commit: add STORM-822 to changelog
add STORM-822 to changelog Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/96d190e8 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/96d190e8 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/96d190e8 Branch: refs/heads/1.x-branch Commit: 96d190e8c2aeaecca55b55a57bbf56bf75fb970d Parents: cf9f837 Author: P. Taylor Goetz Authored: Fri Apr 1 14:42:08 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 14:42:08 2016 -0400 -- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/96d190e8/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index b3f5a00..9106fbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 1.0.0 + * STORM-822: Kafka Spout New Consumer API * STORM-1673: log4j2/worker.xml refers old package of LoggerMetricsConsumer * STORM-1632 Disable event logging by default * STORM-1667: Log the IO exception when deleting worker pid dir
[1/4] storm git commit: STORM-822: Kafka Spout New Consumer API - Refactored code to avoid keeping records data inside spout state - Refactored code to specify output fields per stream and build tuple
Repository: storm
Updated Branches:
refs/heads/1.x-branch 45b7ecac3 -> 96d190e8c
STORM-822: Kafka Spout New Consumer API - Refactored code to avoid keeping
records data inside spout state - Refactored code to specify output fields per
stream and build tuples per topic - Implement exponential backoff retry
strategy - Send one tuple per call to nextTuple
Signed-off-by: P. Taylor Goetz
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9d9d1427
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9d9d1427
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9d9d1427
Branch: refs/heads/1.x-branch
Commit: 9d9d142791aab04a19cafd5f32f1155458feb3ee
Parents: d988d44
Author: Hugo Louro
Authored: Mon Mar 21 14:42:50 2016 -0700
Committer: P. Taylor Goetz
Committed: Fri Apr 1 14:35:56 2016 -0400
--
.../kafka/spout/KafkaRecordTupleBuilder.java| 44 ---
.../apache/storm/kafka/spout/KafkaSpout.java| 160 +++
.../storm/kafka/spout/KafkaSpoutConfig.java | 87 +++---
.../storm/kafka/spout/KafkaSpoutMessageId.java | 16 +-
.../KafkaSpoutRetryExponentialBackoff.java | 281 +++
.../kafka/spout/KafkaSpoutRetryService.java | 72 +
.../storm/kafka/spout/KafkaSpoutStream.java | 14 +-
.../storm/kafka/spout/KafkaSpoutStreams.java| 26 +-
.../kafka/spout/KafkaSpoutTupleBuilder.java | 34 ++-
.../kafka/spout/KafkaSpoutTuplesBuilder.java| 82 ++
.../kafka/spout/test/KafkaSpoutTestBolt.java| 50
.../spout/test/KafkaSpoutTopologyMain.java | 37 ++-
.../storm/kafka/spout/test/KafkaTestBolt.java | 52
.../spout/test/TopicTest2TupleBuilder.java | 40 +++
.../test/TopicsTest0Test1TupleBuilder.java | 42 +++
15 files changed, 798 insertions(+), 239 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/9d9d1427/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaRecordTupleBuilder.java
--
diff --git
a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaRecordTupleBuilder.java
b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaRecordTupleBuilder.java
deleted file mode 100644
index 4d67632..000
---
a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaRecordTupleBuilder.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.kafka.spout;
-
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Values;
-
-import java.util.List;
-
-public class KafkaRecordTupleBuilder implements
KafkaSpoutTupleBuilder {
-@Override
-public List buildTuple(final ConsumerRecord consumerRecord,
KafkaSpoutStreams kafkaSpoutStreams) {
-final Fields outputFields =
kafkaSpoutStreams.getOutputFields(consumerRecord.topic());
-if (outputFields != null) {
-if (outputFields.size() == 3) {
-return new Values(consumerRecord.topic(),
-consumerRecord.partition(),
-consumerRecord.offset());
-} else if (outputFields.size() == 5) {
-return new Values(consumerRecord.topic(),
-consumerRecord.partition(),
-consumerRecord.offset(),
-consumerRecord.key(),
-consumerRecord.value());
-}
-}
-throw new RuntimeException("Failed to build tuple. " + consumerRecord
+ " " + kafkaSpoutStreams);
-}
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/9d9d1427/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
--
diff --git
a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
b/external/storm-kafka-client/src/main/java/org/apache/st
[2/4] storm git commit: STORM-822: Kafka Spout New Consumer API
STORM-822: Kafka Spout New Consumer API
Signed-off-by: P. Taylor Goetz
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d988d44e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d988d44e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d988d44e
Branch: refs/heads/1.x-branch
Commit: d988d44e4d571428a270e7585341cc8af0943335
Parents: 45b7eca
Author: Hugo Louro
Authored: Mon Dec 14 10:16:42 2015 -0800
Committer: P. Taylor Goetz
Committed: Fri Apr 1 14:35:56 2016 -0400
--
examples/storm-starter/pom.xml | 16 +-
external/flux/flux-examples/pom.xml | 13 +-
external/flux/pom.xml | 13 +-
external/sql/storm-sql-kafka/pom.xml| 16 +-
external/storm-kafka-client/README.md | 9 +
external/storm-kafka-client/pom.xml | 86
.../kafka/spout/KafkaRecordTupleBuilder.java| 44 ++
.../apache/storm/kafka/spout/KafkaSpout.java| 503 +++
.../storm/kafka/spout/KafkaSpoutConfig.java | 298 +++
.../storm/kafka/spout/KafkaSpoutMessageId.java | 111
.../storm/kafka/spout/KafkaSpoutStream.java | 66 +++
.../storm/kafka/spout/KafkaSpoutStreams.java| 162 ++
.../kafka/spout/KafkaSpoutTupleBuilder.java | 28 ++
.../spout/test/KafkaSpoutTopologyMain.java | 120 +
.../storm/kafka/spout/test/KafkaTestBolt.java | 52 ++
external/storm-kafka/pom.xml| 16 +-
external/storm-solr/pom.xml | 8 +-
pom.xml | 63 ++-
storm-dist/binary/src/main/assembly/binary.xml | 14 +
19 files changed, 1550 insertions(+), 88 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/d988d44e/examples/storm-starter/pom.xml
--
diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml
index 6053595..fb2b2bb 100644
--- a/examples/storm-starter/pom.xml
+++ b/examples/storm-starter/pom.xml
@@ -148,26 +148,12 @@
org.apache.kafka
- kafka_2.10
- 0.8.2.1
-
+ ${kafka.artifact.id}
provided
-
-
- org.apache.zookeeper
- zookeeper
-
-
- log4j
- log4j
-
-
org.apache.kafka
kafka-clients
- 0.8.2.1
- provided
org.apache.storm
http://git-wip-us.apache.org/repos/asf/storm/blob/d988d44e/external/flux/flux-examples/pom.xml
--
diff --git a/external/flux/flux-examples/pom.xml
b/external/flux/flux-examples/pom.xml
index 926f8d0..c72f6d5 100644
--- a/external/flux/flux-examples/pom.xml
+++ b/external/flux/flux-examples/pom.xml
@@ -95,18 +95,7 @@
org.apache.kafka
-kafka_2.10
-0.8.1.1
-
-
-org.apache.zookeeper
-zookeeper
-
-
-log4j
-log4j
-
-
+${kafka.artifact.id}
http://git-wip-us.apache.org/repos/asf/storm/blob/d988d44e/external/flux/pom.xml
--
diff --git a/external/flux/pom.xml b/external/flux/pom.xml
index 7b38161..cc4539d 100644
--- a/external/flux/pom.xml
+++ b/external/flux/pom.xml
@@ -78,19 +78,8 @@
org.apache.kafka
-kafka_2.10
-0.8.1.1
+${kafka.artifact.id}
test
-
-
-org.apache.zookeeper
-zookeeper
-
-
-log4j
-log4j
-
-
junit
http://git-wip-us.apache.org/repos/asf/storm/blob/d988d44e/external/sql/storm-sql-kafka/pom.xml
--
diff --git a/external/sql/storm-sql-kafka/pom.xml
b/external/sql/storm-sql-kafka/pom.xml
index fb754e7..b6a72dc 100644
--- a/external/sql/storm-sql-kafka/pom.xml
+++ b/external/sql/storm-sql-kafka/pom.xml
@@ -63,26 +63,12 @@
org.apache.kafka
-kafka_2.10
-0.8.2.1
-
+${kafka.artifact.id}
provided
-
-
-org.apache.zookeeper
-zookeeper
-
-
-log4j
-log4j
-
-
org.apache.kafka
kafka
[3/4] storm git commit: add commons-cli version to pom
add commons-cli version to pom Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cf9f837e Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cf9f837e Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cf9f837e Branch: refs/heads/1.x-branch Commit: cf9f837e8689be8eda7e6baea7ed8bf8c88c7370 Parents: 9d9d142 Author: P. Taylor Goetz Authored: Fri Apr 1 14:41:44 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 14:41:44 2016 -0400 -- pom.xml | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/cf9f837e/pom.xml -- diff --git a/pom.xml b/pom.xml index 5cf28ab..cb01c9f 100644 --- a/pom.xml +++ b/pom.xml @@ -208,6 +208,7 @@ 1.1 1.2.1 1.6 +1.3.1 0.8.0 2.10.0 1.1
[3/3] storm git commit: add STORM-1673 to changelog
add STORM-1673 to changelog Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/175beec7 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/175beec7 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/175beec7 Branch: refs/heads/master Commit: 175beec7e5b787f51fe4bc772c48bdb4ec41fc20 Parents: e9f6a84 Author: P. Taylor Goetz Authored: Fri Apr 1 13:01:44 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 13:01:44 2016 -0400 -- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/175beec7/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b0339b..120aae0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,7 @@ * STORM-1521: When using Kerberos login from keytab with multiple bolts/executors ticket is not renewed in hbase bolt. ## 1.0.0 + * STORM-1673: log4j2/worker.xml refers old package of LoggerMetricsConsumer * STORM-1669: Fix SolrUpdateBolt flush bug * STORM-1573: Add batch support for MongoInsertBolt * STORM-1660: remove flux gitignore file and move rules to top level gitignore
[2/3] storm git commit: Merge branch 'STORM-1673-1.x' of github.com:HeartSaVioR/storm into 1.x-branch
Merge branch 'STORM-1673-1.x' of github.com:HeartSaVioR/storm into 1.x-branch Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f574f380 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f574f380 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f574f380 Branch: refs/heads/1.x-branch Commit: f574f3800b34b17ea25fa9393ae57989cdc43a17 Parents: 3d19c64 dd3d0b2 Author: P. Taylor Goetz Authored: Fri Apr 1 12:58:20 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 12:58:20 2016 -0400 -- log4j2/worker.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) --
[1/3] storm git commit: STORM-1673 log4j2/worker.xml refers old package of LoggerMetricsConsumer
Repository: storm Updated Branches: refs/heads/1.x-branch 3d19c644f -> 45b7ecac3 STORM-1673 log4j2/worker.xml refers old package of LoggerMetricsConsumer * fix missing package changes Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/dd3d0b20 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/dd3d0b20 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/dd3d0b20 Branch: refs/heads/1.x-branch Commit: dd3d0b2000c41b587cb22ecf16ddfd971c0c1d00 Parents: 97e7d25 Author: Jungtaek Lim Authored: Fri Apr 1 12:25:48 2016 +0900 Committer: Jungtaek Lim Committed: Fri Apr 1 12:25:48 2016 +0900 -- log4j2/worker.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/dd3d0b20/log4j2/worker.xml -- diff --git a/log4j2/worker.xml b/log4j2/worker.xml index 630132a..a9aeaed 100644 --- a/log4j2/worker.xml +++ b/log4j2/worker.xml @@ -77,7 +77,7 @@ - +
[3/3] storm git commit: add STORM-1673 to changelog
add STORM-1673 to changelog Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/45b7ecac Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/45b7ecac Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/45b7ecac Branch: refs/heads/1.x-branch Commit: 45b7ecac3b11fc147fa9efe060d6cadb20a55862 Parents: f574f38 Author: P. Taylor Goetz Authored: Fri Apr 1 12:59:37 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 12:59:37 2016 -0400 -- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/45b7ecac/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index 000a8f8..b3f5a00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 1.0.0 + * STORM-1673: log4j2/worker.xml refers old package of LoggerMetricsConsumer * STORM-1632 Disable event logging by default * STORM-1667: Log the IO exception when deleting worker pid dir * STORM-1669: Fix SolrUpdateBolt flush bug
[2/3] storm git commit: Merge branch 'STORM-1673' of https://github.com/HeartSaVioR/storm
Merge branch 'STORM-1673' of https://github.com/HeartSaVioR/storm Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e9f6a842 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e9f6a842 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e9f6a842 Branch: refs/heads/master Commit: e9f6a842cfaa13f7a81ceb9b6bc0ce87ca267fd1 Parents: 3de3afc 96da31e Author: P. Taylor Goetz Authored: Fri Apr 1 13:00:47 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 13:00:47 2016 -0400 -- log4j2/worker.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) --
[1/3] storm git commit: STORM-1673 log4j2/worker.xml refers old package of LoggerMetricsConsumer
Repository: storm Updated Branches: refs/heads/master 3de3afc37 -> 175beec7e STORM-1673 log4j2/worker.xml refers old package of LoggerMetricsConsumer * fix missing package changes Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/96da31ed Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/96da31ed Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/96da31ed Branch: refs/heads/master Commit: 96da31ed0068dde3b4309ee417382be5084ce941 Parents: 2a435b0 Author: Jungtaek Lim Authored: Fri Apr 1 12:22:38 2016 +0900 Committer: Jungtaek Lim Committed: Fri Apr 1 12:22:38 2016 +0900 -- log4j2/worker.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/96da31ed/log4j2/worker.xml -- diff --git a/log4j2/worker.xml b/log4j2/worker.xml index 630132a..a9aeaed 100644 --- a/log4j2/worker.xml +++ b/log4j2/worker.xml @@ -77,7 +77,7 @@ - +
[2/5] storm git commit: Enable/disable Debug button on UI if eventloggers are enabled. Also show tooltip on Debug button
Enable/disable Debug button on UI if eventloggers are enabled. Also show
tooltip on Debug button
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3354cc73
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3354cc73
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3354cc73
Branch: refs/heads/1.x-branch
Commit: 3354cc7369a494ef9645df08d0803dedfb3d9192
Parents: a33bc45
Author: Roshan Naik
Authored: Wed Mar 30 16:30:43 2016 -0700
Committer: Roshan Naik
Committed: Wed Mar 30 16:30:43 2016 -0700
--
storm-core/src/ui/public/js/script.js | 4 ++--
storm-core/src/ui/public/templates/topology-page-template.html | 2 +-
storm-core/src/ui/public/topology.html | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/3354cc73/storm-core/src/ui/public/js/script.js
--
diff --git a/storm-core/src/ui/public/js/script.js
b/storm-core/src/ui/public/js/script.js
index a880205..c25195d 100644
--- a/storm-core/src/ui/public/js/script.js
+++ b/storm-core/src/ui/public/js/script.js
@@ -210,7 +210,7 @@ function renderToggleSys(div) {
}
}
-function topologyActionJson(id, encodedId, name, status, msgTimeout, debug,
samplingPct) {
+function topologyActionJson(id, encodedId, name, status, msgTimeout,
loggersTotal, debug, samplingPct) {
var jsonData = {};
jsonData["id"] = id;
jsonData["encodedId"] = encodedId;
@@ -220,7 +220,7 @@ function topologyActionJson(id, encodedId, name, status,
msgTimeout, debug, samp
jsonData["deactivateStatus"] = (status === "ACTIVE") ? "enabled" :
"disabled";
jsonData["rebalanceStatus"] = (status === "ACTIVE" || status ===
"INACTIVE" ) ? "enabled" : "disabled";
jsonData["killStatus"] = (status !== "KILLED") ? "enabled" : "disabled";
-jsonData["startDebugStatus"] = (status === "ACTIVE" && !debug) ? "enabled"
: "disabled";
+jsonData["startDebugStatus"] = (status === "ACTIVE" && loggersTotal!=null
&& loggersTotal!=0 && !debug) ? "enabled" : "disabled";
jsonData["stopDebugStatus"] = (status === "ACTIVE" && debug) ? "enabled" :
"disabled";
jsonData["currentSamplingPct"] = samplingPct;
return jsonData;
http://git-wip-us.apache.org/repos/asf/storm/blob/3354cc73/storm-core/src/ui/public/templates/topology-page-template.html
--
diff --git a/storm-core/src/ui/public/templates/topology-page-template.html
b/storm-core/src/ui/public/templates/topology-page-template.html
index 02b3c76..a72effc 100644
--- a/storm-core/src/ui/public/templates/topology-page-template.html
+++ b/storm-core/src/ui/public/templates/topology-page-template.html
@@ -434,7 +434,7 @@
-
+
http://git-wip-us.apache.org/repos/asf/storm/blob/3354cc73/storm-core/src/ui/public/topology.html
--
diff --git a/storm-core/src/ui/public/topology.html
b/storm-core/src/ui/public/topology.html
index feb81f8..061fd91 100644
--- a/storm-core/src/ui/public/topology.html
+++ b/storm-core/src/ui/public/topology.html
@@ -285,7 +285,7 @@ $(document).ready(function() {
var searchForm = $("#search-form")
var formattedConfig = formatConfigData(response["configuration"]);
var buttonJsonData =
topologyActionJson(response["id"],response["encodedId"],response["name"],response["status"]
-
,response["msgTimeout"],response["debug"],response["samplingPct"]);
+
,response["msgTimeout"],response["configuration"]["topology.eventlogger.executors"],response["debug"],response["samplingPct"]);
$.ajax ({url: "/templates/topology-page-template.html", success:
function(template) {
toggleChangeLogLevel = function (data) {
renderLogLevelForm (template, data);
[4/5] storm git commit: Merge branch 'STORM-1632' of github.com:roshannaik/storm into 1.x-branch
Merge branch 'STORM-1632' of github.com:roshannaik/storm into 1.x-branch Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b15e6ab7 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b15e6ab7 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b15e6ab7 Branch: refs/heads/1.x-branch Commit: b15e6ab79a671590b878530b9734233d3a77f59b Parents: dc006da 7d7b3c2 Author: P. Taylor Goetz Authored: Fri Apr 1 12:35:54 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 12:35:54 2016 -0400 -- conf/defaults.yaml| 2 +- storm-core/src/ui/public/component.html | 18 +- storm-core/src/ui/public/js/script.js | 10 ++ .../public/templates/component-page-template.html | 8 +++- .../public/templates/topology-page-template.html | 8 +++- storm-core/src/ui/public/topology.html| 2 +- 6 files changed, 39 insertions(+), 9 deletions(-) --
[5/5] storm git commit: add STORM-1632 to changelog
add STORM-1632 to changelog Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3d19c644 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3d19c644 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3d19c644 Branch: refs/heads/1.x-branch Commit: 3d19c644fd0f6310743279aa8b57da5e2500fa48 Parents: b15e6ab Author: P. Taylor Goetz Authored: Fri Apr 1 12:36:55 2016 -0400 Committer: P. Taylor Goetz Committed: Fri Apr 1 12:36:55 2016 -0400 -- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/3d19c644/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a1d748..000a8f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 1.0.0 + * STORM-1632 Disable event logging by default * STORM-1667: Log the IO exception when deleting worker pid dir * STORM-1669: Fix SolrUpdateBolt flush bug * STORM-1668: Flux silently fails while setting a non-existent property.
[3/5] storm git commit: Showing tooltip on Debug button only when disabled and event loggers are disabled. Also enable/disable of Debug button along with toltip in component page
Showing tooltip on Debug button only when disabled and event loggers are
disabled. Also enable/disable of Debug button along with toltip in component
page
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7d7b3c20
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7d7b3c20
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7d7b3c20
Branch: refs/heads/1.x-branch
Commit: 7d7b3c2086a0172757af4bfc2a1a9dd9feee696c
Parents: 3354cc7
Author: Roshan Naik
Authored: Thu Mar 31 18:16:52 2016 -0700
Committer: Roshan Naik
Committed: Thu Mar 31 18:16:52 2016 -0700
--
storm-core/src/ui/public/component.html | 18 +-
storm-core/src/ui/public/js/script.js | 6 --
.../public/templates/component-page-template.html | 8 +++-
.../public/templates/topology-page-template.html | 8 +++-
4 files changed, 35 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/7d7b3c20/storm-core/src/ui/public/component.html
--
diff --git a/storm-core/src/ui/public/component.html
b/storm-core/src/ui/public/component.html
index 6d5465f..37ee525 100644
--- a/storm-core/src/ui/public/component.html
+++ b/storm-core/src/ui/public/component.html
@@ -192,10 +192,26 @@ $(document).ready(function() {
$('#ui-user [data-toggle="tooltip"]').tooltip()
});
+var topologyUrl = "/api/v1/topology/"+topologyId;
+
+var eventLoggers = (function() {
+ $.ajaxSetup({
+ async: false
+ });
+ var eventLoggers;
+ $.getJSON(topologyUrl, function(response, status, jqXHR) {
+ eventLoggers =
response["configuration"]["topology.eventlogger.executors"];
+ });
+ $.ajaxSetup({
+ async: true
+ })
+ return eventLoggers;
+ })();
+
var componentSummary = $("#component-summary");
var componentActions = $("#component-actions");
var buttonJsonData =
componentActionJson(response["encodedTopologyId"], response["encodedId"],
response["id"],
- response["topologyStatus"],
response["debug"], response["samplingPct"]);
+ response["topologyStatus"],
eventLoggers, response["debug"], response["samplingPct"]);
var componentStatsDetail = $("#component-stats-detail")
var inputStats = $("#component-input-stats");
var outputStats = $("#component-output-stats");
http://git-wip-us.apache.org/repos/asf/storm/blob/7d7b3c20/storm-core/src/ui/public/js/script.js
--
diff --git a/storm-core/src/ui/public/js/script.js
b/storm-core/src/ui/public/js/script.js
index c25195d..ff3c525 100644
--- a/storm-core/src/ui/public/js/script.js
+++ b/storm-core/src/ui/public/js/script.js
@@ -222,17 +222,19 @@ function topologyActionJson(id, encodedId, name, status,
msgTimeout, loggersTota
jsonData["killStatus"] = (status !== "KILLED") ? "enabled" : "disabled";
jsonData["startDebugStatus"] = (status === "ACTIVE" && loggersTotal!=null
&& loggersTotal!=0 && !debug) ? "enabled" : "disabled";
jsonData["stopDebugStatus"] = (status === "ACTIVE" && debug) ? "enabled" :
"disabled";
+jsonData["loggersDisabled"] = loggersTotal==null || loggersTotal==0;
jsonData["currentSamplingPct"] = samplingPct;
return jsonData;
}
-function componentActionJson(encodedTopologyId, encodedId, componentName,
status, debug, samplingPct) {
+function componentActionJson(encodedTopologyId, encodedId, componentName,
status, loggersTotal, debug, samplingPct) {
var jsonData = {};
jsonData["encodedTopologyId"] = encodedTopologyId;
jsonData["encodedId"] = encodedId;
jsonData["componentName"] = componentName;
-jsonData["startDebugStatus"] = (status === "ACTIVE" && !debug) ? "enabled"
: "disabled";
+jsonData["startDebugStatus"] = (status === "ACTIVE" && loggersTotal!=null
&& loggersTotal!=0 && !debug) ? "enabled" : "disabled";
jsonData["stopDebugStatus"] = (status === "ACTIVE" && debug) ? "enabled" :
"disabled";
+jsonData["loggersDisabled"] = loggersTotal==null || loggersTotal==0;
jsonData["currentSamplingPct"] = samplingPct;
return jsonData;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7d7b3c20/storm-core/src/ui/public/templates/component-page-template.html
--
diff --git a/storm-core/src/ui/public/templates/component-page-template.html
b/storm-core/src/ui/public/templates/component-page-template.html
index 3626291
[1/5] storm git commit: STORM-1632) Disable event logging by default
Repository: storm Updated Branches: refs/heads/1.x-branch dc006daae -> 3d19c644f STORM-1632) Disable event logging by default Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a33bc455 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a33bc455 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a33bc455 Branch: refs/heads/1.x-branch Commit: a33bc455db2c4cce0d355c9c0fa4da60471b859b Parents: f0abfff Author: Roshan Naik Authored: Tue Mar 15 21:26:12 2016 -0700 Committer: Roshan Naik Committed: Tue Mar 15 21:26:12 2016 -0700 -- conf/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/a33bc455/conf/defaults.yaml -- diff --git a/conf/defaults.yaml b/conf/defaults.yaml index 7128cfe..3f6a1c9 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -222,7 +222,7 @@ topology.enable.message.timeouts: true topology.debug: false topology.workers: 1 topology.acker.executors: null -topology.eventlogger.executors: null +topology.eventlogger.executors: 0 topology.tasks: null # maximum amount of time a message has to complete before it's considered failed topology.message.timeout.secs: 30
[1/4] storm git commit: [STORM-1667] Log the IO exception when deleting worker pid dir
Repository: storm Updated Branches: refs/heads/1.x-branch 11f23153a -> dc006daae [STORM-1667] Log the IO exception when deleting worker pid dir Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c5416241 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c5416241 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c5416241 Branch: refs/heads/1.x-branch Commit: c54162418a4a46d99ac360661f78cc5e10cb654a Parents: 652d2f6 Author: zhuol Authored: Wed Mar 30 17:21:41 2016 -0500 Committer: zhuol Committed: Wed Mar 30 17:21:41 2016 -0500 -- .../src/clj/org/apache/storm/daemon/supervisor.clj | 13 +++-- 1 file changed, 7 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/c5416241/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj -- diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj index 7be1421..ced8e7e 100644 --- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj @@ -269,10 +269,7 @@ (catch IOException e (log-warn-error e "Failed to cleanup worker " id ". Will retry later")) (catch RuntimeException e -(log-warn-error e "Failed to cleanup worker " id ". Will retry later") -) - (catch java.io.FileNotFoundException e (log-message (.getMessage e))) -)) +(log-warn-error e "Failed to cleanup worker " id ". Will retry later" (defn shutdown-worker [supervisor id] (log-message "Shutting down " (:supervisor-id supervisor) ":" id) @@ -296,11 +293,15 @@ (worker-launcher-and-wait conf user ["signal" pid "9"] :log-prefix (str "kill -9 " pid)) (force-kill-process pid)) (if as-user -(rmr-as-user conf id (worker-pid-path conf id pid)) (try + (rmr-as-user conf id (worker-pid-path conf id pid)) (rmpath (worker-pid-path conf id pid)) (rmpath (worker-tmp-root conf id pid)) - (catch Exception e ;; on windows, the supervisor may still holds the lock on the worker directory + (catch IOException e +(log-warn-error e "Failed to cleanup pid dir: " pid " for worker " id". Will retry later")) + (catch RuntimeException e +(log-warn-error e "Failed to cleanup pid dir: " pid " for worker " id". Will retry later") + ;; on windows, the supervisor may still holds the lock on the worker directory (try-cleanup-worker conf id)) (log-message "Shut down " (:supervisor-id supervisor) ":" id))
[4/4] storm git commit: Putting STORM-1667 in CHANGELOG.md
Putting STORM-1667 in CHANGELOG.md Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/dc006daa Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/dc006daa Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/dc006daa Branch: refs/heads/1.x-branch Commit: dc006daae603f971535967365fd5ecb5151b3607 Parents: de719aa Author: Kyle Nusbaum Authored: Fri Apr 1 10:11:43 2016 -0500 Committer: Kyle Nusbaum Committed: Fri Apr 1 10:11:43 2016 -0500 -- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/dc006daa/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a044c6..9a1d748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 1.0.0 + * STORM-1667: Log the IO exception when deleting worker pid dir * STORM-1669: Fix SolrUpdateBolt flush bug * STORM-1668: Flux silently fails while setting a non-existent property. * STORM-1573: Add batch support for MongoInsertBolt
[2/4] storm git commit: Back to ignore all exception
Back to ignore all exception Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b3b934a7 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b3b934a7 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b3b934a7 Branch: refs/heads/1.x-branch Commit: b3b934a777e6a2f1ec505e63336c12549cd949c1 Parents: c541624 Author: zhuol Authored: Thu Mar 31 11:12:25 2016 -0500 Committer: zhuol Committed: Thu Mar 31 11:12:25 2016 -0500 -- storm-core/src/clj/org/apache/storm/daemon/supervisor.clj | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/b3b934a7/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj -- diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj index ced8e7e..b8242ca 100644 --- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj @@ -297,9 +297,7 @@ (rmr-as-user conf id (worker-pid-path conf id pid)) (rmpath (worker-pid-path conf id pid)) (rmpath (worker-tmp-root conf id pid)) - (catch IOException e -(log-warn-error e "Failed to cleanup pid dir: " pid " for worker " id". Will retry later")) - (catch RuntimeException e + (catch Exception e (log-warn-error e "Failed to cleanup pid dir: " pid " for worker " id". Will retry later") ;; on windows, the supervisor may still holds the lock on the worker directory (try-cleanup-worker conf id))
[3/4] storm git commit: Merge branch 'Storm-1667-1.x-branch' of https://github.com/zhuoliu/storm into 1.x-branch
Merge branch 'Storm-1667-1.x-branch' of https://github.com/zhuoliu/storm into 1.x-branch Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/de719aa1 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/de719aa1 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/de719aa1 Branch: refs/heads/1.x-branch Commit: de719aa19d0808690751edade4d7907c96ab723a Parents: 11f2315 b3b934a Author: Kyle Nusbaum Authored: Fri Apr 1 10:10:28 2016 -0500 Committer: Kyle Nusbaum Committed: Fri Apr 1 10:10:28 2016 -0500 -- .../src/clj/org/apache/storm/daemon/supervisor.clj | 11 +-- 1 file changed, 5 insertions(+), 6 deletions(-) --
[28/35] storm git commit: Merge branch 'master' into supervisor
Merge branch 'master' into supervisor Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d46ed8fb Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d46ed8fb Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d46ed8fb Branch: refs/heads/master Commit: d46ed8fbe6bbd8116ceb60769f4b0b569bfd2d0d Parents: 7d0551d 3812b2f Author: xiaojian.fxj Authored: Wed Mar 30 17:06:30 2016 +0800 Committer: xiaojian.fxj Committed: Wed Mar 30 17:06:30 2016 +0800 -- CHANGELOG.md| 9 + bin/storm.py| 2 +- docs/Documentation.md | 50 -- docs/Kestrel-and-Storm.md | 2 +- docs/README.md | 64 +- docs/Resource_Aware_Scheduler_overview.md | 2 + docs/Trident-RAS-API.md | 52 ++ docs/index.md | 18 +- docs/storm-cassandra.md | 255 +++ docs/storm-elasticsearch.md | 105 +++ docs/storm-mongodb.md | 199 + docs/storm-mqtt.md | 379 ++ docs/windows-users-guide.md | 21 + examples/storm-starter/pom.xml | 13 +- .../TridentHBaseWindowingStoreTopology.java | 93 +++ .../TridentWindowingInmemoryStoreTopology.java | 98 +++ .../main/java/org/apache/storm/flux/Flux.java | 22 +- .../storm/hbase/bolt/AbstractHBaseBolt.java | 1 - .../org/apache/storm/hbase/bolt/HBaseBolt.java | 18 +- .../trident/windowing/HBaseWindowsStore.java| 273 +++ .../windowing/HBaseWindowsStoreFactory.java | 55 ++ .../org/apache/storm/hive/bolt/HiveBolt.java| 147 ++-- .../apache/storm/hive/common/HiveOptions.java | 8 +- .../org/apache/storm/hive/common/HiveUtils.java | 11 +- .../apache/storm/hive/common/HiveWriter.java| 127 ++-- .../apache/storm/hive/trident/HiveState.java| 38 +- .../storm/hive/trident/HiveStateFactory.java| 1 + .../apache/storm/hive/trident/HiveUpdater.java | 1 + .../storm/hive/bolt/BucketTestHiveTopology.java | 190 + .../apache/storm/hive/bolt/HiveTopology.java| 6 +- .../apache/storm/hive/bolt/TestHiveBolt.java| 11 +- .../storm/hive/common/TestHiveWriter.java | 13 +- .../storm/hive/trident/TridentHiveTopology.java | 2 +- pom.xml | 6 +- storm-core/pom.xml | 2 +- .../apache/storm/daemon/local_supervisor.clj| 4 +- .../org/apache/storm/pacemaker/pacemaker.clj| 242 -- storm-core/src/jvm/org/apache/storm/Config.java | 10 +- .../cluster/PaceMakerStateStorageFactory.java | 3 +- .../storm/daemon/supervisor/StateHeartbeat.java | 2 +- .../storm/daemon/supervisor/Supervisor.java | 5 +- .../storm/metric/StormMetricsRegistry.java | 12 +- .../org/apache/storm/pacemaker/Pacemaker.java | 217 ++ .../jvm/org/apache/storm/scheduler/Cluster.java | 23 +- .../org/apache/storm/scheduler/Topologies.java | 11 +- .../scheduler/resource/ClusterStateData.java| 101 --- .../resource/ResourceAwareScheduler.java| 138 ++-- .../scheduler/resource/SchedulingState.java | 56 ++ .../apache/storm/scheduler/resource/User.java | 24 +- .../eviction/DefaultEvictionStrategy.java | 10 +- .../strategies/eviction/IEvictionStrategy.java | 9 +- .../DefaultSchedulingPriorityStrategy.java | 9 +- .../priority/ISchedulingPriorityStrategy.java | 9 +- .../DefaultResourceAwareStrategy.java | 86 +-- .../strategies/scheduling/IStrategy.java| 6 +- .../serialization/BlowfishTupleSerializer.java | 24 +- .../jvm/org/apache/storm/trident/Stream.java| 207 +- .../apache/storm/trident/TridentTopology.java | 4 + .../storm/trident/fluent/UniqueIdGen.java | 14 +- .../storm/trident/operation/builtin/Debug.java | 4 +- .../windowing/AbstractTridentWindowManager.java | 238 ++ .../windowing/ITridentWindowManager.java| 59 ++ .../windowing/InMemoryTridentWindowManager.java | 72 ++ .../trident/windowing/InMemoryWindowsStore.java | 200 + .../windowing/InMemoryWindowsStoreFactory.java | 46 ++ .../StoreBasedTridentWindowManager.java | 217 ++ .../trident/windowing/TridentBatchTuple.java| 42 ++ .../windowing/WindowTridentProcessor.java | 265 +++ .../storm/trident/windowing/WindowsState.java | 52 ++ .../trident/windowing/WindowsStateFactory.java | 40 + .../trident/windowing/WindowsStateUpdater.java | 81 ++ .../storm/trident/windowing/WindowsStore.java | 78 ++ .../trident/windowing/WindowsStoreFactory.java | 35 + .../windowing/config/BaseWindowConfig.java | 48 ++ .../windowing/config/SlidingCountWindow.java|
[17/35] storm git commit: add the plugin to use for manager worker
http://git-wip-us.apache.org/repos/asf/storm/blob/a1e47352/storm-core/test/clj/org/apache/storm/supervisor_test.clj -- diff --git a/storm-core/test/clj/org/apache/storm/supervisor_test.clj b/storm-core/test/clj/org/apache/storm/supervisor_test.clj index d3d7344..8f11f8a 100644 --- a/storm-core/test/clj/org/apache/storm/supervisor_test.clj +++ b/storm-core/test/clj/org/apache/storm/supervisor_test.clj @@ -22,7 +22,8 @@ (:import [org.apache.storm.testing TestWordCounter TestWordSpout TestGlobalCount TestAggregatesCounter TestPlannerSpout] [org.apache.storm.daemon.supervisor SupervisorUtils SyncProcessEvent SupervisorData] [java.util ArrayList Arrays HashMap] - [org.apache.storm.testing.staticmocking MockedSupervisorUtils]) + [org.apache.storm.testing.staticmocking MockedSupervisorUtils] + [org.apache.storm.daemon.supervisor.workermanager DefaultWorkerManager]) (:import [org.apache.storm.scheduler ISupervisor]) (:import [org.apache.storm.utils Time Utils$UptimeComputer ConfigUtils]) (:import [org.apache.storm.generated RebalanceOptions WorkerResources]) @@ -367,17 +368,19 @@ (setWorkerUserWSEImpl [conf worker-id user] nil) (workerRootImpl [conf] "/tmp/workers") (workerArtifactsRootImpl [conf] "/tmp/workers-artifacts")) + worker-manager (proxy [DefaultWorkerManager] [] + (jlp [stormRoot conf] "")) process-proxy (proxy [SyncProcessEvent] [] - (jlp [stormRoot conf] "") (writeLogMetadata [stormconf user workerId stormId port conf] nil) (createBlobstoreLinks [conf stormId workerId] nil))] (with-open [_ (ConfigUtilsInstaller. cu-proxy) _ (UtilsInstaller. utils-spy)] -(.launchWorker process-proxy mock-supervisor nil +(.prepareWorker worker-manager mock-supervisor nil) +(.launchDistributedWorker process-proxy worker-manager mock-supervisor nil "" mock-storm-id mock-port mock-worker-id - (WorkerResources.) nil nil) + (WorkerResources.) nil) (. (Mockito/verify utils-spy) (launchProcessImpl (Matchers/eq exp-args) (Matchers/any) @@ -405,17 +408,19 @@ (addToClasspathImpl [classpath paths] mock-cp) (launchProcessImpl [& _] nil)) Mockito/spy) + worker-manager (proxy [DefaultWorkerManager] [] + (jlp [stormRoot conf] "")) process-proxy (proxy [SyncProcessEvent] [] - (jlp [stormRoot conf] "") (writeLogMetadata [stormconf user workerId stormId port conf] nil) (createBlobstoreLinks [conf stormId workerId] nil))] (with-open [_ (ConfigUtilsInstaller. cu-proxy) _ (UtilsInstaller. utils-spy)] - (.launchWorker process-proxy mock-supervisor nil + (.prepareWorker worker-manager mock-supervisor nil) + (.launchDistributedWorker process-proxy worker-manager mock-supervisor nil "" mock-storm-id mock-port mock-worker-id -(WorkerResources.) nil nil) +(WorkerResources.) nil) (. (Mockito/verify utils-spy) (launchProcessImpl (Matchers/eq exp-args) (Matchers/any) @@ -441,17 +446,19 @@ (str Utils/FILE_PATH_SEPARATOR "base")) (launchProcessImpl [& _] nil)) Mockito/spy) + worker-manager (proxy [DefaultWorkerManager] [] + (jlp [stormRoot conf] "")) process-proxy (proxy [SyncProcessEvent] [] - (jlp [stormRoot conf] "") (writeLogMetadata [stormconf user workerId stormId port conf] nil) (createBlobstoreLinks [conf stormId workerId] nil))] (with-open [_ (ConfigUtilsInstaller. cu-proxy) _ (UtilsInstaller. utils-spy)] - (.launchWorker process-proxy mock-supervisor nil + (.prepareWorker worker-manager mock-supervisor nil) + (.launchDistributedWorker p
[06/35] storm git commit: update test codes about supervisor
update test codes about supervisor
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/19fcafbd
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/19fcafbd
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/19fcafbd
Branch: refs/heads/master
Commit: 19fcafbd0fe1cbee49e797824c47ba1f6b727270
Parents: b281c73
Author: xiaojian.fxj
Authored: Wed Mar 2 09:00:37 2016 +0800
Committer: xiaojian.fxj
Committed: Fri Mar 4 09:21:09 2016 +0800
--
bin/storm.cmd |2 +-
bin/storm.py|2 +-
.../org/apache/storm/command/kill_workers.clj | 14 +-
.../apache/storm/daemon/local_supervisor.clj| 61 +
.../clj/org/apache/storm/daemon/logviewer.clj |8 +-
.../clj/org/apache/storm/daemon/supervisor.clj | 1356 --
storm-core/src/clj/org/apache/storm/testing.clj | 57 +-
.../storm/daemon/supervisor/ShutdownWork.java | 11 +-
.../daemon/supervisor/StandaloneSupervisor.java |7 +-
.../storm/daemon/supervisor/Supervisor.java | 196 +++
.../storm/daemon/supervisor/SupervisorData.java |5 +-
.../daemon/supervisor/SupervisorServer.java | 201 ---
.../daemon/supervisor/SupervisorUtils.java | 108 +-
.../daemon/supervisor/SyncProcessEvent.java | 246 ++--
.../daemon/supervisor/SyncSupervisorEvent.java | 11 +-
.../supervisor/timer/RunProfilerActions.java|2 -
.../supervisor/timer/SupervisorHeartbeat.java | 12 +-
.../staticmocking/MockedSupervisorUtils.java| 31 +
.../src/jvm/org/apache/storm/utils/Utils.java |4 +-
.../clj/org/apache/storm/logviewer_test.clj | 36 +-
.../clj/org/apache/storm/supervisor_test.clj| 300 ++--
21 files changed, 775 insertions(+), 1895 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/19fcafbd/bin/storm.cmd
--
diff --git a/bin/storm.cmd b/bin/storm.cmd
index 1ef1e42..e84bfb3 100644
--- a/bin/storm.cmd
+++ b/bin/storm.cmd
@@ -214,7 +214,7 @@
goto :eof
:supervisor
- set CLASS=org.apache.storm.daemon.supervisor
+ set CLASS=org.apache.storm.daemon.supervisor.Supervisor
"%JAVA%" -client -Dstorm.options= -Dstorm.conf.file= -cp "%CLASSPATH%"
org.apache.storm.command.ConfigValue supervisor.childopts > %CMD_TEMP_FILE%
FOR /F "delims=" %%i in (%CMD_TEMP_FILE%) do (
FOR /F "tokens=1,* delims= " %%a in ("%%i") do (
http://git-wip-us.apache.org/repos/asf/storm/blob/19fcafbd/bin/storm.py
--
diff --git a/bin/storm.py b/bin/storm.py
index 94d6143..a669783 100755
--- a/bin/storm.py
+++ b/bin/storm.py
@@ -552,7 +552,7 @@ def pacemaker(klass="org.apache.storm.pacemaker.pacemaker"):
extrajars=cppaths,
jvmopts=jvmopts)
-def supervisor(klass="org.apache.storm.daemon.supervisor"):
+def supervisor(klass="org.apache.storm.daemon.supervisor.Supervisor"):
"""Syntax: [storm supervisor]
Launches the supervisor daemon. This command should be run
http://git-wip-us.apache.org/repos/asf/storm/blob/19fcafbd/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
--
diff --git a/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
b/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
index 4e713f9..a7de176 100644
--- a/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
+++ b/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
@@ -14,11 +14,10 @@
;; See the License for the specific language governing permissions and
;; limitations under the License.
(ns org.apache.storm.command.kill-workers
- (:import [java.io File])
+ (:import [java.io File]
+ [org.apache.storm.daemon.supervisor SupervisorUtils
StandaloneSupervisor SupervisorData ShutdownWork])
(:use [org.apache.storm.daemon common])
(:use [org.apache.storm util config])
- (:require [org.apache.storm.daemon
- [supervisor :as supervisor]])
(:import [org.apache.storm.utils ConfigUtils])
(:gen-class))
@@ -27,8 +26,9 @@
[& args]
(let [conf (clojurify-structure (ConfigUtils/readStormConfig))
conf (assoc conf STORM-LOCAL-DIR (. (File. (conf STORM-LOCAL-DIR))
getCanonicalPath))
-isupervisor (supervisor/standalone-supervisor)
-supervisor-data (supervisor/supervisor-data conf nil isupervisor)
-ids (supervisor/my-worker-ids conf)]
+isupervisor (StandaloneSupervisor.)
+supervisor-data (SupervisorData. conf nil isupervisor)
+ids (SupervisorUtils/myWorkerIds conf)
+shut-workers (ShutdownWork.)]
(doseq [id ids]
- (supervisor/shutdown-worker supervisor-data id
+ (.shutWor
[12/35] storm git commit: Merge branch 'master' into supervisor
Merge branch 'master' into supervisor Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/65ce9d2e Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/65ce9d2e Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/65ce9d2e Branch: refs/heads/master Commit: 65ce9d2e03be5f5c4defa8342bfbefe9f59adcf9 Parents: 184dc4a 81fb727 Author: xiaojian.fxj Authored: Thu Mar 10 22:57:01 2016 +0800 Committer: xiaojian.fxj Committed: Thu Mar 10 22:57:01 2016 +0800 -- CHANGELOG.md| 6 + conf/log4j2.xml | 2 +- .../travis/print-errors-from-test-reports.py| 4 + .../org/apache/storm/starter/ManualDRPC.java| 53 ++- .../src/clj/org/apache/storm/LocalDRPC.clj | 56 --- .../org/apache/storm/command/kill_workers.clj | 2 +- .../src/clj/org/apache/storm/daemon/common.clj | 13 +- .../src/clj/org/apache/storm/daemon/drpc.clj| 221 +--- .../apache/storm/daemon/local_supervisor.clj| 2 +- .../clj/org/apache/storm/daemon/logviewer.clj | 27 +- .../src/clj/org/apache/storm/daemon/nimbus.clj | 155 storm-core/src/clj/org/apache/storm/testing.clj | 5 +- .../clj/org/apache/storm/trident/testing.clj| 2 - storm-core/src/clj/org/apache/storm/ui/core.clj | 81 +++-- .../src/clj/org/apache/storm/ui/helpers.clj | 10 +- .../src/jvm/org/apache/storm/LocalDRPC.java | 72 .../jvm/org/apache/storm/daemon/DrpcServer.java | 357 +++ .../storm/daemon/supervisor/ShutdownWork.java | 124 --- .../daemon/supervisor/StandaloneSupervisor.java | 1 - .../storm/daemon/supervisor/Supervisor.java | 35 +- .../storm/daemon/supervisor/SupervisorData.java | 18 - .../daemon/supervisor/SupervisorManger.java | 3 - .../daemon/supervisor/SupervisorUtils.java | 3 - .../daemon/supervisor/SyncProcessEvent.java | 4 +- .../daemon/supervisor/SyncSupervisorEvent.java | 24 +- .../supervisor/timer/SupervisorHealthCheck.java | 1 - .../daemon/supervisor/timer/UpdateBlobs.java| 1 - .../storm/metric/StormMetricsRegistry.java | 86 + .../auth/AbstractSaslClientCallbackHandler.java | 76 .../auth/AbstractSaslServerCallbackHandler.java | 94 + .../auth/digest/ClientCallbackHandler.java | 60 +--- .../auth/digest/ServerCallbackHandler.java | 61 +--- .../auth/plain/PlainClientCallbackHandler.java | 31 ++ .../auth/plain/PlainSaslTransportPlugin.java| 71 .../auth/plain/PlainServerCallbackHandler.java | 55 +++ .../security/auth/plain/SaslPlainServer.java| 158 .../test/clj/org/apache/storm/drpc_test.clj | 27 +- .../storm/security/auth/drpc_auth_test.clj | 5 +- 38 files changed, 1237 insertions(+), 769 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/65ce9d2e/storm-core/src/clj/org/apache/storm/command/kill_workers.clj -- diff --cc storm-core/src/clj/org/apache/storm/command/kill_workers.clj index 4ddc993,4e713f9..aadc9fd --- a/storm-core/src/clj/org/apache/storm/command/kill_workers.clj +++ b/storm-core/src/clj/org/apache/storm/command/kill_workers.clj @@@ -14,10 -14,11 +14,10 @@@ ;; See the License for the specific language governing permissions and ;; limitations under the License. (ns org.apache.storm.command.kill-workers - (:import [java.io File]) + (:import [java.io File] -[org.apache.storm.daemon.supervisor SupervisorUtils StandaloneSupervisor SupervisorData ShutdownWork]) ++ [org.apache.storm.daemon.supervisor SupervisorUtils StandaloneSupervisor SupervisorData]) (:use [org.apache.storm.daemon common]) (:use [org.apache.storm util config]) - (:require [org.apache.storm.daemon - [supervisor :as supervisor]]) (:import [org.apache.storm.utils ConfigUtils]) (:gen-class)) http://git-wip-us.apache.org/repos/asf/storm/blob/65ce9d2e/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj -- diff --cc storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj index 2361817,000..c8ae2d6 mode 100644,00..100644 --- a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj +++ b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj @@@ -1,60 -1,0 +1,60 @@@ +;; Licensed to the Apache Software Foundation (ASF) under one +;; or more contributor license agreements. See the NOTICE file +;; distributed with this work for additional information +;; regarding copyright ownership. The ASF licenses this file +;; to you under the Apache License, Version 2.0 (the +;; "License"); you may not use this file except in compliance +;; with the L
[01/35] storm git commit: port Supervisor to java
Repository: storm
Updated Branches:
refs/heads/master b74320497 -> 3de3afc37
http://git-wip-us.apache.org/repos/asf/storm/blob/08934e29/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncProcessEvent.java
--
diff --git
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncProcessEvent.java
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncProcessEvent.java
new file mode 100644
index 000..af454b9
--- /dev/null
+++
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncProcessEvent.java
@@ -0,0 +1,674 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import clojure.lang.IFn;
+import clojure.lang.RT;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.Config;
+import org.apache.storm.ProcessSimulator;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.daemon.Shutdownable;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.WorkerResources;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * 1. to kill are those in allocated that are dead or disallowed 2. kill the
ones that should be dead - read pids, kill -9 and individually remove file - rmr
+ * heartbeat dir, rmdir pid dir, rmdir id dir (catch exception and log) 3. of
the rest, figure out what assignments aren't yet satisfied 4. generate new
worker
+ * ids, write new "approved workers" to LS 5. create local dir for worker id
5. launch new workers (give worker-id, port, and supervisor-id) 6. wait for
workers
+ * launch
+ */
+public class SyncProcessEvent extends ShutdownWork implements Runnable {
+
+private static Logger LOG =
LoggerFactory.getLogger(SyncProcessEvent.class);
+
+private final LocalState localState;
+
+private IStormClusterState stormClusterState;
+
+private SupervisorData supervisorData;
+
+private class ProcessExitCallback implements Utils.ExitCodeCallable {
+private final String logPrefix;
+private final String workerId;
+
+public ProcessExitCallback(String logPrefix, String workerId) {
+this.logPrefix = logPrefix;
+this.workerId = workerId;
+}
+
+@Override
+public Object call() throws Exception {
+return null;
+}
+
+@Override
+public Object call(int exitCode) {
+LOG.info("{} exited with code: {}", logPrefix, exitCode);
+supervisorData.getDeadWorkers().add(workerId);
+return null;
+}
+}
+
+public SyncProcessEvent(SupervisorData supervisorData) {
+
+this.supervisorData = supervisorData;
+
+this.localState = supervisorData.getLocalState();
+
+this.stormClusterState = supervisorData.getStormClusterState();
+}
+
+/**
+ * 1. to kill are those in allocated that are dead or disallowed 2. kill
the ones that should be dead - read pids, kill -9 and individually remove file -
+ * rmr heartbeat dir, rmdir pid dir, rmdir id dir (catch exception and
log) 3. of the rest, figure out what assignments aren't yet satisfied 4.
generate new
+ * worker ids, write new "approved workers" to LS 5. create local dir for
worker id 5. launch new workers (give worker-id, port, and supervisor-id) 6.
wait
+ * for workers launch
+ */
+@Override
+public void run() {
+LOG.debug("Syncing processes");
+try {
+Map conf = supervisorData.getConf();
+Map assignedExecutors =
localState.getLocalAssignmentsMap();
+if (assignedExecutors == null) {
+assignedExecutors = new HashMap<>();
+}
+int now = Time.curre
[03/35] storm git commit: update supervisor's structure
update supervisor's structure
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b281c735
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b281c735
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b281c735
Branch: refs/heads/master
Commit: b281c735f0089d24407af67586a1b41de45ac382
Parents: 08934e2
Author: xiaojian.fxj
Authored: Fri Feb 26 13:15:56 2016 +0800
Committer: xiaojian.fxj
Committed: Fri Feb 26 13:15:56 2016 +0800
--
.../daemon/supervisor/RunProfilerActions.java | 221 --
.../daemon/supervisor/SupervisorHeartbeat.java | 84 ---
.../daemon/supervisor/SupervisorServer.java | 23 +-
.../storm/daemon/supervisor/UpdateBlobs.java| 103 -
.../supervisor/timer/RunProfilerActions.java| 223 +++
.../supervisor/timer/SupervisorHealthCheck.java | 57 +
.../supervisor/timer/SupervisorHeartbeat.java | 85 +++
.../daemon/supervisor/timer/UpdateBlobs.java| 105 +
8 files changed, 476 insertions(+), 425 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/b281c735/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunProfilerActions.java
--
diff --git
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunProfilerActions.java
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunProfilerActions.java
deleted file mode 100644
index 209c067..000
---
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunProfilerActions.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.supervisor;
-
-import org.apache.storm.Config;
-import org.apache.storm.cluster.IStormClusterState;
-import org.apache.storm.generated.ProfileAction;
-import org.apache.storm.generated.ProfileRequest;
-import org.apache.storm.utils.ConfigUtils;
-import org.apache.storm.utils.Utils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.*;
-
-public class RunProfilerActions implements Runnable {
-private static Logger LOG =
LoggerFactory.getLogger(RunProfilerActions.class);
-
-private Map conf;
-private IStormClusterState stormClusterState;
-private String hostName;
-private String stormHome;
-
-private String profileCmd;
-
-private SupervisorData supervisorData;
-
-private class ActionExitCallback implements Utils.ExitCodeCallable {
-private String stormId;
-private ProfileRequest profileRequest;
-private String logPrefix;
-
-public ActionExitCallback(String stormId, ProfileRequest
profileRequest, String logPrefix) {
-this.stormId = stormId;
-this.profileRequest = profileRequest;
-this.logPrefix = logPrefix;
-}
-
-@Override
-public Object call() throws Exception {
-return null;
-}
-
-@Override
-public Object call(int exitCode) {
-LOG.info("{} profile-action exited for {}", logPrefix, exitCode);
-try {
-stormClusterState.deleteTopologyProfileRequests(stormId,
profileRequest);
-} catch (Exception e) {
-LOG.warn("failed delete profileRequest: " + profileRequest);
-}
-return null;
-}
-}
-
-public RunProfilerActions(SupervisorData supervisorData) {
-this.conf = supervisorData.getConf();
-this.stormClusterState = supervisorData.getStormClusterState();
-this.hostName = supervisorData.getHostName();
-this.stormHome = System.getProperty("storm.home");
-this.profileCmd = (String) (conf.get(Config.WORKER_PROFILER_COMMAND));
-this.supervisorData = supervisorData;
-}
-
-@Override
-public void run() {
-Map> stormIdToActions =
supervisorData.getStormIdToProfileActions()
[25/35] storm git commit: Update SyncSupervisorEvent.java
Update SyncSupervisorEvent.java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7d0551d0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7d0551d0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7d0551d0
Branch: refs/heads/master
Commit: 7d0551d0501bf625a9ae8f592df71150bd160540
Parents: afd2e3f 7536489
Author: xiaojian.fxj
Authored: Fri Mar 25 13:19:27 2016 +0800
Committer: xiaojian.fxj
Committed: Fri Mar 25 13:37:36 2016 +0800
--
.../org/apache/storm/daemon/local_supervisor.clj| 2 +-
.../apache/storm/daemon/supervisor/Supervisor.java | 2 +-
.../storm/daemon/supervisor/SyncProcessEvent.java | 7 +++
.../daemon/supervisor/SyncSupervisorEvent.java | 16 +---
.../supervisor/timer/SupervisorHeartbeat.java | 4 ++--
5 files changed, 12 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/7d0551d0/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncProcessEvent.java
--
diff --cc
storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncProcessEvent.java
index 41fa01d,feb8e03..fb4e7ab
---
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncProcessEvent.java
+++
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncProcessEvent.java
@@@ -82,8 -82,8 +82,7 @@@ public class SyncProcessEvent implement
public SyncProcessEvent(SupervisorData supervisorData) {
init(supervisorData);
}
--
--//TODO: initData is intended to local supervisor, so we will remove them
after porting worker.clj to java
++
public void init(SupervisorData supervisorData){
this.supervisorData = supervisorData;
this.localState = supervisorData.getLocalState();
http://git-wip-us.apache.org/repos/asf/storm/blob/7d0551d0/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java
--
diff --cc
storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java
index 4549d4d,4f33c85..b53db06
---
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java
+++
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java
@@@ -204,17 -204,11 +204,11 @@@ public class SyncSupervisorEvent implem
for (Integer port : intersectAssignment.keySet()) {
List existExecutors =
existingAssignment.get(port).get_executors();
List newExecutors =
newAssignment.get(port).get_executors();
- if (newExecutors.size() != existExecutors.size()) {
- syncProcesses.shutWorker(supervisorData,
supervisorData.getWorkerManager(), vaildPortToWorkerIds.get(port));
- continue;
+ Set setExitExecutors = new
HashSet<>(existExecutors);
+ Set setNewExecutors = new HashSet<>(newExecutors);
-if (setExitExecutors != setNewExecutors){
++if (!setExitExecutors.equals(setNewExecutors)){
+ syncProcesses.killWorker(supervisorData,
supervisorData.getWorkerManager(), vaildPortToWorkerIds.get(port));
}
- for (ExecutorInfo executorInfo : newExecutors) {
- if (!existExecutors.contains(executorInfo)) {
- syncProcesses.shutWorker(supervisorData,
supervisorData.getWorkerManager(), vaildPortToWorkerIds.get(port));
- break;
- }
- }
-
}
}
[29/35] storm git commit: Merge branch 'master' of github.com:apache/storm
Merge branch 'master' of github.com:apache/storm Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8be5417c Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8be5417c Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8be5417c Branch: refs/heads/master Commit: 8be5417c8db89ec74dae1f41589a95d95c5c6aac Parents: 3812b2f 2f1411d Author: xiaojian.fxj Authored: Thu Mar 31 09:21:47 2016 +0800 Committer: xiaojian.fxj Committed: Thu Mar 31 09:21:47 2016 +0800 -- .gitignore | 33 +++-- CHANGELOG.md | 1 + external/flux/.gitignore | 15 --- 3 files changed, 24 insertions(+), 25 deletions(-) --
[05/35] storm git commit: update test codes about supervisor
http://git-wip-us.apache.org/repos/asf/storm/blob/19fcafbd/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
--
diff --git
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
new file mode 100644
index 000..9df7ec1
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.MetricRegistry;
+import org.apache.commons.io.FileUtils;
+import org.apache.storm.Config;
+import org.apache.storm.StormTimer;
+import org.apache.storm.daemon.metrics.MetricsUtils;
+import org.apache.storm.daemon.metrics.reporters.PreparableReporter;
+import org.apache.storm.daemon.supervisor.timer.RunProfilerActions;
+import org.apache.storm.daemon.supervisor.timer.SupervisorHealthCheck;
+import org.apache.storm.daemon.supervisor.timer.SupervisorHeartbeat;
+import org.apache.storm.daemon.supervisor.timer.UpdateBlobs;
+import org.apache.storm.event.EventManagerImp;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.messaging.IContext;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.Utils;
+import org.apache.storm.utils.VersionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.InterruptedIOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class Supervisor {
+private static Logger LOG = LoggerFactory.getLogger(Supervisor.class);
+
+//TODO: to be removed after porting worker.clj. localSyncProcess is
intended to start local supervisor
+private SyncProcessEvent localSyncProcess;
+
+public void setLocalSyncProcess(SyncProcessEvent localSyncProcess) {
+this.localSyncProcess = localSyncProcess;
+}
+
+
+/**
+ * in local state, supervisor stores who its current assignments are
another thread launches events to restart any dead processes if necessary
+ *
+ * @param conf
+ * @param sharedContext
+ * @param iSupervisor
+ * @return
+ * @throws Exception
+ */
+public SupervisorManger mkSupervisor(final Map conf, IContext
sharedContext, ISupervisor iSupervisor) throws Exception {
+SupervisorManger supervisorManger = null;
+try {
+LOG.info("Starting Supervisor with conf {}", conf);
+iSupervisor.prepare(conf,
ConfigUtils.supervisorIsupervisorDir(conf));
+String path = ConfigUtils.supervisorTmpDir(conf);
+FileUtils.cleanDirectory(new File(path));
+
+final SupervisorData supervisorData = new SupervisorData(conf,
sharedContext, iSupervisor);
+Localizer localizer = supervisorData.getLocalizer();
+
+SupervisorHeartbeat hb = new SupervisorHeartbeat(conf,
supervisorData);
+hb.run();
+// should synchronize supervisor so it doesn't launch anything
after being down (optimization)
+Integer heartbeatFrequency =
Utils.getInt(conf.get(Config.SUPERVISOR_HEARTBEAT_FREQUENCY_SECS));
+supervisorData.getHeartbeatTimer().scheduleRecurring(0,
heartbeatFrequency, hb);
+
+Set downdedStormId =
SupervisorUtils.readDownLoadedStormIds(conf);
+for (String stormId : downdedStormId) {
+SupervisorUtils.addBlobReferences(localizer, stormId, conf);
+}
+// do this after adding the references so we don't try to clean
things being used
+localizer.startCleaner();
+
+EventManagerImp syncSupEventManager = new EventManagerImp(false);
+EventManagerImp syncProcessManager = new EventManagerImp(false);
+
+SyncProcessEvent syncProcessEvent = null;
+if (ConfigUtils.isLocalMode(conf)){
+localSyncProcess.init(supervisorData);
+syncProcessEvent = localSyncProcess;
+
[08/35] storm git commit: Merge branch 'master' into supervisor
Merge branch 'master' into supervisor
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/42bacde2
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/42bacde2
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/42bacde2
Branch: refs/heads/master
Commit: 42bacde20ea86867b874395532aa034cfad4f120
Parents: b09b412 96f81d7
Author: xiaojian.fxj
Authored: Sun Mar 6 16:05:14 2016 +0800
Committer: xiaojian.fxj
Committed: Sun Mar 6 16:17:47 2016 +0800
--
CHANGELOG.md| 6 ++
.../src/clj/org/apache/storm/MockAutoCred.clj | 58 ---
.../storm/cluster/StormClusterStateImpl.java| 7 +-
.../storm/daemon/metrics/MetricsUtils.java | 2 +-
.../storm/daemon/supervisor/ShutdownWork.java | 1 +
.../daemon/supervisor/SyncProcessEvent.java | 4 ++
.../daemon/supervisor/SyncSupervisorEvent.java | 45 +++-
.../jvm/org/apache/storm/drpc/DRPCSpout.java| 2 +
.../jvm/org/apache/storm/utils/ConfigUtils.java | 10 +++
.../test/clj/org/apache/storm/nimbus_test.clj | 10 +--
.../clj/org/apache/storm/supervisor_test.clj| 10 ++-
.../test/jvm/org/apache/storm/MockAutoCred.java | 75
12 files changed, 162 insertions(+), 68 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/42bacde2/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ShutdownWork.java
--
diff --cc
storm-core/src/jvm/org/apache/storm/daemon/supervisor/ShutdownWork.java
index 19328e5,000..5018ce1
mode 100644,00..100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ShutdownWork.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ShutdownWork.java
@@@ -1,124 -1,0 +1,125 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.Config;
+import org.apache.storm.ProcessSimulator;
+import org.apache.storm.daemon.Shutdownable;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+public class ShutdownWork implements Shutdownable {
+
+private static Logger LOG = LoggerFactory.getLogger(ShutdownWork.class);
+
+public void shutWorker(SupervisorData supervisorData, String workerId)
throws IOException, InterruptedException {
+LOG.info("Shutting down {}:{}", supervisorData.getSupervisorId(),
workerId);
+Map conf = supervisorData.getConf();
+Collection pids =
Utils.readDirContents(ConfigUtils.workerPidsRoot(conf, workerId));
+Integer shutdownSleepSecs =
Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS));
+Boolean asUser =
Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false);
+String user = ConfigUtils.getWorkerUser(conf, workerId);
+String threadPid =
supervisorData.getWorkerThreadPidsAtom().get(workerId);
+if (StringUtils.isNotBlank(threadPid)) {
+ProcessSimulator.killProcess(threadPid);
+}
+
+for (String pid : pids) {
+if (asUser) {
+List commands = new ArrayList<>();
+commands.add("signal");
+commands.add(pid);
+commands.add("15");
+String logPrefix = "kill - 15 " + pid;
+SupervisorUtils.workerLauncherAndWait(conf, user, commands,
null, logPrefix);
+} else {
+Utils.killProcessWithSigTerm(pid);
+}
+}
+
+if (pids.size() > 0) {
+LOG.info("Sleep {} seconds for execution of cleanup threads on
worker.", shutdownSleepSecs);
+Time.sleepSecs(shutdownSleepSecs);
+}
+
+for (String pi
[22/35] storm git commit: update
update
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/724f5d2c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/724f5d2c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/724f5d2c
Branch: refs/heads/master
Commit: 724f5d2cea8debea8c6fb6a0d42d275880636834
Parents: f03b8be
Author: xiaojian.fxj
Authored: Thu Mar 24 01:29:20 2016 +0800
Committer: xiaojian.fxj
Committed: Thu Mar 24 01:29:20 2016 +0800
--
.../src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/724f5d2c/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
--
diff --git
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
index 1dd44a9..2b5078b 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
@@ -156,7 +156,7 @@ public class Supervisor {
}
private void registerWorkerNumGauge(String name, final Map conf) {
-StormMetricsRegistry.registerGauge("supervisor:num-slots-used-gauge",
new Callable() {
+StormMetricsRegistry.registerGauge(name, new Callable() {
@Override
public Integer call() throws Exception {
Collection pids =
SupervisorUtils.supervisorWorkerIds(conf);
[20/35] storm git commit: start worker successfully
start worker successfully
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/42928c21
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/42928c21
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/42928c21
Branch: refs/heads/master
Commit: 42928c2182cf2b755c6f98ad039b2e858787dfe4
Parents: b49c995
Author: xiaojian.fxj
Authored: Tue Mar 15 00:16:19 2016 +0800
Committer: xiaojian.fxj
Committed: Tue Mar 15 00:16:19 2016 +0800
--
.../src/clj/org/apache/storm/daemon/nimbus.clj | 4 +--
.../workermanager/DefaultWorkerManager.java | 33 +++-
2 files changed, 21 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/42928c21/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
--
diff --git a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
index e6fd0a2..5820ee9 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
@@ -1353,8 +1353,8 @@
(str "Failed to submit topology. Topology requests more than "
workers-allowed " workers."))
(defn nimbus-topology-bases [storm-cluster-state]
- map-val #(clojurify-storm-base %) (clojurify-structure
-(StormCommon/topologyBases
storm-cluster-state)))
+ (map-val #(clojurify-storm-base %) (clojurify-structure
+(StormCommon/topologyBases
storm-cluster-state
(defn- set-logger-timeouts [log-config]
(let [timeout-secs (.get_reset_log_level_timeout_secs log-config)
http://git-wip-us.apache.org/repos/asf/storm/blob/42928c21/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
--
diff --git
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
index b19fd89..a73a9bd 100644
---
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
+++
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
@@ -340,7 +340,7 @@ public class DefaultWorkerManager implements IWorkerManager
{
String os = System.getProperty("os.name").replaceAll("\\s+", "_");
String arch = System.getProperty("os.arch");
String archResourceRoot = resourceRoot + Utils.FILE_PATH_SEPARATOR +
os + "-" + arch;
-String ret = archResourceRoot + Utils.FILE_PATH_SEPARATOR +
resourceRoot + Utils.FILE_PATH_SEPARATOR + conf.get(Config.JAVA_LIBRARY_PATH);
+String ret = archResourceRoot + Utils.CLASS_PATH_SEPARATOR +
resourceRoot + Utils.CLASS_PATH_SEPARATOR + conf.get(Config.JAVA_LIBRARY_PATH);
return ret;
}
@@ -373,23 +373,28 @@ public class DefaultWorkerManager implements
IWorkerManager {
List rets = new ArrayList<>();
if (value instanceof String) {
String string = (String) value;
-string = string.replace("%ID%", String.valueOf(port));
-string = string.replace("%WORKER-ID%", workerId);
-string = string.replace("%TOPOLOGY-ID%", stormId);
-string = string.replace("%WORKER-PORT%", String.valueOf(port));
-string = string.replace("%HEAP-MEM%", String.valueOf(memOnheap));
-String[] strings = string.split("\\s+");
-rets.addAll(Arrays.asList(strings));
+if (StringUtils.isNotBlank(string)){
+string = string.replace("%ID%", String.valueOf(port));
+string = string.replace("%WORKER-ID%", workerId);
+string = string.replace("%TOPOLOGY-ID%", stormId);
+string = string.replace("%WORKER-PORT%", String.valueOf(port));
+string = string.replace("%HEAP-MEM%",
String.valueOf(memOnheap));
+String[] strings = string.split("\\s+");
+rets.addAll(Arrays.asList(strings));
+}
+
} else if (value instanceof List) {
List objects = (List) value;
for (Object object : objects) {
String str = (String) object;
-str = str.replace("%ID%", String.valueOf(port));
-str = str.replace("%WORKER-ID%", workerId);
-str = str.replace("%TOPOLOGY-ID%", stormId);
-str = str.replace("%WORKER-PORT%", String.valueOf(port));
-str = str.replace("%HEAP-MEM%", String.valueOf(memOnheap));
-rets.add(str);
+if (StringUtils.isN
[35/35] storm git commit: Added STORM-1279 to Changelog
Added STORM-1279 to Changelog Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3de3afc3 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3de3afc3 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3de3afc3 Branch: refs/heads/master Commit: 3de3afc37498ebf007c0f8d07146a2afe286a507 Parents: 20d05e3 Author: Robert (Bobby) Evans Authored: Fri Apr 1 08:19:19 2016 -0500 Committer: Robert (Bobby) Evans Committed: Fri Apr 1 08:19:19 2016 -0500 -- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/storm/blob/3de3afc3/CHANGELOG.md -- diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b05797..3b0339b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 2.0.0 + * STORM-1279: port backtype.storm.daemon.supervisor to java * STORM-1668: Flux silently fails while setting a non-existent property. * STORM-1271: Port backtype.storm.daemon.task to java * STORM-822: Kafka Spout New Consumer API
[11/35] storm git commit: Merge branch 'master' into supervisor
http://git-wip-us.apache.org/repos/asf/storm/blob/65ce9d2e/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java
--
diff --cc
storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java
index e96395f,000..4c08014
mode 100644,00..100644
---
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java
+++
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java
@@@ -1,641 -1,0 +1,631 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.storm.Config;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.cluster.IStateStorage;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.event.EventManager;
+import org.apache.storm.generated.*;
+import org.apache.storm.localizer.LocalResource;
+import org.apache.storm.localizer.LocalizedResource;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.utils.*;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.JarURLConnection;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class SyncSupervisorEvent implements Runnable {
+
+private static final Logger LOG =
LoggerFactory.getLogger(SyncSupervisorEvent.class);
+
+private EventManager syncSupEventManager;
+private EventManager syncProcessManager;
-
+private IStormClusterState stormClusterState;
-
+private LocalState localState;
-
+private SyncProcessEvent syncProcesses;
+private SupervisorData supervisorData;
+
+public SyncSupervisorEvent(SupervisorData supervisorData,
SyncProcessEvent syncProcesses, EventManager syncSupEventManager,
+EventManager syncProcessManager) {
+
+this.syncProcesses = syncProcesses;
+this.syncSupEventManager = syncSupEventManager;
+this.syncProcessManager = syncProcessManager;
+this.stormClusterState = supervisorData.getStormClusterState();
+this.localState = supervisorData.getLocalState();
+this.supervisorData = supervisorData;
+}
+
+@Override
+public void run() {
+try {
+Map conf = supervisorData.getConf();
+Runnable syncCallback = new EventManagerPushCallback(this,
syncSupEventManager);
+List stormIds =
stormClusterState.assignments(syncCallback);
+Map> assignmentsSnapshot =
+getAssignmentsSnapshot(stormClusterState, stormIds,
supervisorData.getAssignmentVersions().get(), syncCallback);
+Map> stormIdToProfilerActions =
getProfileActions(stormClusterState, stormIds);
+
+Set allDownloadedTopologyIds =
SupervisorUtils.readDownLoadedStormIds(conf);
+Map stormcodeMap =
readStormCodeLocations(assignmentsSnapshot);
+Map existingAssignment =
localState.getLocalAssignmentsMap();
- if (existingAssignment == null){
++if (existingAssignment == null) {
+existingAssignment = new HashMap<>();
+}
+
+Map allAssignment =
+readAssignments(assignmentsSnapshot, existingAssignment,
supervisorData.getAssignmentId(), supervisorData.getSyncRetry());
+
-
+Map newAssignment = new HashMap<>();
+Set assignedStormIds = new HashSet<>();
+
+for (Map.Entry entry :
allAssignment.entrySet()) {
+if
(supervisorData.getiSupervisor().confirmAssigned(entry.getKey())) {
+newAssignment.put(entry.getKey(), entry.getValue());
+assignedStormIds.add(entry.getValue().get_topology_id());
+}
+
[34/35] storm git commit: Merge branch 'supervisor' of https://github.com/hustfxj/storm into STORM-1279
Merge branch 'supervisor' of https://github.com/hustfxj/storm into STORM-1279 STORM-1279: port backtype.storm.daemon.supervisor to java Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/20d05e30 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/20d05e30 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/20d05e30 Branch: refs/heads/master Commit: 20d05e30f956b2905a0816d5dc6aad79dfc2d75f Parents: b743204 ac9942c Author: Robert (Bobby) Evans Authored: Fri Apr 1 08:18:52 2016 -0500 Committer: Robert (Bobby) Evans Committed: Fri Apr 1 08:18:52 2016 -0500 -- bin/storm.cmd |2 +- bin/storm.py|2 +- conf/defaults.yaml |4 + .../org/apache/storm/command/kill_workers.clj | 20 +- .../apache/storm/daemon/local_supervisor.clj| 64 + .../clj/org/apache/storm/daemon/logviewer.clj |6 +- .../clj/org/apache/storm/daemon/supervisor.clj | 1391 -- storm-core/src/clj/org/apache/storm/testing.clj | 64 +- storm-core/src/jvm/org/apache/storm/Config.java |7 + .../DefaultUncaughtExceptionHandler.java| 31 + .../supervisor/EventManagerPushCallback.java| 37 + .../daemon/supervisor/StandaloneSupervisor.java | 83 ++ .../apache/storm/daemon/supervisor/State.java | 22 + .../storm/daemon/supervisor/StateHeartbeat.java | 45 + .../storm/daemon/supervisor/Supervisor.java | 177 +++ .../daemon/supervisor/SupervisorDaemon.java | 28 + .../storm/daemon/supervisor/SupervisorData.java | 234 +++ .../daemon/supervisor/SupervisorManager.java| 92 ++ .../daemon/supervisor/SupervisorUtils.java | 286 .../daemon/supervisor/SyncProcessEvent.java | 427 ++ .../daemon/supervisor/SyncSupervisorEvent.java | 633 .../supervisor/timer/RunProfilerActions.java| 211 +++ .../supervisor/timer/SupervisorHealthCheck.java | 52 + .../supervisor/timer/SupervisorHeartbeat.java | 87 ++ .../daemon/supervisor/timer/UpdateBlobs.java| 105 ++ .../workermanager/DefaultWorkerManager.java | 401 + .../workermanager/IWorkerManager.java | 35 + .../storm/metric/StormMetricsRegistry.java |2 +- .../staticmocking/MockedSupervisorUtils.java| 31 + .../src/jvm/org/apache/storm/utils/Utils.java |4 +- .../clj/org/apache/storm/logviewer_test.clj | 36 +- .../clj/org/apache/storm/supervisor_test.clj| 327 ++-- 32 files changed, 3359 insertions(+), 1587 deletions(-) --
[26/35] storm git commit: Merge branch 'master' of github.com:apache/storm
Merge branch 'master' of github.com:apache/storm Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a48e9535 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a48e9535 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a48e9535 Branch: refs/heads/master Commit: a48e9535fd8bcd6de195a70af6102535b60d56d6 Parents: 0100898 31db7dc Author: xiaojian.fxj Authored: Mon Mar 28 13:37:42 2016 +0800 Committer: xiaojian.fxj Committed: Mon Mar 28 13:37:42 2016 +0800 -- CHANGELOG.md| 5 + docs/Documentation.md | 50 --- docs/Kestrel-and-Storm.md | 2 +- docs/README.md | 50 +++ docs/index.md | 17 +- docs/storm-cassandra.md | 255 + docs/storm-elasticsearch.md | 105 + docs/storm-mongodb.md | 199 ++ docs/storm-mqtt.md | 379 +++ docs/windows-users-guide.md | 21 + examples/storm-starter/pom.xml | 13 +- .../TridentHBaseWindowingStoreTopology.java | 93 + .../TridentWindowingInmemoryStoreTopology.java | 98 + .../main/java/org/apache/storm/flux/Flux.java | 22 +- .../storm/hbase/bolt/AbstractHBaseBolt.java | 1 - .../org/apache/storm/hbase/bolt/HBaseBolt.java | 18 +- .../trident/windowing/HBaseWindowsStore.java| 273 + .../windowing/HBaseWindowsStoreFactory.java | 55 +++ pom.xml | 6 +- storm-core/pom.xml | 2 +- storm-core/src/jvm/org/apache/storm/Config.java | 8 + .../serialization/BlowfishTupleSerializer.java | 24 +- .../jvm/org/apache/storm/trident/Stream.java| 207 -- .../apache/storm/trident/TridentTopology.java | 4 + .../storm/trident/fluent/UniqueIdGen.java | 14 +- .../storm/trident/operation/builtin/Debug.java | 4 +- .../windowing/AbstractTridentWindowManager.java | 238 .../windowing/ITridentWindowManager.java| 59 +++ .../windowing/InMemoryTridentWindowManager.java | 72 .../trident/windowing/InMemoryWindowsStore.java | 200 ++ .../windowing/InMemoryWindowsStoreFactory.java | 46 +++ .../StoreBasedTridentWindowManager.java | 217 +++ .../trident/windowing/TridentBatchTuple.java| 42 ++ .../windowing/WindowTridentProcessor.java | 265 + .../storm/trident/windowing/WindowsState.java | 52 +++ .../trident/windowing/WindowsStateFactory.java | 40 ++ .../trident/windowing/WindowsStateUpdater.java | 81 .../storm/trident/windowing/WindowsStore.java | 78 .../trident/windowing/WindowsStoreFactory.java | 35 ++ .../windowing/config/BaseWindowConfig.java | 48 +++ .../windowing/config/SlidingCountWindow.java| 43 +++ .../windowing/config/SlidingDurationWindow.java | 44 +++ .../windowing/config/TumblingCountWindow.java | 43 +++ .../config/TumblingDurationWindow.java | 42 ++ .../trident/windowing/config/WindowConfig.java | 57 +++ .../windowing/strategy/BaseWindowStrategy.java | 32 ++ .../strategy/SlidingCountWindowStrategy.java| 59 +++ .../strategy/SlidingDurationWindowStrategy.java | 60 +++ .../strategy/TumblingCountWindowStrategy.java | 60 +++ .../TumblingDurationWindowStrategy.java | 60 +++ .../windowing/strategy/WindowStrategy.java | 45 +++ .../apache/storm/windowing/TriggerHandler.java | 2 +- .../storm/trident/TridentWindowingTest.java | 105 + 53 files changed, 3910 insertions(+), 140 deletions(-) --
[19/35] storm git commit: Merge branch 'master' into supervisor
Merge branch 'master' into supervisor Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b49c9954 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b49c9954 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b49c9954 Branch: refs/heads/master Commit: b49c99541ae9c2c3f86d9823c64d30765f7716c6 Parents: a1e4735 c2cf3be Author: xiaojian.fxj Authored: Mon Mar 14 18:56:59 2016 +0800 Committer: xiaojian.fxj Committed: Mon Mar 14 20:30:16 2016 +0800 -- .gitignore | 1 + CHANGELOG.md| 5 + bin/storm | 19 ++- bin/storm.py| 2 +- conf/defaults.yaml | 3 +- pom.xml | 2 +- .../daemon/supervisor/SyncSupervisorEvent.java | 7 +- .../apache/storm/messaging/netty/Client.java| 34 ++ .../clj/org/apache/storm/serialization_test.clj | 85 ++ .../org/apache/storm/TestConfigValidate.java| 20 .../storm/serialization/SerializationTest.java | 115 +++ 11 files changed, 196 insertions(+), 97 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/b49c9954/bin/storm.py -- http://git-wip-us.apache.org/repos/asf/storm/blob/b49c9954/conf/defaults.yaml -- diff --cc conf/defaults.yaml index da25ef8,215a84d..9423ed7 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@@ -287,12 -287,10 +287,14 @@@ storm.daemon.metrics.reporter.plugins - "org.apache.storm.daemon.metrics.reporters.JmxPreparableReporter" storm.resource.isolation.plugin: "org.apache.storm.container.cgroup.CgroupManager" + # Also determines whether the unit tests for cgroup runs. + # If storm.resource.isolation.plugin.enable is set to false the unit tests for cgroups will not run storm.resource.isolation.plugin.enable: false + +# Default plugin to use for manager worker +storm.supervisor.worker.manager.plugin: org.apache.storm.daemon.supervisor.workermanager.DefaultWorkerManager + # Configs for CGroup support storm.cgroup.hierarchy.dir: "/cgroup/storm_resources" storm.cgroup.resources: http://git-wip-us.apache.org/repos/asf/storm/blob/b49c9954/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java -- diff --cc storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java index 47cf440,000..4549d4d mode 100644,00..100644 --- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java +++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SyncSupervisorEvent.java @@@ -1,637 -1,0 +1,632 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.storm.daemon.supervisor; + +import org.apache.commons.io.FileUtils; +import org.apache.storm.Config; +import org.apache.storm.blobstore.BlobStore; +import org.apache.storm.blobstore.ClientBlobStore; +import org.apache.storm.cluster.IStateStorage; +import org.apache.storm.cluster.IStormClusterState; +import org.apache.storm.event.EventManager; +import org.apache.storm.generated.*; +import org.apache.storm.localizer.LocalResource; +import org.apache.storm.localizer.LocalizedResource; +import org.apache.storm.localizer.Localizer; +import org.apache.storm.utils.*; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.JarURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +public class SyncSupervisorEvent implements Runnable
commits@storm.apache.org
Merge branch 'master' into supervisor and update supervisor based STORM-1631&STORM-1636 Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f03b8bec Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f03b8bec Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f03b8bec Branch: refs/heads/master Commit: f03b8bec105e88282211bf3e7dd4be4aeed484d8 Parents: 42928c2 2886737 Author: xiaojian.fxj Authored: Wed Mar 23 13:53:00 2016 +0800 Committer: xiaojian.fxj Committed: Wed Mar 23 14:40:22 2016 +0800 -- .gitignore |1 + CHANGELOG.md| 18 +- README.markdown |4 +- docs/Acking-framework-implementation.md | 36 + docs/Clojure-DSL.md | 266 + docs/Command-line-client.md | 104 + docs/Common-patterns.md | 100 + docs/Concepts.md| 115 + docs/Configuration.md | 31 + docs/Contributing-to-Storm.md | 33 + docs/Creating-a-new-Storm-project.md| 25 + docs/DSLs-and-multilang-adapters.md | 10 + docs/Daemon-Fault-Tolerance.md | 30 + ...Defining-a-non-jvm-language-dsl-for-storm.md | 38 + docs/Distributed-RPC.md | 199 + docs/Documentation.md | 50 + docs/FAQ.md | 127 + docs/Fault-tolerance.md | 28 + docs/Guaranteeing-message-processing.md | 181 + docs/Hooks.md |9 + docs/Implementation-docs.md | 13 + docs/Installing-native-dependencies.md | 38 + docs/Kestrel-and-Storm.md | 200 + docs/Lifecycle-of-a-topology.md | 82 + docs/Local-mode.md | 29 + docs/Logs.md| 30 + docs/Maven.md | 22 + docs/Message-passing-implementation.md | 30 + docs/Metrics.md | 36 + docs/Multilang-protocol.md | 287 + docs/Pacemaker.md | 113 + docs/Powered-By.md | 1028 +++ docs/Project-ideas.md |6 + docs/README.md | 61 + docs/Rationale.md | 33 + docs/Resource_Aware_Scheduler_overview.md | 232 + ...unning-topologies-on-a-production-cluster.md | 77 + docs/SECURITY.md| 478 ++ docs/STORM-UI-REST-API.md | 1017 +++ docs/Serialization-(prior-to-0.6.0).md | 50 + docs/Serialization.md | 62 + docs/Serializers.md |4 + docs/Setting-up-a-Storm-cluster.md | 117 + docs/Setting-up-a-Storm-project-in-Eclipse.md |1 + docs/Setting-up-development-environment.md | 33 + docs/Spout-implementations.md | 10 + docs/State-checkpointing.md | 160 + ...guage-protocol-(versions-0.7.0-and-below).md | 122 + docs/Structure-of-the-codebase.md | 134 + docs/Support-for-non-java-languages.md |9 + docs/Transactional-topologies.md| 361 + docs/Trident-API-Overview.md| 525 ++ docs/Trident-spouts.md | 44 + docs/Trident-state.md | 331 + docs/Trident-tutorial.md| 254 + docs/Troubleshooting.md | 182 + docs/Tutorial.md| 320 + ...nding-the-parallelism-of-a-Storm-topology.md | 123 + docs/Using-non-JVM-languages-with-Storm.md | 53 + docs/Windowing.md | 239 + docs/_config.yml| 18 + docs/_includes/footer.html | 55 + docs/_includes/head.html| 34 + docs/_includes/header.html | 59 + docs/_layouts/about.html| 43 + docs/_layouts/default.html | 18 + docs/_layouts/documentation.html|9 + docs/_layouts/page.html |5 + docs/_layouts/post.html | 61 + docs/_plugins/releases.rb | 84 + docs/_sass/_syntax-highlighting.scss| 70 + docs/assets/css/bootstrap-theme.css | 470 ++ docs/assets/css/bootstrap-theme.css.map |1 + docs/assets/css/bootstrap-theme.min.css |5 + docs/assets/css/bootstrap.css
[32/35] storm git commit: Merge branch 'master' of github.com:apache/storm
Merge branch 'master' of github.com:apache/storm Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1b4edf42 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1b4edf42 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1b4edf42 Branch: refs/heads/master Commit: 1b4edf4225cadae99d0a88d51d1b74dc7a9347ce Parents: 8be5417 33e4994 Author: xiaojian.fxj Authored: Fri Apr 1 09:03:01 2016 +0800 Committer: xiaojian.fxj Committed: Fri Apr 1 09:03:01 2016 +0800 -- CHANGELOG.md| 6 + docs/Trident-API-Overview.md| 100 examples/storm-starter/pom.xml | 16 +- external/flux/flux-examples/pom.xml | 13 +- external/flux/pom.xml | 13 +- external/sql/storm-sql-kafka/pom.xml| 16 +- external/storm-kafka-client/README.md | 9 + external/storm-kafka-client/pom.xml | 86 +++ .../apache/storm/kafka/spout/KafkaSpout.java| 547 +++ .../storm/kafka/spout/KafkaSpoutConfig.java | 309 +++ .../storm/kafka/spout/KafkaSpoutMessageId.java | 101 .../KafkaSpoutRetryExponentialBackoff.java | 281 ++ .../kafka/spout/KafkaSpoutRetryService.java | 72 +++ .../storm/kafka/spout/KafkaSpoutStream.java | 70 +++ .../storm/kafka/spout/KafkaSpoutStreams.java| 158 ++ .../kafka/spout/KafkaSpoutTupleBuilder.java | 58 ++ .../kafka/spout/KafkaSpoutTuplesBuilder.java| 82 +++ .../kafka/spout/test/KafkaSpoutTestBolt.java| 50 ++ .../spout/test/KafkaSpoutTopologyMain.java | 133 + .../spout/test/TopicTest2TupleBuilder.java | 40 ++ .../test/TopicsTest0Test1TupleBuilder.java | 42 ++ external/storm-kafka/pom.xml| 16 +- .../storm/mongodb/bolt/MongoInsertBolt.java | 74 ++- .../storm/mongodb/bolt/MongoUpdateBolt.java | 3 +- .../storm/mongodb/common/MongoDBClient.java | 20 +- .../storm/mongodb/trident/state/MongoState.java | 2 +- external/storm-solr/pom.xml | 8 +- pom.xml | 57 +- storm-core/pom.xml | 6 - storm-core/src/clj/org/apache/storm/config.clj | 27 - .../clj/org/apache/storm/daemon/executor.clj| 222 +++- .../src/clj/org/apache/storm/daemon/nimbus.clj | 3 +- .../src/clj/org/apache/storm/daemon/task.clj| 190 --- .../org/apache/storm/daemon/GrouperFactory.java | 243 .../src/jvm/org/apache/storm/daemon/Task.java | 247 + .../daemon/metrics/BuiltinMetricsUtil.java | 8 +- .../apache/storm/hooks/info/BoltAckInfo.java| 8 + .../storm/hooks/info/BoltExecuteInfo.java | 8 + .../apache/storm/hooks/info/BoltFailInfo.java | 8 + .../org/apache/storm/hooks/info/EmitInfo.java | 9 + .../apache/storm/hooks/info/SpoutAckInfo.java | 9 + .../apache/storm/hooks/info/SpoutFailInfo.java | 9 + .../jvm/org/apache/storm/stats/StatsUtil.java | 23 +- .../jvm/org/apache/storm/utils/ConfigUtils.java | 35 +- .../test/clj/org/apache/storm/grouping_test.clj | 19 +- storm-dist/binary/src/main/assembly/binary.xml | 14 + 46 files changed, 2979 insertions(+), 491 deletions(-) --
[15/35] storm git commit: resolve conflict when merge with master
resolve conflict when merge with master Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/95bf6734 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/95bf6734 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/95bf6734 Branch: refs/heads/master Commit: 95bf67347cad7c11aeaf55b7588e627be298d1c2 Parents: 69c8b3c Author: xiaojian.fxj Authored: Thu Mar 10 23:49:52 2016 +0800 Committer: xiaojian.fxj Committed: Thu Mar 10 23:49:52 2016 +0800 -- storm-core/src/clj/org/apache/storm/testing.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/95bf6734/storm-core/src/clj/org/apache/storm/testing.clj -- diff --git a/storm-core/src/clj/org/apache/storm/testing.clj b/storm-core/src/clj/org/apache/storm/testing.clj index 4cec39a..d2d2671 100644 --- a/storm-core/src/clj/org/apache/storm/testing.clj +++ b/storm-core/src/clj/org/apache/storm/testing.clj @@ -296,7 +296,7 @@ [(:nimbus cluster-map)] ; because a worker may already be dead workers)] -(while-timeout timeout-ms (or (not (every? (memfn waiting?) daemons)) +(while-timeout timeout-ms (or (not (every? (memfn isWaiting?) daemons)) (not (every? is-supervisor-waiting supervisors))) (Thread/sleep (rand-int 20)) ;; (doseq [d daemons]
[31/35] storm git commit: update supervisor based on revans2 and longdafeng
update supervisor based on revans2 and longdafeng
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/dba69b52
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/dba69b52
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/dba69b52
Branch: refs/heads/master
Commit: dba69b528860b29fed435c85ee1f76b09f982105
Parents: d46ed8f 8be5417
Author: xiaojian.fxj
Authored: Thu Mar 31 09:22:13 2016 +0800
Committer: xiaojian.fxj
Committed: Thu Mar 31 13:24:15 2016 +0800
--
.gitignore | 33 ++
CHANGELOG.md| 1 +
external/flux/.gitignore| 15 -
.../storm/daemon/supervisor/SupervisorData.java | 46 +++---
.../daemon/supervisor/SupervisorManager.java| 19 ++
.../daemon/supervisor/SupervisorUtils.java | 27 +++--
.../daemon/supervisor/SyncProcessEvent.java | 2 +-
.../daemon/supervisor/SyncSupervisorEvent.java | 41 +++--
.../supervisor/timer/RunProfilerActions.java| 63 +---
.../supervisor/timer/SupervisorHealthCheck.java | 14 +
.../workermanager/DefaultWorkerManager.java | 41 ++---
.../workermanager/IWorkerManager.java | 11 ++--
.../supervisor/workermanager/IWorkerResult.java | 21 ---
13 files changed, 148 insertions(+), 186 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/dba69b52/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorData.java
--
diff --cc
storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorData.java
index 213457d,000..da4102c
mode 100644,00..100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorData.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorData.java
@@@ -1,234 -1,0 +1,234 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.storm.Config;
+import org.apache.storm.StormTimer;
+import org.apache.storm.cluster.ClusterStateContext;
+import org.apache.storm.cluster.ClusterUtils;
+import org.apache.storm.cluster.DaemonType;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.daemon.supervisor.workermanager.IWorkerManager;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.messaging.IContext;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.apache.storm.utils.VersionInfo;
+import org.apache.zookeeper.data.ACL;
+import org.eclipse.jetty.util.ConcurrentHashSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class SupervisorData {
+
+private static final Logger LOG =
LoggerFactory.getLogger(SupervisorData.class);
+
+private final Map conf;
+private final IContext sharedContext;
+private volatile boolean active;
- private ISupervisor iSupervisor;
- private Utils.UptimeComputer upTime;
- private String stormVersion;
- private ConcurrentHashMap workerThreadPids; // for local
mode
- private IStormClusterState stormClusterState;
- private LocalState localState;
- private String supervisorId;
- private String assignmentId;
- private String hostName;
++private final ISupervisor iSupervisor;
++private final Utils.UptimeComputer upTime;
++private final String stormVersion;
++private final ConcurrentHashM
[33/35] storm git commit: fix about RunProfilerActions
fix about RunProfilerActions Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ac9942cf Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ac9942cf Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ac9942cf Branch: refs/heads/master Commit: ac9942cfee18dfb29dae1787c3f36f978cdcfa85 Parents: dba69b5 1b4edf4 Author: xiaojian.fxj Authored: Fri Apr 1 09:03:46 2016 +0800 Committer: xiaojian.fxj Committed: Fri Apr 1 09:21:47 2016 +0800 -- CHANGELOG.md| 6 + docs/Trident-API-Overview.md| 100 examples/storm-starter/pom.xml | 16 +- external/flux/flux-examples/pom.xml | 13 +- external/flux/pom.xml | 13 +- external/sql/storm-sql-kafka/pom.xml| 16 +- external/storm-kafka-client/README.md | 9 + external/storm-kafka-client/pom.xml | 86 +++ .../apache/storm/kafka/spout/KafkaSpout.java| 547 +++ .../storm/kafka/spout/KafkaSpoutConfig.java | 309 +++ .../storm/kafka/spout/KafkaSpoutMessageId.java | 101 .../KafkaSpoutRetryExponentialBackoff.java | 281 ++ .../kafka/spout/KafkaSpoutRetryService.java | 72 +++ .../storm/kafka/spout/KafkaSpoutStream.java | 70 +++ .../storm/kafka/spout/KafkaSpoutStreams.java| 158 ++ .../kafka/spout/KafkaSpoutTupleBuilder.java | 58 ++ .../kafka/spout/KafkaSpoutTuplesBuilder.java| 82 +++ .../kafka/spout/test/KafkaSpoutTestBolt.java| 50 ++ .../spout/test/KafkaSpoutTopologyMain.java | 133 + .../spout/test/TopicTest2TupleBuilder.java | 40 ++ .../test/TopicsTest0Test1TupleBuilder.java | 42 ++ external/storm-kafka/pom.xml| 16 +- .../storm/mongodb/bolt/MongoInsertBolt.java | 74 ++- .../storm/mongodb/bolt/MongoUpdateBolt.java | 3 +- .../storm/mongodb/common/MongoDBClient.java | 20 +- .../storm/mongodb/trident/state/MongoState.java | 2 +- external/storm-solr/pom.xml | 8 +- pom.xml | 57 +- storm-core/pom.xml | 6 - storm-core/src/clj/org/apache/storm/config.clj | 27 - .../clj/org/apache/storm/daemon/executor.clj| 222 +++- .../src/clj/org/apache/storm/daemon/nimbus.clj | 3 +- .../src/clj/org/apache/storm/daemon/task.clj| 190 --- .../org/apache/storm/daemon/GrouperFactory.java | 243 .../src/jvm/org/apache/storm/daemon/Task.java | 247 + .../daemon/metrics/BuiltinMetricsUtil.java | 8 +- .../supervisor/timer/RunProfilerActions.java| 15 +- .../apache/storm/hooks/info/BoltAckInfo.java| 8 + .../storm/hooks/info/BoltExecuteInfo.java | 8 + .../apache/storm/hooks/info/BoltFailInfo.java | 8 + .../org/apache/storm/hooks/info/EmitInfo.java | 9 + .../apache/storm/hooks/info/SpoutAckInfo.java | 9 + .../apache/storm/hooks/info/SpoutFailInfo.java | 9 + .../jvm/org/apache/storm/stats/StatsUtil.java | 23 +- .../jvm/org/apache/storm/utils/ConfigUtils.java | 35 +- .../test/clj/org/apache/storm/grouping_test.clj | 19 +- storm-dist/binary/src/main/assembly/binary.xml | 14 + 47 files changed, 2985 insertions(+), 500 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/ac9942cf/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/RunProfilerActions.java -- diff --cc storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/RunProfilerActions.java index 04467c2,000..3e1e34d mode 100644,00..100644 --- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/RunProfilerActions.java +++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/RunProfilerActions.java @@@ -1,214 -1,0 +1,211 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.storm.daemon.supervisor.timer; + +import com.google.
[09/35] storm git commit: xxxx
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/465a4b89
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/465a4b89
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/465a4b89
Branch: refs/heads/master
Commit: 465a4b89521a4ac15b81969009133bdfa12d0655
Parents: 42bacde
Author: xiaojian.fxj
Authored: Thu Mar 10 20:12:18 2016 +0800
Committer: xiaojian.fxj
Committed: Thu Mar 10 20:12:18 2016 +0800
--
.../org/apache/storm/command/kill_workers.clj | 5 +-
.../apache/storm/daemon/local_supervisor.clj| 5 +-
.../storm/daemon/supervisor/ShutdownWork.java | 7 +-
.../daemon/supervisor/StandaloneSupervisor.java | 2 -
.../apache/storm/daemon/supervisor/State.java | 2 +-
.../storm/daemon/supervisor/Supervisor.java | 9 +-
.../storm/daemon/supervisor/SupervisorData.java | 112 ---
.../daemon/supervisor/SupervisorManger.java | 5 +-
.../daemon/supervisor/SupervisorUtils.java | 101 +++--
.../daemon/supervisor/SyncProcessEvent.java | 33 +++---
.../daemon/supervisor/SyncSupervisorEvent.java | 17 ++-
.../supervisor/timer/RunProfilerActions.java| 2 +-
.../supervisor/timer/SupervisorHealthCheck.java | 4 +-
.../supervisor/timer/SupervisorHeartbeat.java | 14 +--
.../daemon/supervisor/timer/UpdateBlobs.java| 5 +-
15 files changed, 168 insertions(+), 155 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/465a4b89/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
--
diff --git a/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
b/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
index a7de176..4ddc993 100644
--- a/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
+++ b/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
@@ -28,7 +28,6 @@
conf (assoc conf STORM-LOCAL-DIR (. (File. (conf STORM-LOCAL-DIR))
getCanonicalPath))
isupervisor (StandaloneSupervisor.)
supervisor-data (SupervisorData. conf nil isupervisor)
-ids (SupervisorUtils/myWorkerIds conf)
-shut-workers (ShutdownWork.)]
+ids (SupervisorUtils/supervisorWorkerIds conf)]
(doseq [id ids]
- (.shutWorker shut-workers supervisor-data id
+ (SupervisorUtils/shutWorker supervisor-data id
http://git-wip-us.apache.org/repos/asf/storm/blob/465a4b89/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
--
diff --git a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
index 3dfed6f..70c280a 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
@@ -14,7 +14,7 @@
;; See the License for the specific language governing permissions and
;; limitations under the License.
(ns org.apache.storm.daemon.local-supervisor
- (:import [org.apache.storm.daemon.supervisor SyncProcessEvent SupervisorData
ShutdownWork Supervisor]
+ (:import [org.apache.storm.daemon.supervisor SyncProcessEvent SupervisorData
ShutdownWork Supervisor SupervisorUtils]
[org.apache.storm.utils Utils ConfigUtils]
[org.apache.storm ProcessSimulator])
(:use [org.apache.storm.daemon common]
@@ -38,9 +38,8 @@
))
(defn shutdown-local-worker [supervisorData workerId]
- (let [shut-workers (ShutdownWork.)]
(log-message "shutdown-local-worker")
-(.shutWorker shut-workers supervisorData workerId)))
+(SupervisorUtils/shutWorker supervisorData workerId))
(defn local-process []
"Create a local process event"
http://git-wip-us.apache.org/repos/asf/storm/blob/465a4b89/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ShutdownWork.java
--
diff --git
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ShutdownWork.java
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ShutdownWork.java
index 5018ce1..ec69980 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ShutdownWork.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ShutdownWork.java
@@ -26,7 +26,6 @@ import org.apache.storm.utils.Time;
import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
import java.util.*;
@@ -42,7 +41,7 @@ public class ShutdownWork implements Shutdownable {
Integer shutdownSleepSecs =
Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS));
Boolean asUser =
Utils.getBoolean(conf.get(Config
[30/35] storm git commit: update supervisor based on revans2 and longdafeng
http://git-wip-us.apache.org/repos/asf/storm/blob/dba69b52/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/RunProfilerActions.java
--
diff --cc
storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/RunProfilerActions.java
index 6b294f2,000..04467c2
mode 100644,00..100644
---
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/RunProfilerActions.java
+++
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/RunProfilerActions.java
@@@ -1,221 -1,0 +1,214 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.supervisor.timer;
+
++import com.google.common.collect.Lists;
+import org.apache.storm.Config;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.daemon.supervisor.SupervisorData;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.*;
+
+public class RunProfilerActions implements Runnable {
+private static Logger LOG =
LoggerFactory.getLogger(RunProfilerActions.class);
+
+private Map conf;
+private IStormClusterState stormClusterState;
+private String hostName;
+
+private String profileCmd;
+
+private SupervisorData supervisorData;
+
+private class ActionExitCallback implements Utils.ExitCodeCallable {
+private String stormId;
+private ProfileRequest profileRequest;
+private String logPrefix;
++private boolean stop;
+
- public ActionExitCallback(String stormId, ProfileRequest
profileRequest, String logPrefix) {
++public ActionExitCallback(String stormId, ProfileRequest
profileRequest, String logPrefix, boolean stop) {
+this.stormId = stormId;
+this.profileRequest = profileRequest;
+this.logPrefix = logPrefix;
++this.stop = stop;
+}
+
+@Override
+public Object call() throws Exception {
+return null;
+}
+
+@Override
+public Object call(int exitCode) {
+LOG.info("{} profile-action exited for {}", logPrefix, exitCode);
+try {
- stormClusterState.deleteTopologyProfileRequests(stormId,
profileRequest);
++if (stop)
++stormClusterState.deleteTopologyProfileRequests(stormId,
profileRequest);
+} catch (Exception e) {
+LOG.warn("failed delete profileRequest: " + profileRequest);
+}
+return null;
+}
+}
+
+public RunProfilerActions(SupervisorData supervisorData) {
+this.conf = supervisorData.getConf();
+this.stormClusterState = supervisorData.getStormClusterState();
+this.hostName = supervisorData.getHostName();
+this.profileCmd = (String) (conf.get(Config.WORKER_PROFILER_COMMAND));
+this.supervisorData = supervisorData;
+}
+
+@Override
+public void run() {
- Map> stormIdToActions =
supervisorData.getStormIdToProfileActions().get();
++Map> stormIdToActions =
supervisorData.getStormIdToProfilerActions().get();
+try {
+for (Map.Entry> entry :
stormIdToActions.entrySet()) {
+String stormId = entry.getKey();
+List requests = entry.getValue();
+if (requests != null) {
+for (ProfileRequest profileRequest : requests) {
+if
(profileRequest.get_nodeInfo().get_node().equals(hostName)) {
- boolean stop = System.currentTimeMillis() >
profileRequest.get_time_stamp() ? true : false;
++boolean stop = System.currentTimeMillis() >
profileRequest.get_time_stamp();
+
[13/35] storm git commit: remove setLocalizer
remove setLocalizer
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f78c36d7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f78c36d7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f78c36d7
Branch: refs/heads/master
Commit: f78c36d7cc9ca82c6aa4e073f07279650a14fd45
Parents: 65ce9d2
Author: xiaojian.fxj
Authored: Thu Mar 10 23:20:33 2016 +0800
Committer: xiaojian.fxj
Committed: Thu Mar 10 23:20:33 2016 +0800
--
.../jvm/org/apache/storm/daemon/supervisor/SupervisorData.java | 5 -
1 file changed, 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/f78c36d7/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorData.java
--
diff --git
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorData.java
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorData.java
index be79847..8c17edc 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorData.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorData.java
@@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.UnknownHostException;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -222,10 +221,6 @@ public class SupervisorData {
return localizer;
}
-public void setLocalizer(Localizer localizer) {
-this.localizer = localizer;
-}
-
public AtomicInteger getSyncRetry() {
return syncRetry;
}
[07/35] storm git commit: Merge branch 'master' into supervisor
Merge branch 'master' into supervisor Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b09b4129 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b09b4129 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b09b4129 Branch: refs/heads/master Commit: b09b4129d845aff6be285ea1748b842499c40e0b Parents: 19fcafb 672c895 Author: xiaojian.fxj Authored: Fri Mar 4 12:14:41 2016 +0800 Committer: xiaojian.fxj Committed: Fri Mar 4 13:33:09 2016 +0800 -- .gitignore | 1 + CHANGELOG.md| 9 ++ README.markdown | 4 + bin/storm.cmd | 2 +- bin/storm.py| 4 +- .../storm/starter/ThroughputVsLatency.java | 2 +- .../apache/storm/sql/compiler/CompilerUtil.java | 7 +- .../apache/storm/sql/compiler/ExprCompiler.java | 32 - .../backends/standalone/RelNodeCompiler.java| 6 +- .../apache/storm/sql/parser/StormParser.java| 5 + .../test/org/apache/storm/sql/TestStormSql.java | 64 +- .../storm/sql/compiler/TestCompilerUtils.java | 62 - .../storm/sql/compiler/TestExprSemantic.java| 18 +++ .../backends/standalone/TestPlanCompiler.java | 20 +++ .../backends/trident/TestPlanCompiler.java | 4 +- .../test/org/apache/storm/sql/TestUtils.java| 32 - pom.xml | 23 .../apache/storm/command/upload_credentials.clj | 35 - .../src/clj/org/apache/storm/daemon/acker.clj | 108 .../src/clj/org/apache/storm/daemon/common.clj | 17 ++- .../apache/storm/daemon/local_supervisor.clj| 2 +- storm-core/src/clj/org/apache/storm/testing.clj | 7 +- .../storm/blobstore/LocalFsBlobStore.java | 2 +- .../src/jvm/org/apache/storm/command/List.java | 50 .../apache/storm/command/ListTopologies.java| 52 .../apache/storm/command/UploadCredentials.java | 61 + .../src/jvm/org/apache/storm/daemon/Acker.java | 128 +++ .../storm/daemon/supervisor/Supervisor.java | 4 +- .../daemon/supervisor/SyncProcessEvent.java | 11 +- .../apache/storm/security/auth/AuthUtils.java | 40 ++ .../storm/security/auth/kerberos/AutoTGT.java | 64 -- .../auth/kerberos/AutoTGTKrb5LoginModule.java | 8 +- .../apache/storm/topology/TopologyBuilder.java | 13 +- .../jvm/org/apache/storm/utils/ConfigUtils.java | 8 +- .../src/jvm/org/apache/storm/utils/Utils.java | 13 +- .../security/auth/auto_login_module_test.clj| 24 +++- .../clj/org/apache/storm/supervisor_test.clj| 52 .../storm/topology/TopologyBuilderTest.java | 65 ++ 38 files changed, 734 insertions(+), 325 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/b09b4129/bin/storm.cmd -- http://git-wip-us.apache.org/repos/asf/storm/blob/b09b4129/bin/storm.py -- http://git-wip-us.apache.org/repos/asf/storm/blob/b09b4129/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj -- diff --cc storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj index 65cf907,000..3dfed6f mode 100644,00..100644 --- a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj +++ b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj @@@ -1,61 -1,0 +1,61 @@@ +;; Licensed to the Apache Software Foundation (ASF) under one +;; or more contributor license agreements. See the NOTICE file +;; distributed with this work for additional information +;; regarding copyright ownership. The ASF licenses this file +;; to you under the Apache License, Version 2.0 (the +;; "License"); you may not use this file except in compliance +;; with the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +(ns org.apache.storm.daemon.local-supervisor + (:import [org.apache.storm.daemon.supervisor SyncProcessEvent SupervisorData ShutdownWork Supervisor] + [org.apache.storm.utils Utils ConfigUtils] + [org.apache.storm ProcessSimulator]) + (:use [org.apache.storm.daemon common] +[org.apache.storm log]) + (:require [org.apa
[14/35] storm git commit: Merge branch 'master' into supervisor
Merge branch 'master' into supervisor Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/69c8b3c3 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/69c8b3c3 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/69c8b3c3 Branch: refs/heads/master Commit: 69c8b3c31d4ee528aea58f716b092c24ba6b0b1a Parents: f78c36d 6390d18 Author: xiaojian.fxj Authored: Thu Mar 10 23:26:42 2016 +0800 Committer: xiaojian.fxj Committed: Thu Mar 10 23:26:42 2016 +0800 -- CHANGELOG.md| 1 + .../src/clj/org/apache/storm/converter.clj | 15 + .../src/clj/org/apache/storm/daemon/common.clj | 350 +--- .../clj/org/apache/storm/daemon/executor.clj| 24 +- .../clj/org/apache/storm/daemon/logviewer.clj | 2 +- .../src/clj/org/apache/storm/daemon/nimbus.clj | 63 +-- .../src/clj/org/apache/storm/daemon/task.clj| 5 +- .../src/clj/org/apache/storm/daemon/worker.clj | 24 +- storm-core/src/clj/org/apache/storm/testing.clj | 98 ++-- storm-core/src/clj/org/apache/storm/ui/core.clj | 15 +- .../org/apache/storm/daemon/DaemonCommon.java | 22 + .../org/apache/storm/daemon/StormCommon.java| 537 +++ .../storm/utils/StormCommonInstaller.java | 43 ++ .../src/jvm/org/apache/storm/utils/Utils.java | 16 + .../org/apache/storm/integration_test.clj | 6 +- .../test/clj/org/apache/storm/nimbus_test.clj | 121 +++-- .../apache/storm/security/auth/auth_test.clj| 3 +- .../clj/org/apache/storm/supervisor_test.clj| 11 +- 18 files changed, 833 insertions(+), 523 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/69c8b3c3/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj -- http://git-wip-us.apache.org/repos/asf/storm/blob/69c8b3c3/storm-core/src/clj/org/apache/storm/testing.clj -- diff --cc storm-core/src/clj/org/apache/storm/testing.clj index dd1f2df,bda09ee..4cec39a --- a/storm-core/src/clj/org/apache/storm/testing.clj +++ b/storm-core/src/clj/org/apache/storm/testing.clj @@@ -290,13 -286,13 +291,13 @@@ ([cluster-map timeout-ms] ;; wait until all workers, supervisors, and nimbus is waiting (let [supervisors @(:supervisors cluster-map) - workers (filter (partial satisfies? common/DaemonCommon) (clojurify-structure (ProcessSimulator/getAllProcessHandles))) + workers (filter (partial instance? DaemonCommon) (clojurify-structure (ProcessSimulator/getAllProcessHandles))) daemons (concat [(:nimbus cluster-map)] - supervisors ; because a worker may already be dead workers)] -(while-timeout timeout-ms (not (every? (memfn isWaiting) daemons)) +(while-timeout timeout-ms (or (not (every? (memfn waiting?) daemons)) +(not (every? is-supervisor-waiting supervisors))) (Thread/sleep (rand-int 20)) ;; (doseq [d daemons] ;;(if-not ((memfn waiting?) d) @@@ -377,26 -373,27 +378,28 @@@ (defn submit-mocked-assignment [nimbus storm-cluster-state storm-name conf topology task->component executor->node+port worker->resources] - (with-var-roots [common/storm-task-info (fn [& ignored] task->component) -nimbus/compute-new-scheduler-assignments (mocked-compute-new-scheduler-assignments) -nimbus/convert-assignments-to-worker->resources (mocked-convert-assignments-to-worker->resources - storm-cluster-state - storm-name - worker->resources) -nimbus/compute-new-topology->executor->node+port (mocked-compute-new-topology->executor->node+port - storm-cluster-state - storm-name - executor->node+port)] - (submit-local-topology nimbus storm-name conf topology))) + (let [fake-common (proxy [StormCommon] [] + (stormTaskInfoImpl [_] task->component))] + (with-open [- (StormCommonInstaller. fake-common)] + (with-var-roots [nimbus/compute-new-scheduler-assignments (mocked-compute-new-scheduler-assignments) +nimbus/convert-assignments-to-worker->resources (mocked-convert-assignments-to-worker->resources +
[24/35] storm git commit: Merge branch 'master' into supervisor
Merge branch 'master' into supervisor Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/afd2e3f5 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/afd2e3f5 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/afd2e3f5 Branch: refs/heads/master Commit: afd2e3f5a53d2a086de1d1d62048377486ab1992 Parents: f03b8be 0100898 Author: xiaojian.fxj Authored: Fri Mar 25 11:37:48 2016 +0800 Committer: xiaojian.fxj Committed: Fri Mar 25 11:37:48 2016 +0800 -- CHANGELOG.md| 3 + bin/storm.py| 5 +- .../storm/starter/tools/SlotBasedCounter.java | 12 +-- external/sql/storm-sql-core/pom.xml | 18 external/storm-hbase/pom.xml| 2 +- .../jvm/org/apache/storm/kafka/KafkaUtils.java | 4 + .../org/apache/storm/daemon/builtin_metrics.clj | 97 .../clj/org/apache/storm/daemon/executor.clj| 37 .../src/clj/org/apache/storm/daemon/task.clj| 8 +- .../src/jvm/org/apache/storm/StormTimer.java| 1 + .../daemon/metrics/BuiltinBoltMetrics.java | 78 .../storm/daemon/metrics/BuiltinMetrics.java| 33 +++ .../daemon/metrics/BuiltinMetricsUtil.java | 79 .../daemon/metrics/BuiltinSpoutMetrics.java | 64 + .../daemon/metrics/SpoutThrottlingMetrics.java | 57 .../jvm/org/apache/storm/drpc/DRPCSpout.java| 51 +++--- .../jvm/org/apache/storm/stats/StatsUtil.java | 4 +- .../test/clj/org/apache/storm/drpc_test.clj | 69 -- storm-dist/binary/pom.xml | 10 -- storm-dist/binary/src/main/assembly/binary.xml | 4 +- 20 files changed, 474 insertions(+), 162 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/afd2e3f5/bin/storm.py --
[02/35] storm git commit: port Supervisor to java
port Supervisor to java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/08934e29
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/08934e29
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/08934e29
Branch: refs/heads/master
Commit: 08934e29982d3936c9e247a8d7bac563053f869f
Parents: 73312ad
Author: xiaojian.fxj
Authored: Fri Feb 26 12:38:23 2016 +0800
Committer: xiaojian.fxj
Committed: Fri Feb 26 12:38:23 2016 +0800
--
.../storm/daemon/supervisor/DaemonCommon.java | 22 +
.../DefaultUncaughtExceptionHandler.java| 31 +
.../supervisor/EventManagerPushCallback.java| 37 +
.../daemon/supervisor/RunProfilerActions.java | 221 ++
.../storm/daemon/supervisor/ShutdownWork.java | 125
.../daemon/supervisor/StandaloneSupervisor.java | 82 +++
.../apache/storm/daemon/supervisor/State.java | 22 +
.../storm/daemon/supervisor/StateHeartbeat.java | 45 ++
.../daemon/supervisor/SupervisorDaemon.java | 28 +
.../storm/daemon/supervisor/SupervisorData.java | 340 ++
.../daemon/supervisor/SupervisorHeartbeat.java | 84 +++
.../daemon/supervisor/SupervisorManger.java | 101 +++
.../daemon/supervisor/SupervisorServer.java | 212 ++
.../daemon/supervisor/SupervisorUtils.java | 173 +
.../daemon/supervisor/SyncProcessEvent.java | 674 +++
.../daemon/supervisor/SyncSupervisorEvent.java | 592
.../storm/daemon/supervisor/UpdateBlobs.java| 103 +++
17 files changed, 2892 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/08934e29/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DaemonCommon.java
--
diff --git
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DaemonCommon.java
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DaemonCommon.java
new file mode 100644
index 000..3b7a18e
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DaemonCommon.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+public interface DaemonCommon {
+boolean isWaiting();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/08934e29/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
--
diff --git
a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
new file mode 100644
index 000..8785f86
--- /dev/null
+++
b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DefaultUncaughtExceptionHandler implements
Thread.UncaughtExceptionHandler {
+private static final Logger LOG =
LoggerFactory.getLogger(DefaultUncaughtExceptionHandler.class);
+@Override
+public void uncaughtException(Thread t, Throwable
[04/35] storm git commit: update test codes about supervisor
http://git-wip-us.apache.org/repos/asf/storm/blob/19fcafbd/storm-core/test/clj/org/apache/storm/supervisor_test.clj
--
diff --git a/storm-core/test/clj/org/apache/storm/supervisor_test.clj
b/storm-core/test/clj/org/apache/storm/supervisor_test.clj
index cdd66e4..b367fce 100644
--- a/storm-core/test/clj/org/apache/storm/supervisor_test.clj
+++ b/storm-core/test/clj/org/apache/storm/supervisor_test.clj
@@ -19,7 +19,10 @@
(:use [conjure core])
(:require [clojure.contrib [string :as contrib-str]])
(:require [clojure [string :as string] [set :as set]])
- (:import [org.apache.storm.testing TestWordCounter TestWordSpout
TestGlobalCount TestAggregatesCounter TestPlannerSpout])
+ (:import [org.apache.storm.testing TestWordCounter TestWordSpout
TestGlobalCount TestAggregatesCounter TestPlannerSpout]
+ [org.apache.storm.daemon.supervisor SupervisorUtils
SyncProcessEvent SupervisorData]
+ [java.util ArrayList Arrays HashMap]
+ [org.apache.storm.testing.staticmocking MockedSupervisorUtils])
(:import [org.apache.storm.scheduler ISupervisor])
(:import [org.apache.storm.utils Time Utils$UptimeComputer ConfigUtils])
(:import [org.apache.storm.generated RebalanceOptions WorkerResources])
@@ -36,7 +39,7 @@
(:import [java.nio.file.attribute FileAttribute])
(:use [org.apache.storm config testing util log converter])
(:use [org.apache.storm.daemon common])
- (:require [org.apache.storm.daemon [worker :as worker] [supervisor :as
supervisor]])
+ (:require [org.apache.storm.daemon [worker :as worker] [local-supervisor :as
local-supervisor]])
(:use [conjure core])
(:require [clojure.java.io :as io]))
@@ -60,7 +63,7 @@
))
(defn heartbeat-worker [supervisor port storm-id executors]
- (let [conf (.get-conf supervisor)]
+ (let [conf (.getConf supervisor)]
(worker/do-heartbeat {:conf conf
:port port
:storm-id storm-id
@@ -294,53 +297,61 @@
(deftest test-worker-launch-command
(testing "*.worker.childopts configuration"
-(let [mock-port "42"
+(let [mock-port 42
mock-storm-id "fake-storm-id"
mock-worker-id "fake-worker-id"
mock-cp (str Utils/FILE_PATH_SEPARATOR "base"
Utils/CLASS_PATH_SEPARATOR Utils/FILE_PATH_SEPARATOR "stormjar.jar")
mock-sensitivity "S3"
mock-cp "/base:/stormjar.jar"
exp-args-fn (fn [opts topo-opts classpath]
- (concat [(supervisor/java-cmd) "-cp" classpath
- (str "-Dlogfile.name=" "worker.log")
- "-Dstorm.home="
- (str "-Dworkers.artifacts="
"/tmp/workers-artifacts")
- (str "-Dstorm.id=" mock-storm-id)
- (str "-Dworker.id=" mock-worker-id)
- (str "-Dworker.port=" mock-port)
- "-Dstorm.log.dir=/logs"
- "-Dlog4j.configurationFile=/log4j2/worker.xml"
-
"-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector"
- "org.apache.storm.LogWriter"]
- [(supervisor/java-cmd) "-server"]
- opts
- topo-opts
- ["-Djava.library.path="
-(str "-Dlogfile.name=" "worker.log")
-"-Dstorm.home="
-"-Dworkers.artifacts=/tmp/workers-artifacts"
-"-Dstorm.conf.file="
-"-Dstorm.options="
-(str "-Dstorm.log.dir="
Utils/FILE_PATH_SEPARATOR "logs")
-(str "-Dlogging.sensitivity=" mock-sensitivity)
-(str "-Dlog4j.configurationFile="
Utils/FILE_PATH_SEPARATOR "log4j2" Utils/FILE_PATH_SEPARATOR "worker.xml")
-
"-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector"
-(str "-Dstorm.id=" mock-storm-id)
-(str "-Dworker.id=" mock-worker-id)
-(str "-Dworker.port=" mock-port)
-"-cp" classpath
-"org.apache.storm.daemon.worker"
-mock-storm-id
-mock-port
-mock-worker-id]))]
+(let [file-prefix (let [os (System/getProperty
"os.name")]
+(if (.startsWith os "Windows")
(str "file:///")
+(str "")))
+ sequences
[27/35] storm git commit: Merge branch 'master' of github.com:apache/storm
Merge branch 'master' of github.com:apache/storm Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3812b2fa Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3812b2fa Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3812b2fa Branch: refs/heads/master Commit: 3812b2fa04cc7eb86035d402f3c407af3b4daffe Parents: a48e953 e14f429 Author: xiaojian.fxj Authored: Wed Mar 30 16:56:12 2016 +0800 Committer: xiaojian.fxj Committed: Wed Mar 30 16:56:12 2016 +0800 -- CHANGELOG.md| 4 + bin/storm.py| 2 +- docs/README.md | 14 +- docs/Resource_Aware_Scheduler_overview.md | 2 + docs/Trident-RAS-API.md | 52 ++ docs/index.md | 1 + .../org/apache/storm/hive/bolt/HiveBolt.java| 147 ++-- .../apache/storm/hive/common/HiveOptions.java | 8 +- .../org/apache/storm/hive/common/HiveUtils.java | 11 +- .../apache/storm/hive/common/HiveWriter.java| 127 ++-- .../apache/storm/hive/trident/HiveState.java| 38 +- .../storm/hive/trident/HiveStateFactory.java| 1 + .../apache/storm/hive/trident/HiveUpdater.java | 1 + .../storm/hive/bolt/BucketTestHiveTopology.java | 190 + .../apache/storm/hive/bolt/HiveTopology.java| 6 +- .../apache/storm/hive/bolt/TestHiveBolt.java| 11 +- .../storm/hive/common/TestHiveWriter.java | 13 +- .../storm/hive/trident/TridentHiveTopology.java | 2 +- .../org/apache/storm/pacemaker/pacemaker.clj| 242 -- storm-core/src/jvm/org/apache/storm/Config.java | 2 +- .../cluster/PaceMakerStateStorageFactory.java | 3 +- .../storm/metric/StormMetricsRegistry.java | 16 +- .../org/apache/storm/pacemaker/Pacemaker.java | 217 ++ .../jvm/org/apache/storm/scheduler/Cluster.java | 23 +- .../org/apache/storm/scheduler/Topologies.java | 11 +- .../scheduler/resource/ClusterStateData.java| 101 --- .../resource/ResourceAwareScheduler.java| 138 ++-- .../scheduler/resource/SchedulingState.java | 56 ++ .../apache/storm/scheduler/resource/User.java | 24 +- .../eviction/DefaultEvictionStrategy.java | 10 +- .../strategies/eviction/IEvictionStrategy.java | 9 +- .../DefaultSchedulingPriorityStrategy.java | 9 +- .../priority/ISchedulingPriorityStrategy.java | 9 +- .../DefaultResourceAwareStrategy.java | 86 +-- .../strategies/scheduling/IStrategy.java| 6 +- .../jvm/org/apache/storm/utils/ConfigUtils.java | 20 +- .../storm/pacemaker_state_factory_test.clj | 151 .../clj/org/apache/storm/pacemaker_test.clj | 242 -- .../scheduler/resource_aware_scheduler_test.clj | 738 --- .../storm/PaceMakerStateStorageFactoryTest.java | 136 .../jvm/org/apache/storm/PacemakerTest.java | 242 ++ .../resource/TestResourceAwareScheduler.java| 725 +- .../TestUtilsForResourceAwareScheduler.java | 43 +- 43 files changed, 2069 insertions(+), 1820 deletions(-) --
[18/35] storm git commit: add the plugin to use for manager worker
add the plugin to use for manager worker Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a1e47352 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a1e47352 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a1e47352 Branch: refs/heads/master Commit: a1e473526b5d9074ae1f9ff98162ddc78e426a73 Parents: cc95d4f Author: xiaojian.fxj Authored: Mon Mar 14 16:54:36 2016 +0800 Committer: xiaojian.fxj Committed: Mon Mar 14 18:55:57 2016 +0800 -- conf/defaults.yaml | 4 + .../org/apache/storm/command/kill_workers.clj | 11 +- .../apache/storm/daemon/local_supervisor.clj| 16 +- storm-core/src/clj/org/apache/storm/testing.clj | 16 +- storm-core/src/jvm/org/apache/storm/Config.java | 7 + .../storm/daemon/supervisor/DaemonCommon.java | 22 - .../daemon/supervisor/StandaloneSupervisor.java | 1 - .../storm/daemon/supervisor/Supervisor.java | 14 +- .../storm/daemon/supervisor/SupervisorData.java | 24 +- .../daemon/supervisor/SupervisorManager.java| 103 + .../daemon/supervisor/SupervisorManger.java | 97 - .../daemon/supervisor/SupervisorUtils.java | 105 + .../daemon/supervisor/SyncProcessEvent.java | 274 + .../daemon/supervisor/SyncSupervisorEvent.java | 16 +- .../supervisor/timer/RunProfilerActions.java| 2 +- .../supervisor/timer/SupervisorHealthCheck.java | 8 +- .../workermanager/DefaultWorkerManager.java | 397 +++ .../workermanager/IWorkerManager.java | 38 ++ .../supervisor/workermanager/IWorkerResult.java | 21 + .../clj/org/apache/storm/supervisor_test.clj| 84 ++-- 20 files changed, 706 insertions(+), 554 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/a1e47352/conf/defaults.yaml -- diff --git a/conf/defaults.yaml b/conf/defaults.yaml index 9817161..da25ef8 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -289,6 +289,10 @@ storm.daemon.metrics.reporter.plugins: storm.resource.isolation.plugin: "org.apache.storm.container.cgroup.CgroupManager" storm.resource.isolation.plugin.enable: false + +# Default plugin to use for manager worker +storm.supervisor.worker.manager.plugin: org.apache.storm.daemon.supervisor.workermanager.DefaultWorkerManager + # Configs for CGroup support storm.cgroup.hierarchy.dir: "/cgroup/storm_resources" storm.cgroup.resources: http://git-wip-us.apache.org/repos/asf/storm/blob/a1e47352/storm-core/src/clj/org/apache/storm/command/kill_workers.clj -- diff --git a/storm-core/src/clj/org/apache/storm/command/kill_workers.clj b/storm-core/src/clj/org/apache/storm/command/kill_workers.clj index aadc9fd..08de3ed 100644 --- a/storm-core/src/clj/org/apache/storm/command/kill_workers.clj +++ b/storm-core/src/clj/org/apache/storm/command/kill_workers.clj @@ -28,6 +28,13 @@ conf (assoc conf STORM-LOCAL-DIR (. (File. (conf STORM-LOCAL-DIR)) getCanonicalPath)) isupervisor (StandaloneSupervisor.) supervisor-data (SupervisorData. conf nil isupervisor) -ids (SupervisorUtils/supervisorWorkerIds conf)] +worker-manager (.getWorkerManager supervisor-data) +ids (SupervisorUtils/supervisorWorkerIds conf) +supervisor-id (.getSupervisorId supervisor-data) +worker-pids (.getWorkerThreadPids supervisor-data) +dead-workers (.getDeadWorkers supervisor-data)] (doseq [id ids] - (SupervisorUtils/shutWorker supervisor-data id + (.shutdownWorker worker-manager supervisor-id id worker-pids) + (if (.cleanupWorker worker-manager id) +(.remove dead-workers id)) + ))) http://git-wip-us.apache.org/repos/asf/storm/blob/a1e47352/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj -- diff --git a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj index c8ae2d6..b28ae08 100644 --- a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj +++ b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj @@ -36,17 +36,21 @@ (ProcessSimulator/registerProcess pid worker) (.put (.getWorkerThreadPids supervisorData) workerId pid) )) - -(defn shutdown-local-worker [supervisorData workerId] -(log-message "shutdown-local-worker") -(SupervisorUtils/shutWorker supervisorData workerId)) +(defn shutdown-local-worker [supervisorData worker-manager workerId] + (log-message "shutdown-local-worker") + (let [supervisor-id (.getSupervisorId supervisorData) +worker-pids (.getWorkerThreadPids superv
[16/35] storm git commit: sdf
sdf Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cc95d4f7 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cc95d4f7 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cc95d4f7 Branch: refs/heads/master Commit: cc95d4f708efa123e5fc908bea15545f7139655b Parents: 95bf673 Author: xiaojian.fxj Authored: Fri Mar 11 08:03:00 2016 +0800 Committer: xiaojian.fxj Committed: Fri Mar 11 08:03:00 2016 +0800 -- storm-core/src/clj/org/apache/storm/testing.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/cc95d4f7/storm-core/src/clj/org/apache/storm/testing.clj -- diff --git a/storm-core/src/clj/org/apache/storm/testing.clj b/storm-core/src/clj/org/apache/storm/testing.clj index d2d2671..7804747 100644 --- a/storm-core/src/clj/org/apache/storm/testing.clj +++ b/storm-core/src/clj/org/apache/storm/testing.clj @@ -296,7 +296,7 @@ [(:nimbus cluster-map)] ; because a worker may already be dead workers)] -(while-timeout timeout-ms (or (not (every? (memfn isWaiting?) daemons)) +(while-timeout timeout-ms (or (not (every? (memfn isWaiting) daemons)) (not (every? is-supervisor-waiting supervisors))) (Thread/sleep (rand-int 20)) ;; (doseq [d daemons]
[10/35] storm git commit: sdf
sdf Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/184dc4a5 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/184dc4a5 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/184dc4a5 Branch: refs/heads/master Commit: 184dc4a5c3fa8c9662ab224a82f33cc687b95c4b Parents: 465a4b8 Author: xiaojian.fxj Authored: Thu Mar 10 22:17:06 2016 +0800 Committer: xiaojian.fxj Committed: Thu Mar 10 22:17:06 2016 +0800 -- storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/184dc4a5/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj -- diff --git a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj index 70c280a..2361817 100644 --- a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj +++ b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj @@ -34,7 +34,7 @@ workerId)] (ConfigUtils/setWorkerUserWSE conf workerId "") (ProcessSimulator/registerProcess pid worker) -(.put (.getWorkerThreadPidsAtom supervisorData) workerId pid) +(.put (.getWorkerThreadPids supervisorData) workerId pid) )) (defn shutdown-local-worker [supervisorData workerId]
[23/35] storm git commit: update
update Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/75364892 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/75364892 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/75364892 Branch: refs/heads/master Commit: 753648927bb2c82443ede9525200bb6197f8d3b6 Parents: 724f5d2 0100898 Author: xiaojian.fxj Authored: Fri Mar 25 10:08:57 2016 +0800 Committer: xiaojian.fxj Committed: Fri Mar 25 10:48:05 2016 +0800 -- CHANGELOG.md| 3 + bin/storm.py| 5 +- .../storm/starter/tools/SlotBasedCounter.java | 12 +-- external/sql/storm-sql-core/pom.xml | 18 external/storm-hbase/pom.xml| 2 +- .../jvm/org/apache/storm/kafka/KafkaUtils.java | 4 + .../org/apache/storm/daemon/builtin_metrics.clj | 97 .../clj/org/apache/storm/daemon/executor.clj| 37 .../apache/storm/daemon/local_supervisor.clj| 2 +- .../src/clj/org/apache/storm/daemon/task.clj| 8 +- .../src/jvm/org/apache/storm/StormTimer.java| 1 + .../daemon/metrics/BuiltinBoltMetrics.java | 78 .../storm/daemon/metrics/BuiltinMetrics.java| 33 +++ .../daemon/metrics/BuiltinMetricsUtil.java | 79 .../daemon/metrics/BuiltinSpoutMetrics.java | 64 + .../daemon/metrics/SpoutThrottlingMetrics.java | 57 .../daemon/supervisor/SyncProcessEvent.java | 4 +- .../daemon/supervisor/SyncSupervisorEvent.java | 16 +--- .../supervisor/timer/SupervisorHeartbeat.java | 4 +- .../jvm/org/apache/storm/drpc/DRPCSpout.java| 51 +++--- .../jvm/org/apache/storm/stats/StatsUtil.java | 4 +- .../test/clj/org/apache/storm/drpc_test.clj | 69 -- storm-dist/binary/pom.xml | 10 -- storm-dist/binary/src/main/assembly/binary.xml | 4 +- 24 files changed, 484 insertions(+), 178 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/75364892/bin/storm.py -- http://git-wip-us.apache.org/repos/asf/storm/blob/75364892/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj -- diff --cc storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj index b28ae08,000..ba3c87e mode 100644,00..100644 --- a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj +++ b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj @@@ -1,64 -1,0 +1,64 @@@ +;; Licensed to the Apache Software Foundation (ASF) under one +;; or more contributor license agreements. See the NOTICE file +;; distributed with this work for additional information +;; regarding copyright ownership. The ASF licenses this file +;; to you under the Apache License, Version 2.0 (the +;; "License"); you may not use this file except in compliance +;; with the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +(ns org.apache.storm.daemon.local-supervisor + (:import [org.apache.storm.daemon.supervisor SyncProcessEvent SupervisorData Supervisor SupervisorUtils] + [org.apache.storm.utils Utils ConfigUtils] + [org.apache.storm ProcessSimulator]) + (:use [org.apache.storm.daemon common] +[org.apache.storm log]) + (:require [org.apache.storm.daemon [worker :as worker] ]) + (:require [clojure.string :as str]) + (:gen-class)) + +(defn launch-local-worker [supervisorData stormId port workerId resources] + (let [conf (.getConf supervisorData) + pid (Utils/uuid) +worker (worker/mk-worker conf + (.getSharedContext supervisorData) + stormId + (.getAssignmentId supervisorData) + (int port) + workerId)] +(ConfigUtils/setWorkerUserWSE conf workerId "") +(ProcessSimulator/registerProcess pid worker) +(.put (.getWorkerThreadPids supervisorData) workerId pid) +)) +(defn shutdown-local-worker [supervisorData worker-manager workerId] + (log-message "shutdown-local-worker") + (let [supervisor-id (.getSupervisorId supervisorData) +worker-pids (.getWorkerThreadPids supervisorData) +dead-workers (.getDeadWorkers supervisorData)] +(.shutdownWorker worker-manager supervisor-id workerId wo
