Repository: storm Updated Branches: refs/heads/master 175beec7e -> 68cc70518
[STORM-1671] Enable logviewer to delete a dir without yaml Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/82102837 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/82102837 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/82102837 Branch: refs/heads/master Commit: 821028374fd09d88d97b3ca3e283da2192f0f2ae Parents: 3de3afc Author: zhuol <[email protected]> Authored: Thu Mar 31 13:25:58 2016 -0500 Committer: zhuol <[email protected]> Committed: Fri Apr 1 11:07:09 2016 -0500 ---------------------------------------------------------------------- .../clj/org/apache/storm/daemon/logviewer.clj | 7 ++++--- .../apache/storm/daemon/DirectoryCleaner.java | 2 +- .../clj/org/apache/storm/logviewer_test.clj | 20 ++++++++++---------- 3 files changed, 15 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/82102837/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj index 53e8527..2b80fd8 100644 --- a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj +++ b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj @@ -151,9 +151,10 @@ (defn identify-worker-log-dirs [log-dirs] "return the workerid to worker-log-dir map" (into {} (for [logdir log-dirs - :let [metaFile (get-metadata-file-for-wroker-logdir logdir)] - :when metaFile] - {(get-worker-id-from-metadata-file metaFile) logdir}))) + :let [metaFile (get-metadata-file-for-wroker-logdir logdir)]] + (if metaFile + {(get-worker-id-from-metadata-file metaFile) logdir} + {"" logdir})))) ;; an old directory that has no yaml file will be treated as a dead dir for deleting (defn get-alive-ids [conf now-secs] http://git-wip-us.apache.org/repos/asf/storm/blob/82102837/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java b/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java index a51a49e..047e024 100644 --- a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java +++ b/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java @@ -138,7 +138,7 @@ public class DirectoryCleaner { while (!stack.isEmpty() && toDeleteSize > 0) { File file = stack.pop(); toDeleteSize -= file.length(); - LOG.info("Delete file: {}, size: {}, lastModified: {}", file.getName(), file.length(), file.lastModified()); + LOG.info("Delete file: {}, size: {}, lastModified: {}", file.getCanonicalPath(), file.length(), file.lastModified()); file.delete(); deletedFiles++; } http://git-wip-us.apache.org/repos/asf/storm/blob/82102837/storm-core/test/clj/org/apache/storm/logviewer_test.clj ---------------------------------------------------------------------- diff --git a/storm-core/test/clj/org/apache/storm/logviewer_test.clj b/storm-core/test/clj/org/apache/storm/logviewer_test.clj index d06c11c..d161f3f 100644 --- a/storm-core/test/clj/org/apache/storm/logviewer_test.clj +++ b/storm-core/test/clj/org/apache/storm/logviewer_test.clj @@ -246,24 +246,24 @@ (. (Mockito/when (.readWorkerHeartbeatsImpl supervisor-util (Mockito/any))) (thenReturn nil)) (is (= expected (logviewer/identify-worker-log-dirs [port1-dir])))))))) - - (deftest test-get-dead-worker-dirs - (testing "removes any files of workers that are still alive" + (testing "return directories for workers that are not alive" (let [conf {SUPERVISOR-WORKER-TIMEOUT-SECS 5} - hb (let[lwb (LSWorkerHeartbeat.)] + hb (let [lwb (LSWorkerHeartbeat.)] (.set_time_secs lwb (int 1)) lwb) id->hb {"42" hb} now-secs 2 - unexpected-dir (mk-mock-File {:name "dir1" :type :directory}) - expected-dir (mk-mock-File {:name "dir2" :type :directory}) - log-dirs #{unexpected-dir expected-dir} + unexpected-dir1 (mk-mock-File {:name "dir1" :type :directory}) + expected-dir2 (mk-mock-File {:name "dir2" :type :directory}) + expected-dir3 (mk-mock-File {:name "dir3" :type :directory}) + log-dirs #{unexpected-dir1 expected-dir2 expected-dir3} supervisor-util (Mockito/mock SupervisorUtils)] (with-open [_ (MockedSupervisorUtils. supervisor-util)] - (stubbing [logviewer/identify-worker-log-dirs {"42" unexpected-dir, - "007" expected-dir}] + (stubbing [logviewer/identify-worker-log-dirs {"42" unexpected-dir1, + "007" expected-dir2, + "" expected-dir3}] ;; this tests a directory with no yaml file thus no worker id (. (Mockito/when (.readWorkerHeartbeatsImpl supervisor-util (Mockito/any))) (thenReturn id->hb)) - (is (= #{expected-dir} + (is (= #{expected-dir2 expected-dir3} (logviewer/get-dead-worker-dirs conf now-secs log-dirs)))))))) (deftest test-cleanup-fn
