Repository: storm Updated Branches: refs/heads/1.x-branch 3399a4a46 -> 9c6e6dfd2
STORM-1704 When logviewer_search.html opens daemon file, next search always show no result * ensures that is-daemon parameter is passed around multiple searches * set logviewerUrl to '/daemonlog' when search is done with is-daemon=yes Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/449394a0 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/449394a0 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/449394a0 Branch: refs/heads/1.x-branch Commit: 449394a0c66f459da33253cb5ecbf22274dc0e40 Parents: 6af0d10 Author: Jungtaek Lim <[email protected]> Authored: Mon Apr 11 16:40:58 2016 +0900 Committer: Jungtaek Lim <[email protected]> Committed: Mon Apr 11 16:40:58 2016 +0900 ---------------------------------------------------------------------- .../clj/org/apache/storm/daemon/logviewer.clj | 77 +++++++++++++------- storm-core/src/ui/public/logviewer_search.html | 4 +- .../logviewer-search-page-template.html | 5 +- .../clj/org/apache/storm/logviewer_test.clj | 57 +++++++++++++-- 4 files changed, 106 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/449394a0/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj index 7baeae7..fec313b 100644 --- a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj +++ b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj @@ -526,13 +526,31 @@ (int (/ (alength needle) -2)))) ;; Addition :length default-bytes-per-page}))) +(defn url-to-match-centered-in-log-page-daemon-file + [needle fname offset port] + (let [host (local-hostname) + port (logviewer-port) + fname (clojure.string/join file-path-separator (take-last 1 (split fname (re-pattern file-path-separator))))] + (url (str "http://"; host ":" port "/daemonlog") + {:file fname + :start (max 0 + (- offset + (int (/ default-bytes-per-page 2)) + (int (/ (alength needle) -2)))) ;; Addition + :length default-bytes-per-page}))) + (defnk mk-match-data [^bytes needle ^ByteBuffer haystack haystack-offset file-offset fname - :before-bytes nil :after-bytes nil] - (let [url (url-to-match-centered-in-log-page needle - fname - file-offset - (*STORM-CONF* LOGVIEWER-PORT)) + :is-daemon false :before-bytes nil :after-bytes nil] + (let [url (if is-daemon + (url-to-match-centered-in-log-page-daemon-file needle + fname + file-offset + (*STORM-CONF* LOGVIEWER-PORT)) + (url-to-match-centered-in-log-page needle + fname + file-offset + (*STORM-CONF* LOGVIEWER-PORT))) haystack-bytes (.array haystack) before-string (if (>= haystack-offset grep-context-size) (String. haystack-bytes @@ -628,7 +646,7 @@ "As the file is read into a buffer, 1/2 the buffer's size at a time, we search the buffer for matches of the substring and return a list of zero or more matches." - [file file-len offset-to-buf init-buf-offset stream bytes-skipped + [is-daemon file file-len offset-to-buf init-buf-offset stream bytes-skipped bytes-read ^ByteBuffer haystack ^bytes needle initial-matches num-matches ^bytes before-bytes] (loop [buf-offset init-buf-offset @@ -653,6 +671,7 @@ offset file-offset (.getCanonicalPath file) + :is-daemon is-daemon :before-bytes before-arg :after-bytes after-arg)))) (let [before-str-to-offset (min (.limit haystack) @@ -709,7 +728,7 @@ context lines. Other information is included to be useful for progressively searching through a file for display in a UI. The search string must grep-max-search-size bytes or fewer when decoded with UTF-8." - [file ^String search-string :num-matches 10 :start-byte-offset 0] + [file ^String search-string :is-daemon false :num-matches 10 :start-byte-offset 0] {:pre [(not (empty? search-string)) (<= (count (.getBytes search-string "UTF-8")) grep-max-search-size)]} (let [zip-file? (.endsWith (.getName file) ".gz") @@ -744,7 +763,9 @@ byte-offset start-byte-offset before-bytes nil] (let [[matches new-byte-offset new-before-bytes] - (buffer-substring-search! file + (buffer-substring-search! + is-daemon + file file-len byte-offset init-buf-offset @@ -771,16 +792,17 @@ new-buf-offset new-byte-offset new-before-bytes)) - (mk-grep-response search-bytes - start-byte-offset - matches - (if-not (and (< (count matches) num-matches) - (>= @total-bytes-read file-len)) - (let [next-byte-offset (+ (get (last matches) - "byteOffset") - (alength search-bytes))] - (if (> file-len next-byte-offset) - next-byte-offset))))))))) + (merge {"isDaemon" (if is-daemon "yes" "no")} + (mk-grep-response search-bytes + start-byte-offset + matches + (if-not (and (< (count matches) num-matches) + (>= @total-bytes-read file-len)) + (let [next-byte-offset (+ (get (last matches) + "byteOffset") + (alength search-bytes))] + (if (> file-len next-byte-offset) + next-byte-offset)))))))))) (defn- try-parse-int-param [nam value] @@ -793,7 +815,7 @@ throw)))) (defn search-log-file - [fname user ^String root-dir search num-matches offset callback origin] + [fname user ^String root-dir is-daemon search num-matches offset callback origin] (let [file (.getCanonicalFile (File. root-dir fname))] (if (.exists file) (if (or (blank? (*STORM-CONF* UI-FILTER)) @@ -807,10 +829,13 @@ (if (and (not (empty? search)) <= (count (.getBytes search "UTF-8")) grep-max-search-size) (json-response - (substring-search file - search - :num-matches num-matches-int - :start-byte-offset offset-int) + (merge {"isDaemon" (if is-daemon "yes" "no")} + (substring-search file + search + :is-daemon is-daemon + :num-matches num-matches-int + :start-byte-offset offset-int)) + callback :headers {"Access-Control-Allow-Origin" origin "Access-Control-Allow-Credentials" "true"}) @@ -1078,10 +1103,12 @@ ;; :keys list, or this rule could stop working when an authentication ;; filter is configured. (try - (let [user (.getUserName http-creds-handler servlet-request)] + (let [user (.getUserName http-creds-handler servlet-request) + is-daemon (= (:is-daemon m) "yes")] (search-log-file (url-decode file) user - (if (= (:is-daemon m) "yes") daemonlog-root log-root) + (if is-daemon daemonlog-root log-root) + is-daemon (:search-string m) (:num-matches m) (:start-byte-offset m) http://git-wip-us.apache.org/repos/asf/storm/blob/449394a0/storm-core/src/ui/public/logviewer_search.html ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/logviewer_search.html b/storm-core/src/ui/public/logviewer_search.html index f878ab9..f981499 100644 --- a/storm-core/src/ui/public/logviewer_search.html +++ b/storm-core/src/ui/public/logviewer_search.html @@ -47,12 +47,12 @@ $(document).ready(function() { var file = $.url("?file"); var search = $.url("?search"); var offset = $.url("?offset") || 0; - var isDaemon = $.url("?is-daemon"); + var isDaemon = $.url("?is-daemon") || "no"; file = decodeURIComponent(file); search = decodeURIComponent(search); $.get("/templates/logviewer-search-page-template.html", function(template) { - $("#search-form").append(Mustache.render($(template).filter("#search-single-file").html(),{file: file, search: search})); + $("#search-form").append(Mustache.render($(template).filter("#search-single-file").html(),{file: file, search: search, isDaemon: isDaemon})); var result = $("#result"); var url = "/search/"+encodeURIComponent(file)+"?search-string="+search+"&start-byte-offset="+offset+"&is-daemon="+isDaemon; http://git-wip-us.apache.org/repos/asf/storm/blob/449394a0/storm-core/src/ui/public/templates/logviewer-search-page-template.html ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/templates/logviewer-search-page-template.html b/storm-core/src/ui/public/templates/logviewer-search-page-template.html index 9d208d3..e132936 100644 --- a/storm-core/src/ui/public/templates/logviewer-search-page-template.html +++ b/storm-core/src/ui/public/templates/logviewer-search-page-template.html @@ -16,7 +16,7 @@ --> <script id="logviewer-search-result-template" type="text/html"> {{#nextByteOffset}} -<a href="/logviewer_search.html?file={{file}}&search={{searchString}}&offset={{nextByteOffset}}" class="btn btn-default enabled">Next</a> +<a href="/logviewer_search.html?file={{file}}&search={{searchString}}&offset={{nextByteOffset}}&is-daemon={{isDaemon}}" class="btn btn-default enabled">Next</a> {{/nextByteOffset}} <table id="search-result-table" class="table table-striped compact"> <thead><tr><th>File offset</th><th>Match</th></tr></thead> @@ -30,7 +30,7 @@ </tbody> </table> {{#nextByteOffset}} -<a href="/logviewer_search.html?file={{file}}&search={{searchString}}&offset={{nextByteOffset}}" class="btn btn-default enabled">Next</a> +<a href="/logviewer_search.html?file={{file}}&search={{searchString}}&offset={{nextByteOffset}}&is-daemon={{isDaemon}}" class="btn btn-default enabled">Next</a> {{/nextByteOffset}} </script> <script id="search-single-file" type="text/html"> @@ -38,6 +38,7 @@ Search {{file}}: <input type="text" name="search" value="{{search}}"> <input type="hidden" name="file" value="{{file}}"> + <input type="hidden" name="is-daemon" value="{{isDaemon}}"> <input type="submit" value="Search"> </form> </script> http://git-wip-us.apache.org/repos/asf/storm/blob/449394a0/storm-core/test/clj/org/apache/storm/logviewer_test.clj ---------------------------------------------------------------------- diff --git a/storm-core/test/clj/org/apache/storm/logviewer_test.clj b/storm-core/test/clj/org/apache/storm/logviewer_test.clj index 1617c34..4d32a06 100644 --- a/storm-core/test/clj/org/apache/storm/logviewer_test.clj +++ b/storm-core/test/clj/org/apache/storm/logviewer_test.clj @@ -380,10 +380,26 @@ 27526 8888))))) + (testing "Logviewer link centers the match in the page (daemon)" + (let [expected-fname "foobar.log"] + (is (= (str "http://"; + expected-host + ":" + expected-port + "/daemonlog?file=" + expected-fname + "&start=1947&length=" + logviewer/default-bytes-per-page) + (logviewer/url-to-match-centered-in-log-page-daemon-file (byte-array 42) + expected-fname + 27526 + 8888))))) + (let [file (->> "logviewer-search-context-tests.log.test" (clojure.java.io/file "src" "dev"))] (testing "returns correct before/after context" - (is (= {"searchString" pattern + (is (= {"isDaemon" "no" + "searchString" pattern "startByteOffset" 0 "matches" [{"byteOffset" 0 "beforeString" "" @@ -434,7 +450,8 @@ (let [file (clojure.java.io/file "src" "dev" "small-worker.log.test")] (testing "a really small log file" - (is (= {"searchString" pattern + (is (= {"isDaemon" "no" + "searchString" pattern "startByteOffset" 0 "matches" [{"byteOffset" 7 "beforeString" "000000 " @@ -449,10 +466,29 @@ "&start=0&length=51200")}]} (logviewer/substring-search file pattern))))) + (let [file (clojure.java.io/file "src" "dev" "small-worker.log.test")] + (testing "a really small log file (daemon)" + (is (= {"isDaemon" "yes" + "searchString" pattern + "startByteOffset" 0 + "matches" [{"byteOffset" 7 + "beforeString" "000000 " + "afterString" " 000000\n" + "matchString" pattern + "logviewerURL" (str "http://"; + expected-host + ":" + expected-port + "/daemonlog?file=" + (.getName file) + "&start=0&length=51200")}]} + (logviewer/substring-search file pattern :is-daemon true))))) + (let [file (clojure.java.io/file "src" "dev" "test-3072.log.test")] (testing "no offset returned when file ends on buffer offset" (let [expected - {"searchString" pattern + {"isDaemon" "no" + "searchString" pattern "startByteOffset" 0 "matches" [{"byteOffset" 3066 "beforeString" (->> @@ -499,7 +535,8 @@ (is (= num-matches-found (count (get result "matches"))))))) (is - (= {"nextByteOffset" 6252 + (= {"isDaemon" "no" + "nextByteOffset" 6252 "searchString" pattern "startByteOffset" 0 "matches" [ @@ -585,7 +622,8 @@ (testing "Correct match offset is returned when skipping bytes" (let [start-byte-offset 3197] - (is (= {"nextByteOffset" 6252 + (is (= {"isDaemon" "no" + "nextByteOffset" 6252 "searchString" pattern "startByteOffset" start-byte-offset "matches" [{"byteOffset" 6246 @@ -606,7 +644,8 @@ (let [pattern (clojure.string/join (repeat 1024 'X))] (is - (= {"nextByteOffset" 6183 + (= {"isDaemon" "no" + "nextByteOffset" 6183 "searchString" pattern "startByteOffset" 0 "matches" [ @@ -637,7 +676,8 @@ (let [pattern "ððð"] (is - (= {"nextByteOffset" 7176 + (= {"isDaemon" "no" + "nextByteOffset" 7176 "searchString" pattern "startByteOffset" 0 "matches" [ @@ -657,7 +697,8 @@ (testing "Returns 0 matches for unseen pattern" (let [pattern "Not There"] - (is (= {"searchString" pattern + (is (= {"isDaemon" "no" + "searchString" pattern "startByteOffset" 0 "matches" []} (logviewer/substring-search file
