[STORM-902] log search, init commit Continue on logviewer search
lv compile passed, to do html and then manual test, unit test Add html image files Now able to search, except the magnifying lens stuff (related to topo history) Support TopologyHistory (just a couple of topoId for a user) Debug, thrift, topo history, now working well for deep search. Unit test and add related resurces files Fix nimbus test Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/22cf0acb Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/22cf0acb Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/22cf0acb Branch: refs/heads/master Commit: 22cf0acbe8233edfc1fa397e5592e5fb6bf10120 Parents: c12e28c Author: zhuol <[email protected]> Authored: Fri Oct 30 14:35:32 2015 -0500 Committer: zhuol <[email protected]> Committed: Mon Nov 9 11:43:58 2015 -0600 ---------------------------------------------------------------------- storm-core/src/clj/backtype/storm/config.clj | 18 + .../src/clj/backtype/storm/daemon/logviewer.clj | 468 +++++++- .../src/clj/backtype/storm/daemon/nimbus.clj | 73 +- .../src/clj/backtype/storm/local_state.clj | 24 + storm-core/src/clj/backtype/storm/ui/core.clj | 37 +- .../src/clj/backtype/storm/ui/helpers.clj | 5 + storm-core/src/clj/backtype/storm/util.clj | 2 + .../src/dev/logviewer-search-context-tests.log | 1 + .../dev/logviewer-search-context-tests.log.gz | Bin 0 -> 72 bytes storm-core/src/dev/small-worker.log | 1 + storm-core/src/dev/test-3072.log | 3 + storm-core/src/dev/test-worker.log | 380 +++++++ .../backtype/storm/generated/LSTopoHistory.java | 805 ++++++++++++++ .../storm/generated/LSTopoHistoryList.java | 460 ++++++++ .../jvm/backtype/storm/generated/Nimbus.java | 1010 +++++++++++++++++- .../storm/generated/TopologyHistoryInfo.java | 461 ++++++++ storm-core/src/py/storm/Nimbus-remote | 7 + storm-core/src/py/storm/Nimbus.py | 213 +++- storm-core/src/py/storm/ttypes.py | 295 ++++- storm-core/src/storm.thrift | 16 + .../src/ui/public/deep_search_result.html | 155 +++ storm-core/src/ui/public/images/search.png | Bin 0 -> 2354 bytes .../src/ui/public/js/typeahead.jquery.min.js | 7 + storm-core/src/ui/public/logviewer_search.html | 65 ++ storm-core/src/ui/public/search_result.html | 100 ++ .../deep-search-result-page-template.html | 66 ++ .../logviewer-search-page-template.html | 44 + .../templates/search-result-page-template.html | 60 ++ .../templates/topology-page-template.html | 11 + .../src/ui/public/templates/user-template.html | 17 +- storm-core/src/ui/public/topology.html | 8 +- .../test/clj/backtype/storm/logviewer_test.clj | 418 ++++++++ .../test/clj/backtype/storm/nimbus_test.clj | 3 +- 33 files changed, 5181 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/clj/backtype/storm/config.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/config.clj b/storm-core/src/clj/backtype/storm/config.clj index c9e151a..1a5cb51 100644 --- a/storm-core/src/clj/backtype/storm/config.clj +++ b/storm-core/src/clj/backtype/storm/config.clj @@ -181,6 +181,10 @@ [conf] (LocalState. (str (supervisor-local-dir conf) file-path-separator "localstate"))) +(defn ^LocalState nimbus-topo-history-state + [conf] + (LocalState. (str (master-local-dir conf) file-path-separator "history"))) + (defn read-supervisor-storm-conf [conf storm-id] (let [stormroot (supervisor-stormdist-root conf storm-id) @@ -268,3 +272,17 @@ (defn ^LocalState worker-state [conf id] (LocalState. (worker-heartbeats-root conf id))) + +(defn get-topo-logs-users + [topology-conf] + (sort (distinct (remove nil? + (concat + (topology-conf LOGS-USERS) + (topology-conf TOPOLOGY-USERS)))))) + +(defn get-topo-logs-groups + [topology-conf] + (sort (distinct (remove nil? + (concat + (topology-conf LOGS-GROUPS) + (topology-conf TOPOLOGY-GROUPS)))))) http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/clj/backtype/storm/daemon/logviewer.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj index e4d518d..16d2fe7 100644 --- a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj +++ b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj @@ -22,17 +22,19 @@ (:use [backtype.storm.ui helpers]) (:import [backtype.storm.utils Utils]) (:import [org.slf4j LoggerFactory]) - (:import [java.io File FileFilter FileInputStream InputStream]) + (:import [java.util Arrays]) (:import [java.util.zip GZIPInputStream]) (:import [org.apache.logging.log4j LogManager]) (:import [org.apache.logging.log4j.core Appender LoggerContext]) (:import [org.apache.logging.log4j.core.appender RollingFileAppender]) + (:import [java.io BufferedInputStream File FileFilter FileInputStream + InputStream InputStreamReader]) + (:import [java.nio ByteBuffer]) (:import [org.yaml.snakeyaml Yaml] [org.yaml.snakeyaml.constructor SafeConstructor]) (:import [backtype.storm.ui InvalidRequestException] [backtype.storm.security.auth AuthUtils]) (:require [backtype.storm.daemon common [supervisor :as supervisor]]) - (:import [java.io File FileFilter]) (:require [compojure.route :as route] [compojure.handler :as handler] [ring.middleware.keyword-params] @@ -322,8 +324,7 @@ (defn log-root-dir "Given an appender name, as configured, get the parent directory of the appender's log file. - -Note that if anything goes wrong, this will throw an Error and exit." + Note that if anything goes wrong, this will throw an Error and exit." [appender-name] (let [appender (.getAppender (.getConfiguration (LogManager/getContext)) appender-name)] (if (and appender-name appender (instance? RollingFileAppender appender)) @@ -337,6 +338,13 @@ Note that if anything goes wrong, this will throw an Error and exit." [:a {:href (java.net.URI. url) :class (str "btn btn-default " (if enabled "enabled" "disabled"))} text]) +(defn search-file-form [fname] + [[:form {:action "logviewer_search.html" :id "search-box"} + "Search this file:" + [:input {:type "text" :name "search"}] + [:input {:type "hidden" :name "file" :value fname}] + [:input {:type "submit" :value "Search"}]]]) + (defn log-file-selection-form [log-files type] [[:form {:action type :id "list-of-files"} (drop-down "file" log-files ) @@ -416,7 +424,8 @@ Note that if anything goes wrong, this will throw an Error and exit." (string/join "\n")) log-string)]) (let [pager-data (if (is-txt-file fname) (pager-links fname start length file-length) nil)] - (html (concat (log-file-selection-form reordered-files-str "log") ; list all files for this topology + (html (concat (search-file-form fname) + (log-file-selection-form reordered-files-str "log") ; list all files for this topology pager-data (download-link fname) [[:pre#logContent log-string]] @@ -484,6 +493,405 @@ Note that if anything goes wrong, this will throw an Error and exit." (-> (resp/response "Page not found") (resp/status 404))))) +(def grep-max-search-size 1024) +(def grep-buf-size 2048) +(def grep-context-size 128) + +(defn logviewer-port + [] + (int (*STORM-CONF* LOGVIEWER-PORT))) + +(defn url-to-match-centered-in-log-page + [needle fname offset port] + (let [host (local-hostname) + port (logviewer-port) + fname (clojure.string/join file-path-separator (take-last 3 (split fname (re-pattern file-path-separator))))] + (url (str "http://"; host ":" port "/log") + {:file fname + :start (max 0 + (- offset + (int (/ default-bytes-per-page 2)) + (int (/ (alength needle) -2)))) ;; Addition + :length default-bytes-per-page}))) + +(defnk mk-match-data + [^bytes needle ^ByteBuffer haystack haystack-offset file-offset fname + :before-bytes nil :after-bytes nil] + (let [url (url-to-match-centered-in-log-page needle + fname + file-offset + (*STORM-CONF* LOGVIEWER-PORT)) + haystack-bytes (.array haystack) + before-string (if (>= haystack-offset grep-context-size) + (String. haystack-bytes + (- haystack-offset grep-context-size) + grep-context-size + "UTF-8") + (let [num-desired (max 0 (- grep-context-size + haystack-offset)) + before-size (if before-bytes + (alength before-bytes) + 0) + num-expected (min before-size num-desired)] + (if (pos? num-expected) + (str (String. before-bytes + (- before-size num-expected) + num-expected + "UTF-8") + (String. haystack-bytes + 0 + haystack-offset + "UTF-8")) + (String. haystack-bytes + 0 + haystack-offset + "UTF-8")))) + after-string (let [needle-size (alength needle) + after-offset (+ haystack-offset needle-size) + haystack-size (.limit haystack)] + (if (< (+ after-offset grep-context-size) haystack-size) + (String. haystack-bytes + after-offset + grep-context-size + "UTF-8") + (let [num-desired (- grep-context-size + (- haystack-size after-offset)) + after-size (if after-bytes + (alength after-bytes) + 0) + num-expected (min after-size num-desired)] + (if (pos? num-expected) + (str (String. haystack-bytes + after-offset + (- haystack-size after-offset) + "UTF-8") + (String. after-bytes 0 num-expected "UTF-8")) + (String. haystack-bytes + after-offset + (- haystack-size after-offset) + "UTF-8")))))] + {"byteOffset" file-offset + "beforeString" before-string + "afterString" after-string + "matchString" (String. needle "UTF-8") + "logviewerURL" url})) + +(defn- try-read-ahead! + "Tries once to read ahead in the stream to fill the context and resets the + stream to its position before the call." + [^BufferedInputStream stream haystack offset file-len bytes-read] + (let [num-expected (min (- file-len bytes-read) + grep-context-size) + after-bytes (byte-array num-expected)] + (.mark stream num-expected) + ;; Only try reading once. + (.read stream after-bytes 0 num-expected) + (.reset stream) + after-bytes)) + +(defn offset-of-bytes + "Searches a given byte array for a match of a sub-array of bytes. Returns + the offset to the byte that matches, or -1 if no match was found." + [^bytes buf ^bytes value init-offset] + {:pre [(> (alength value) 0) + (not (neg? init-offset))]} + (loop [offset init-offset + candidate-offset init-offset + val-offset 0] + (if-not (pos? (- (alength value) val-offset)) + ;; Found + candidate-offset + (if (>= offset (alength buf)) + ;; We ran out of buffer for the search. + -1 + (if (not= (aget value val-offset) (aget buf offset)) + ;; The match at this candidate offset failed, so start over with the + ;; next candidate byte from the buffer. + (let [new-offset (inc candidate-offset)] + (recur new-offset new-offset 0)) + ;; So far it matches. Keep going... + (recur (inc offset) candidate-offset (inc val-offset))))))) + +(defn- buffer-substring-search! + "As the file is read into a buffer, 1/2 the buffer's size at a time, we + search the buffer for matches of the substring and return a list of zero or + more matches." + [file file-len offset-to-buf init-buf-offset stream bytes-skipped + bytes-read ^ByteBuffer haystack ^bytes needle initial-matches num-matches + ^bytes before-bytes] + (loop [buf-offset init-buf-offset + matches initial-matches] + (let [offset (offset-of-bytes (.array haystack) needle buf-offset)] + (if (and (< (count matches) num-matches) (not (neg? offset))) + (let [file-offset (+ offset-to-buf offset) + bytes-needed-after-match (- (.limit haystack) + grep-context-size + (alength needle)) + before-arg (if (< offset grep-context-size) before-bytes) + after-arg (if (> offset bytes-needed-after-match) + (try-read-ahead! stream + haystack + offset + file-len + bytes-read))] + (recur (+ offset (alength needle)) + (conj matches + (mk-match-data needle + haystack + offset + file-offset + (.getCanonicalPath file) + :before-bytes before-arg + :after-bytes after-arg)))) + (let [before-str-to-offset (min (.limit haystack) + grep-max-search-size) + before-str-from-offset (max 0 (- before-str-to-offset + grep-context-size)) + new-before-bytes (Arrays/copyOfRange (.array haystack) + before-str-from-offset + before-str-to-offset) + ;; It's OK if new-byte-offset is negative. This is normal if + ;; we are out of bytes to read from a small file. + new-byte-offset (if (>= (count matches) num-matches) + (+ (get (last matches) "byteOffset") + (alength needle)) + (+ bytes-skipped + bytes-read + (- grep-max-search-size)))] + [matches new-byte-offset new-before-bytes]))))) + +(defn- mk-grep-response + "This response data only includes a next byte offset if there is more of the + file to read." + [search-bytes offset matches next-byte-offset] + (merge {"searchString" (String. search-bytes "UTF-8") + "startByteOffset" offset + "matches" matches} + (and next-byte-offset {"nextByteOffset" next-byte-offset}))) + +(defn rotate-grep-buffer! + [^ByteBuffer buf ^BufferedInputStream stream total-bytes-read file file-len] + (let [buf-arr (.array buf)] + ;; Copy the 2nd half of the buffer to the first half. + (System/arraycopy buf-arr + grep-max-search-size + buf-arr + 0 + grep-max-search-size) + ;; Zero-out the 2nd half to prevent accidental matches. + (Arrays/fill buf-arr + grep-max-search-size + (count buf-arr) + (byte 0)) + ;; Fill the 2nd half with new bytes from the stream. + (let [bytes-read (.read stream + buf-arr + grep-max-search-size + (min file-len grep-max-search-size))] + (.limit buf (+ grep-max-search-size bytes-read)) + (swap! total-bytes-read + bytes-read)))) + +(defnk substring-search + "Searches for a substring in a log file, starting at the given offset, + returning the given number of matches, surrounded by the given number of + context lines. Other information is included to be useful for progressively + searching through a file for display in a UI. The search string must + grep-max-search-size bytes or fewer when decoded with UTF-8." + [file ^String search-string :num-matches 10 :start-byte-offset 0] + {:pre [(not (empty? search-string)) + (<= (count (.getBytes search-string "UTF-8")) grep-max-search-size)]} + (let [zip-file? (.endsWith (.getName file) ".gz") + f-input-steam (FileInputStream. file) + gzipped-input-stream (if zip-file? + (GZIPInputStream. f-input-steam) + f-input-steam) + stream ^BufferedInputStream (BufferedInputStream. + gzipped-input-stream) + file-len (if zip-file? (Utils/zipFileSize file) (.length file)) + buf ^ByteBuffer (ByteBuffer/allocate grep-buf-size) + buf-arr ^bytes (.array buf) + string nil + total-bytes-read (atom 0) + matches [] + search-bytes ^bytes (.getBytes search-string "UTF-8") + num-matches (or num-matches 10) + start-byte-offset (or start-byte-offset 0)] + ;; Start at the part of the log file we are interested in. + ;; Allow searching when start-byte-offset == file-len so it doesn't blow up on 0-length files + (if (> start-byte-offset file-len) + (throw + (InvalidRequestException. "Cannot search past the end of the file"))) + (when (> start-byte-offset 0) + (skip-bytes stream start-byte-offset)) + (java.util.Arrays/fill buf-arr (byte 0)) + (let [bytes-read (.read stream buf-arr 0 (min file-len grep-buf-size))] + (.limit buf bytes-read) + (swap! total-bytes-read + bytes-read)) + (loop [initial-matches [] + init-buf-offset 0 + byte-offset start-byte-offset + before-bytes nil] + (let [[matches new-byte-offset new-before-bytes] + (buffer-substring-search! file + file-len + byte-offset + init-buf-offset + stream + start-byte-offset + @total-bytes-read + buf + search-bytes + initial-matches + num-matches + before-bytes)] + (if (and (< (count matches) num-matches) + (< (+ @total-bytes-read start-byte-offset) file-len)) + (let [;; The start index is positioned to find any possible + ;; occurrence search string that did not quite fit in the + ;; buffer on the previous read. + new-buf-offset (- (min (.limit ^ByteBuffer buf) + grep-max-search-size) + (alength search-bytes))] + (rotate-grep-buffer! buf stream total-bytes-read file file-len) + (when (< @total-bytes-read 0) + (throw (InvalidRequestException. "Cannot search past the end of the file"))) + (recur matches + new-buf-offset + new-byte-offset + new-before-bytes)) + (mk-grep-response search-bytes + start-byte-offset + matches + (if-not (and (< (count matches) num-matches) + (>= @total-bytes-read file-len)) + (let [next-byte-offset (+ (get (last matches) + "byteOffset") + (alength search-bytes))] + (if (> file-len next-byte-offset) + next-byte-offset))))))))) + +(defn- try-parse-int-param + [nam value] + (try + (Integer/parseInt value) + (catch java.lang.NumberFormatException e + (-> + (str "Could not parse " nam " to an integer") + (InvalidRequestException. e) + throw)))) + +(defn search-log-file + [fname user ^String root-dir search num-matches offset callback origin] + (let [file (.getCanonicalFile (File. root-dir fname))] + (if (.exists file) + (if (or (blank? (*STORM-CONF* UI-FILTER)) + (authorized-log-user? user fname *STORM-CONF*)) + (let [num-matches-int (if num-matches + (try-parse-int-param "num-matches" + num-matches)) + offset-int (if offset + (try-parse-int-param "start-byte-offset" offset))] + (try + (if (and (not (empty? search)) + <= (count (.getBytes search "UTF-8")) grep-max-search-size) + (json-response + (substring-search file + search + :num-matches num-matches-int + :start-byte-offset offset-int) + callback + :headers {"Access-Control-Allow-Origin" origin + "Access-Control-Allow-Credentials" "true"}) + (throw + (InvalidRequestException. + (str "Search substring must be between 1 and 1024 UTF-8 " + "bytes in size (inclusive)")))) + (catch Exception ex + (json-response (exception->json ex) callback :status 500)))) + (json-response (unauthorized-user-json user) callback :status 401)) + (json-response {"error" "Not Found" + "errorMessage" "The file was not found on this node."} + callback + :status 404)))) + +(defn find-n-matches [logs n file-offset offset search] + (let [logs (drop file-offset logs) + wrap-matches-fn (fn [matches] + {"fileOffset" file-offset + "searchString" search + "matches" matches})] + (loop [matches [] + logs logs + offset offset + file-offset file-offset + match-count 0] + (if (empty? logs) + (wrap-matches-fn matches) + (let [these-matches (try + (log-debug "Looking through " (first logs)) + (substring-search (first logs) + search + :num-matches (- n match-count) + :start-byte-offset offset) + (catch InvalidRequestException e + (log-error e "Can't search past end of file.") + {})) + file-name (get-topo-port-workerlog (first logs)) + new-matches (conj matches + (merge these-matches + { "fileName" file-name + "port" (first (take-last 2 (split (.getCanonicalPath (first logs)) (re-pattern file-path-separator))))})) + new-count (+ match-count (count (these-matches "matches")))] + (if (empty? these-matches) + (recur matches (rest logs) 0 (+ file-offset 1) match-count) + (if (>= new-count n) + (wrap-matches-fn new-matches) + (recur new-matches (rest logs) 0 (+ file-offset 1) new-count)))))))) + +(defn logs-for-port + "Get the filtered, authorized, sorted log files for a port." + [user port-dir] + (let [filter-authorized-fn (fn [user logs] + (filter #(or + (blank? (*STORM-CONF* UI-FILTER)) + (authorized-log-user? user (get-topo-port-workerlog %) *STORM-CONF*)) logs))] + (sort #(compare (.lastModified %2) (.lastModified %1)) + (filter-authorized-fn + user + (filter #(re-find worker-log-filename-pattern (.getName %)) (.listFiles port-dir)))))) + +(defn deep-search-logs-for-topology + [topology-id user ^String root-dir search num-matches port file-offset offset search-archived? callback origin] + (json-response + (if (or (not search) (not (.exists (File. (str root-dir file-path-separator topology-id))))) + [] + (let [file-offset (if file-offset (Integer/parseInt file-offset) 0) + offset (if offset (Integer/parseInt offset) 0) + num-matches (or (Integer/parseInt num-matches) 1) + port-dirs (vec (.listFiles (File. (str root-dir file-path-separator topology-id)))) + logs-for-port-fn (partial logs-for-port user)] + (if (or (not port) (= "*" port)) + ;; Check for all ports + (let [filtered-logs (filter (comp not empty?) (map logs-for-port-fn port-dirs))] + (if search-archived? + (map #(find-n-matches % num-matches 0 0 search) + filtered-logs) + (map #(find-n-matches % num-matches 0 0 search) + (map (comp vector first) filtered-logs)))) + ;; Check just the one port + (if (not (contains? (into #{} (map str (*STORM-CONF* SUPERVISOR-SLOTS-PORTS))) port)) + [] + (let [port-dir (File. (str root-dir file-path-separator topology-id file-path-separator port))] + (if (or (not (.exists port-dir)) (empty? (logs-for-port user port-dir))) + [] + (let [filtered-logs (logs-for-port user port-dir)] + (if search-archived? + (find-n-matches filtered-logs num-matches file-offset offset search) + (find-n-matches [(first filtered-logs)] num-matches 0 offset search))))))))) + callback + :headers {"Access-Control-Allow-Origin" origin + "Access-Control-Allow-Credentials" "true"})) + (defn log-template ([body] (log-template body nil nil)) ([body fname user] @@ -632,6 +1040,56 @@ Note that if anything goes wrong, this will throw an Error and exit." (catch InvalidRequestException ex (log-error ex) (ring-response-from-exception ex)))) + (GET "/search/:file" [:as {:keys [servlet-request servlet-response log-root]} file & m] + ;; We do not use servlet-response here, but do not remove it from the + ;; :keys list, or this rule could stop working when an authentication + ;; filter is configured. + (try + (let [user (.getUserName http-creds-handler servlet-request)] + (search-log-file (url-decode file) + user + log-root + (:search-string m) + (:num-matches m) + (:start-byte-offset m) + (:callback m) + (.getHeader servlet-request "Origin"))) + (catch InvalidRequestException ex + (log-error ex) + (json-response (exception->json ex) (:callback m) :status 400)))) + (GET "/deepSearch/:topo-id" [:as {:keys [servlet-request servlet-response log-root]} topo-id & m] + ;; We do not use servlet-response here, but do not remove it from the + ;; :keys list, or this rule could stop working when an authentication + ;; filter is configured. + (try + (let [user (.getUserName http-creds-handler servlet-request)] + (deep-search-logs-for-topology topo-id + user + log-root + (:search-string m) + (:num-matches m) + (:port m) + (:start-file-offset m) + (:start-byte-offset m) + (:search-archived m) + (:callback m) + (.getHeader servlet-request "Origin"))) + (catch InvalidRequestException ex + (log-error ex) + (json-response (exception->json ex) (:callback m) :status 400)))) + (GET "/searchLogs" [:as req & m] + (try + (let [servlet-request (:servlet-request req) + user (.getUserName http-creds-handler servlet-request)] + (list-log-files user + (:topoId m) + (:port m) + (:log-root req) + (:callback m) + (.getHeader servlet-request "Origin"))) + (catch InvalidRequestException ex + (log-error ex) + (json-response (exception->json ex) (:callback m) :status 400)))) (GET "/listLogs" [:as req & m] (try (mark! logviewer:num-list-logs-http-requests) http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/clj/backtype/storm/daemon/nimbus.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj index 75b2c80..dddfbc9 100644 --- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj +++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj @@ -34,12 +34,12 @@ BufferFileInputStream]) (:import [backtype.storm.generated NotAliveException AlreadyAliveException StormTopology ErrorInfo ExecutorInfo InvalidTopologyException Nimbus$Iface Nimbus$Processor SubmitOptions TopologyInitialStatus - KillOptions RebalanceOptions ClusterSummary SupervisorSummary TopologySummary TopologyInfo + KillOptions RebalanceOptions ClusterSummary SupervisorSummary TopologySummary TopologyInfo TopologyHistoryInfo ExecutorSummary AuthorizationException GetInfoOptions NumErrorsChoice ComponentPageInfo TopologyPageInfo LogConfig LogLevel LogLevelAction ProfileRequest ProfileAction NodeInfo]) (:import [backtype.storm.daemon Shutdownable]) - (:use [backtype.storm util config log timer zookeeper]) + (:use [backtype.storm util config log timer zookeeper local-state]) (:require [backtype.storm [cluster :as cluster] [converter :as converter] [stats :as stats] @@ -156,6 +156,8 @@ :id->sched-status (atom {}) :id->resources (atom {}) :cred-renewers (AuthUtils/GetCredentialRenewers conf) + :topology-history-lock (Object.) + :topo-history-state (nimbus-topo-history-state conf) :nimbus-autocred-plugins (AuthUtils/getNimbusAutoCredPlugins conf) })) @@ -1015,6 +1017,17 @@ (log-error "Cleaning inbox ... error deleting: " (.getName f)) )))) +(defn clean-topology-history + "Deletes topologies from history older than minutes." + [mins nimbus] + (locking (:topology-history-lock nimbus) + (let [cutoff-age (- (current-time-secs) (* mins 60)) + topo-history-state (:topo-history-state nimbus) + curr-history (vec (ls-topo-hist topo-history-state)) + new-history (vec (filter (fn [line] + (> (line :timestamp) cutoff-age)) curr-history))] + (ls-topo-hist! topo-history-state new-history)))) + (defn cleanup-corrupt-topologies! [nimbus] (if (is-leader nimbus :throw-exception false) (let [storm-cluster-state (:storm-cluster-state nimbus) @@ -1088,6 +1101,46 @@ ) ) +(defn add-topology-to-history-log + [storm-id nimbus topology-conf] + (log-message "Adding topo to history log: " storm-id) + (locking (:topology-history-lock nimbus) + (let [topo-history-state (:topo-history-state nimbus) + users (get-topo-logs-users topology-conf) + groups (get-topo-logs-groups topology-conf) + curr-history (vec (ls-topo-hist topo-history-state)) + new-history (conj curr-history {:topoid storm-id :timestamp (current-time-secs) + :users users :groups groups})] + (ls-topo-hist! topo-history-state new-history)))) + +(defn igroup-mapper + [storm-conf] + (AuthUtils/GetGroupMappingServiceProviderPlugin storm-conf)) + +(defn user-groups + [user storm-conf] + (if (clojure.string/blank? user) [] (.getGroups (igroup-mapper storm-conf) user))) + +(defn does-users-group-intersect? + "Check to see if any of the users groups intersect with the list of groups passed in" + [user groups-to-check storm-conf] + (let [groups (user-groups user storm-conf)] + (> (.size (set/intersection (set groups) (set groups-to-check))) 0))) + +(defn read-topology-history + [nimbus user admin-users] + (let [topo-history-state (:topo-history-state nimbus) + curr-history (vec (ls-topo-hist topo-history-state)) + topo-user-can-access (fn [line user storm-conf] + (if (nil? user) + (line :topoid) + (if (or (some #(= % user) admin-users) + (does-users-group-intersect? user (line :groups) storm-conf) + (some #(= % user) (line :users))) + (line :topoid) + nil)))] + (remove nil? (map #(topo-user-can-access % user (:conf nimbus)) curr-history)))) + (defn renew-credentials [nimbus] (if (is-leader nimbus :throw-exception false) (let [storm-cluster-state (:storm-cluster-state nimbus) @@ -1140,6 +1193,7 @@ (log-message "Starting Nimbus with conf " conf) (let [nimbus (nimbus-data conf inimbus) principal-to-local (AuthUtils/GetPrincipalToLocalPlugin conf) + admin-users (or (.get conf NIMBUS-ADMINS) []) get-common-topo-info (fn [^String storm-id operation] (let [storm-cluster-state (:storm-cluster-state nimbus) @@ -1771,6 +1825,21 @@ (.set_eventlog_port port)))) comp-page-info)) + (^TopologyHistoryInfo getTopologyHistory [this ^String user] + (let [storm-cluster-state (:storm-cluster-state nimbus) + bases (topology-bases storm-cluster-state) + assigned-topology-ids (.assignments storm-cluster-state nil) + user-group-match-fn (fn [topo-id user conf] + (let [topology-conf (try-read-storm-conf conf topo-id) + groups (get-topo-logs-groups topology-conf)] + (or (nil? user) + (some #(= % user) admin-users) + (does-users-group-intersect? user groups conf) + (some #(= % user) (get-topo-logs-users topology-conf))))) + active-ids-for-user (filter #(user-group-match-fn % user (:conf nimbus)) assigned-topology-ids) + topo-history-list (read-topology-history nimbus user admin-users)] + (TopologyHistoryInfo. (distinct (concat active-ids-for-user topo-history-list))))) + Shutdownable (shutdown [this] (mark! nimbus:num-shutdown-calls) http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/clj/backtype/storm/local_state.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/local_state.clj b/storm-core/src/clj/backtype/storm/local_state.clj index 4eaae15..bf9567d 100644 --- a/storm-core/src/clj/backtype/storm/local_state.clj +++ b/storm-core/src/clj/backtype/storm/local_state.clj @@ -20,6 +20,7 @@ LSSupervisorId LSApprovedWorkers LSSupervisorAssignments LocalAssignment ExecutorInfo LSWorkerHeartbeat + LSTopoHistory LSTopoHistoryList WorkerResources]) (:import [backtype.storm.utils LocalState])) @@ -27,6 +28,29 @@ (def LS-ID "supervisor-id") (def LS-LOCAL-ASSIGNMENTS "local-assignments") (def LS-APPROVED-WORKERS "approved-workers") +(def LS-TOPO-HISTORY "topo-hist") + +(defn ->LSTopoHistory + [{topoid :topoid timestamp :timestamp users :users groups :groups}] + (LSTopoHistory. topoid timestamp users groups)) + +(defn ->topo-history + [thrift-topo-hist] + { + :topoid (.get_topology_id thrift-topo-hist) + :timestamp (.get_time_stamp thrift-topo-hist) + :users (.get_users thrift-topo-hist) + :groups (.get_groups thrift-topo-hist)}) + +(defn ls-topo-hist! + [^LocalState local-state hist-list] + (.put local-state LS-TOPO-HISTORY + (LSTopoHistoryList. (map ->LSTopoHistory hist-list)))) + +(defn ls-topo-hist + [^LocalState local-state] + (if-let [thrift-hist-list (.get local-state LS-TOPO-HISTORY)] + (map ->topo-history (.get_topo_history thrift-hist-list)))) (defn ls-supervisor-id! [^LocalState local-state ^String id] http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/clj/backtype/storm/ui/core.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj index ecf9b2c..e0bd669 100644 --- a/storm-core/src/clj/backtype/storm/ui/core.clj +++ b/storm-core/src/clj/backtype/storm/ui/core.clj @@ -283,6 +283,12 @@ (map (fn [row] {:row row}) (partition 4 4 nil streams)))) +(defn- get-topology-info + ([^Nimbus$Client nimbus id] + (.getTopologyInfo nimbus id)) + ([^Nimbus$Client nimbus id options] + (.getTopologyInfoWithOpts nimbus id options))) + (defn mk-visualization-data [id window include-sys?] (thrift/with-configured-nimbus-connection @@ -344,6 +350,15 @@ (thrift/with-configured-nimbus-connection nimbus (.getNimbusConf ^Nimbus$Client nimbus))) +(defn topology-history-info + ([user] + (thrift/with-configured-nimbus-connection nimbus + (topology-history-info (.getTopologyHistory ^Nimbus$Client nimbus user) user))) + ([history user] + {"topo-history" + (for [^String s (.get_topo_ids history)] + {"host" s})})) + (defn cluster-summary ([user] (thrift/with-configured-nimbus-connection nimbus @@ -584,6 +599,18 @@ "samplingPct" (or samplingPct 10) "replicationCount" (.get_replication_count topo-info)})) +(defn exec-host-port + [executors] + (for [^ExecutorSummary e executors] + {"host" (.get_host e) + "port" (.get_port e)})) + +(defn worker-host-port + "Get the set of all worker host/ports" + [id] + (thrift/with-configured-nimbus-connection nimbus + (distinct (exec-host-port (.get_executors (get-topology-info nimbus id)))))) + (defn topology-page [id window include-sys? user secure?] (thrift/with-configured-nimbus-connection nimbus (let [window (if window window ":all-time") @@ -901,16 +928,24 @@ (populate-context! servlet-request) (assert-authorized-user "getClusterInfo") (json-response (nimbus-summary) (:callback m))) + (GET "/api/v1/history/summary" [:as {:keys [cookies servlet-request]} & m] + (let [user (.getUserName http-creds-handler servlet-request)] + (json-response (topology-history-info user) (:callback m)))) (GET "/api/v1/supervisor/summary" [:as {:keys [cookies servlet-request]} & m] (mark! ui:num-supervisor-summary-http-requests) (populate-context! servlet-request) (assert-authorized-user "getClusterInfo") - (json-response (supervisor-summary) (:callback m))) + (json-response (assoc (supervisor-summary) + "logviewerPort" (*STORM-CONF* LOGVIEWER-PORT)) (:callback m))) (GET "/api/v1/topology/summary" [:as {:keys [cookies servlet-request]} & m] (mark! ui:num-all-topologies-summary-http-requests) (populate-context! servlet-request) (assert-authorized-user "getClusterInfo") (json-response (all-topologies-summary) (:callback m))) + (GET "/api/v1/topology-workers/:id" [:as {:keys [cookies servlet-request]} id & m] + (let [id (url-decode id)] + (json-response {"hostPortList" (worker-host-port id) + "logviewerPort" (*STORM-CONF* LOGVIEWER-PORT)} (:callback m)))) (GET "/api/v1/topology/:id" [:as {:keys [cookies servlet-request scheme]} id & m] (mark! ui:num-topology-page-http-requests) (populate-context! servlet-request) http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/clj/backtype/storm/ui/helpers.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/ui/helpers.clj b/storm-core/src/clj/backtype/storm/ui/helpers.clj index 43cfa22..e0db5c8 100644 --- a/storm-core/src/clj/backtype/storm/ui/helpers.clj +++ b/storm-core/src/clj/backtype/storm/ui/helpers.clj @@ -113,6 +113,11 @@ (defn pretty-executor-info [^ExecutorInfo e] (str "[" (.get_task_start e) "-" (.get_task_end e) "]")) +(defn unauthorized-user-json + [user] + {"error" "No Authorization" + "errorMessage" (str "User " user " is not authorized.")}) + (defn unauthorized-user-html [user] [[:h2 "User '" (escape-html user) "' is not authorized."]]) http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/clj/backtype/storm/util.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/util.clj b/storm-core/src/clj/backtype/storm/util.clj index dc8ba2d..5cdcfe7 100644 --- a/storm-core/src/clj/backtype/storm/util.clj +++ b/storm-core/src/clj/backtype/storm/util.clj @@ -1046,6 +1046,8 @@ [storm-id port] (str storm-id file-path-separator port file-path-separator "worker.log")) +(def worker-log-filename-pattern #"^worker.log(.*)") + (defn event-logs-filename [storm-id port] (str storm-id file-path-separator port file-path-separator "events.log")) http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/dev/logviewer-search-context-tests.log ---------------------------------------------------------------------- diff --git a/storm-core/src/dev/logviewer-search-context-tests.log b/storm-core/src/dev/logviewer-search-context-tests.log new file mode 100644 index 0000000..6e4d4af --- /dev/null +++ b/storm-core/src/dev/logviewer-search-context-tests.log @@ -0,0 +1 @@ +needle needle000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000needle needle http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/dev/logviewer-search-context-tests.log.gz ---------------------------------------------------------------------- diff --git a/storm-core/src/dev/logviewer-search-context-tests.log.gz b/storm-core/src/dev/logviewer-search-context-tests.log.gz new file mode 100644 index 0000000..5cf2a06 Binary files /dev/null and b/storm-core/src/dev/logviewer-search-context-tests.log.gz differ http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/dev/small-worker.log ---------------------------------------------------------------------- diff --git a/storm-core/src/dev/small-worker.log b/storm-core/src/dev/small-worker.log new file mode 100644 index 0000000..27d61d1 --- /dev/null +++ b/storm-core/src/dev/small-worker.log @@ -0,0 +1 @@ +000000 needle 000000 http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/dev/test-3072.log ---------------------------------------------------------------------- diff --git a/storm-core/src/dev/test-3072.log b/storm-core/src/dev/test-3072.log new file mode 100644 index 0000000..56dc6f1 --- /dev/null +++ b/storm-core/src/dev/test-3072.log @@ -0,0 +1,3 @@ +This is a test log file of size 3072. + +..................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................... ..................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................... ..................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................... ....................................needle \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/dev/test-worker.log ---------------------------------------------------------------------- diff --git a/storm-core/src/dev/test-worker.log b/storm-core/src/dev/test-worker.log new file mode 100644 index 0000000..8fb4c53 --- /dev/null +++ b/storm-core/src/dev/test-worker.log @@ -0,0 +1,380 @@ +Test needle is near the beginning of the file. +This file assumes a buffer size of 2048 bytes, a max search string size of 1024 bytes, and a context length of 128 UTF-8 characters. +The early match tests the case when we find a match too close to the start of the file to give the normal before context strings. + +padding 5 +padding 6 +padding 7 +padding 8 +padding 9 +padding 10 +padding 11 +padding 12 +padding 13 +padding 14 +padding 15 +padding 16 +padding 17 +padding 18 +padding 19 +padding 20 +padding 21 +padding 22 +padding 23 +padding 24 +padding 25 +padding 26 +padding 27 +padding 28 +padding 29 +padding 30 +padding 31 +padding 32 +padding 33 +padding 34 +padding 35 +padding 36 +padding 37 +padding 38 +padding 39 +padding 40 +padding 41 +padding 42 +padding 43 +padding 44 +padding 45 +padding 46 +padding 47 +padding 48 +padding 49 +padding 50 +padding 51 +padding 52 +padding 53 +padding 54 +padding 55 +padding 56 +padding 57 +padding 58 +padding 59 +padding 60 +padding 61 +padding 62 +padding 63 +padding 64 +padding 65 +padding 66 +padding 67 +padding 68 +padding 69 +padding 70 +padding 71 +padding 72 +padding 73 +padding 74 +padding 75 +padding 76 +padding 77 +padding 78 +padding 79 +padding 80 +padding 81 +padding 82 +padding 83 +padding 84 +padding 85 +padding 86 +padding 87 +padding 88 +padding 89 +padding 90 +padding 91 +padding 92 +padding 93 +padding 94 +padding 95 +padding 96 +padding 97 +padding 98 +padding 99 +padding 100 +padding 101 +padding 102 +padding 103 +padding 104 +padding 105 +padding 106 +padding 107 +padding 108 +padding 109 +padding 110 +padding 111 +padding 112 +padding 113 +padding 114 +padding 115 +padding 116 +padding 117 +padding 118 +padding 119 +padding 120 +padding 121 +padding 122 +padding 123 +padding 124 +padding 125 +padding 126 +padding 127 +padding 128 +padding 129 +padding 130 +padding 131 +padding 132 +padding 133 +padding 134 +padding 135 +padding 136 +padding 137 +padding 138 +padding 139 +padding 140 +padding 141 +padding 142 +padding 143 +padding 144 +padding 145 +padding 146 +padding 147 +padding 148 +padding 149 +padding 150 +padding 151 +padding 152 +padding 153 +Near the end of a 1024 byte block, a needle. +A needle that straddles a 1024 byte boundary should also be detected. + +padding 157 +padding 158 +padding 159 +padding 160 +padding 161 +padding 162 +padding 163 +padding 164 +padding 165 +padding 166 +padding 167 +padding 168 +padding 169 +padding 170 +padding 171 +padding 172 +padding 173 +padding 174 +padding 175 +padding 176 +padding 177 +padding 178 +padding 179 +padding 180 +padding 181 +padding 182 +padding 183 +padding 184 +padding 185 +padding 186 +padding 187 +padding 188 +padding 189 +padding 190 +padding 191 +padding 192 +padding 193 +padding 194 +padding 195 +padding 196 +padding 197 +padding 198 +padding 199 +padding 200 +padding 201 +padding 202 +padding 203 +padding 204 +padding 205 +padding 206 +padding 207 +padding 208 +padding 209 +padding 210 +padding 211 +padding 212 +padding 213 +padding 214 +padding 215 +padding 216 +padding 217 +padding 218 +padding 219 +padding 220 +padding 221 +padding 222 +padding 223 +padding 224 +padding 225 +padding 226 +padding 227 +padding 228 +padding 229 +padding 230 +padding 231 +padding 232 +padding 233 +padding 234 +padding 235 + + +Here a needle occurs just after a 1024 byte boundary. It should have the correct context. + +Text with two adjoining matches: needleneedle + +padding 243 +padding 244 +padding 245 +padding 246 +padding 247 +padding 248 +padding 249 +padding 250 +padding 251 +padding 252 +padding 253 +padding 254 +padding 255 +padding 256 +padding 257 +padding 258 +padding 259 +padding 260 +padding 261 +padding 262 +padding 263 +padding 264 +padding 265 +padding 266 +padding 267 +padding 268 +padding 269 +padding 270 +padding 271 +padding 272 +padding 273 +padding 274 +padding 275 +padding 276 +padding 277 +padding 278 +padding 279 +padding 280 +padding 281 +padding 282 +padding 283 +padding 284 +padding 285 +padding 286 +padding 287 +padding 288 +padding 289 +padding 290 +padding 291 +padding 292 +padding 293 +padding 294 +padding 295 +padding 296 +padding 297 +padding 298 +padding 299 +padding 300 +padding 301 +padding 302 +padding 303 +padding 304 + +The following match of 1024 bytes completely fills half the byte buffer. It is a search substring of the maximum size...... + +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXX +The following max-size match straddles a 1024 byte buffer. +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXX + +Here are four non-ascii 1-byte UTF-8 characters: αβγδε + +needle + +Here are four printable 2-byte UTF-8 characters: ¡¢£¤¥ + +needle + + + +Here are four printable 3-byte UTF-8 characters: à¤à¤ à¤à¤à¤ + +needle + +Here are four printable 4-byte UTF-8 characters: ððððð + +needle + +Here are four of the same invalid UTF-8 characters: ���������������� + +needle + +padding 332 +padding 333 +padding 334 +padding 335 +padding 336 +padding 337 +padding 338 +padding 339 +padding 340 +padding 341 +padding 342 +padding 343 +padding 344 +padding 345 +padding 346 +padding 347 +padding 348 +padding 349 +padding 350 +padding 351 +padding 352 +padding 353 +padding 354 +padding 355 +padding 356 +padding 357 +padding 358 +padding 359 +padding 360 +padding 361 +padding 362 +padding 363 +padding 364 +padding 365 +padding 366 +padding 367 +padding 368 +padding 369 +padding 370 +padding 371 +padding 372 +padding 373 +padding 374 +padding 375 + +The following tests multibyte UTF-8 Characters straddling the byte boundary: ððð + +needle \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm/blob/22cf0acb/storm-core/src/jvm/backtype/storm/generated/LSTopoHistory.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/generated/LSTopoHistory.java b/storm-core/src/jvm/backtype/storm/generated/LSTopoHistory.java new file mode 100644 index 0000000..75fde1e --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/generated/LSTopoHistory.java @@ -0,0 +1,805 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Autogenerated by Thrift Compiler (0.9.2) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package backtype.storm.generated; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.thrift.server.AbstractNonblockingServer.*; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import javax.annotation.Generated; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2015-11-4") +public class LSTopoHistory implements org.apache.thrift.TBase<LSTopoHistory, LSTopoHistory._Fields>, java.io.Serializable, Cloneable, Comparable<LSTopoHistory> { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("LSTopoHistory"); + + private static final org.apache.thrift.protocol.TField TOPOLOGY_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("topology_id", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField TIME_STAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("time_stamp", org.apache.thrift.protocol.TType.I64, (short)2); + private static final org.apache.thrift.protocol.TField USERS_FIELD_DESC = new org.apache.thrift.protocol.TField("users", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField GROUPS_FIELD_DESC = new org.apache.thrift.protocol.TField("groups", org.apache.thrift.protocol.TType.LIST, (short)4); + + private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new LSTopoHistoryStandardSchemeFactory()); + schemes.put(TupleScheme.class, new LSTopoHistoryTupleSchemeFactory()); + } + + private String topology_id; // required + private long time_stamp; // required + private List<String> users; // required + private List<String> groups; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TOPOLOGY_ID((short)1, "topology_id"), + TIME_STAMP((short)2, "time_stamp"), + USERS((short)3, "users"), + GROUPS((short)4, "groups"); + + private static final Map<String, _Fields> byName = new HashMap<String, _Fields>(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TOPOLOGY_ID + return TOPOLOGY_ID; + case 2: // TIME_STAMP + return TIME_STAMP; + case 3: // USERS + return USERS; + case 4: // GROUPS + return GROUPS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TIME_STAMP_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TOPOLOGY_ID, new org.apache.thrift.meta_data.FieldMetaData("topology_id", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.TIME_STAMP, new org.apache.thrift.meta_data.FieldMetaData("time_stamp", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.USERS, new org.apache.thrift.meta_data.FieldMetaData("users", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.GROUPS, new org.apache.thrift.meta_data.FieldMetaData("groups", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(LSTopoHistory.class, metaDataMap); + } + + public LSTopoHistory() { + } + + public LSTopoHistory( + String topology_id, + long time_stamp, + List<String> users, + List<String> groups) + { + this(); + this.topology_id = topology_id; + this.time_stamp = time_stamp; + set_time_stamp_isSet(true); + this.users = users; + this.groups = groups; + } + + /** + * Performs a deep copy on <i>other</i>. + */ + public LSTopoHistory(LSTopoHistory other) { + __isset_bitfield = other.__isset_bitfield; + if (other.is_set_topology_id()) { + this.topology_id = other.topology_id; + } + this.time_stamp = other.time_stamp; + if (other.is_set_users()) { + List<String> __this__users = new ArrayList<String>(other.users); + this.users = __this__users; + } + if (other.is_set_groups()) { + List<String> __this__groups = new ArrayList<String>(other.groups); + this.groups = __this__groups; + } + } + + public LSTopoHistory deepCopy() { + return new LSTopoHistory(this); + } + + @Override + public void clear() { + this.topology_id = null; + set_time_stamp_isSet(false); + this.time_stamp = 0; + this.users = null; + this.groups = null; + } + + public String get_topology_id() { + return this.topology_id; + } + + public void set_topology_id(String topology_id) { + this.topology_id = topology_id; + } + + public void unset_topology_id() { + this.topology_id = null; + } + + /** Returns true if field topology_id is set (has been assigned a value) and false otherwise */ + public boolean is_set_topology_id() { + return this.topology_id != null; + } + + public void set_topology_id_isSet(boolean value) { + if (!value) { + this.topology_id = null; + } + } + + public long get_time_stamp() { + return this.time_stamp; + } + + public void set_time_stamp(long time_stamp) { + this.time_stamp = time_stamp; + set_time_stamp_isSet(true); + } + + public void unset_time_stamp() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TIME_STAMP_ISSET_ID); + } + + /** Returns true if field time_stamp is set (has been assigned a value) and false otherwise */ + public boolean is_set_time_stamp() { + return EncodingUtils.testBit(__isset_bitfield, __TIME_STAMP_ISSET_ID); + } + + public void set_time_stamp_isSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TIME_STAMP_ISSET_ID, value); + } + + public int get_users_size() { + return (this.users == null) ? 0 : this.users.size(); + } + + public java.util.Iterator<String> get_users_iterator() { + return (this.users == null) ? null : this.users.iterator(); + } + + public void add_to_users(String elem) { + if (this.users == null) { + this.users = new ArrayList<String>(); + } + this.users.add(elem); + } + + public List<String> get_users() { + return this.users; + } + + public void set_users(List<String> users) { + this.users = users; + } + + public void unset_users() { + this.users = null; + } + + /** Returns true if field users is set (has been assigned a value) and false otherwise */ + public boolean is_set_users() { + return this.users != null; + } + + public void set_users_isSet(boolean value) { + if (!value) { + this.users = null; + } + } + + public int get_groups_size() { + return (this.groups == null) ? 0 : this.groups.size(); + } + + public java.util.Iterator<String> get_groups_iterator() { + return (this.groups == null) ? null : this.groups.iterator(); + } + + public void add_to_groups(String elem) { + if (this.groups == null) { + this.groups = new ArrayList<String>(); + } + this.groups.add(elem); + } + + public List<String> get_groups() { + return this.groups; + } + + public void set_groups(List<String> groups) { + this.groups = groups; + } + + public void unset_groups() { + this.groups = null; + } + + /** Returns true if field groups is set (has been assigned a value) and false otherwise */ + public boolean is_set_groups() { + return this.groups != null; + } + + public void set_groups_isSet(boolean value) { + if (!value) { + this.groups = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TOPOLOGY_ID: + if (value == null) { + unset_topology_id(); + } else { + set_topology_id((String)value); + } + break; + + case TIME_STAMP: + if (value == null) { + unset_time_stamp(); + } else { + set_time_stamp((Long)value); + } + break; + + case USERS: + if (value == null) { + unset_users(); + } else { + set_users((List<String>)value); + } + break; + + case GROUPS: + if (value == null) { + unset_groups(); + } else { + set_groups((List<String>)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TOPOLOGY_ID: + return get_topology_id(); + + case TIME_STAMP: + return Long.valueOf(get_time_stamp()); + + case USERS: + return get_users(); + + case GROUPS: + return get_groups(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TOPOLOGY_ID: + return is_set_topology_id(); + case TIME_STAMP: + return is_set_time_stamp(); + case USERS: + return is_set_users(); + case GROUPS: + return is_set_groups(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof LSTopoHistory) + return this.equals((LSTopoHistory)that); + return false; + } + + public boolean equals(LSTopoHistory that) { + if (that == null) + return false; + + boolean this_present_topology_id = true && this.is_set_topology_id(); + boolean that_present_topology_id = true && that.is_set_topology_id(); + if (this_present_topology_id || that_present_topology_id) { + if (!(this_present_topology_id && that_present_topology_id)) + return false; + if (!this.topology_id.equals(that.topology_id)) + return false; + } + + boolean this_present_time_stamp = true; + boolean that_present_time_stamp = true; + if (this_present_time_stamp || that_present_time_stamp) { + if (!(this_present_time_stamp && that_present_time_stamp)) + return false; + if (this.time_stamp != that.time_stamp) + return false; + } + + boolean this_present_users = true && this.is_set_users(); + boolean that_present_users = true && that.is_set_users(); + if (this_present_users || that_present_users) { + if (!(this_present_users && that_present_users)) + return false; + if (!this.users.equals(that.users)) + return false; + } + + boolean this_present_groups = true && this.is_set_groups(); + boolean that_present_groups = true && that.is_set_groups(); + if (this_present_groups || that_present_groups) { + if (!(this_present_groups && that_present_groups)) + return false; + if (!this.groups.equals(that.groups)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List<Object> list = new ArrayList<Object>(); + + boolean present_topology_id = true && (is_set_topology_id()); + list.add(present_topology_id); + if (present_topology_id) + list.add(topology_id); + + boolean present_time_stamp = true; + list.add(present_time_stamp); + if (present_time_stamp) + list.add(time_stamp); + + boolean present_users = true && (is_set_users()); + list.add(present_users); + if (present_users) + list.add(users); + + boolean present_groups = true && (is_set_groups()); + list.add(present_groups); + if (present_groups) + list.add(groups); + + return list.hashCode(); + } + + @Override + public int compareTo(LSTopoHistory other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(is_set_topology_id()).compareTo(other.is_set_topology_id()); + if (lastComparison != 0) { + return lastComparison; + } + if (is_set_topology_id()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.topology_id, other.topology_id); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(is_set_time_stamp()).compareTo(other.is_set_time_stamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (is_set_time_stamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.time_stamp, other.time_stamp); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(is_set_users()).compareTo(other.is_set_users()); + if (lastComparison != 0) { + return lastComparison; + } + if (is_set_users()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.users, other.users); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(is_set_groups()).compareTo(other.is_set_groups()); + if (lastComparison != 0) { + return lastComparison; + } + if (is_set_groups()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.groups, other.groups); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("LSTopoHistory("); + boolean first = true; + + sb.append("topology_id:"); + if (this.topology_id == null) { + sb.append("null"); + } else { + sb.append(this.topology_id); + } + first = false; + if (!first) sb.append(", "); + sb.append("time_stamp:"); + sb.append(this.time_stamp); + first = false; + if (!first) sb.append(", "); + sb.append("users:"); + if (this.users == null) { + sb.append("null"); + } else { + sb.append(this.users); + } + first = false; + if (!first) sb.append(", "); + sb.append("groups:"); + if (this.groups == null) { + sb.append("null"); + } else { + sb.append(this.groups); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (!is_set_topology_id()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'topology_id' is unset! Struct:" + toString()); + } + + if (!is_set_time_stamp()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'time_stamp' is unset! Struct:" + toString()); + } + + if (!is_set_users()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'users' is unset! Struct:" + toString()); + } + + if (!is_set_groups()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'groups' is unset! Struct:" + toString()); + } + + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class LSTopoHistoryStandardSchemeFactory implements SchemeFactory { + public LSTopoHistoryStandardScheme getScheme() { + return new LSTopoHistoryStandardScheme(); + } + } + + private static class LSTopoHistoryStandardScheme extends StandardScheme<LSTopoHistory> { + + public void read(org.apache.thrift.protocol.TProtocol iprot, LSTopoHistory struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TOPOLOGY_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.topology_id = iprot.readString(); + struct.set_topology_id_isSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // TIME_STAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.time_stamp = iprot.readI64(); + struct.set_time_stamp_isSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // USERS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list656 = iprot.readListBegin(); + struct.users = new ArrayList<String>(_list656.size); + String _elem657; + for (int _i658 = 0; _i658 < _list656.size; ++_i658) + { + _elem657 = iprot.readString(); + struct.users.add(_elem657); + } + iprot.readListEnd(); + } + struct.set_users_isSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // GROUPS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list659 = iprot.readListBegin(); + struct.groups = new ArrayList<String>(_list659.size); + String _elem660; + for (int _i661 = 0; _i661 < _list659.size; ++_i661) + { + _elem660 = iprot.readString(); + struct.groups.add(_elem660); + } + iprot.readListEnd(); + } + struct.set_groups_isSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, LSTopoHistory struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.topology_id != null) { + oprot.writeFieldBegin(TOPOLOGY_ID_FIELD_DESC); + oprot.writeString(struct.topology_id); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TIME_STAMP_FIELD_DESC); + oprot.writeI64(struct.time_stamp); + oprot.writeFieldEnd(); + if (struct.users != null) { + oprot.writeFieldBegin(USERS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.users.size())); + for (String _iter662 : struct.users) + { + oprot.writeString(_iter662); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.groups != null) { + oprot.writeFieldBegin(GROUPS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.groups.size())); + for (String _iter663 : struct.groups) + { + oprot.writeString(_iter663); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class LSTopoHistoryTupleSchemeFactory implements SchemeFactory { + public LSTopoHistoryTupleScheme getScheme() { + return new LSTopoHistoryTupleScheme(); + } + } + + private static class LSTopoHistoryTupleScheme extends TupleScheme<LSTopoHistory> { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, LSTopoHistory struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeString(struct.topology_id); + oprot.writeI64(struct.time_stamp); + { + oprot.writeI32(struct.users.size()); + for (String _iter664 : struct.users) + { + oprot.writeString(_iter664); + } + } + { + oprot.writeI32(struct.groups.size()); + for (String _iter665 : struct.groups) + { + oprot.writeString(_iter665); + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, LSTopoHistory struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.topology_id = iprot.readString(); + struct.set_topology_id_isSet(true); + struct.time_stamp = iprot.readI64(); + struct.set_time_stamp_isSet(true); + { + org.apache.thrift.protocol.TList _list666 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.users = new ArrayList<String>(_list666.size); + String _elem667; + for (int _i668 = 0; _i668 < _list666.size; ++_i668) + { + _elem667 = iprot.readString(); + struct.users.add(_elem667); + } + } + struct.set_users_isSet(true); + { + org.apache.thrift.protocol.TList _list669 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.groups = new ArrayList<String>(_list669.size); + String _elem670; + for (int _i671 = 0; _i671 < _list669.size; ++_i671) + { + _elem670 = iprot.readString(); + struct.groups.add(_elem670); + } + } + struct.set_groups_isSet(true); + } + } + +} +
