Github user d2r commented on a diff in the pull request:
https://github.com/apache/storm/pull/845#discussion_r45099492
--- Diff: storm-core/src/clj/backtype/storm/daemon/nimbus.clj ---
@@ -390,53 +444,98 @@
[(.getNodeId slot) (.getPort slot)]
)))
+(defn- get-version-for-key [key nimbus-host-port-info conf]
+ (let [version (KeyVersion. key nimbus-host-port-info)]
+ (.getKeyVersion version conf)))
+
+(defn get-key-seq-from-blob-store [blob-store]
+ (let [key-iter (.listKeys blob-store nimbus-subject)]
+ (iterator-seq key-iter)))
+
(defn- setup-storm-code [nimbus conf storm-id tmp-jar-location storm-conf
topology]
- (let [stormroot (master-stormdist-root conf storm-id)]
- (log-message "nimbus file location:" stormroot)
- (FileUtils/forceMkdir (File. stormroot))
- (FileUtils/cleanDirectory (File. stormroot))
- (setup-jar conf tmp-jar-location stormroot)
- (FileUtils/writeByteArrayToFile (File. (master-stormcode-path
stormroot)) (Utils/serialize topology))
- (FileUtils/writeByteArrayToFile (File. (master-stormconf-path
stormroot)) (Utils/toCompressedJsonConf storm-conf))
- (if (:code-distributor nimbus) (.upload (:code-distributor nimbus)
stormroot storm-id))
- ))
+ (let [subject (get-subject)
+ storm-cluster-state (:storm-cluster-state nimbus)
+ blob-store (:blob-store nimbus)
+ jar-key (master-stormjar-key storm-id)
+ code-key (master-stormcode-key storm-id)
+ conf-key (master-stormconf-key storm-id)
+ nimbus-host-port-info (:nimbus-host-port-info nimbus)]
+ (when tmp-jar-location ;;in local mode there is no jar
+ (.createBlob blob-store jar-key (FileInputStream. tmp-jar-location)
(SettableBlobMeta. BlobStoreAclHandler/DEFAULT) subject)
+ (if (instance? LocalFsBlobStore blob-store)
+ (.setup-blobstore! storm-cluster-state jar-key
nimbus-host-port-info (get-version-for-key jar-key nimbus-host-port-info
conf))))
+ (.createBlob blob-store conf-key (Utils/toCompressedJsonConf
storm-conf) (SettableBlobMeta. BlobStoreAclHandler/DEFAULT) subject)
+ (if (instance? LocalFsBlobStore blob-store)
+ (.setup-blobstore! storm-cluster-state conf-key
nimbus-host-port-info (get-version-for-key conf-key nimbus-host-port-info
conf)))
+ (.createBlob blob-store code-key (Utils/serialize topology)
(SettableBlobMeta. BlobStoreAclHandler/DEFAULT) subject)
+ (if (instance? LocalFsBlobStore blob-store)
+ (.setup-blobstore! storm-cluster-state code-key
nimbus-host-port-info (get-version-for-key code-key nimbus-host-port-info
conf)))))
+
+(defn- read-storm-topology [storm-id blob-store]
+ (Utils/deserialize
+ (.readBlob blob-store (master-stormcode-key storm-id) (get-subject))
StormTopology))
+
+(defn- get-blob-replication-count [blob-key nimbus]
+ (if (:blob-store nimbus)
+ (-> (:blob-store nimbus)
+ (.getBlobReplication blob-key nimbus-subject))))
(defn- wait-for-desired-code-replication [nimbus conf storm-id]
(let [min-replication-count (conf TOPOLOGY-MIN-REPLICATION-COUNT)
max-replication-wait-time (conf
TOPOLOGY-MAX-REPLICATION-WAIT-TIME-SEC)
- total-wait-time (atom 0)
- current-replication-count (atom (if (:code-distributor nimbus)
(.getReplicationCount (:code-distributor nimbus) storm-id) 0))]
- (if (:code-distributor nimbus)
- (while (and (> min-replication-count @current-replication-count)
- (or (= -1 max-replication-wait-time)
- (< @total-wait-time max-replication-wait-time)))
+ current-replication-count-jar (if (not (local-mode? conf))
+ (atom (get-blob-replication-count
(master-stormjar-key storm-id) nimbus))
+ (atom min-replication-count))
+ current-replication-count-code (atom (get-blob-replication-count
(master-stormcode-key storm-id) nimbus))
+ current-replication-count-conf (atom (get-blob-replication-count
(master-stormconf-key storm-id) nimbus))
+ total-wait-time (atom 0)]
+ (if (:blob-store nimbus)
+ (while (and
+ (or (> min-replication-count @current-replication-count-jar)
+ (> min-replication-count
@current-replication-count-code)
+ (> min-replication-count
@current-replication-count-conf))
+ (or (= -1 max-replication-wait-time)
+ (< @total-wait-time max-replication-wait-time)))
--- End diff --
Can we test for `(neg? max-replication-wait-time)`, as per my previous
comment, or is there a reason we want to check specifically for `-1`?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
