storm git commit: STORM-3026: Upgrade ZK instance for security
Repository: storm Updated Branches: refs/heads/1.0.x-branch 59e2539fa -> a6bf3e421 STORM-3026: Upgrade ZK instance for security Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a6bf3e42 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a6bf3e42 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a6bf3e42 Branch: refs/heads/1.0.x-branch Commit: a6bf3e421d3d37a797e3bb374fcd20a00189feb4 Parents: 59e2539 Author: Robert Evans Authored: Fri Apr 13 16:51:10 2018 -0500 Committer: Robert Evans Committed: Fri Apr 13 16:51:10 2018 -0500 -- conf/defaults.yaml | 2 + storm-core/src/clj/org/apache/storm/cluster.clj | 61 ++-- .../cluster_state/zookeeper_state_factory.clj | 4 +- .../apache/storm/command/shell_submission.clj | 24 +- .../src/clj/org/apache/storm/daemon/nimbus.clj | 38 ++- .../src/clj/org/apache/storm/zookeeper.clj | 8 +- storm-core/src/jvm/org/apache/storm/Config.java | 15 + .../apache/storm/blobstore/BlobStoreUtils.java | 9 +- .../storm/blobstore/BlobSynchronizer.java | 3 +- .../storm/blobstore/KeySequenceNumber.java | 5 +- .../storm/blobstore/LocalFsBlobStore.java | 8 +- .../org/apache/storm/cluster/ClusterUtils.java | 30 +- .../storm/cluster/IStormClusterState.java | 14 +- .../storm/cluster/StormClusterStateImpl.java| 38 ++- .../apache/storm/cluster/ZKStateStorage.java| 14 +- .../daemon/supervisor/SupervisorUtils.java | 5 +- .../transactional/state/TransactionalState.java | 4 +- .../topology/state/TransactionalState.java | 4 +- .../src/jvm/org/apache/storm/utils/Utils.java | 69 +++-- .../apache/storm/zookeeper/AclEnforcement.java | 301 +++ .../org/apache/storm/zookeeper/Zookeeper.java | 37 +-- .../test/clj/org/apache/storm/cluster_test.clj | 10 +- .../test/clj/org/apache/storm/utils_test.clj| 2 +- 23 files changed, 566 insertions(+), 139 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/a6bf3e42/conf/defaults.yaml -- diff --git a/conf/defaults.yaml b/conf/defaults.yaml index ceabd59..e24b879 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -47,6 +47,8 @@ storm.messaging.transport: "org.apache.storm.messaging.netty.Context" storm.nimbus.retry.times: 5 storm.nimbus.retry.interval.millis: 2000 storm.nimbus.retry.intervalceiling.millis: 6 +storm.nimbus.zookeeper.acls.check: true +storm.nimbus.zookeeper.acls.fixup: true storm.auth.simple-white-list.users: [] storm.auth.simple-acl.users: [] storm.auth.simple-acl.users.commands: [] http://git-wip-us.apache.org/repos/asf/storm/blob/a6bf3e42/storm-core/src/clj/org/apache/storm/cluster.clj -- diff --git a/storm-core/src/clj/org/apache/storm/cluster.clj b/storm-core/src/clj/org/apache/storm/cluster.clj index 810b3c3..f1a0412 100644 --- a/storm-core/src/clj/org/apache/storm/cluster.clj +++ b/storm-core/src/clj/org/apache/storm/cluster.clj @@ -30,13 +30,22 @@ (:require [org.apache.storm [zookeeper :as zk]]) (:require [org.apache.storm.daemon [common :as common]])) -(defn mk-topo-only-acls - [topo-conf] +(defn mk-topo-acls + [topo-conf type] (let [payload (.get topo-conf STORM-ZOOKEEPER-TOPOLOGY-AUTH-PAYLOAD)] (when (Utils/isZkAuthenticationConfiguredTopology topo-conf) [(first ZooDefs$Ids/CREATOR_ALL_ACL) - (ACL. ZooDefs$Perms/READ (Id. "digest" (DigestAuthenticationProvider/generateDigest payload)))]))) - + (ACL. type (Id. "digest" (DigestAuthenticationProvider/generateDigest payload)))]))) + +(defn mk-topo-read-write-acls + [topo-conf] + (mk-topo-acls topo-conf ZooDefs$Perms/ALL)) + +(defn mk-topo-read-only-acls + [topo-conf] + [topo-conf] + (mk-topo-acls topo-conf ZooDefs$Perms/READ)) + (defnk mk-distributed-cluster-state [conf :auth-conf nil :acls nil :context (ClusterStateContext.)] (let [clazz (Class/forName (or (conf STORM-CLUSTER-STATE-STORE) @@ -68,26 +77,27 @@ (executor-beats [this storm-id executor->node+port]) (supervisors [this callback]) (supervisor-info [this supervisor-id]) ;; returns nil if doesn't exist - (setup-heartbeats! [this storm-id]) + (setup-heartbeats! [this storm-id topo-conf]) (teardown-heartbeats! [this storm-id]) (teardown-topology-errors! [this storm-id]) (heartbeat-storms [this]) (error-topologies [this]) (backpressure-topologies [this]) - (set-topology-log-config! [this storm-id log-config]) + (set-topology-log-config! [this storm-id log-config topo-conf]) (topology-log-config [this storm-id cb]) (worker-heartbeat! [this storm-id node port info]) (remove-worker-heartbeat! [this storm-id no
storm git commit: STORM-3026: Upgrade ZK instance for security
Repository: storm Updated Branches: refs/heads/1.1.x-branch 58f7aefb8 -> 22a962073 STORM-3026: Upgrade ZK instance for security Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/22a96207 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/22a96207 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/22a96207 Branch: refs/heads/1.1.x-branch Commit: 22a962073c5f12dc5ab281a15d93eb5efc31ab6b Parents: 58f7aef Author: Robert Evans Authored: Fri Apr 13 16:46:42 2018 -0500 Committer: Robert Evans Committed: Fri Apr 13 16:46:42 2018 -0500 -- conf/defaults.yaml | 2 + storm-core/src/clj/org/apache/storm/cluster.clj | 61 ++-- .../cluster_state/zookeeper_state_factory.clj | 4 +- .../apache/storm/command/shell_submission.clj | 24 +- .../src/clj/org/apache/storm/daemon/nimbus.clj | 41 ++- .../src/clj/org/apache/storm/zookeeper.clj | 8 +- storm-core/src/jvm/org/apache/storm/Config.java | 15 + .../apache/storm/blobstore/BlobStoreUtils.java | 9 +- .../storm/blobstore/BlobSynchronizer.java | 3 +- .../storm/blobstore/KeySequenceNumber.java | 5 +- .../storm/blobstore/LocalFsBlobStore.java | 8 +- .../org/apache/storm/cluster/ClusterUtils.java | 30 +- .../storm/cluster/IStormClusterState.java | 14 +- .../storm/cluster/StormClusterStateImpl.java| 38 ++- .../apache/storm/cluster/ZKStateStorage.java| 14 +- .../daemon/supervisor/SupervisorUtils.java | 5 +- .../transactional/state/TransactionalState.java | 4 +- .../topology/state/TransactionalState.java | 4 +- .../src/jvm/org/apache/storm/utils/Utils.java | 69 +++-- .../apache/storm/zookeeper/AclEnforcement.java | 301 +++ .../org/apache/storm/zookeeper/Zookeeper.java | 37 +-- .../test/clj/org/apache/storm/cluster_test.clj | 10 +- .../test/clj/org/apache/storm/utils_test.clj| 2 +- 23 files changed, 568 insertions(+), 140 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/22a96207/conf/defaults.yaml -- diff --git a/conf/defaults.yaml b/conf/defaults.yaml index 46a4d87..d8029ff 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -48,6 +48,8 @@ storm.messaging.transport: "org.apache.storm.messaging.netty.Context" storm.nimbus.retry.times: 5 storm.nimbus.retry.interval.millis: 2000 storm.nimbus.retry.intervalceiling.millis: 6 +storm.nimbus.zookeeper.acls.check: true +storm.nimbus.zookeeper.acls.fixup: true storm.auth.simple-white-list.users: [] storm.auth.simple-acl.users: [] storm.auth.simple-acl.users.commands: [] http://git-wip-us.apache.org/repos/asf/storm/blob/22a96207/storm-core/src/clj/org/apache/storm/cluster.clj -- diff --git a/storm-core/src/clj/org/apache/storm/cluster.clj b/storm-core/src/clj/org/apache/storm/cluster.clj index 810b3c3..f1a0412 100644 --- a/storm-core/src/clj/org/apache/storm/cluster.clj +++ b/storm-core/src/clj/org/apache/storm/cluster.clj @@ -30,13 +30,22 @@ (:require [org.apache.storm [zookeeper :as zk]]) (:require [org.apache.storm.daemon [common :as common]])) -(defn mk-topo-only-acls - [topo-conf] +(defn mk-topo-acls + [topo-conf type] (let [payload (.get topo-conf STORM-ZOOKEEPER-TOPOLOGY-AUTH-PAYLOAD)] (when (Utils/isZkAuthenticationConfiguredTopology topo-conf) [(first ZooDefs$Ids/CREATOR_ALL_ACL) - (ACL. ZooDefs$Perms/READ (Id. "digest" (DigestAuthenticationProvider/generateDigest payload)))]))) - + (ACL. type (Id. "digest" (DigestAuthenticationProvider/generateDigest payload)))]))) + +(defn mk-topo-read-write-acls + [topo-conf] + (mk-topo-acls topo-conf ZooDefs$Perms/ALL)) + +(defn mk-topo-read-only-acls + [topo-conf] + [topo-conf] + (mk-topo-acls topo-conf ZooDefs$Perms/READ)) + (defnk mk-distributed-cluster-state [conf :auth-conf nil :acls nil :context (ClusterStateContext.)] (let [clazz (Class/forName (or (conf STORM-CLUSTER-STATE-STORE) @@ -68,26 +77,27 @@ (executor-beats [this storm-id executor->node+port]) (supervisors [this callback]) (supervisor-info [this supervisor-id]) ;; returns nil if doesn't exist - (setup-heartbeats! [this storm-id]) + (setup-heartbeats! [this storm-id topo-conf]) (teardown-heartbeats! [this storm-id]) (teardown-topology-errors! [this storm-id]) (heartbeat-storms [this]) (error-topologies [this]) (backpressure-topologies [this]) - (set-topology-log-config! [this storm-id log-config]) + (set-topology-log-config! [this storm-id log-config topo-conf]) (topology-log-config [this storm-id cb]) (worker-heartbeat! [this storm-id node port info]) (remove-worker-heartbeat! [this storm-id no
storm git commit: STORM-3026: Upgrade ZK instance for security
Repository: storm Updated Branches: refs/heads/1.x-branch df9525914 -> e3652b44a STORM-3026: Upgrade ZK instance for security Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e3652b44 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e3652b44 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e3652b44 Branch: refs/heads/1.x-branch Commit: e3652b44a377436256f77a2749ed133bbafd2fbf Parents: df95259 Author: Robert Evans Authored: Fri Apr 13 16:37:48 2018 -0500 Committer: Robert Evans Committed: Fri Apr 13 16:37:48 2018 -0500 -- conf/defaults.yaml | 2 + storm-core/src/clj/org/apache/storm/cluster.clj | 61 ++-- .../cluster_state/zookeeper_state_factory.clj | 4 +- .../apache/storm/command/shell_submission.clj | 29 +- .../src/clj/org/apache/storm/daemon/nimbus.clj | 32 +- .../src/clj/org/apache/storm/zookeeper.clj | 4 +- storm-core/src/jvm/org/apache/storm/Config.java | 15 + .../apache/storm/blobstore/BlobStoreUtils.java | 5 +- .../storm/blobstore/LocalFsBlobStore.java | 8 +- .../org/apache/storm/cluster/ClusterUtils.java | 30 +- .../storm/cluster/IStormClusterState.java | 14 +- .../storm/cluster/StormClusterStateImpl.java| 38 ++- .../apache/storm/cluster/ZKStateStorage.java| 14 +- .../daemon/supervisor/SupervisorUtils.java | 5 +- .../transactional/state/TransactionalState.java | 4 +- .../topology/state/TransactionalState.java | 4 +- .../src/jvm/org/apache/storm/utils/Utils.java | 70 +++-- .../apache/storm/zookeeper/AclEnforcement.java | 301 +++ .../org/apache/storm/zookeeper/Zookeeper.java | 27 +- .../test/clj/org/apache/storm/cluster_test.clj | 10 +- .../test/clj/org/apache/storm/utils_test.clj| 2 +- 21 files changed, 544 insertions(+), 135 deletions(-) -- http://git-wip-us.apache.org/repos/asf/storm/blob/e3652b44/conf/defaults.yaml -- diff --git a/conf/defaults.yaml b/conf/defaults.yaml index ec8ccd6..3a8db94 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -48,6 +48,8 @@ storm.messaging.transport: "org.apache.storm.messaging.netty.Context" storm.nimbus.retry.times: 5 storm.nimbus.retry.interval.millis: 2000 storm.nimbus.retry.intervalceiling.millis: 6 +storm.nimbus.zookeeper.acls.check: true +storm.nimbus.zookeeper.acls.fixup: true storm.auth.simple-white-list.users: [] storm.auth.simple-acl.users: [] storm.auth.simple-acl.users.commands: [] http://git-wip-us.apache.org/repos/asf/storm/blob/e3652b44/storm-core/src/clj/org/apache/storm/cluster.clj -- diff --git a/storm-core/src/clj/org/apache/storm/cluster.clj b/storm-core/src/clj/org/apache/storm/cluster.clj index 731a0b9..becc34e 100644 --- a/storm-core/src/clj/org/apache/storm/cluster.clj +++ b/storm-core/src/clj/org/apache/storm/cluster.clj @@ -31,13 +31,22 @@ (:require [org.apache.storm [zookeeper :as zk]]) (:require [org.apache.storm.daemon [common :as common]])) -(defn mk-topo-only-acls - [topo-conf] +(defn mk-topo-acls + [topo-conf type] (let [payload (.get topo-conf STORM-ZOOKEEPER-TOPOLOGY-AUTH-PAYLOAD)] (when (Utils/isZkAuthenticationConfiguredTopology topo-conf) [(first ZooDefs$Ids/CREATOR_ALL_ACL) - (ACL. ZooDefs$Perms/READ (Id. "digest" (DigestAuthenticationProvider/generateDigest payload)))]))) - + (ACL. type (Id. "digest" (DigestAuthenticationProvider/generateDigest payload)))]))) + +(defn mk-topo-read-write-acls + [topo-conf] + (mk-topo-acls topo-conf ZooDefs$Perms/ALL)) + +(defn mk-topo-read-only-acls + [topo-conf] + [topo-conf] + (mk-topo-acls topo-conf ZooDefs$Perms/READ)) + (defnk mk-distributed-cluster-state [conf :auth-conf nil :acls nil :context (ClusterStateContext.)] (let [clazz (Class/forName (or (conf STORM-CLUSTER-STATE-STORE) @@ -69,26 +78,27 @@ (executor-beats [this storm-id executor->node+port]) (supervisors [this callback]) (supervisor-info [this supervisor-id]) ;; returns nil if doesn't exist - (setup-heartbeats! [this storm-id]) + (setup-heartbeats! [this storm-id topo-conf]) (teardown-heartbeats! [this storm-id]) (teardown-topology-errors! [this storm-id]) (heartbeat-storms [this]) (error-topologies [this]) (backpressure-topologies [this]) - (set-topology-log-config! [this storm-id log-config]) + (set-topology-log-config! [this storm-id log-config topo-conf]) (topology-log-config [this storm-id cb]) (worker-heartbeat! [this storm-id node port info]) (remove-worker-heartbeat! [this storm-id node port]) (supervisor-heartbeat! [this supervisor-id info]) (worker-backpressure! [this storm-id node port info])
storm git commit: STORM-3026: Upgrade ZK instance for security
Repository: storm
Updated Branches:
refs/heads/master c5988e83d -> 8ffa920d3
STORM-3026: Upgrade ZK instance for security
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8ffa920d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8ffa920d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8ffa920d
Branch: refs/heads/master
Commit: 8ffa920d3894634aa078f0fdf6b02d270262caf4
Parents: c5988e8
Author: Robert Evans
Authored: Fri Apr 13 16:31:57 2018 -0500
Committer: Robert Evans
Committed: Fri Apr 13 16:31:57 2018 -0500
--
conf/defaults.yaml | 2 +
.../org/apache/storm/cluster/ClusterUtils.java | 30 +-
.../org/apache/storm/cluster/DaemonType.java| 10 +-
.../storm/cluster/IStormClusterState.java | 14 +-
.../storm/cluster/StormClusterStateImpl.java| 42 +--
.../apache/storm/cluster/ZKStateStorage.java| 21 +-
.../transactional/state/TransactionalState.java | 5 +-
.../topology/state/TransactionalState.java | 5 +-
.../org/apache/storm/utils/CuratorUtils.java| 38 ++-
.../src/jvm/org/apache/storm/utils/Utils.java | 25 +-
.../apache/storm/zookeeper/ClientZookeeper.java | 34 +-
.../apache/storm/utils/CuratorUtilsTest.java| 4 +-
.../apache/storm/command/shell_submission.clj | 31 +-
.../test/clj/org/apache/storm/cluster_test.clj | 13 +-
.../test/clj/org/apache/storm/nimbus_test.clj | 4 +-
.../java/org/apache/storm/DaemonConfig.java | 15 +
.../apache/storm/blobstore/BlobStoreUtils.java | 6 +-
.../storm/blobstore/LocalFsBlobStore.java | 3 +-
.../org/apache/storm/daemon/nimbus/Nimbus.java | 23 +-
.../daemon/supervisor/SupervisorUtils.java | 2 -
.../apache/storm/zookeeper/AclEnforcement.java | 321 +++
21 files changed, 519 insertions(+), 129 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/storm/blob/8ffa920d/conf/defaults.yaml
--
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index c985c12..0e67957 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -48,6 +48,8 @@ storm.messaging.transport:
"org.apache.storm.messaging.netty.Context"
storm.nimbus.retry.times: 5
storm.nimbus.retry.interval.millis: 2000
storm.nimbus.retry.intervalceiling.millis: 6
+storm.nimbus.zookeeper.acls.check: true
+storm.nimbus.zookeeper.acls.fixup: true
storm.auth.simple-white-list.users: []
storm.cluster.state.store: "org.apache.storm.cluster.ZKStateStorageFactory"
storm.meta.serialization.delegate:
"org.apache.storm.serialization.GzipThriftSerializationDelegate"
http://git-wip-us.apache.org/repos/asf/storm/blob/8ffa920d/storm-client/src/jvm/org/apache/storm/cluster/ClusterUtils.java
--
diff --git a/storm-client/src/jvm/org/apache/storm/cluster/ClusterUtils.java
b/storm-client/src/jvm/org/apache/storm/cluster/ClusterUtils.java
index b3dfc7d..147f586 100644
--- a/storm-client/src/jvm/org/apache/storm/cluster/ClusterUtils.java
+++ b/storm-client/src/jvm/org/apache/storm/cluster/ClusterUtils.java
@@ -46,7 +46,6 @@ public class ClusterUtils {
public static final String ZK_SEPERATOR = "/";
public static final String ASSIGNMENTS_ROOT = "assignments";
-public static final String CODE_ROOT = "code";
public static final String STORMS_ROOT = "storms";
public static final String SUPERVISORS_ROOT = "supervisors";
public static final String WORKERBEATS_ROOT = "workerbeats";
@@ -98,15 +97,38 @@ public class ClusterUtils {
_instance = INSTANCE;
}
-public static List mkTopoOnlyAcls(Map topoConf)
throws NoSuchAlgorithmException {
+/**
+ * Get ZK ACLs for a topology to have read/write access.
+ * @param topoConf the topology config.
+ * @return the ACLs.
+ */
+public static List mkTopoReadWriteAcls(Map topoConf) {
+return mkTopoAcls(topoConf, ZooDefs.Perms.ALL);
+}
+
+/**
+ * Get ZK ACLs for a topology to have read only access.
+ * @param topoConf the topology config.
+ * @return the ACLs.
+ */
+public static List mkTopoReadOnlyAcls(Map topoConf) {
+return mkTopoAcls(topoConf, ZooDefs.Perms.READ);
+}
+
+private static List mkTopoAcls(Map topoConf, int
perms) {
List aclList = null;
String payload = (String)
topoConf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD);
if (Utils.isZkAuthenticationConfiguredTopology(topoConf)) {
aclList = new ArrayList<>();
ACL acl1 = ZooDefs.Ids.CREATOR_ALL_ACL.get(0);
aclList.add(acl1);
-ACL acl2 = new ACL(ZooDefs.Perms.READ, new Id("digest",
DigestAuthenticationProvider.generateDigest(payload)));
