[
https://issues.apache.org/jira/browse/STORM-3354?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Stig Rohde Døssing updated STORM-3354:
--------------------------------------
Description:
Nimbus' LeaderElector is not shut down when Nimbus shuts down. This can cause
test flakiness because the elector callback may be called after the Zookeeper
client is closed.
Additionally the LeaderListenerCallback may in some cases quit the leadership
election if Nimbus isn't ready to become master, but I don't see any code to
re-enter the election.
{quote}java.lang.IllegalStateException: Client is not started
at
org.apache.storm.shade.org.apache.curator.shaded.com.google.common.base.Preconditions.checkState(Preconditions.java:444)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:139)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:602)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1191)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1158)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.connection.StandardConnectionHandlingPolicy.callWithRetry(StandardConnectionHandlingPolicy.java:64)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:100)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.pathInForeground(CreateBuilderImpl.java:1155)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.protectedPathInForeground(CreateBuilderImpl.java:605)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.forPath(CreateBuilderImpl.java:595)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:360)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:308)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.zookeeper.ClientZookeeper.createNode(ClientZookeeper.java:98)
~[storm-client-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.nimbus.LeaderListenerCallback.setUpNimbusInfo(LeaderListenerCallback.java:154)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.nimbus.LeaderListenerCallback.leaderCallBack(LeaderListenerCallback.java:96)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.zookeeper.Zookeeper$1.isLeader(Zookeeper.java:123)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:665)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:661)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:93)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.shaded.com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:435)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:85)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.setLeadership(LeaderLatch.java:660)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.checkLeadership(LeaderLatch.java:539)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.access$700(LeaderLatch.java:65)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$7.processResult(LeaderLatch.java:590)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.sendToBackgroundCallback(CuratorFrameworkImpl.java:865)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:635)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.WatcherRemovalFacade.processBackgroundOperation(WatcherRemovalFacade.java:152)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl$2.processResult(GetChildrenBuilderImpl.java:187)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:590)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
{quote}
was:
Nimbus' LeaderElector is not shut down when Nimbus shuts down. This can cause
test flakiness because the elector callback may be called after the Zookeeper
client is closed.
{quote}
java.lang.IllegalStateException: Client is not started
at
org.apache.storm.shade.org.apache.curator.shaded.com.google.common.base.Preconditions.checkState(Preconditions.java:444)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:139)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:602)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1191)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1158)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.connection.StandardConnectionHandlingPolicy.callWithRetry(StandardConnectionHandlingPolicy.java:64)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:100)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.pathInForeground(CreateBuilderImpl.java:1155)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.protectedPathInForeground(CreateBuilderImpl.java:605)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.forPath(CreateBuilderImpl.java:595)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:360)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:308)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.zookeeper.ClientZookeeper.createNode(ClientZookeeper.java:98)
~[storm-client-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.nimbus.LeaderListenerCallback.setUpNimbusInfo(LeaderListenerCallback.java:154)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.nimbus.LeaderListenerCallback.leaderCallBack(LeaderListenerCallback.java:96)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.zookeeper.Zookeeper$1.isLeader(Zookeeper.java:123)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:665)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:661)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:93)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.shaded.com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:435)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:85)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.setLeadership(LeaderLatch.java:660)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.checkLeadership(LeaderLatch.java:539)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.access$700(LeaderLatch.java:65)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$7.processResult(LeaderLatch.java:590)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.sendToBackgroundCallback(CuratorFrameworkImpl.java:865)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:635)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.WatcherRemovalFacade.processBackgroundOperation(WatcherRemovalFacade.java:152)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl$2.processResult(GetChildrenBuilderImpl.java:187)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:590)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
{quote}
> LeaderElector is not shut down properly
> ---------------------------------------
>
> Key: STORM-3354
> URL: https://issues.apache.org/jira/browse/STORM-3354
> Project: Apache Storm
> Issue Type: Bug
> Affects Versions: 2.0.0
> Reporter: Stig Rohde Døssing
> Assignee: Stig Rohde Døssing
> Priority: Major
>
> Nimbus' LeaderElector is not shut down when Nimbus shuts down. This can cause
> test flakiness because the elector callback may be called after the Zookeeper
> client is closed.
> Additionally the LeaderListenerCallback may in some cases quit the leadership
> election if Nimbus isn't ready to become master, but I don't see any code to
> re-enter the election.
>
> {quote}java.lang.IllegalStateException: Client is not started
> at
> org.apache.storm.shade.org.apache.curator.shaded.com.google.common.base.Preconditions.checkState(Preconditions.java:444)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:139)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:602)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1191)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1158)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.connection.StandardConnectionHandlingPolicy.callWithRetry(StandardConnectionHandlingPolicy.java:64)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:100)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.pathInForeground(CreateBuilderImpl.java:1155)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.protectedPathInForeground(CreateBuilderImpl.java:605)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.forPath(CreateBuilderImpl.java:595)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:360)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:308)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.zookeeper.ClientZookeeper.createNode(ClientZookeeper.java:98)
> ~[storm-client-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.nimbus.LeaderListenerCallback.setUpNimbusInfo(LeaderListenerCallback.java:154)
> ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.nimbus.LeaderListenerCallback.leaderCallBack(LeaderListenerCallback.java:96)
> ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.zookeeper.Zookeeper$1.isLeader(Zookeeper.java:123)
> ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:665)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:661)
> ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:93)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.shaded.com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:435)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:85)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.setLeadership(LeaderLatch.java:660)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.checkLeadership(LeaderLatch.java:539)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.access$700(LeaderLatch.java:65)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$7.processResult(LeaderLatch.java:590)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.sendToBackgroundCallback(CuratorFrameworkImpl.java:865)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:635)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.WatcherRemovalFacade.processBackgroundOperation(WatcherRemovalFacade.java:152)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl$2.processResult(GetChildrenBuilderImpl.java:187)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:590)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at
> org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
> [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> {quote}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
