[
https://issues.apache.org/jira/browse/ZOOKEEPER-1858?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13861931#comment-13861931
]
angeline shen commented on ZOOKEEPER-1858:
------------------------------------------
2014-01-02 23:53:22,474 DEBUG [o.a.c.e.o.NetworkOrchestrator]
(Network-Scavenger-1:ctx-8cae1bef) Sending network shutdown to VirtualRouter
2014-01-02 23:53:22,478 DEBUG [c.c.n.r.VirtualNetworkApplianceManagerImpl]
(Network-Scavenger-1:ctx-8cae1bef) Stopping router VM[DomainRouter|r-8-VM]
2014-01-02 23:53:22,479 WARN [o.a.c.e.o.NetworkOrchestrator]
(Network-Scavenger-1:ctx-8cae1bef) Unable to complete shutdown of the network
elements due to element: VirtualRouter
java.lang.NullPointerException
at
com.cloud.vm.VirtualMachineManagerImpl.advanceStop(VirtualMachineManagerImpl.java:1268)
at
com.cloud.network.router.VirtualNetworkApplianceManagerImpl.stop(VirtualNetworkApplianceManagerImpl.java:2702)
at
com.cloud.network.router.VirtualNetworkApplianceManagerImpl.stop(VirtualNetworkApplianceManagerImpl.java:139)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at sun.proxy.$Proxy240.stop(Unknown Source)
at
com.cloud.network.element.VirtualRouterElement.shutdown(VirtualRouterElement.java:665)
at
org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.shutdownNetworkElementsAndResources(NetworkOrchestrator.java:2052)
at
org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.shutdownNetwork(NetworkOrchestrator.java:1965)
at
org.apache.cloudstack.engine.orchestration.NetworkOrchestrator$NetworkGarbageCollector.reallyRun(NetworkOrchestrator.java:2305)
at
org.apache.cloudstack.engine.orchestration.NetworkOrchestrator$NetworkGarbageCollector.runInContext(NetworkOrchestrator.java:2248)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at
java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:165)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:267)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
2014-01-02 23:53:22,488 DEBUG [o.a.c.e.o.NetworkOrchestrator]
(Network-Scavenger-1:ctx-8cae1bef) Lock is released for network
Ntwk[205|Guest|8] as a part of network shutdown
2014-01-02 23:53:23,112 WARN [c.c.u.n.Link] (AgentManager-Selector:null) SSL:
Fail to find the generated keystore. Loading fail-safe one to continue.
2014-01-02 23:53:27,528 DEBUG [c.c.s.StatsCollector]
(StatsCollector-2:ctx-0a4e0b12) VmStatsCollector is running...
2014-01-02 23:53:27,550 DEBUG [c.c.a.m.DirectAgentAttache]
(DirectAgent-406:ctx-d13f6f69) Seq 1-77595884: Executing request
2014-01-02 23:53:27,898 DEBUG [c.c.h.v.m.HostMO] (DirectAgent-406:ctx-d13f6f69
10.223.51.4) find VM i-2-12-VM on host
2014-01-02 23:13:22,188 DEBUG [o.a.c.e.o.NetworkOrchestrator]
(Network-Scavenger-1:ctx-fbd8a699) Sending network shutdown to VirtualRouter
2014-01-02 23:13:22,192 DEBUG [c.c.n.r.VirtualNetworkApplianceManagerImpl]
(Network-Scavenger-1:ctx-fbd8a699) Stopping router VM[DomainRouter|r-8-VM]
2014-01-02 23:13:22,192 WARN [o.a.c.e.o.NetworkOrchestrator]
(Network-Scavenger-1:ctx-fbd8a699) Unable to complete shutdown of the network
elements due to element: VirtualRouter
java.lang.NullPointerException
at
com.cloud.vm.VirtualMachineManagerImpl.advanceStop(VirtualMachineManagerImpl.java:1268)
at
com.cloud.network.router.VirtualNetworkApplianceManagerImpl.stop(VirtualNetworkApplianceManagerImpl.java:2702)
at
com.cloud.network.router.VirtualNetworkApplianceManagerImpl.stop(VirtualNetworkApplianceManagerImpl.java:139)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at sun.proxy.$Proxy240.stop(Unknown Source)
at
com.cloud.network.element.VirtualRouterElement.shutdown(VirtualRouterElement.java:665)
at
org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.shutdownNetworkElementsAndResources(NetworkOrchestrator.java:2052)
at
org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.shutdownNetwork(NetworkOrchestrator.java:1965)
at
org.apache.cloudstack.engine.orchestration.NetworkOrchestrator$NetworkGarbageCollector.reallyRun(NetworkOrchestrator.java:2305)
at
org.apache.cloudstack.engine.orchestration.NetworkOrchestrator$NetworkGarbageCollector.runInContext(NetworkOrchestrator.java:2248)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at
java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:165)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:267)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
2014-01-02 23:13:22,202 DEBUG [o.a.c.e.o.NetworkOrchestrator]
(Network-Scavenger-1:ctx-fbd8a699) Lock is released for network
Ntwk[205|Guest|8] as a part of network shutdown
2014-01-02 23:13:23,299 WARN [c.c.u.n.Link] (AgentManager-Selector:null) SSL:
Fail to find the generated keystore. Loading fail-safe one to continue.
2014-01-02 23:13:27,734 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentManager-Handler-4:null) SeqA 3-2432: Processing Seq 3-2432: { Cmd ,
MgmtId: -1, via: 3, Ver: v1, Flags: 11,
[{"com.cloud.agent.api.ConsoleProxyLoadReportC
ommand":{"_proxyVmId":1,"_loadInfo":"{\n \"connections\": []\n}","wait":0}}] }
> JMX checks - potential race conditions while stopping and starting server
> -------------------------------------------------------------------------
>
> Key: ZOOKEEPER-1858
> URL: https://issues.apache.org/jira/browse/ZOOKEEPER-1858
> Project: ZooKeeper
> Issue Type: Sub-task
> Reporter: Rakesh R
> Assignee: Rakesh R
> Labels: test
> Fix For: 3.4.6, 3.5.0
>
> Attachments: ZOOKEEPER-1858-br3.4.patch, ZOOKEEPER-1858.patch
>
>
> I've noticed one potential case, where previously created zkclient session
> immediately reconnected and publishing those beans while starting back the
> zkserver and affecting zk#startup jmx checks.
> Say, before stopping the server, there is a zk client session
> 0x143576544c50000 exists. While starting back the server, there could be
> possibility of seeing the client sessions in jmx. Following is one such case.
> Please see below logs which has taken from build
> https://builds.apache.org/job/ZooKeeper-trunk-WinVS2008_java/642/
> {code} [junit] 2014-01-03 09:18:12,809 [myid:] - INFO
> [main-SendThread(127.0.0.1:11222):ClientCnxn$SendThread@1228] - Session
> establishment complete on server 127.0.0.1/127.0.0.1:11222, sessionid =
> 0x143576544c50000, negotiated timeout = 30000
> [junit] 2014-01-03 09:18:12,809 [myid:] - INFO
> [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:11222:ZooKeeperServer@617] -
> Established session 0x143576544c50000 with negotiated timeout 30000 for
> client /127.0.0.1:55377{code}
> {code} [junit] 2014-01-03 09:18:12,391 [myid:] - INFO [main:JMXEnv@135] -
> ensureOnly:[]
> [junit] 2014-01-03 09:18:12,395 [myid:] - INFO [main:ClientBase@438] -
> STARTING server
> [junit] 2014-01-03 09:18:12,395 [myid:] - INFO [main:ClientBase@359] -
> CREATING server instance 127.0.0.1:11222
> [junit] 2014-01-03 09:18:12,395 [myid:] - INFO
> [main:NIOServerCnxnFactory@94] - binding to port 0.0.0.0/0.0.0.0:11222
> [junit] 2014-01-03 09:18:12,395 [myid:] - INFO [main:ClientBase@334] -
> STARTING server instance 127.0.0.1:11222
> [junit] 2014-01-03 09:18:19,030 [myid:] - INFO [main:JMXEnv@142] -
> unexpected:org.apache.ZooKeeperService:name0=StandaloneServer_port-1,name1=Connections,name2=127.0.0.1,name3=0x143576544c50000
> [junit] 2014-01-03 09:18:19,030 [myid:] - INFO [main:JMXEnv@142] -
> unexpected:org.apache.ZooKeeperService:name0=StandaloneServer_port-1
> [junit] 2014-01-03 09:18:19,030 [myid:] - INFO
> [main:JUnit4ZKTestRunner$LoggedInvokeMethod@62] - TEST METHOD FAILED
> testDefaultWatcherAutoResetWithChroot
> [junit] junit.framework.AssertionFailedError: expected:<0> but was:<2>
> [junit] at junit.framework.Assert.fail(Assert.java:47)
> [junit] at junit.framework.Assert.failNotEquals(Assert.java:283)
> [junit] at junit.framework.Assert.assertEquals(Assert.java:64)
> [junit] at junit.framework.Assert.assertEquals(Assert.java:195)
> [junit] at junit.framework.Assert.assertEquals(Assert.java:201)
> [junit] at org.apache.zookeeper.test.JMXEnv.ensureOnly(JMXEnv.java:144)
> [junit] at
> org.apache.zookeeper.test.ClientBase.startServer(ClientBase.java:443)
> [junit] at
> org.apache.zookeeper.test.DisconnectedWatcherTest.testDefaultWatcherAutoResetWithChroot(DisconnectedWatcherTest.java:123)
> [junit] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> {code}
--
This message was sent by Atlassian JIRA
(v6.1.5#6160)
