See
<https://builds.apache.org/job/HBase-1.3/jdk=latest1.7,label=Hadoop/231/changes>
Changes:
[stack] HBASE-14545 TestMasterFailover often times out
------------------------------------------
[...truncated 49052 lines...]
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007fab424d0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2024)
"IPC Server handler 0 on 43534" daemon prio=10 tid=0x00007fb158d83800
nid=0x2551 waiting on condition [0x00007fb10e690000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007fab424d0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2024)
"IPC Server listener on 43534" daemon prio=10 tid=0x00007fb158d82800 nid=0x2550
runnable [0x00007fb10e791000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0x0000000753027178> (a sun.nio.ch.Util$2)
- locked <0x0000000753027168> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000753026cd0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
at org.apache.hadoop.ipc.Server$Listener.run(Server.java:693)
"IPC Server Responder" daemon prio=10 tid=0x00007fb1588a7800 nid=0x254f
runnable [0x00007fb10e892000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0x0000000753027c20> (a sun.nio.ch.Util$2)
- locked <0x0000000753027c10> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007530279f8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:881)
at org.apache.hadoop.ipc.Server$Responder.run(Server.java:864)
"org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@3fb7edd3"
daemon prio=10 tid=0x00007fb1588a7000 nid=0x254e waiting on condition
[0x00007fb10e993000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor.run(BlockManager.java:3114)
at java.lang.Thread.run(Thread.java:662)
"org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@445ed853"
daemon prio=10 tid=0x00007fb15884c000 nid=0x254d waiting on condition
[0x00007fb10ea94000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:295)
at java.lang.Thread.run(Thread.java:662)
"org.apache.hadoop.hdfs.server.blockmanagement.DecommissionManager$Monitor@1d70ec24"
daemon prio=10 tid=0x00007fb15884b800 nid=0x254c waiting on condition
[0x00007fb10eb95000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.hdfs.server.blockmanagement.DecommissionManager$Monitor.run(DecommissionManager.java:76)
at java.lang.Thread.run(Thread.java:662)
"org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor@35a631cc"
daemon prio=10 tid=0x00007fb158888800 nid=0x254b waiting on condition
[0x00007fb10ec96000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:221)
at java.lang.Thread.run(Thread.java:662)
"Socket Reader #1 for port 43534" prio=10 tid=0x00007fb158888000 nid=0x2544
runnable [0x00007fb10ed97000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0x00000007fad5e780> (a sun.nio.ch.Util$2)
- locked <0x00000007fad5e770> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007fad5ced8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
at
org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:580)
- locked <0x0000000753027388> (a
org.apache.hadoop.ipc.Server$Listener$Reader)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:566)
"Timer-0" daemon prio=10 tid=0x00007fb15882a800 nid=0x2543 in Object.wait()
[0x00007fb10ee98000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x000000075304ea78> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:509)
- locked <0x000000075304ea78> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:462)
"752166762@qtp-673532189-1 - Acceptor0 SelectChannelConnector@localhost:52167"
daemon prio=10 tid=0x00007fb158d77000 nid=0x2542 runnable [0x00007fb10ef99000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0x00000007530e8d88> (a sun.nio.ch.Util$2)
- locked <0x00000007530e8d78> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007530e88c0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
at
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
at
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
"1186010055@qtp-673532189-0" daemon prio=10 tid=0x00007fb158d44800 nid=0x2541
in Object.wait() [0x00007fb10f1a5000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007530e8410> (a
org.mortbay.thread.QueuedThreadPool$PoolThread)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
- locked <0x00000007530e8410> (a
org.mortbay.thread.QueuedThreadPool$PoolThread)
"Timer for 'NameNode' metrics system" daemon prio=10 tid=0x00007fb158812000
nid=0x2540 in Object.wait() [0x00007fb10f2a6000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:509)
- locked <0x0000000753107008> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:462)
"Low Memory Detector" daemon prio=10 tid=0x00007fb1580ca800 nid=0x2539 runnable
[0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" daemon prio=10 tid=0x00007fb1580c8000 nid=0x2538 waiting
on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" daemon prio=10 tid=0x00007fb1580c5000 nid=0x2537 waiting
on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=10 tid=0x00007fb1580c3000 nid=0x2536 runnable
[0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=10 tid=0x00007fb1580a7000 nid=0x2535 in Object.wait()
[0x00007fb1545f4000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
- locked <0x00000007531a10e0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:171)
"Reference Handler" daemon prio=10 tid=0x00007fb1580a5000 nid=0x2534 in
Object.wait() [0x00007fb1546f5000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:485)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
- locked <0x00000007fcbd2778> (a java.lang.ref.Reference$Lock)
"main" prio=10 tid=0x00007fb158008000 nid=0x2525 in Object.wait()
[0x00007fb15edac000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at org.apache.hadoop.hbase.ipc.RpcClient.call(RpcClient.java:1453)
- locked <0x00000007e3700470> (a
org.apache.hadoop.hbase.ipc.RpcClient$Call)
at
org.apache.hadoop.hbase.ipc.RpcClient.callBlockingMethod(RpcClient.java:1662)
at
org.apache.hadoop.hbase.ipc.RpcClient$BlockingRpcChannelImplementation.callBlockingMethod(RpcClient.java:1720)
at
org.apache.hadoop.hbase.protobuf.generated.MasterProtos$MasterService$BlockingStub.isMasterRunning(MasterProtos.java:45501)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$MasterServiceState.isMasterRunning(HConnectionManager.java:1618)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.isKeepAliveMasterConnectedAndRunning(HConnectionManager.java:2274)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getKeepAliveMasterService(HConnectionManager.java:1969)
- locked <0x00000007fc19a5d0> (a java.lang.Object)
at
org.apache.hadoop.hbase.client.HBaseAdmin$MasterCallable.prepare(HBaseAdmin.java:3452)
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:114)
- locked <0x00000007e3700328> (a
org.apache.hadoop.hbase.client.RpcRetryingCaller)
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:91)
- locked <0x00000007e3700328> (a
org.apache.hadoop.hbase.client.RpcRetryingCaller)
at
org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3479)
at
org.apache.hadoop.hbase.client.HBaseAdmin.deleteTable(HBaseAdmin.java:657)
at
org.apache.hadoop.hbase.HBaseTestingUtility.deleteTable(HBaseTestingUtility.java:1581)
at
org.apache.hadoop.hbase.client.TestSnapshotCloneIndependence.tearDown(TestSnapshotCloneIndependence.java:108)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at org.junit.runners.Suite.runChild(Suite.java:127)
at org.junit.runners.Suite.runChild(Suite.java:26)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at org.junit.runner.JUnitCore.run(JUnitCore.java:160)
at org.junit.runner.JUnitCore.run(JUnitCore.java:138)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.createRequestAndRun(JUnitCoreWrapper.java:108)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.executeEager(JUnitCoreWrapper.java:78)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:54)
at
org.apache.maven.surefire.junitcore.JUnitCoreProvider.invoke(JUnitCoreProvider.java:144)
at
org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:203)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:155)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103)
"VM Thread" prio=10 tid=0x00007fb15809e800 nid=0x2533 runnable
"GC task thread#0 (ParallelGC)" prio=10 tid=0x00007fb15801b800 nid=0x2526
runnable
"GC task thread#1 (ParallelGC)" prio=10 tid=0x00007fb15801d000 nid=0x2527
runnable
"GC task thread#2 (ParallelGC)" prio=10 tid=0x00007fb15801f000 nid=0x2528
runnable
"GC task thread#3 (ParallelGC)" prio=10 tid=0x00007fb158021000 nid=0x2529
runnable
"GC task thread#4 (ParallelGC)" prio=10 tid=0x00007fb158022800 nid=0x252a
runnable
"GC task thread#5 (ParallelGC)" prio=10 tid=0x00007fb158024800 nid=0x252b
runnable
"GC task thread#6 (ParallelGC)" prio=10 tid=0x00007fb158026800 nid=0x252c
runnable
"GC task thread#7 (ParallelGC)" prio=10 tid=0x00007fb158028000 nid=0x252d
runnable
"GC task thread#8 (ParallelGC)" prio=10 tid=0x00007fb15802a000 nid=0x252e
runnable
"GC task thread#9 (ParallelGC)" prio=10 tid=0x00007fb15802c000 nid=0x252f
runnable
"GC task thread#10 (ParallelGC)" prio=10 tid=0x00007fb15802d800 nid=0x2530
runnable
"GC task thread#11 (ParallelGC)" prio=10 tid=0x00007fb15802f800 nid=0x2531
runnable
"GC task thread#12 (ParallelGC)" prio=10 tid=0x00007fb158031000 nid=0x2532
runnable
"VM Periodic Task Thread" prio=10 tid=0x00007fb1580d5000 nid=0x253a waiting on
condition
JNI global references: 1577
+ echo '************ END zombies jstack extract'
************ END zombies jstack extract
+ JIRA_COMMENT='
{color:red}-1 core zombie tests{color}. There are 1 zombie test(s):
at
org.apache.hadoop.hbase.client.TestSnapshotCloneIndependence.testOnlineSnapshotRegionOperationsIndependent(TestSnapshotCloneIndependence.java:173)'
+ BAD=1
+ jps
+ grep surefirebooter
+ cut -d ' ' -f 1
+ xargs kill -9
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 0
Archiving artifacts
