See
<https://builds.apache.org/job/HBase-1.4/jdk=JDK_1_8,label=yahoo-not-h2/416/changes>
Changes:
[stack] HBASE-16388 Prevent client threads being blocked by only one slow region
------------------------------------------
[...truncated 47964 lines...]
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000716bc9978> (a sun.nio.ch.Util$3)
- locked <0x0000000716bc9968> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000716bc9840> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:683)
- locked <0x0000000716bc9828> (a
org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
at
org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:670)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"HBase-Metrics2-1" #236 daemon prio=5 os_prio=0 tid=0x00007fe7c80f6800
nid=0x136c waiting on condition [0x00007fe7921e0000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000717d08b80> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"RS:0;asf907:46692-EventThread" #235 daemon prio=5 os_prio=0
tid=0x00007fe7c80bd000 nid=0x1367 waiting on condition [0x00007fe7922e1000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000717d16358> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
"RS:0;asf907:46692-SendThread(localhost:62240)" #234 daemon prio=5 os_prio=0
tid=0x00007fe7c80bb000 nid=0x1366 waiting on condition [0x00007fe7923e2000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
"asf907:36074.activeMasterManager-EventThread" #232 daemon prio=5 os_prio=0
tid=0x00007fe7c018a800 nid=0x1356 waiting on condition [0x00007fe7925e4000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000716df6c50> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
"asf907:36074.activeMasterManager-SendThread(localhost:62240)" #231 daemon
prio=5 os_prio=0 tid=0x00007fe7c0155800 nid=0x1355 waiting on condition
[0x00007fe7926e5000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
"asf907:36074.activeMasterManager-EventThread" #230 daemon prio=5 os_prio=0
tid=0x00007fe7c0153800 nid=0x1353 waiting on condition [0x00007fe7928e7000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000716df3d40> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
"asf907:36074.activeMasterManager-SendThread(localhost:62240)" #229 daemon
prio=5 os_prio=0 tid=0x00007fe7c0151800 nid=0x1352 waiting on condition
[0x00007fe7929e8000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
"snapshot-hfile-cleaner-cache-refresher" #228 daemon prio=5 os_prio=0
tid=0x00007fe7c014e000 nid=0x1346 in Object.wait() [0x00007fe792ae9000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x0000000717bc0b40> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"asf907:36074.activeMasterManager-EventThread" #227 daemon prio=5 os_prio=0
tid=0x00007fe7c0145800 nid=0x1337 waiting on condition [0x00007fe792bea000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000717bbf760> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
"asf907:36074.activeMasterManager-SendThread(localhost:62240)" #226 daemon
prio=5 os_prio=0 tid=0x00007fe7c0144000 nid=0x1336 waiting on condition
[0x00007fe792ceb000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at
org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
"org.apache.hadoop.hdfs.PeerCache@7e6cd138" #176 daemon prio=5 os_prio=0
tid=0x00007fe7c001d000 nid=0x11dc waiting on condition [0x00007fe80d0d1000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:244)
at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:41)
at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:119)
at java.lang.Thread.run(Thread.java:745)
"Service Thread" #17 daemon prio=9 os_prio=0 tid=0x00007fe9ac2cd000 nid=0x667
runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread11" #16 daemon prio=9 os_prio=0 tid=0x00007fe9ac2c8000
nid=0x660 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread10" #15 daemon prio=9 os_prio=0 tid=0x00007fe9ac2c5800
nid=0x65f waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread9" #14 daemon prio=9 os_prio=0 tid=0x00007fe9ac2c3800
nid=0x65e waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread8" #13 daemon prio=9 os_prio=0 tid=0x00007fe9ac2c1800
nid=0x65a waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread7" #12 daemon prio=9 os_prio=0 tid=0x00007fe9ac2bf800
nid=0x659 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread6" #11 daemon prio=9 os_prio=0 tid=0x00007fe9ac2bd800
nid=0x658 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread5" #10 daemon prio=9 os_prio=0 tid=0x00007fe9ac2bb800
nid=0x657 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread4" #9 daemon prio=9 os_prio=0 tid=0x00007fe9ac2b9800
nid=0x655 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x00007fe9ac2b7800
nid=0x652 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x00007fe9ac2b5000
nid=0x64d waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x00007fe9ac2b3800
nid=0x635 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x00007fe9ac2b1000
nid=0x630 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x00007fe9ac2af800 nid=0x62e
runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007fe9ac27c800 nid=0x61d in
Object.wait() [0x00007fe969cdb000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x0000000711042dc0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007fe9ac278000
nid=0x61b in Object.wait() [0x00007fe969ddc000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
- locked <0x00000007110a1de8> (a java.lang.ref.Reference$Lock)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)
"main" #1 prio=5 os_prio=0 tid=0x00007fe9ac009000 nid=0x5e8 in Object.wait()
[0x00007fe9b4de1000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Thread.join(Thread.java:1249)
- locked <0x0000000717dea2e0> (a
org.apache.hadoop.hbase.util.JVMClusterUtil$RegionServerThread)
at java.lang.Thread.join(Thread.java:1323)
at
org.apache.hadoop.hbase.regionserver.TestClusterId.tearDown(TestClusterId.java:65)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.createRequestAndRun(JUnitCoreWrapper.java:108)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.executeEager(JUnitCoreWrapper.java:78)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:54)
at
org.apache.maven.surefire.junitcore.JUnitCoreProvider.invoke(JUnitCoreProvider.java:144)
at
org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:203)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:155)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103)
"VM Thread" os_prio=0 tid=0x00007fe9ac270800 nid=0x619 runnable
"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x00007fe9ac01e000 nid=0x5f8
runnable
"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x00007fe9ac020000 nid=0x5f9
runnable
"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x00007fe9ac021800 nid=0x5fa
runnable
"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x00007fe9ac023800 nid=0x5fb
runnable
"GC task thread#4 (ParallelGC)" os_prio=0 tid=0x00007fe9ac025000 nid=0x5fc
runnable
"GC task thread#5 (ParallelGC)" os_prio=0 tid=0x00007fe9ac027000 nid=0x5fd
runnable
"GC task thread#6 (ParallelGC)" os_prio=0 tid=0x00007fe9ac028800 nid=0x5ff
runnable
"GC task thread#7 (ParallelGC)" os_prio=0 tid=0x00007fe9ac02a800 nid=0x600
runnable
"GC task thread#8 (ParallelGC)" os_prio=0 tid=0x00007fe9ac02c000 nid=0x601
runnable
"GC task thread#9 (ParallelGC)" os_prio=0 tid=0x00007fe9ac02e000 nid=0x602
runnable
"GC task thread#10 (ParallelGC)" os_prio=0 tid=0x00007fe9ac02f800 nid=0x603
runnable
"GC task thread#11 (ParallelGC)" os_prio=0 tid=0x00007fe9ac031800 nid=0x604
runnable
"GC task thread#12 (ParallelGC)" os_prio=0 tid=0x00007fe9ac033000 nid=0x605
runnable
"VM Periodic Task Thread" os_prio=0 tid=0x00007fe9ac2d0000 nid=0x66b waiting on
condition
JNI global references: 364
+ exit 1
POST BUILD TASK : FAILURE
END OF POST BUILD TASK : 0
Archiving artifacts
Setting JDK_1_7_LATEST__HOME=/home/jenkins/tools/java/latest1.7
Setting JDK_1_8_LATEST__HOME=/home/jenkins/tools/java/latest1.8
No prior successful build to compare, so performing full copy of artifacts
Recording test results
Setting JDK_1_7_LATEST__HOME=/home/jenkins/tools/java/latest1.7
Setting JDK_1_8_LATEST__HOME=/home/jenkins/tools/java/latest1.8
Setting JDK_1_7_LATEST__HOME=/home/jenkins/tools/java/latest1.7
Setting JDK_1_8_LATEST__HOME=/home/jenkins/tools/java/latest1.8
